Clean up delphi_utils (take 2) (#2014)

* add pyproject.toml

* made changes based on suggestion

* update ci config

* separate new line

Co-authored-by: Dmitry Shemetov <[email protected]>

* fix: add explicit boto and moto dependencies to indicators

* trying to get package data to show

* installing dev package before testing in ci

* adding dependency

* move comment to right package

* Update _delphi_utils_python/pyproject.toml

Co-authored-by: Dmitry Shemetov <[email protected]>

* more suggestion

* adding pyproject.toml to template

* Update changehc/Makefile

* Update claims_hosp/Makefile

* Update doctor_visits/Makefile

* Update google_symptoms/Makefile

* Update hhs_hosp/Makefile

* Update nchs_mortality/Makefile

* Update nssp/Makefile

* Update nwss_wastewater/Makefile

* Update quidel_covidtest/Makefile

* Update sir_complainsalot/Makefile

* fix: dependence bugs and cleanup
* pandas[excel]<2 doesn't work, go back to xlrd and comment
* add mock to changehc
* add requests
* remove setup.py from template

* build: fix Jenkins build script

---------

Co-authored-by: Amaris Sim <[email protected]>
Co-authored-by: aysim319 <[email protected]>
Co-authored-by: Dmitry Shemetov <[email protected]>

.github/workflows/python-ci.yml

@@ -51,7 +51,7 @@ jobs:
         with:
           python-version: 3.8
           cache: "pip"
-          cache-dependency-path: "setup.py"
+          cache-dependency-path: "pyproject.toml"
       - name: Install testing dependencies
         run: |
           python -m pip install --upgrade pip

_delphi_utils_python/.bumpversion.cfg

@@ -4,6 +4,6 @@ commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False
 
-[bumpversion:file:setup.py]
+[bumpversion:file:pyproject.toml]
 
 [bumpversion:file:delphi_utils/__init__.py]

_delphi_utils_python/DEVELOP.md

@@ -9,7 +9,7 @@ To install the module in your default version of Python, run the
 following from this directory:
 
 ```
-pip install .
+pip install -e '.[dev]'
 ```
 
 As described in each of the indicator code directories, you will want to install

_delphi_utils_python/Makefile

@@ -6,12 +6,12 @@ venv:
 install: venv
 	. env/bin/activate; \
 	pip install wheel ; \
-	pip install -e .
+	pip install -e '.[dev]'
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
-	pip install .
+	pip install 'build[virtualenv]' pylint pytest pydocstyle wheel twine ; \
+	pip install '.[dev]'
 
 lint:
 	. env/bin/activate; pylint delphi_utils --rcfile=../pyproject.toml
@@ -30,4 +30,5 @@ clean:
 
 release: lint test
 	. env/bin/activate ; \
-	python setup.py sdist bdist_wheel
+	pip install 'build[virtualenv]' ; \
+	python -m build --sdist --wheel

_delphi_utils_python/pyproject.toml

@@ -0,0 +1,57 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "delphi-utils"
+version = "0.3.24"
+description = "Shared Utility Functions for Indicators"
+readme = "README.md"
+requires-python = "== 3.8.*"
+license = { text = "MIT License" }
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.8",
+    "License :: MIT",
+]
+dependencies = [
+    "boto3",
+    "covidcast",
+    "cvxpy",
+    "epiweeks",
+    "gitpython",
+    "importlib_resources>=1.3",
+    "numpy",
+    "pandas>=1.1.0",
+    "requests",
+    "slackclient",
+    "scs<3.2.6",                # TODO: remove this ; it is a cvxpy dependency, and the excluded version appears to break our jenkins build. see: https://github.com/cvxgrp/scs/issues/283
+    "structlog",
+    "xlrd",                     # needed by Pandas to read Excel files
+]
+
+[project.urls]
+Homepage = "https://github.com/cmu-delphi/covidcast-indicators"
+
+[project.optional-dependencies]
+dev = [
+    "darker[isort]~=2.1.1",
+    "pylint==2.8.3",
+    "pytest",
+    "pydocstyle",
+    "pytest-cov",
+    "mock",
+    "moto~=4.2.14",
+    "requests-mock",
+    "freezegun",
+]
+flash = ["scipy"]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["delphi_utils"]
+namespaces = true
+
+[tool.setuptools.package-data]
+"delphi_utils.data" = ["20*/*.csv"]

_template_python/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "delphi_NAME"
+version = "0.1.0"
+description = "SHORT DESCRIPTION"
+readme = "README.md"
+requires-python = "== 3.8.*"
+license = { text = "MIT License" }
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.8",
+    "License :: MIT",
+]
+dependencies = [
+    "delphi-utils",
+    "numpy",
+    "pandas>=1.1.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/cmu-delphi/covidcast-indicators"
+
+[project.optional-dependencies]
+dev = [
+    "darker[isort]~=2.1.1",
+    "pylint==2.8.3",
+    "pytest",
+    "pydocstyle",
+    "pytest-cov",
+    "mock",
+    "moto~=4.2.14",
+    "requests-mock",
+    "freezegun",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+namespaces = true

changehc/setup.py

@@ -6,6 +6,7 @@
     "covidcast",
     "darker[isort]~=2.1.1",
     "delphi-utils",
+    "mock",
     "moto~=4.2.14",
     "numpy",
     "pandas",

jenkins/build-indicator.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Jenkins build 
+# Jenkins build
 #
 
 set -eo pipefail
@@ -17,4 +17,4 @@ source env/bin/activate
 pip install pip==23.0.1 --retries 10 --timeout 20
 pip install numpy --retries 10 --timeout 20
 pip install ../_delphi_utils_python/. --retries 10 --timeout 20
-[ ! -f setup.py ] || pip install . --retries 10 --timeout 20
+[ ! -f pyproject.toml ] || pip install . --retries 10 --timeout 20

nchs_mortality/setup.py

@@ -2,10 +2,12 @@
 from setuptools import find_packages
 
 required = [
+    "boto3",
     "darker[isort]~=2.1.1",
     "delphi-utils",
     "epiweeks",
     "freezegun",
+    "moto~=4.2.14",
     "numpy",
     "pandas",
     "pydocstyle",

quidel_covidtest/setup.py

@@ -2,6 +2,7 @@
 from setuptools import find_packages
 
 required = [
+    "boto3",
     "covidcast",
     "darker[isort]~=2.1.1",
     "delphi-utils",
@@ -14,7 +15,7 @@
     "pylint==2.8.3",
     "pytest-cov",
     "pytest",
-    "xlrd==1.2.0",
+    "xlrd==1.2.0", # needed by Pandas to read Excel files
 ]
 
 setup(