From c4a166c4dcc44c743c4d98272a0994d288e05b0c Mon Sep 17 00:00:00 2001 From: Ken Kroenlein Date: Wed, 17 Apr 2024 15:21:30 -0600 Subject: [PATCH 1/4] Migrate to github actions --- .github/actions/initialize/action.yml | 31 +++++++++++++++++ .github/pull_request_template.md | 18 ++++++++++ .github/workflows/build-docs.yml | 41 +++++++++++++++++++++++ .github/workflows/deploy.yml | 19 +++++++++++ .github/workflows/pr-checks.yml | 45 +++++++++++++++++++++++++ .github/workflows/pr-tests.yml | 48 +++++++++++++++++++++++++++ gemd/__version__.py | 2 +- scripts/build_docs.sh | 1 - scripts/validate_version_bump.py | 24 ++++++-------- setup.py | 27 ++++++++------- 10 files changed, 228 insertions(+), 28 deletions(-) create mode 100644 .github/actions/initialize/action.yml create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/build-docs.yml create mode 100644 .github/workflows/deploy.yml create mode 100644 .github/workflows/pr-checks.yml create mode 100644 .github/workflows/pr-tests.yml diff --git a/.github/actions/initialize/action.yml b/.github/actions/initialize/action.yml new file mode 100644 index 00000000..716211c7 --- /dev/null +++ b/.github/actions/initialize/action.yml @@ -0,0 +1,31 @@ +name: 'Initialize' +description: 'Checkout repo and install dependencies' +inputs: + latest: + description: 'If true, ignore requirements.txt and the versions pinned there.' + default: 'false' + documentation: + description: 'If true, install documentation build frameworks.' + default: 'false' +runs: + using: "composite" + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Upgrade pip + run: python -m pip install --upgrade pip + shell: bash + - name: Install minimum-version runtime dependencies + GEMD + run: python -m pip install --only-binary ':all:' -r requirements.txt + shell: bash + if: ${{ inputs.latest == 'false' }} + - name: Install test dependencies + run: python -m pip install --only-binary ':all:' -r test_requirements.txt + shell: bash + - name: Install documentation building framework + run: python -m pip install --only-binary ':all:' -r doc_requirements.txt + shell: bash + if: ${{ inputs.documentation == 'true' }} + - name: Install gemd-python, along with the latest version of any outstanding dependencies + run: python -m pip install --only-binary ':all:' -e . + shell: bash diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..9d27ff64 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,18 @@ +# GEMD Python PR + +## Description +Please briefly explain the goal of the changes/this PR. +The reviewer should be able to understand why the change is being made by reading this description +and its links (e.g. JIRA tickets). + +### PR Type: +- [ ] Breaking change (fix or feature that would cause existing functionality to change) +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Maintenance (non-breaking change to assist developers) + +### Adherence to team decisions +- [ ] I have added tests for 100% coverage +- [ ] I have written Numpy-style docstrings for every method and class. +- [ ] I have communicated the downstream consequences of the PR to others. +- [ ] I have bumped the version in __version\__.py diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml new file mode 100644 index 00000000..e09a33a3 --- /dev/null +++ b/.github/workflows/build-docs.yml @@ -0,0 +1,41 @@ +name: Build and Deploy Docs + +on: + release: + types: [published] + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Initialize the environment + uses: ./.github/actions/initialize + with: + documentation: 'true' + - name: Build Docs + run: bash scripts/build_docs.sh + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: 'docs/_build/html' + - name: Setup Pages + uses: actions/configure-pages@v4 + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..1f00dea3 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,19 @@ +name: Deploy to PyPI + +on: + release: + types: [published] + +jobs: + publish: + name: Publish package to PyPI + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Build + run: python setup.py sdist bdist_wheel + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml new file mode 100644 index 00000000..5cc27328 --- /dev/null +++ b/.github/workflows/pr-checks.yml @@ -0,0 +1,45 @@ +name: PR Checks + +on: + pull_request: + branches: + - main + - 'release/**' + +jobs: + check-version: + name: Check version bumped + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Initialize the environment + uses: ./.github/actions/initialize + - name: Check version + run: python scripts/validate_version_bump.py + linting: + name: Run linting with flake8 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Initialize the environment + uses: ./.github/actions/initialize + - name: Lint the source directory + run: flake8 gemd + check-deprecated: + name: Find code marked for removal in this version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Initialize the environment + uses: ./.github/actions/initialize + - name: Deprecated check + run: derp . gemd/__version__.py + check-docs: + name: Check docs for warnings + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Initialize the environment + uses: ./.github/actions/initialize + - name: Build Docs + run: make -C docs/ html SPHINXOPTS='-W --keep-going' diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml new file mode 100644 index 00000000..0d9f29b1 --- /dev/null +++ b/.github/workflows/pr-tests.yml @@ -0,0 +1,48 @@ +name: PR Tests + +on: + pull_request: + branches: + - main + - 'release/**' + +jobs: + run-tests: + name: Execute unit tests + runs-on: ubuntu-latest + continue-on-error: true + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v4 + - name: Initialize the environment + uses: ./.github/actions/initialize + - name: Execute unit tests + run: pytest --cov=src --cov-report term-missing:skip-covered --cov-config=tox.ini --no-cov-on-fail --cov-fail-under=100 tests/ + run-tests-against-latest: + # These runs are intended to confirm the latest minor version of our dependencies we claim to + # support don't break with our latest changes. Since they're not the versions we directly state + # you should use (i.e. in requirements.txt), they argubly aren't critical, hence not blocking. + name: Non-blocking - Execute unit tests against latest version of dependencies + runs-on: ubuntu-latest + continue-on-error: true + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v4 + - name: Initialize the environment + uses: ./.github/actions/initialize + with: + latest: 'true' + - name: Execute unit tests + run: pytest tests/ diff --git a/gemd/__version__.py b/gemd/__version__.py index 58039f50..4eabd0b3 100644 --- a/gemd/__version__.py +++ b/gemd/__version__.py @@ -1 +1 @@ -__version__ = "2.1.1" +__version__ = "2.1.2" diff --git a/scripts/build_docs.sh b/scripts/build_docs.sh index 97bd975c..e42c72bc 100755 --- a/scripts/build_docs.sh +++ b/scripts/build_docs.sh @@ -1,4 +1,3 @@ -pip install -r doc_requirements.txt cd docs make html touch _build/html/.nojekyll diff --git a/scripts/validate_version_bump.py b/scripts/validate_version_bump.py index 2a3b2b50..5a3ea3b7 100755 --- a/scripts/validate_version_bump.py +++ b/scripts/validate_version_bump.py @@ -4,22 +4,20 @@ from packaging.version import Version import re import sys -from typing import TextIO def main(): - repo_dir = popen("git rev-parse --show-toplevel", mode="r").read().rstrip() - version_path = relpath(f'{repo_dir}/gemd/__version__.py', getcwd()) - try: + repo_dir = popen("git rev-parse --show-toplevel", mode="r").read().rstrip() + version_path = relpath(f'{repo_dir}/gemd/__version__.py', getcwd()) with open(version_path, "r") as fh: - new_version = extract_version(fh) + new_version = extract_version(fh.read()) except Exception as e: - raise ValueError(f"Couldn't extract version from {version_path}") from e + raise ValueError(f"Couldn't extract version from working directory") from e try: - with popen(f"git show main:gemd/__version__.py", mode="r") as fh: - old_version = extract_version(fh) + with popen("git show main:gemd/__version__.py", mode="r") as fh: + old_version = extract_version(fh.read()) except Exception as e: raise ValueError(f"Couldn't extract version from main branch") from e @@ -47,12 +45,10 @@ def main(): return 4 -def extract_version(handle: TextIO) -> Version: - text = handle.read() - if not re.search(r"\S", text): - raise ValueError(f"No content") - match = re.search(r"""^\s*_*version_*\s*=\s*(['"])(\S+)\1""", text, re.MULTILINE) - if match: +def extract_version(text: str) -> Version: + version_re = r'''^\s*__version__\s*=\s*(['"])([\w\.]+)\1$''' + + if match := re.search(version_re, text, re.MULTILINE): return Version(match.group(2)) else: raise ValueError(f"No version found\n{text}") diff --git a/setup.py b/setup.py index 8aac4cc8..6a10e4af 100644 --- a/setup.py +++ b/setup.py @@ -1,23 +1,20 @@ from setuptools import setup, find_packages -from os import path -from packaging.version import Version +from pathlib import Path import re packages = find_packages() -this_directory = path.abspath(path.dirname(__file__)) -version_file = path.join(this_directory, 'gemd', '__version__.py') -version_re = r'''^__version__\s*=\s*(['"])([\w\.]+)\1$''' -with open(version_file, 'r') as f: - mo = re.search(version_re, f.read(), re.M) - if mo: - version = Version(mo.group(2)) - else: - raise RuntimeError(f"Unable to find version string in {version_file}") +this_directory = Path(__file__).parent.absolute() +version_file = this_directory / 'gemd' / '__version__.py' +version_re = r'''^\s*__version__\s*=\s*(['"])([\w\.]+)\1$''' +if mo := re.search(version_re, version_file.read_text(), re.M): + version = mo.group(2) +else: + raise RuntimeError(f"Unable to find version string in {version_file}") setup(name='gemd', # Update this in gemd/__version__.py - version=str(version), + version=version, python_requires='>=3.8', url='http://github.com/CitrineInformatics/gemd-python', description="Python binding for Citrine's GEMD data model", @@ -42,6 +39,12 @@ "importlib-resources>=5.3,<7" ], extras_require={ + "scripts": [ + "packaging" + "sphinx==5.0.0", + "sphinx-rtd-theme==1.0.0", + "sphinxcontrib-apidoc==0.3.0", + ], "tests": [ "pytest>=8.0.0,<9" ], From 01a9e1d0f478efab5821500c97fc5d7f98d036e1 Mon Sep 17 00:00:00 2001 From: Ken Kroenlein Date: Wed, 17 Apr 2024 17:06:47 -0600 Subject: [PATCH 2/4] Fix environment errors for Actions --- .github/workflows/pr-checks.yml | 2 ++ .github/workflows/pr-tests.yml | 2 +- scripts/validate_version_bump.py | 5 +++-- tests/__init__.py | 0 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 tests/__init__.py diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 5cc27328..49b30be2 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -41,5 +41,7 @@ jobs: - uses: actions/checkout@v4 - name: Initialize the environment uses: ./.github/actions/initialize + with: + documentation: 'true' - name: Build Docs run: make -C docs/ html SPHINXOPTS='-W --keep-going' diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 0d9f29b1..3b74761b 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -23,7 +23,7 @@ jobs: - name: Initialize the environment uses: ./.github/actions/initialize - name: Execute unit tests - run: pytest --cov=src --cov-report term-missing:skip-covered --cov-config=tox.ini --no-cov-on-fail --cov-fail-under=100 tests/ + run: pytest --cov=gemd --cov-report term-missing:skip-covered --cov-config=tox.ini --no-cov-on-fail --cov-fail-under=100 tests/ run-tests-against-latest: # These runs are intended to confirm the latest minor version of our dependencies we claim to # support don't break with our latest changes. Since they're not the versions we directly state diff --git a/scripts/validate_version_bump.py b/scripts/validate_version_bump.py index 5a3ea3b7..d6ddc957 100755 --- a/scripts/validate_version_bump.py +++ b/scripts/validate_version_bump.py @@ -1,5 +1,5 @@ #!python -from os import getcwd, popen +from os import getcwd, popen, system from os.path import relpath from packaging.version import Version import re @@ -16,7 +16,8 @@ def main(): raise ValueError(f"Couldn't extract version from working directory") from e try: - with popen("git show main:gemd/__version__.py", mode="r") as fh: + system("git fetch origin main") + with popen("git show origin/main:gemd/__version__.py", mode="r") as fh: old_version = extract_version(fh.read()) except Exception as e: raise ValueError(f"Couldn't extract version from main branch") from e diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b From ec7e66a18218cd79a068c732837f60f9f63a5916 Mon Sep 17 00:00:00 2001 From: Ken Kroenlein Date: Wed, 17 Apr 2024 18:33:13 -0600 Subject: [PATCH 3/4] Make doc build warnings non-blocking --- .github/workflows/pr-checks.yml | 1 + docs/source/conf.py | 6 +++--- docs/source/index.rst | 8 ++------ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 49b30be2..8df4054f 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -37,6 +37,7 @@ jobs: check-docs: name: Check docs for warnings runs-on: ubuntu-latest + continue-on-error: true steps: - uses: actions/checkout@v4 - name: Initialize the environment diff --git a/docs/source/conf.py b/docs/source/conf.py index 94b60730..eeb44e80 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,7 +13,7 @@ import gemd import os import sys -sys.path.insert(0, os.path.abspath('../..')) +sys.path.insert(0, os.path.abspath('../../gemd')) # -- Project information ----------------------------------------------------- @@ -36,7 +36,8 @@ extensions = [ 'sphinxcontrib.apidoc', 'sphinx.ext.napoleon', - 'sphinx.ext.intersphinx' + 'sphinx.ext.intersphinx', + 'sphinx_rtd_theme' ] # Use the sphinxcontrib.apidoc extension to wire in the sphinx-apidoc invocation @@ -48,7 +49,6 @@ apidoc_output_dir = 'reference' apidoc_excluded_paths = ['tests', '*impl*'] apidoc_separate_modules = True -apidoc_toc_file = False # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/source/index.rst b/docs/source/index.rst index a49d07df..23b4f7c8 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -26,7 +26,7 @@ or a specific version can be installed, for example: .. code:: bash - pip install gemd==1.17.1 + pip install gemd==2.1.1 Table of Contents ----------------- @@ -37,6 +37,7 @@ Table of Contents depth/unit_parsing depth/serialization + API Reference Indices ------------------ @@ -44,8 +45,3 @@ Indices * :ref:`genindex` * :ref:`modindex` * :ref:`search` - -.. toctree:: - :maxdepth: 1 - - reference/gemd From 93c49fbb767975c6f57647dbd1fe1b5781689c67 Mon Sep 17 00:00:00 2001 From: Ken Kroenlein Date: Fri, 19 Apr 2024 10:13:21 -0600 Subject: [PATCH 4/4] Correct document building issues --- .github/workflows/build-docs.yml | 4 +++- .github/workflows/pr-checks.yml | 2 +- .github/workflows/pr-tests.yml | 4 ++-- CONTRIBUTING.md | 41 +++++++++++++++++++++++++++----- README.md | 2 +- docs/source/conf.py | 3 +++ scripts/build_docs.sh | 1 + 7 files changed, 46 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index e09a33a3..ea747731 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -29,7 +29,9 @@ jobs: with: documentation: 'true' - name: Build Docs - run: bash scripts/build_docs.sh + run: | + make -C docs/ html + touch docs/_build/html/.nojekyll - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 8df4054f..5c595329 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -37,7 +37,6 @@ jobs: check-docs: name: Check docs for warnings runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@v4 - name: Initialize the environment @@ -45,4 +44,5 @@ jobs: with: documentation: 'true' - name: Build Docs + continue-on-error: true run: make -C docs/ html SPHINXOPTS='-W --keep-going' diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 3b74761b..0f190e4c 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -10,8 +10,8 @@ jobs: run-tests: name: Execute unit tests runs-on: ubuntu-latest - continue-on-error: true strategy: + fail-fast: false matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: @@ -27,7 +27,7 @@ jobs: run-tests-against-latest: # These runs are intended to confirm the latest minor version of our dependencies we claim to # support don't break with our latest changes. Since they're not the versions we directly state - # you should use (i.e. in requirements.txt), they argubly aren't critical, hence not blocking. + # you should use (i.e. in requirements.txt), they arguably aren't critical, hence not blocking. name: Non-blocking - Execute unit tests against latest version of dependencies runs-on: ubuntu-latest continue-on-error: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5fde1e00..0d33f17f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,17 +1,43 @@ # Contributing +## Documentation + +Documentation is built using [Sphinx](https://www.sphinx-doc.org/en/master/) with the +[autodoc](https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#module-sphinx.ext.autodoc) +extension. In order to build the documentation, you can install the necessary packages and then use make: +```shell +pip install -r doc_requirements.txt +make make -C docs/ html +``` + +All documentation source files and configuration are in the `docs/source` directory. +Documentation for all modules is autogenerated and populated in the `reference` subdirectory. +Additional reference material, such as tutorials, should be referenced in `index.rst`. + ## Testing Changes are gated on: - * Passing unit tests - * 100% unit test coverage + * Passing unit tests with minimum supported library versions + * 100% test coverage * PEP8 style compliance, with some exceptions in the [tox file](tox.ini) * Incrementing the package version number in [setup.py](setup.py) -Travis runs the tests in `scripts/run_tests.sh`, which gives a convenient one-line invocation for testing. +Additionally, PRs are checked against (but not blocked by): +* Passing unit tests with latest nominally supported versions of all dependencies +* Documentation builds without warnings + +In order to run tests locally, you'll need to install additional packages: +```shell +pip install -r test_requirements.txt +``` + +A test runner is available in `scripts/run_tests.sh`, which gives a convenient one-line invocation for testing. -As it can be easy to forget to verify these prior to pushing, it's possible to use [git hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) to enforce compliance during normal workflows. -Consider editing `.git/hooks/pre-commit` or `.git/hooks/pre-push` (or adding them and marking them as executable: `chmod +x `). +As it can be easy to forget to verify these prior to pushing, +it's possible to use [git hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) +to enforce compliance during normal workflows. +Consider editing `.git/hooks/pre-commit` or `.git/hooks/pre-push` +(or adding them and marking them as executable: `chmod +x `). For example, you could set your local `.git/hooks/pre-commit` to be ```shell scripts/run_tests.sh --quiet --exitfirst @@ -24,7 +50,10 @@ This project follows [PEP8](https://www.python.org/dev/peps/pep-0008/), with the Additionally: * Type hints are strongly encouraged, but not required. -* Positional arguments are strongly discouraged for methods with multiple arguments. Keyword-only arguments are preferred instead. Every positional argument should be required. +* Positional arguments are strongly discouraged for methods with multiple arguments, + especially for multiple arguments of the same type. + Keyword-only arguments are preferred instead. + Every positional argument should be required. For additional (non-binding) inspiration, check out the [Google Python Style Guide](https://github.com/google/styleguide/blob/gh-pages/pyguide.md). diff --git a/README.md b/README.md index fb0745a0..7e3e332f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # GEMD-python -Python binding for Citrine's nextgen data model, GEMD. +Python binding for Citrine's data model, GEMD. This package provides a framework for storing information about the processes that create materials, the materials themselves, and measurements performed on those materials. diff --git a/docs/source/conf.py b/docs/source/conf.py index eeb44e80..1a33d123 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -79,6 +79,9 @@ autodoc_member_order = 'groupwise' # autodoc_mock_imports allows Sphinx to ignore any external modules listed in the array autodoc_mock_imports = [] +autodoc_default_options = { + 'ignore-module-all': True +} html_favicon = '_static/favicon.png' html_logo = '_static/logo.png' diff --git a/scripts/build_docs.sh b/scripts/build_docs.sh index e42c72bc..97bd975c 100755 --- a/scripts/build_docs.sh +++ b/scripts/build_docs.sh @@ -1,3 +1,4 @@ +pip install -r doc_requirements.txt cd docs make html touch _build/html/.nojekyll