From 4667f94ccaedfb37ae1f9f82f89ffcd0b9336cfe Mon Sep 17 00:00:00 2001 From: Lea Vauchier Date: Wed, 14 Feb 2024 10:59:07 +0100 Subject: [PATCH 1/2] Update pip package config --- .flake8 | 2 +- .github/workflows/cicd.yaml | 23 ++++++++++---------- docs/source/conf.py | 15 ++++++------- docs/source/guides/development.md | 2 +- myria3d/_version.py | 5 +++++ package_metadata.yaml | 6 ------ pyproject.toml | 35 ++++++++++++++++++++++++++++++- setup.py | 17 --------------- 8 files changed, 60 insertions(+), 45 deletions(-) create mode 100644 myria3d/_version.py delete mode 100644 package_metadata.yaml delete mode 100644 setup.py diff --git a/.flake8 b/.flake8 index 1393bbfe..c5e5e004 100644 --- a/.flake8 +++ b/.flake8 @@ -7,4 +7,4 @@ exclude = .git __pycache__ logs/* - .vscode/* \ No newline at end of file + .vscode/* diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 207f2756..f577603b 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -83,18 +83,17 @@ jobs: docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune # docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar - # get version number and date, to tag the image pushed to a private docker registry - - name: get version number - id: tag - run: | - echo "VERSION=$(docker run myria3d grep '__version__' package_metadata.yaml| cut -d\" -f2)" >> $GITHUB_ENV - echo "DATE=$(date '+%Y.%m.%d')" >> $GITHUB_ENV - - # show possible tags, for debugging purpose - - name: Print tags - run: | - echo "${{ env.VERSION }}" - echo "${{ env.DATE }}" + # get version number and date, to tag the image pushed to a private docker registry + - name: get version number + id: tag + run: | + echo "VERSION=$(docker run lidar_prod python -m myria3d._version)" >> $GITHUB_ENV + echo "DATE=$(date '+%Y.%m.%d')" >> $GITHUB_ENV + # show possible tags, for debugging purpose + - name: Print tags + run: | + echo "${{ env.VERSION }}" + echo "${{ env.DATE }}" - name: push main docker on nexus (tagged with a date) # we push on nexus an image from the main branch when it has been updated (push or accepted pull request) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1680326d..72ea44a3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,7 +13,7 @@ import os import sys -import yaml +import tomli from hydra import compose, initialize from omegaconf import OmegaConf @@ -21,15 +21,16 @@ abs_root_path = os.path.abspath(rel_root_path) sys.path.insert(0, abs_root_path) +from myria3d._version import __version__ # noqa: E402 # -- Project information ----------------------------------------------------- -with open(os.path.join(abs_root_path, "package_metadata.yaml"), "r") as f: - pm = yaml.safe_load(f) +with open(os.path.join(abs_root_path, "pyproject.toml"), "rb") as f: + data = tomli.load(f) -release = pm["__version__"] -project = pm["__name__"] -author = pm["__author__"] -copyright = pm["__copyright__"] +release = __version__ +project = data["project"]["name"] +author = ", ".join([a["name"] for a in data["project"]["authors"]]) +copyright = data["metadata"]["copyright"] # -- YAML main to print the config into --------------------------------------------------- # We need to concatenate configs into a single file using hydra diff --git a/docs/source/guides/development.md b/docs/source/guides/development.md index 022d730d..8655b3b1 100644 --- a/docs/source/guides/development.md +++ b/docs/source/guides/development.md @@ -2,7 +2,7 @@ ## Code versionning -Package version follows semantic versionning conventions and is defined in `package_metadata.yaml`. +Package version follows semantic versionning conventions and is defined in `myria3d._version.py`. Releases are created when new high-level functionnality are implemented (e.g. a new step in the production process), with a documentation role. A `prod-release-tag` is created that tracks an _arbitrary_ commit, and serves as a mean to make a few models, model card, and config accessible via its associated [release](https://github.com/IGNF/myria3d/releases/tag/prod-release-tag). diff --git a/myria3d/_version.py b/myria3d/_version.py new file mode 100644 index 00000000..678499e9 --- /dev/null +++ b/myria3d/_version.py @@ -0,0 +1,5 @@ +__version__ = "3.8.3" + + +if __name__ == "__main__": + print(__version__) diff --git a/package_metadata.yaml b/package_metadata.yaml deleted file mode 100644 index 5aabdc13..00000000 --- a/package_metadata.yaml +++ /dev/null @@ -1,6 +0,0 @@ -__version__: "3.8.3" -__name__: "myria3d" -__url__: "https://github.com/IGNF/myria3d" -__description__: "Deep Learning for the Semantic Segmentation of Aerial Lidar Point Clouds" -__author__: "Charles GAYDON" -__copyright__: "2022, Institut National de l'Information Géographique et Forestière" diff --git a/pyproject.toml b/pyproject.toml index 2e9d9db9..a6a6c4ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,35 @@ +[project] +name = "myria3d" +dynamic = ["version"] +description = "Deep Learning for the Semantic Segmentation of Aerial Lidar Point Clouds" +readme = "README.md" +authors = [ + { name = "Charles GAYDON", email = "charles.gaydon@gmail.com" } +] +maintainers = [ + {name = "Charles GAYDON", email = "charles.gaydon@gmail.com"}, + {name = "Michel DAAB"}, + {name = "Léa VAUCHIER", email = "lea.vauchier@ign.fr"} +] +license = {file = "LICENSE"} +dependencies = [] # assume an environment as described in environment.yml + +[metadata] +project_template = "https://github.com/ashleve/lightning-hydra-template" +copyright = "2022, Institut National de l'Information Géographique et Forestière" + +[project.urls] +Documentation = "https://ignf.github.io/myria3d/" +Repository = "https://github.com/IGNF/myria3d" +Issues = "https://github.com/IGNF/myria3d/issues" +Changelog = "https://github.com/IGNF/myria3d/blob/main/CHANGELOG.md" + +[tool.setuptools.dynamic] +version = {attr = "myria3d._version.__version__"} + +[tool.setuptools] +packages = [ "myria3d" ] + [tool.black] line-length = 99 target-version = ['py39'] @@ -28,8 +60,9 @@ filterwarnings = [ "ignore::DeprecationWarning", "ignore::UserWarning" ] + markers = [ - "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "slow: marks tests as slow (deselect with '-m \"not slow\"')" ] [tool.coverage.run] diff --git a/setup.py b/setup.py deleted file mode 100644 index a44d6da3..00000000 --- a/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -import yaml -from setuptools import find_packages, setup - -with open("package_metadata.yaml", "r") as f: - pm = yaml.safe_load(f) - -setup( - name=pm["__name__"], - version=pm["__version__"], - url=pm["__url__"], - description=pm["__description__"], - author=pm["__author__"], - install_requires=[ - # assume an environment as described in ./setup_env/requirements.yml - ], - packages=find_packages(), -) From c5b7ddfab13c381fcb2790dff1d88651ec73d1b1 Mon Sep 17 00:00:00 2001 From: Lea Vauchier Date: Thu, 18 Apr 2024 15:42:29 +0200 Subject: [PATCH 2/2] Enable publishing to pypi --- .github/workflows/cicd.yaml | 212 +++++++++++++--------- docs/source/tutorials/install_on_linux.md | 18 +- environment.yml | 19 +- 3 files changed, 151 insertions(+), 98 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index f577603b..0405f050 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -15,102 +15,146 @@ on: jobs: CICD: runs-on: self-hosted + env: + http_proxy: ${{ secrets.PROXY_URL }} + https_proxy: ${{ secrets.PROXY_URL }} steps: + - name: Checkout branch + uses: actions/checkout@v4 - - name: Checkout branch - uses: actions/checkout@v4 - - - name: Build docker image - run: docker build --build-arg http_proxy=${{ secrets.PROXY_URL }} --build-arg https_proxy=${{ secrets.PROXY_URL }} -t myria3d . - - - name: Run pytest - run: > - docker run - --ipc=host - myria3d - python -m - pytest -rA -v - --ignore=actions-runner - - # IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable - # sufficient shared memory when predicting on large files. - - name: Example inference run via Docker with default config and checkpoint - run: > - docker run - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ - --ipc=host - --shm-size=2gb - myria3d - python run.py - predict.src_las=/inputs/792000_6272000_subset_buildings.las - datamodule.epsg=2154 - predict.output_dir=/outputs/ - task.task_name=predict - - # predict.subtile_overlap specifies overlap between adjacent samples (in meters). - - name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results. - run: > - docker run - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ - --ipc=host - --shm-size=2gb - myria3d - python run.py - --config-path /inputs/ - --config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0 - predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt - datamodule.epsg=2154 - predict.src_las=/inputs/792000_6272000_subset_buildings.las - predict.output_dir=/outputs/ - predict.subtile_overlap=25 - datamodule.batch_size=10 - predict.interpolator.probas_to_save=[building,ground] - task.task_name=predict - - - name: Check code neatness (linter) - run: docker run myria3d python -m flake8 - - # Everything ran so we tag the valid docker image to keep it - # This happens for push events, which are in particular - # triggered when a pull request is merged. - - name: Tag the docker image with branch name - if: github.event_name == 'push' - run: | - docker tag myria3d:latest myria3d:${{github.ref_name}} - docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune - # docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar + - name: Build docker image + run: docker build --build-arg http_proxy=${{ secrets.PROXY_URL }} --build-arg https_proxy=${{ secrets.PROXY_URL }} -t myria3d . + + - name: Run pytest + run: > + docker run + --ipc=host + myria3d + python -m + pytest -rA -v + --ignore=actions-runner + + # IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable + # sufficient shared memory when predicting on large files. + - name: Example inference run via Docker with default config and checkpoint + run: > + docker run + -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ + -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ + --ipc=host + --shm-size=2gb + myria3d + python run.py + predict.src_las=/inputs/792000_6272000_subset_buildings.las + datamodule.epsg=2154 + predict.output_dir=/outputs/ + task.task_name=predict + + # predict.subtile_overlap specifies overlap between adjacent samples (in meters). + - name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results. + run: > + docker run + -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ + -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ + --ipc=host + --shm-size=2gb + myria3d + python run.py + --config-path /inputs/ + --config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0 + predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt + datamodule.epsg=2154 + predict.src_las=/inputs/792000_6272000_subset_buildings.las + predict.output_dir=/outputs/ + predict.subtile_overlap=25 + datamodule.batch_size=10 + predict.interpolator.probas_to_save=[building,ground] + task.task_name=predict + + - name: Check code neatness (linter) + run: docker run myria3d python -m flake8 + + # Everything ran so we tag the valid docker image to keep it + # This happens for push events, which are in particular + # triggered when a pull request is merged. + - name: Tag the docker image with branch name + if: github.event_name == 'push' + run: | + docker tag myria3d:latest myria3d:${{github.ref_name}} + docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune + # docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar # get version number and date, to tag the image pushed to a private docker registry - name: get version number id: tag run: | - echo "VERSION=$(docker run lidar_prod python -m myria3d._version)" >> $GITHUB_ENV + echo "VERSION=$(docker run myria3d python -m myria3d._version)" >> $GITHUB_ENV echo "DATE=$(date '+%Y.%m.%d')" >> $GITHUB_ENV + # show possible tags, for debugging purpose - name: Print tags run: | echo "${{ env.VERSION }}" echo "${{ env.DATE }}" - - name: push main docker on nexus (tagged with a date) - # we push on nexus an image from the main branch when it has been updated (push or accepted pull request) - if: ((github.ref_name == 'main') && (github.event_name == 'push')) - run: | - docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} - docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} - docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} - - - name: push branch docker on nexus (tagged with the branch name) - # we push on nexus an image from a branch when it's pushed - if: ((github.event_name == 'push') && (github.ref_name != 'main')) - run: | - docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} - docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} - docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} - - - name: Clean dangling docker images - if: always() # always do it, even if something failed - run: docker system prune --force # remove dangling docker images, without asking user for confirmation + - name: push main docker on nexus (tagged with a date) + # we push on nexus an image from the main branch when it has been updated (push or accepted pull request) + if: ((github.ref_name == 'main') && (github.event_name == 'push')) + run: | + docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} + docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} + docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} + + - name: push branch docker on nexus (tagged with the branch name) + # we push on nexus an image from a branch when it's pushed + if: ((github.event_name == 'push') && (github.ref_name != 'main')) + run: | + docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} + docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} + docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} + + - name: Clean dangling docker images + if: always() # always do it, even if something failed + run: docker system prune --force # remove dangling docker images, without asking user for confirmation + + publish-pypi: + runs-on: ubuntu-latest + if: ((github.ref_name == 'main') && (github.event_name == 'push')) + environment: + name: pypi + url: https://pypi.org/p/myria3d/ + permissions: + contents: read + packages: write + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - name: Checkout branch + uses: actions/checkout@v4 + + # See https://github.com/marketplace/actions/setup-micromamba + - name: setup-micromamba + uses: mamba-org/setup-micromamba@v1.4.3 + with: + environment-file: environment.yml + environment-name: myria3d # activate the environment + cache-environment: true + cache-downloads: true + generate-run-shell: true + + - name: Run tests with pytest + shell: micromamba-shell {0} + run: python -m pytest ./tests -rA -v + + - name: Clean up before building package + run: rm -rf tmp myria3d.egg-info dist + + - name: Build pip package + shell: micromamba-shell {0} + run: python -m build + + - name: pypi-publish + uses: pypa/gh-action-pypi-publish@v1.8.14 + with: + skip-existing: true diff --git a/docs/source/tutorials/install_on_linux.md b/docs/source/tutorials/install_on_linux.md index 2b9a66f9..50533e44 100644 --- a/docs/source/tutorials/install_on_linux.md +++ b/docs/source/tutorials/install_on_linux.md @@ -4,7 +4,7 @@ ### Prerequisites -We use [anaconda](https://www.anaconda.com/products/individual) to manage virtual environments. +We use [anaconda](https://www.anaconda.com/products/individual) to manage virtual environments. This makes installing pytorch-related libraries way easier than using pure pip installs. We enable CUDA-acceleration in pytorch as part of the defaut virtual environment recipe (see below). @@ -43,9 +43,17 @@ Start by activating the virtual environment with ```bash conda activate myria3d ``` -Then install from a specific branch from github directly. Argument `branch_name` might be `main`, `dev`, or a specific release. + +Then install the latest version from pypi. +**Warning:** activating the environment is required as the public pip package does not +handle its dependencies! ``` -pip install --upgrade https://github.com/IGNF/myria3d/tarball/{branch_name} +pip install myria3d +``` + +Or install from a specific branch from github directly. Argument `branch_name` might be `main`, `dev`, or a specific release. +``` +pip install --upgrade https://github.com/IGNF/myria3d/tarball/{branch_name} ``` Alternatively, you can install from sources directly in editable mode with @@ -53,10 +61,10 @@ Alternatively, you can install from sources directly in editable mode with pip install --editable . ``` - + ## Troubleshooting - *OSError(libcusparse.so.11 cannot open shared object file no such file or directory)* ([**](https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625)) - open the .bashrc file from your Ubuntu home directory and at the end of the file, add the following line (replace anaconda3 with miniconda3 if needed) - export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/myria3d/lib:$LD_LIBRARY_PATH" + export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/myria3d/lib:$LD_LIBRARY_PATH" diff --git a/environment.yml b/environment.yml index 17f3bf24..47f589c4 100644 --- a/environment.yml +++ b/environment.yml @@ -38,7 +38,7 @@ dependencies: - matplotlib # --------- loggers --------- # - comet_ml::comet_ml=3.35 - - conda-forge::urllib3<2 # To solve for https://github.com/GeneralMills/pytrends/issues/591 + - conda-forge::urllib3<2 # To solve for https://github.com/GeneralMills/pytrends/issues/591 # --------- Visualization --------- # - pandas - matplotlib @@ -57,17 +57,18 @@ dependencies: - rich # beautiful text formatting in terminal - sh # for running bash commands in some tests - pudb # debugger + - twine # to publish pip package # # --------- Documentation --------- # - sphinx==4.5.* - recommonmark==0.7.* - sphinx_rtd_theme==1.0.* - docutils==0.17 - - rstcheck==3.3.* # RST Linter + - rstcheck==3.3.* # RST Linter - pip: - - hydra-core==1.1.* - - hydra-colorlog==1.1.* - # --------- Documentation --------- # - - myst_parser==0.17.* - - sphinxnotes-mock==1.0.0b0 # still a beta - - sphinx_paramlinks==0.5.* - - ign-pdal-tools>=1.5.2 \ No newline at end of file + - hydra-core==1.1.* + - hydra-colorlog==1.1.* + # --------- Documentation --------- # + - myst_parser==0.17.* + - sphinxnotes-mock==1.0.0b0 # still a beta + - sphinx_paramlinks==0.5.* + - ign-pdal-tools>=1.5.2