From c5b7ddfab13c381fcb2790dff1d88651ec73d1b1 Mon Sep 17 00:00:00 2001 From: Lea Vauchier Date: Thu, 18 Apr 2024 15:42:29 +0200 Subject: [PATCH] Enable publishing to pypi --- .github/workflows/cicd.yaml | 212 +++++++++++++--------- docs/source/tutorials/install_on_linux.md | 18 +- environment.yml | 19 +- 3 files changed, 151 insertions(+), 98 deletions(-) diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index f577603b..0405f050 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -15,102 +15,146 @@ on: jobs: CICD: runs-on: self-hosted + env: + http_proxy: ${{ secrets.PROXY_URL }} + https_proxy: ${{ secrets.PROXY_URL }} steps: + - name: Checkout branch + uses: actions/checkout@v4 - - name: Checkout branch - uses: actions/checkout@v4 - - - name: Build docker image - run: docker build --build-arg http_proxy=${{ secrets.PROXY_URL }} --build-arg https_proxy=${{ secrets.PROXY_URL }} -t myria3d . - - - name: Run pytest - run: > - docker run - --ipc=host - myria3d - python -m - pytest -rA -v - --ignore=actions-runner - - # IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable - # sufficient shared memory when predicting on large files. - - name: Example inference run via Docker with default config and checkpoint - run: > - docker run - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ - --ipc=host - --shm-size=2gb - myria3d - python run.py - predict.src_las=/inputs/792000_6272000_subset_buildings.las - datamodule.epsg=2154 - predict.output_dir=/outputs/ - task.task_name=predict - - # predict.subtile_overlap specifies overlap between adjacent samples (in meters). - - name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results. - run: > - docker run - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ - -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ - --ipc=host - --shm-size=2gb - myria3d - python run.py - --config-path /inputs/ - --config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0 - predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt - datamodule.epsg=2154 - predict.src_las=/inputs/792000_6272000_subset_buildings.las - predict.output_dir=/outputs/ - predict.subtile_overlap=25 - datamodule.batch_size=10 - predict.interpolator.probas_to_save=[building,ground] - task.task_name=predict - - - name: Check code neatness (linter) - run: docker run myria3d python -m flake8 - - # Everything ran so we tag the valid docker image to keep it - # This happens for push events, which are in particular - # triggered when a pull request is merged. - - name: Tag the docker image with branch name - if: github.event_name == 'push' - run: | - docker tag myria3d:latest myria3d:${{github.ref_name}} - docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune - # docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar + - name: Build docker image + run: docker build --build-arg http_proxy=${{ secrets.PROXY_URL }} --build-arg https_proxy=${{ secrets.PROXY_URL }} -t myria3d . + + - name: Run pytest + run: > + docker run + --ipc=host + myria3d + python -m + pytest -rA -v + --ignore=actions-runner + + # IMPORTANT: Always run images with --ipc=host and --shm-size=2gb (at least) to enable + # sufficient shared memory when predicting on large files. + - name: Example inference run via Docker with default config and checkpoint + run: > + docker run + -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ + -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ + --ipc=host + --shm-size=2gb + myria3d + python run.py + predict.src_las=/inputs/792000_6272000_subset_buildings.las + datamodule.epsg=2154 + predict.output_dir=/outputs/ + task.task_name=predict + + # predict.subtile_overlap specifies overlap between adjacent samples (in meters). + - name: Example inference run via Docker with inference-time subtiles overlap to smooth-out results. + run: > + docker run + -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/inputs/:/inputs/ + -v /var/data/cicd/CICD_github_assets/myria3d_V3.7.0/outputs/:/outputs/ + --ipc=host + --shm-size=2gb + myria3d + python run.py + --config-path /inputs/ + --config-name proto151_V2.0_epoch_100_Myria3DV3.1.0_predict_config_V3.7.0 + predict.ckpt_path=/inputs/proto151_V2.0_epoch_100_Myria3DV3.1.0.ckpt + datamodule.epsg=2154 + predict.src_las=/inputs/792000_6272000_subset_buildings.las + predict.output_dir=/outputs/ + predict.subtile_overlap=25 + datamodule.batch_size=10 + predict.interpolator.probas_to_save=[building,ground] + task.task_name=predict + + - name: Check code neatness (linter) + run: docker run myria3d python -m flake8 + + # Everything ran so we tag the valid docker image to keep it + # This happens for push events, which are in particular + # triggered when a pull request is merged. + - name: Tag the docker image with branch name + if: github.event_name == 'push' + run: | + docker tag myria3d:latest myria3d:${{github.ref_name}} + docker run myria3d:${{github.ref_name}} bash # Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune + # docker save myria3d:${{github.ref_name}} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${github.ref_name}.tar # Save the docker image as myria3d_${github.ref_name}.tar # get version number and date, to tag the image pushed to a private docker registry - name: get version number id: tag run: | - echo "VERSION=$(docker run lidar_prod python -m myria3d._version)" >> $GITHUB_ENV + echo "VERSION=$(docker run myria3d python -m myria3d._version)" >> $GITHUB_ENV echo "DATE=$(date '+%Y.%m.%d')" >> $GITHUB_ENV + # show possible tags, for debugging purpose - name: Print tags run: | echo "${{ env.VERSION }}" echo "${{ env.DATE }}" - - name: push main docker on nexus (tagged with a date) - # we push on nexus an image from the main branch when it has been updated (push or accepted pull request) - if: ((github.ref_name == 'main') && (github.event_name == 'push')) - run: | - docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} - docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} - docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} - - - name: push branch docker on nexus (tagged with the branch name) - # we push on nexus an image from a branch when it's pushed - if: ((github.event_name == 'push') && (github.ref_name != 'main')) - run: | - docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} - docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} - docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} - - - name: Clean dangling docker images - if: always() # always do it, even if something failed - run: docker system prune --force # remove dangling docker images, without asking user for confirmation + - name: push main docker on nexus (tagged with a date) + # we push on nexus an image from the main branch when it has been updated (push or accepted pull request) + if: ((github.ref_name == 'main') && (github.event_name == 'push')) + run: | + docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} + docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} + docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{ env.DATE }} + + - name: push branch docker on nexus (tagged with the branch name) + # we push on nexus an image from a branch when it's pushed + if: ((github.event_name == 'push') && (github.ref_name != 'main')) + run: | + docker tag myria3d ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} + docker login ${{ secrets.DOCKER_REGISTRY }} --username svc_lidarhd --password ${{ secrets.PASSWORD_SVC_LIDARHD }} + docker push ${{ secrets.DOCKER_REGISTRY }}/lidar_hd/myria3d:${{ env.VERSION }}-${{github.ref_name}} + + - name: Clean dangling docker images + if: always() # always do it, even if something failed + run: docker system prune --force # remove dangling docker images, without asking user for confirmation + + publish-pypi: + runs-on: ubuntu-latest + if: ((github.ref_name == 'main') && (github.event_name == 'push')) + environment: + name: pypi + url: https://pypi.org/p/myria3d/ + permissions: + contents: read + packages: write + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - name: Checkout branch + uses: actions/checkout@v4 + + # See https://github.com/marketplace/actions/setup-micromamba + - name: setup-micromamba + uses: mamba-org/setup-micromamba@v1.4.3 + with: + environment-file: environment.yml + environment-name: myria3d # activate the environment + cache-environment: true + cache-downloads: true + generate-run-shell: true + + - name: Run tests with pytest + shell: micromamba-shell {0} + run: python -m pytest ./tests -rA -v + + - name: Clean up before building package + run: rm -rf tmp myria3d.egg-info dist + + - name: Build pip package + shell: micromamba-shell {0} + run: python -m build + + - name: pypi-publish + uses: pypa/gh-action-pypi-publish@v1.8.14 + with: + skip-existing: true diff --git a/docs/source/tutorials/install_on_linux.md b/docs/source/tutorials/install_on_linux.md index 2b9a66f9..50533e44 100644 --- a/docs/source/tutorials/install_on_linux.md +++ b/docs/source/tutorials/install_on_linux.md @@ -4,7 +4,7 @@ ### Prerequisites -We use [anaconda](https://www.anaconda.com/products/individual) to manage virtual environments. +We use [anaconda](https://www.anaconda.com/products/individual) to manage virtual environments. This makes installing pytorch-related libraries way easier than using pure pip installs. We enable CUDA-acceleration in pytorch as part of the defaut virtual environment recipe (see below). @@ -43,9 +43,17 @@ Start by activating the virtual environment with ```bash conda activate myria3d ``` -Then install from a specific branch from github directly. Argument `branch_name` might be `main`, `dev`, or a specific release. + +Then install the latest version from pypi. +**Warning:** activating the environment is required as the public pip package does not +handle its dependencies! ``` -pip install --upgrade https://github.com/IGNF/myria3d/tarball/{branch_name} +pip install myria3d +``` + +Or install from a specific branch from github directly. Argument `branch_name` might be `main`, `dev`, or a specific release. +``` +pip install --upgrade https://github.com/IGNF/myria3d/tarball/{branch_name} ``` Alternatively, you can install from sources directly in editable mode with @@ -53,10 +61,10 @@ Alternatively, you can install from sources directly in editable mode with pip install --editable . ``` - + ## Troubleshooting - *OSError(libcusparse.so.11 cannot open shared object file no such file or directory)* ([**](https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625)) - open the .bashrc file from your Ubuntu home directory and at the end of the file, add the following line (replace anaconda3 with miniconda3 if needed) - export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/myria3d/lib:$LD_LIBRARY_PATH" + export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/myria3d/lib:$LD_LIBRARY_PATH" diff --git a/environment.yml b/environment.yml index 17f3bf24..47f589c4 100644 --- a/environment.yml +++ b/environment.yml @@ -38,7 +38,7 @@ dependencies: - matplotlib # --------- loggers --------- # - comet_ml::comet_ml=3.35 - - conda-forge::urllib3<2 # To solve for https://github.com/GeneralMills/pytrends/issues/591 + - conda-forge::urllib3<2 # To solve for https://github.com/GeneralMills/pytrends/issues/591 # --------- Visualization --------- # - pandas - matplotlib @@ -57,17 +57,18 @@ dependencies: - rich # beautiful text formatting in terminal - sh # for running bash commands in some tests - pudb # debugger + - twine # to publish pip package # # --------- Documentation --------- # - sphinx==4.5.* - recommonmark==0.7.* - sphinx_rtd_theme==1.0.* - docutils==0.17 - - rstcheck==3.3.* # RST Linter + - rstcheck==3.3.* # RST Linter - pip: - - hydra-core==1.1.* - - hydra-colorlog==1.1.* - # --------- Documentation --------- # - - myst_parser==0.17.* - - sphinxnotes-mock==1.0.0b0 # still a beta - - sphinx_paramlinks==0.5.* - - ign-pdal-tools>=1.5.2 \ No newline at end of file + - hydra-core==1.1.* + - hydra-colorlog==1.1.* + # --------- Documentation --------- # + - myst_parser==0.17.* + - sphinxnotes-mock==1.0.0b0 # still a beta + - sphinx_paramlinks==0.5.* + - ign-pdal-tools>=1.5.2