Merge branch 'main' into droumis-patch-3

holoviz-topics · Oct 28, 2024 · 371610e · 371610e
2 parents c0fb02d + 3d556d0
commit 371610e
Show file tree

Hide file tree

Showing 76 changed files with 2,107 additions and 1,934 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -58,10 +58,10 @@ jobs:
     outputs:
       projects: ${{ steps.set-projects.outputs.projects }}
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.12'
     - name: install deps
       # Minimal required deps to run the following script
       run: pip install doit pyyaml

diff --git a/.github/workflows/build_one.yml b/.github/workflows/build_one.yml
@@ -33,10 +33,10 @@ jobs:
     outputs:
       runner: ${{ steps.set-runner.outputs.runner }}
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.12'
     - name: install deps
       # Minimal required deps to run the following script
       run: pip install doit pyyaml
@@ -55,19 +55,14 @@ jobs:
         shell: bash -l {0}
     timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v3
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         miniconda-version: "latest"
         auto-update-conda: true
         auto-activate-base: false
         activate-environment: examples-gallery-manage
         environment-file: envs/environment-${{ needs.infer_runner.outputs.runner }}.lock
-    - name: enable libmamba
-      run: |
-        conda activate base
-        conda install conda-libmamba-solver
-        conda config --set solver libmamba
     - name: list files
       run: doit build_list_existing_files:${{ inputs.project }}
     - name: prepare project

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -50,11 +50,11 @@ jobs:
       changedprojects: ${{ steps.set-projects.outputs.changedprojects }}
       removedprojects: ${{ steps.set-projects.outputs.removedprojects }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         # Needed to compute the diff between the head and the latest commit on main
         fetch-depth: 2
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         miniconda-version: "latest"
         auto-activate-base: false
@@ -71,7 +71,7 @@ jobs:
 
         if [ "$EVENTNAME" == "pull_request" ]; then
 
-          CHANGES=$(doit util_list_changed_dirs_with_last_commit | tail -n -1)
+          CHANGES=$(doit util_list_changed_dirs_with_last_commit --exclude-website-metadata --exclude-test-data | tail -n -1)
 
           CHANGEDPROJECTS=$(echo $CHANGES | jq -c -r '.changed')
           REMOVEDPROJECTS=$(echo $CHANGES | jq -c -r '.removed')

diff --git a/.github/workflows/deploy_one.yml b/.github/workflows/deploy_one.yml
@@ -32,8 +32,8 @@ jobs:
         shell: bash -l {0}
     timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v3
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         miniconda-version: "latest"
         auto-update-conda: false
@@ -57,8 +57,8 @@ jobs:
         shell: bash -l {0}
     timeout-minutes: 15
     steps:
-    - uses: actions/checkout@v3
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         miniconda-version: "latest"
         auto-update-conda: false

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -79,18 +79,19 @@ jobs:
       run:
         shell: bash -el {0}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
-        # Needed to compute the diff between the head and the latest commit on main
-        fetch-depth: 2
-    - uses: hmarr/debug-action@v2
+        # Needed to compute the diff between the head and the latest commit on main (so at least 2)
+        # AND needed to compute last_updated
+        fetch-depth: 0
+    # - uses: hmarr/debug-action@v2
     - name: debug
       run: |
         echo ${{ github.ref_name }}
         echo ${{ github.event_name }}
         echo ${{ inputs.type }}
         echo ${{ inputs.branch }}
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         miniconda-version: "latest"
         auto-activate-base: false
@@ -141,7 +142,7 @@ jobs:
         git config user.name "github-actions"
 
         # git fetch and git diff
-        CHANGES=$(doit util_list_changed_dirs_with_last_commit | tail -n -1)
+        CHANGES=$(doit util_list_changed_dirs_with_last_commit --exclude-website-metadata --exclude-deployments-metadata --exclude-test-data | tail -n -1)
 
         CHANGEDPROJECTS=$(echo $CHANGES | jq -c -r '.changed')
         REMOVEDPROJECTS=$(echo $CHANGES | jq -c -r '.removed')
@@ -293,8 +294,7 @@ jobs:
           ls
         fi
     - name: archive projects
-      run: |
-        doit doc_archive_projects
+      run: doit doc_archive_projects
     - name: move content
       run: doit doc_move_content
     - name: "temp: remove non evaluated projects"
@@ -312,18 +312,18 @@ jobs:
       # ZIP and upload the built site:
       # Only when called from pr_flow.yml. Done as multiple PRs can update the dev website
       # concurrently, this offers a way to download the site and see it locally.
-    - name: tar built site
-      if: inputs.type == 'workflow_call'
-      run: tar czf builtdocs.tar.gz builtdocs/
-    - uses: actions/upload-artifact@v3
-      if: inputs.type == 'workflow_call'
-      with:
-        name: website
-        path: builtdocs.tar.gz
-        retention-days: 3
-    - name: delete zip
-      if: inputs.type == 'workflow_call'
-      run: rm builtdocs.tar.gz
+    # - name: tar built site
+    #   if: inputs.type == 'workflow_call'
+    #   run: tar czf builtdocs.tar.gz builtdocs/
+    # - uses: actions/upload-artifact@v3
+    #   if: inputs.type == 'workflow_call'
+    #   with:
+    #     name: website
+    #     path: builtdocs.tar.gz
+    #     retention-days: 3
+    # - name: delete zip
+    #   if: inputs.type == 'workflow_call'
+    #   run: rm builtdocs.tar.gz
     - name: Deploy dev
       # workflow_call, by pr_flow.yml
       # workflow_dispatch and dev target
@@ -346,9 +346,6 @@ jobs:
         force_orphan: true
     - name: Clean up
       run: doit clean --clean-dep doc_full
-    - name: debug
-      run: |
-        ls -hla
       # TODO: re-enable git diff --quiet --exit-code
     # - name: Check clean up
     #   run: git diff
@@ -357,10 +354,6 @@ jobs:
     - name: Comment PR
       # Only display the comment in a pr_flow context
       if: inputs.type == 'workflow_call'
-      uses: thollander/actions-comment-pull-request@v2
+      uses: thollander/actions-comment-pull-request@v3
       with:
-        message: |
-          Your changes were successfully integrated in the dev site, make sure to review
-          the pages of the projects you touched before merging this PR: https://holoviz-dev.github.io/examples/.
-          You can also download an archive of the site from the workflow summary page which comes in handy
-          when your dev site built was overriden by another PR (we have a single dev site!).
+        message: 'Your changes were successfully integrated in the <a href="https://holoviz-dev.github.io/examples/" target="_blank">dev site</a>, make sure to review the pages of the projects you touched before merging this PR.'
diff --git a/.github/workflows/pr_flow.yml b/.github/workflows/pr_flow.yml
@@ -24,6 +24,12 @@ permissions:
   # To allow the docs workflow to submit a comment using thollander/actions-comment-pull-request
   pull-requests: write
 
+# Project file changes:
+# - Always validate it
+# - Website only metadata -> Skip test/build
+# - Deployment metadata (examples_config.deployments, commands) -> We should redeploy, but skip test/build/doc
+# - Docs is special, it's just to pull the `evaluated` branch, and this is only useful to get the built notebooks
+
 jobs:
   setup:
     runs-on: ubuntu-latest
@@ -34,22 +40,22 @@ jobs:
     outputs:
       changedprojects: ${{ steps.set-list.outputs.changedprojects }}
       removedprojects: ${{ steps.set-list.outputs.removedprojects }}
+      changedprojectsfile: ${{ steps.set-list.outputs.changedprojectsfile }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 100
-    - uses: hmarr/debug-action@v2
-    - uses: actions/setup-python@v4
+    # - uses: hmarr/debug-action@v2
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.12'
     - name: install deps
       # Minimal required deps to run the following script
       run: pip install doit pyyaml
     - name: infer project list
       id: set-list
       run: |
-        CHANGES=$(doit util_list_changed_dirs_with_main | tail -n -1)
-
+        CHANGES=$(doit util_list_changed_dirs_with_main --exclude-website-metadata --exclude-deployments-metadata | tail -n -1)
         CHANGEDPROJECTS=$(echo $CHANGES | jq -c -r '.changed')
         REMOVEDPROJECTS=$(echo $CHANGES | jq -c -r '.removed')
 
@@ -64,6 +70,19 @@ jobs:
 
         echo "CHANGEDPROJECTS=$CHANGEDPROJECTS" >> $GITHUB_OUTPUT
         echo "REMOVEDPROJECTS=$REMOVEDPROJECTS" >> $GITHUB_OUTPUT
+    # There's logic to skip testing/building when only some metadata in the
+    # project file changes. In these cases, we still want to validate the project file.
+    - name: validate project files
+      run: |
+        CHANGESPROJECTSFILE=$(doit util_list_changed_dirs_with_main --only-project-file | tail -n -1)
+        CHANGEDPROJECTSFILE=$(echo $CHANGESPROJECTSFILE | jq -c -r '.changed')
+        echo "Changed projects (project file only): $CHANGEDPROJECTSFILE"
+        if [ "$CHANGEDPROJECTSFILE" != "[]" ]; then
+          items=$(echo $CHANGEDPROJECTSFILE | jq -c -r '.[]')
+          for item in ${items[@]}; do
+            doit validate_project_file:$item
+          done
+        fi
   test:
     needs: setup
     uses: ./.github/workflows/test.yml

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -44,10 +44,10 @@ jobs:
     outputs:
       projects: ${{ steps.set-projects.outputs.projects }}
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.12'
     - name: install deps
       # Minimal required deps to run the following script
       run: pip install doit pyyaml

diff --git a/.github/workflows/test_one.yml b/.github/workflows/test_one.yml
@@ -32,10 +32,10 @@ jobs:
     outputs:
       runner: ${{ steps.set-runner.outputs.runner }}
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.12'
     - name: install deps
       # Minimal required deps to run the following script
       run: pip install doit pyyaml
@@ -57,23 +57,14 @@ jobs:
         shell: bash -l {0}
     timeout-minutes: 60
     steps:
-    - uses: actions/checkout@v3
-    - uses: conda-incubator/setup-miniconda@v2
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         miniconda-version: "latest"
         auto-update-conda: false
         auto-activate-base: false
         activate-environment: examples-gallery-manage
         environment-file: envs/environment-${{ needs.infer_runner.outputs.runner }}.lock
-    - name: enable libmamba
-      run: |
-        conda activate base
-        conda install conda-libmamba-solver
-        conda config --set solver libmamba
-    - name: install nbval
-      # TODO: latest nbval not yet available on conda-forge
-      # when it is, remove this and add it to the lock file
-      run: pip install "nbval >= 0.10.0"
     - name: validate anaconda-project file
       run: doit validate_project_file:${{ inputs.project }}
     - name: validate anaconda-project lock file

diff --git a/.gitignore b/.gitignore
@@ -42,9 +42,6 @@ doc/gallery/*
 jupyter_execute/
 builtdocs/
 
-# Ignore output of the git diff 
-.diff
-
 # Ignore output of test_small_data_setup
 */tmp_catalog.yml
 

diff --git a/MAINTAINING.md b/MAINTAINING.md
@@ -106,3 +106,11 @@ The endpoints should be:
 
 where `<projectname>` is the project name, with underscores turned
 into hyphens.
+
+
+## Monitoring deployments
+
+An AWS Lambda was set up on the HoloViews account to check the deployments and report the output to the `deployments-monitoring` on Discord.
+The code of this Lambda is available on this [Gist](https://gist.github.com/maximlt/f2e29eb7e6dbcaa4da4ee7d68636ac55).
+The command `doit ae5_monitor_deployment` (based on similar code) is available to check the status of the deployment of 
+one or all the projects.