Merge branch 'main' into thanos_persistentVolumeClaimRetentionPolicy

Signed-off-by: François Marceau <[email protected]>

.github/workflows/cd-pipeline.yml

@@ -21,7 +21,7 @@ jobs:
       chart: ${{ steps.get-chart.outputs.chart }}
       result: ${{ steps.get-chart.outputs.result }}
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
         with:
           path: charts
           fetch-depth: 2 # to be able to obtain files changed in the latest commit
@@ -71,7 +71,7 @@ jobs:
       CSP_API_TOKEN: ${{ secrets.CSP_API_TOKEN }}
       VIB_PUBLIC_URL: https://cp.bromelia.vmware.com
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
         name: Checkout Repository
         with:
           path: charts
@@ -95,7 +95,7 @@ jobs:
         with:
           path: ~/artifacts
       # If we perform a checkout of the main branch, we will find conflicts with the submodules
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
         with:
           ref: 'index'
           path: index
@@ -105,9 +105,11 @@ jobs:
         run: |
           HELM_TARBALL="helm-v3.8.1-linux-amd64.tar.gz"
           curl -SsLfO "https://get.helm.sh/${HELM_TARBALL}" && sudo tar xf "$HELM_TARBALL" --strip-components 1 -C /usr/local/bin
+          # Install file plugin
+          helm plugin add https://github.com/zoobab/helm_file_repo
       - id: update-index
         name: Fetch chart and update index
-        env: 
+        env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_PUBLISH_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_PUBLISH_SECRET_ACCESS_KEY }}
           AWS_ASSUME_ROLE_ARN: ${{ secrets.AWS_PUBLISH_ROLE_ARN }}
@@ -132,21 +134,64 @@ jobs:
           is_index_updated=0
           while [[ $attempts -lt $max_attempts && $is_index_updated -eq 0 ]]; do
             attempts=$((attempts + 1))
+
+            # Pull changes from remote
             git fetch origin index
-            git reset --hard origin/index
+            current_commit_id=$(git rev-parse origin/index)
+            git reset --hard $(git commit-tree origin/index^{tree} -m "Update index.yaml")
 
             # Rebuild index
             helm repo index --url https://charts.bitnami.com/bitnami --merge bitnami/index.yaml ../download
+            # Compare size of files
+            if [[ $(stat -c%s bitnami/index.yaml) -gt $(stat -c%s ../download/index.yaml) ]]; then
+              echo "New index.yaml file is shorter than the current one"
+              exit 1
+            fi
+            # Adding tmp file as a helm repo
+            if ! helm repo add cache file://../download/ ; then
+              echo "New index.yaml file can't be indexed"
+              exit 1
+            fi
+
             cp ../download/index.yaml bitnami/index.yaml
+
             # Push changes
-            git add bitnami/index.yaml && git commit -m "${chart_name}-${chart_version}: Update index.yaml" -s
-            git push && is_index_updated=1 || echo "Failed to push during attempt $attempts"
+            git add bitnami/index.yaml && git commit --signoff --amend --no-edit
+            git push origin index --force-with-lease=index:${current_commit_id} && is_index_updated=1 || echo "Failed to push during attempt $attempts"
           done
 
           if [[ $is_index_updated -ne 1 ]]; then
             echo "Could not update the index after $max_attempts attempts"
             exit 1
           fi
+  push-tag:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    needs:
+      - get-chart
+      - vib-publish
+    name: Push tag
+    if: ${{ needs.get-chart.outputs.result == 'ok' }}
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+        with:
+          path: charts
+          fetch-depth: 2 # to be able to obtain files changed in the latest commit
+      - id: push-tag
+        name: 'Push tag'
+        env:
+          CHART: ${{ needs.get-chart.outputs.chart }}
+        run: |
+          cd charts
+          # Get chart version and list of tags
+          chart_version="$(yq e '.version' bitnami/${CHART}/Chart.yaml)"
+          git fetch --tags
+          # If the tag does not exist, create and push it (this allows re-executing the job)
+          if ! git tag | grep ${CHART}/${chart_version}; then
+            git tag ${CHART}/${chart_version}
+            git push --tags
+          fi
   # If the CD Pipeline does not succeed we should notify the interested agents
   slack-notif:
     runs-on: ubuntu-latest

.github/workflows/ci-pipeline.yml

@@ -92,23 +92,72 @@ jobs:
   update-pr:
     runs-on: ubuntu-latest
     needs: [get-chart]
-    name: Automatically update README and CRDs
+    name: Automatically update README, CRDs and CHANGELOG
     permissions:
       contents: write
     outputs:
       result: ${{ steps.update-pr.outputs.result }}
     if: needs.get-chart.outputs.result == 'ok'
     steps:
       - name: Checkout bitnami/charts
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
         with:
           ref: ${{github.event.pull_request.head.ref}}
           repository: ${{github.event.pull_request.head.repo.full_name}}
           token: ${{ secrets.BITNAMI_BOT_TOKEN }}
+          path: charts
+      - name: Clone upstream bitnami/charts repository
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
+        with:
+          path: upstream-charts
       - name: Setup git configuration
         run: |
+          cd $GITHUB_WORKSPACE/charts
           git config user.name "Bitnami Containers"
           git config user.email "[email protected]"
+      # In order to avoid doing a full clone (which would fetch the index branch), we
+      # unshallow the clone only using the main branch. We need to get the tags to
+      # regenerate the changelog too
+      - name: Unshallow main branch and get tags
+        run: |
+          cd $GITHUB_WORKSPACE/upstream-charts
+          git fetch origin main --unshallow
+          git fetch --tags
+      - name: Install conventional-changelog-cli
+        run: npm install -g conventional-changelog-cli
+      - id: generate-changelog
+        name: Generate changelog
+        env:
+          CHART: ${{ needs.get-chart.outputs.chart }}
+        run: |
+          cd $GITHUB_WORKSPACE/upstream-charts
+          # The generator needs the file to exist
+          chart_version="$(yq e '.version' $GITHUB_WORKSPACE/charts/bitnami/${CHART}/Chart.yaml)"
+          changelog_file="$GITHUB_WORKSPACE/charts/bitnami/${CHART}/CHANGELOG.md"
+          changelog_tmp="$GITHUB_WORKSPACE/charts/bitnami/${CHART}/CHANGELOG.md.tmp"
+          touch "$changelog_file"
+          npx conventional-changelog-cli -i ${changelog_file} -s -t ${CHART}/ -r 0 --commit-path bitnami/${CHART}
+          # The tool uses short sha to generate commit links. Sometimes, Github does not offer links with the short sha, so we change all commit links to use the full sha instead
+          for short_sha in $(grep -Eo "/commit/[a-z0-9]+" ${changelog_file} | awk -F/ '{print $3}'); do
+            long_sha="$(git rev-list @ | grep "^$short_sha" | head -n 1)";
+            sed -i "s%/commit/$short_sha%/commit/$long_sha%g" ${changelog_file};
+          done
+          cd $GITHUB_WORKSPACE/charts
+          # Remove unreleased section (includes all intermediate commits in the branch) and create future entry based on PR title
+          # The unreleased section looks like this "## (YYYY-MM-DD)" whereas a released section looks like this "## 0.0.1 (YYYY-MM-DD)"
+          # So we only need to find a released section to start printing in the awk script below
+          awk '/^##[^(]*[0-9]/ {flag=1} flag {print}' ${changelog_file} > ${changelog_tmp}
+          # Remove extra newlines so the changelog file passes the markdown linter
+          sed -i -E -e '/^$/d' ${changelog_tmp} && sed -i -E -e 's/(##.*)/\n\1\n/g' ${changelog_tmp}
+          # Include h1 heading and add entry for the current version. There is no tag for the current version (this will be created once merged), so we need to manually add it.
+          # We know the final squashed commit title, which will be the PR title. We cannot add a link to the commit in the main branch because it has not been
+          # merged yet (this will be corrected once a new version regenerates the changelog). Instead, we add the PR url which contains the exact same information.
+          echo -e -n "# Changelog\n\n## $chart_version ($(date +'%Y-%m-%d'))\n\n* ${{ github.event.pull_request.title }} ([#${{ github.event.number }}](${{ github.server_url }}/${{ github.repository }}/pull/${{ github.event.number }}))\n" > ${changelog_file}
+          cat ${changelog_tmp} >> ${changelog_file}
+          rm ${changelog_tmp}
+          if git status -s | grep "bitnami/${CHART}/CHANGELOG.md"; then
+            git add "bitnami/${CHART}/CHANGELOG.md" && git commit -m "Update CHANGELOG.md" --signoff
+          fi
       - name: Install readme-generator-for-helm
         if: needs.get-chart.outputs.values-updated == 'true'
         run: npm install -g @bitnami/readme-generator-for-helm
@@ -119,6 +168,7 @@ jobs:
           CHART: ${{ needs.get-chart.outputs.chart }}
         run: |
             exit_code=0
+            cd $GITHUB_WORKSPACE/charts
             echo "Validating README.md for bitnami/${CHART}"
             # Validating *.registry parameters
             while read line; do
@@ -165,6 +215,7 @@ jobs:
           CHART: ${{ needs.get-chart.outputs.chart }}
         run: |
           # Updating CRDs stored at 'bitnami/$CHART/crds' and 'bitnami/$CHART/templates/crds'
+          cd $GITHUB_WORKSPACE/charts
           mapfile -t crd_files < <(find "bitnami/${CHART}/crds" "bitnami/${CHART}/templates/crds" -name "*.yaml" -o -name "*.yml" 2>/dev/null || true)
           for file in "${crd_files[@]}"; do
             # Automatically update CRDs that use the '# Source' header
@@ -245,6 +296,7 @@ jobs:
       - id: update-pr
         name: Push changes
         run: |
+          cd $GITHUB_WORKSPACE/charts
           # Push all the new commits, if any
           if [[ $(git cherry -v) ]]; then
             git push
@@ -270,7 +322,7 @@ jobs:
       )
     name: VIB Verify
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
         name: Checkout Repository
         with:
           ref: ${{ github.event.pull_request.head.ref }}

.github/workflows/license-headers.yml

@@ -19,7 +19,7 @@ jobs:
       contents: read
       pull-requests: write
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
         name: Checkout Repository
         with:
           ref: ${{ github.event.pull_request.head.ref }}

.github/workflows/markdown-linter.yml

@@ -8,6 +8,7 @@ on:
       - main
     paths:
       - '**.md'
+      - '!**/CHANGELOG.md'
 # Remove all permissions by default
 permissions: {}
 jobs:
@@ -19,7 +20,7 @@ jobs:
       - name: Install mardownlint
         run: npm install -g [email protected]
       - name: Checkout project
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
       - name: Execute markdownlint
         env:
           DIFF_URL: "${{github.event.pull_request.diff_url}}"
@@ -29,7 +30,7 @@ jobs:
           # This request doesn't consume API calls.
           curl -Lkso $TEMP_FILE $DIFF_URL
           files_changed="$(sed -nr 's/[\-\+]{3} [ab]\/(.*)/\1/p' $TEMP_FILE | sort | uniq)"
-          md_files="$(echo "$files_changed" | grep -o ".*\.md$" | sort | uniq || true)"
+          md_files="$(echo "$files_changed" | grep -v "CHANGELOG.md" | grep -o ".*\.md$" | sort | uniq || true)"
           # Create an empty file, useful when the PR changes ignored files
           touch "${TEMP_OUTPUT}"
           exit_code=0

.github/workflows/pr-review-hack.yml

@@ -20,6 +20,7 @@ jobs:
     outputs:
       author: ${{ steps.get-info.outputs.author }}
       actor: ${{ steps.get-info.outputs.actor }}
+      review_state: ${{ steps.get-info.outputs.review_state }}
       labels: ${{ steps.get-info.outputs.labels }}
       resource_url: ${{ steps.get-info.outputs.resource_url }}
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
@@ -29,10 +30,11 @@ jobs:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
         run: |
           actor="${{ github.event.workflow_run.actor.login }}"
-          download_url="$(gh api "${{ github.event.workflow_run.artifacts_url }}" | jq -cr '.artifacts[] | select(.name == "pull_request_number") | .archive_download_url')"
-          curl -sSL -o pull_request_number.zip -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -H "Authorization: Bearer $GITHUB_TOKEN" $download_url
-          unzip pull_request_number.zip
-          pull_request_number=$(cat pull_request_number)
+          download_url="$(gh api "${{ github.event.workflow_run.artifacts_url }}" | jq -cr '.artifacts[] | select(.name == "pull_request_info.json") | .archive_download_url')"
+          curl -sSL -o pull_request_info.zip -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -H "Authorization: Bearer $GITHUB_TOKEN" $download_url
+          unzip pull_request_info.zip
+          pull_request_number="$(jq -cr '.issue.number' pull_request_info.json)"
+          issue_review_state="$(jq -cr '.review.state' pull_request_info.json)"
           pull_request="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${pull_request_number}")"
           author="$(echo $pull_request | jq -cr '.user.login')"
           author_association="$(echo $pull_request | jq -cr '.author_association')"
@@ -43,6 +45,7 @@ jobs:
           echo "actor=${actor}" >> $GITHUB_OUTPUT
           echo "author=${author}" >> $GITHUB_OUTPUT
           echo "author_association=${author_association}" >> $GITHUB_OUTPUT
+          echo "review_state=${issue_review_state}" >> $GITHUB_OUTPUT
           echo "labels=${labels}" >> $GITHUB_OUTPUT
           echo "resource_url=${resource_url}" >> $GITHUB_OUTPUT
   call-pr-review-comment:
@@ -55,4 +58,5 @@ jobs:
       author: ${{ needs.pr-info.outputs.author }}
       actor: ${{ needs.pr-info.outputs.actor }}
       labels: ${{ needs.pr-info.outputs.labels }}
+      review_state: ${{ needs.pr-info.outputs.review_state }}
       resource_url: ${{ needs.pr-info.outputs.resource_url }}

.github/workflows/pr-reviews.yml

@@ -21,9 +21,9 @@ jobs:
     steps:
       - run: |
           echo "::notice:: Comment on PR #${{ github.event.pull_request.number }}"
-          echo "${{ github.event.pull_request.number }}" > pull_request_number
-      - name: Upload the PR number
+          jq -n --arg issue '${{ github.event.pull_request.number }}' --arg state '${{ github.event.review != null && github.event.review.state || '' }}' '{"issue": {"number": $issue }, "review": { "state": $state }}' > pull_request_info.json
+      - name: Upload the PR info
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
         with:
-          name: pull_request_number
-          path: ./pull_request_number
+          name: pull_request_info.json
+          path: ./pull_request_info.json

.markdownlintignore

@@ -1,2 +1,3 @@
 LICENSE.md
 .github/PULL_REQUEST_TEMPLATE.md
+CHANGELOG.md

.vib/harbor/vib-verify.json

@@ -25,7 +25,7 @@
         "target_platform": {
           "target_platform_id": "{VIB_ENV_TARGET_PLATFORM}",
           "size": {
-            "name": "S4"
+            "name": "M4"
           }
         }
       },

.vib/jupyterhub/cypress/cypress/e2e/jupyterhub.cy.js

@@ -7,43 +7,47 @@
 import { random } from '../support/utils';
 
 it('allows to upload and execute a python notebook', () => {
-  const notebookName = `notebook_template_${random}.ipynb`;
-  const userName = Cypress.env('username');
+  cy.session('test_upload', () => {
+    const notebookName = `notebook_template_${random}.ipynb`;
+    const userName = Cypress.env('username');
 
-  cy.login();
-  cy.visit(`/user/${userName}/tree/tmp`);
-  cy.contains('Upload').should('be.visible');
-  cy.get('[type=file]').selectFile('cypress/fixtures/notebook_template.ipynb', {
-    force: true,
-  });
-  cy.get('.filename_input').clear().type(notebookName);
-  cy.contains('button', 'Upload').click();
-  cy.contains('a', notebookName);
+    cy.login();
+    cy.visit(`/user/${userName}/tree/tmp`);
+    cy.contains('Upload').should('be.visible');
+    cy.get('[type=file]').selectFile('cypress/fixtures/notebook_template.ipynb', {
+      force: true,
+    });
+    cy.get('.filename_input').clear().type(notebookName);
+    cy.contains('button', 'Upload').click();
+    cy.contains('a', notebookName);
 
-  cy.visit(`/user/${userName}/notebooks/tmp/${notebookName}`);
-  cy.contains('button', 'Run').click();
-  cy.contains('Hello World!');
+    cy.visit(`/user/${userName}/notebooks/tmp/${notebookName}`);
+    cy.contains('button', 'Run').click();
+    cy.contains('Hello World!');
+  });
 });
 
 it('allows generating an API token', () => {
-  cy.login();
-  cy.visit('/hub/token');
-  // We need to wait until the background API request is finished
-  cy.contains(/\d+Z/).should('not.exist');
-  cy.contains('button', 'API token').click();
-  cy.get('#token-result')
-    .should('be.visible')
-    .invoke('text')
-    .then((apiToken) => {
-      cy.request({
-        url: '/hub/api/users',
-        method: 'GET',
-        headers: {
-          Authorization: `token ${apiToken}`,
-        },
-      }).then((response) => {
-        expect(response.status).to.eq(200);
-        expect(response.body[0].name).to.eq(Cypress.env('username'));
+  cy.session('test_token', () => {
+    cy.login();
+    cy.visit('/hub/token');
+    // We need to wait until the background API request is finished
+    cy.contains(/\d+Z/).should('not.exist');
+    cy.contains('button', 'API token').click();
+    cy.get('#token-result')
+      .should('be.visible')
+      .invoke('text')
+      .then((apiToken) => {
+        cy.request({
+          url: '/hub/api/users',
+          method: 'GET',
+          headers: {
+            Authorization: `token ${apiToken}`,
+          },
+        }).then((response) => {
+          expect(response.status).to.eq(200);
+          expect(response.body[0].name).to.eq(Cypress.env('username'));
+        });
       });
-    });
+  });
 });