ci.yaml

name: PR E2E C

env:
  TRIVY_SCAN_SKIP_CHART_LIST: "nmstate cert-manager contour ingress-nginx"
  IMAGE_HINT_FILE_NAME: ".relok8s-images.yaml"
  SCHEMA_FILE_NAME: "values.schema.json"

on:
  pull_request_target:
    branches:
      - main

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: check chart config
        run : |
          TMP=`ls charts`
          for ITEM in $TMP ; do
              [ ! -d "charts/${ITEM}" ] && continue
              echo "check config for chart $ITEM "
              [ ! -f "charts/${ITEM}/config" ] && echo "error, miss charts/${ITEM}/config " && exit 1
          
              for VAR in DAOCLOUD_REPO_PROJECT UPGRADE_METHOD ; do
                  VAL=` source charts/${ITEM}/config && echo ${!VAR} `
                  [ -n "$VAL" ] || { echo "error, miss $VAR in charts/${ITEM}/config"; exit 1 ; }
              done
              
              UPGRADE_METHOD=` source charts/${ITEM}/config && echo $UPGRADE_METHOD `
              if [ "$UPGRADE_METHOD" == "pr" ] || [ "$UPGRADE_METHOD" == "issue" ] ; then
                  for VAR in REPO_URL REPO_NAME CHART_NAME VERSION UPGRADE_REVIWER ; do
                      VAL=` source charts/${ITEM}/config && echo ${!VAR} `
                      [ -n "$VAL" ] || { echo "error, miss $VAR in charts/${ITEM}/config"; exit 1 ; }
                  done
              elif [ "$UPGRADE_METHOD" == "none" ] ; then
                  echo ""
              else
                  echo "error, UPGRADE_METHOD=$UPGRADE_METHOD must be 'pr'/'issue'/'none' in charts/${ITEM}/config"
                  exit 1
              fi

          done
          exit 0


  check_change:
    runs-on: ubuntu-latest
    outputs:
      project: ${{ steps.check_change.outputs.test_project }}
      pro_version_changed: ${{ env.PRO_CHANGED_VERSION }}
    steps:
      - name: "Setup go"
        uses: actions/setup-go@v3
        with:
          go-version: '1.19'

      - name: Checkout code
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          
      - name: check change
        id: check_change
        run: |
          set -x
          # Using the Github API to detect the files changed as git merge-base stops working when the branch is behind
          # and jitterbit/get-changed-files does not support pull_request_target
          URL="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"
          files_changed_data=$(curl -s --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -X GET -G "$URL")
          echo "files_changed_data: $files_changed_data"
          files_changed="$(echo $files_changed_data | jq -r '.[] | .filename')"
          echo "files_changed: $files_changed"
          #--------------------
          # Adding || true to avoid "Process exited with code 1" errors
          charts_changed="$(echo "$files_changed" | xargs dirname | grep -o "charts/[^/]*" | sort | uniq | awk -F'/' '{print $2}' | tr '\n' ' ' || true)"
          echo "charts_changed: $charts_changed"
          #--------------------
          # issue: just release one project
          PRO_CHANGED_VERSION=""
          for PRO in ${charts_changed} ; do
            if [ -d "charts/${PRO}" ] ; then
               if grep  "charts/${PRO}/${PRO}/Chart.yaml" <<< "${files_changed}" &>/dev/null ; then
                   PRO_CHANGED_VERSION="${PRO}"
                  break
               fi
            fi
          done
          echo "PRO_CHANGED_VERSION=${PRO_CHANGED_VERSION}"
          echo "PRO_CHANGED_VERSION=${PRO_CHANGED_VERSION}" >> $GITHUB_ENV
          #-------------------
          # for project test script
          additionall_test_project=""
          project_test_dir_changed="$(echo "$files_changed" | xargs dirname |  grep -o "test/[^/]*" | sort | uniq  | awk -F'/' '{print $2}'  || true)"
          for PRO in ${project_test_dir_changed} ; do
              if ! grep " ${PRO} " <<< " ${charts_changed} " &>/dev/null ; then
                  additionall_test_project+=" $PRO "
                  echo "test script changes for ${PRO} "
              fi
          done
          #--------------------
          # when other files changed
          test_dir_changed="$(echo "$files_changed" |  grep -o "test/[^/]*" | sort | uniq  || true)"
          if [ -n "$test_dir_changed" ] && [ -z "${additionall_test_project}" ] && [ -z "$charts_changed" ] ; then
              # add spiderppool for testing test changes
              charts_changed=` echo "$charts_changed spiderpool" | tr -s ' ' | tr ' ' '\n' | sort | uniq | tr '\n' ' ' || true`
              echo "detect test changes, add spiderpool, charts_changed: $charts_changed"
          fi
          charts_changed=` echo $charts_changed `
          RES=""
          for PRO in ${charts_changed} ; do
            if [ -d "charts/${PRO}" ] ; then
              RES+="${PRO} "
            fi
          done
          echo ::set-output name=changed_project::${RES}
          ALL_PRO=" ${RES} ${additionall_test_project} "
          echo ::set-output name=test_project::${ALL_PRO}

      - name: check chart override
        run: |
          BASE=$PWD
          cd cmd/chart-override-check
          for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do
              PROJECT_PATH=${BASE}/charts/${PROJECT}
              go run main.go -path ${PROJECT_PATH}/${PROJECT} -skipCPath ${PROJECT_PATH}/skip-check.yaml
          done

      - name: check image hint file
        run: |
          BASE=$PWD
          for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do
              source ${BASE}/charts/${PROJECT}/config && [ "${NO_IMAGE}" == "true" ] && continue
              CHART_PATH=${BASE}/charts/${PROJECT}/${PROJECT}
              [ -f "${CHART_PATH}/${{ env.IMAGE_HINT_FILE_NAME }}" ] \
                  || { echo "error, miss ${{ env.IMAGE_HINT_FILE_NAME }} file in chart ${PROJECT} " ; exit 1 ; }
              ! grep -E '\-[[:space:]]+[^"].*[^"]' ${CHART_PATH}/${{ env.IMAGE_HINT_FILE_NAME }} \
                  || { echo "error, must use Double quotation in ${PROJECT}/.relok8s-images.yaml " ; exit 1 ; }
          done

      - name: check Chart yaml
        run: |
          BASE=$PWD
          for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do
              CHART_PATH=${BASE}/charts/${PROJECT}/${PROJECT}
              if ! grep "keywords:" ${CHART_PATH}/Chart.yaml &>/dev/null ;then
                  echo "error, please input 'keywords' into the Chart.yaml of ${PROJECT} "
                  exit 1
              fi
              NAME=` yq '.name' ${CHART_PATH}/Chart.yaml `
              echo "chart NAME '${NAME}' in project  '${PROJECT}' "
              [ "${NAME}" == "${PROJECT}" ] || { echo "error, chart name is not ${PROJECT}" ; exit 1 ; }
          done

      - name: check image repo
        run: |
          BASE=$PWD
          for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do
              source ${BASE}/charts/${PROJECT}/config && [ "${NO_IMAGE}" == "true" ] && continue
              CHART_PATH=${BASE}/charts/${PROJECT}/${PROJECT}
              if helm template test ${CHART_PATH} | grep ' image: ' | cut -d ":" -f2,3 | grep -v "daocloud" &>/dev/null ; then
                  echo "error, ${PROJECT} image is not in daocloud repo"
                  helm template test ${CHART_PATH} | grep ' image: ' | cut -d ":" -f2,3 | grep -v "daocloud"
                  exit 1
              fi
          done

      - name: check resource limit
        run: |
          BASE=$PWD
          for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do
              CHART_PATH=${BASE}/charts/${PROJECT}/${PROJECT}
              if helm template test ${CHART_PATH}  | grep -E "kind: DaemonSet|kind: Deployment|kind: Job|kind: CronJob" &>/dev/null ; then
                  echo "check resource limit and request"
                  if ! ( helm template test ${CHART_PATH} | grep ' limits:' &>/dev/null ) ; then
                      echo "error, did not find resource limits settings in ${PROJECT}"
                      exit 1
                  fi
                  if ! ( helm template test ${CHART_PATH} | grep ' requests:' &>/dev/null ) ; then
                      echo "error, did not find resource requests settings in ${PROJECT}"
                      exit 1
                  fi
              fi
          done

      - name: check schema file
        run: |
          BASE=$PWD
          for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do
              CHART_PATH=${BASE}/charts/${PROJECT}/${PROJECT}
              # skip schema check
              source ${BASE}/charts/${PROJECT}/config && [ "${SKIP_SCHEMA}" == "true" ] && echo "ignore schema check" && continue
              [ -f "${CHART_PATH}/${{ env.SCHEMA_FILE_NAME }}" ] \
                  || { echo "error, miss ${{ env.SCHEMA_FILE_NAME }} file in chart ${PROJECT} " ; exit 1 ; }
          done

      - name: check chart trivy
        run: |
          BASE=$PWD
          for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do
              # skip chart trivy
              source ${BASE}/charts/${PROJECT}/config && [ "${NO_TRIVY}" == "true" ] && echo "ignore trivy" && continue
              docker run --rm \
                  -v /tmp/trivy:/root/trivy.cache/  \
                    -v ${BASE}:/tmp/src  \
                    aquasec/trivy:latest config --exit-code 1  --severity CRITICAL /tmp/src/charts/${PROJECT}/${PROJECT}
              (( $? == 0 )) || { echo "error, failed to check chart trivy" && exit 1 ; }
              echo "chart trivy check: pass charts/${PROJECT}/${PROJECT} "
          done
          
      - name: check upgrade
        run: |
          for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do
               make build_chart -e PROJECT=${PROJECT} \
                  || { echo "error, failed to call 'make build_chart' to update ${PROJECT} " ; exit 1 ; }
              if ! git diff --exit-code --quiet ; then
                  git status
                  git diff
                  echo "error, after 'make build_chart', chart is not updated"
                  exit 1
              fi
          done
      - name: run validate schema test
        run: |
          BASE=$PWD
          python -m unittest ${BASE}/scripts/validate_schema_keys/test_validate.py -v || { echo "error, failed to run test script" ; exit 1 ; }
      - name: validate schema for value
        run: |
          BASE=$PWD
          for PROJECT in ${{ steps.check_change.outputs.changed_project }} ; do
              CHART_PATH=charts/${PROJECT}/${PROJECT}
              SCHEMA_FILE=${CHART_PATH}/${{ env.SCHEMA_FILE_NAME }}
              PARENT_SCHEMA_FILE=charts/${PROJECT}/parent/${{ env.SCHEMA_FILE_NAME }}
              [ -f ${PARENT_SCHEMA_FILE} ] && SCHEMA_FILE=${PARENT_SCHEMA_FILE}
          
              python ${BASE}/scripts/validate_schema_keys/validate.py ${SCHEMA_FILE} ${CHART_PATH}/values.yaml
              [ $? -eq 0 ] || { echo "error, failed to validate ${CHART_PATH}/values.yaml with ${SCHEMA_FILE}" ; exit 1 ; } 
          
              PROJECT_SRC_CONFIG_PATH=charts/${PROJECT}/config
              source $PROJECT_SRC_CONFIG_PATH
              if [ -n "${APPEND_VALUES_FILE}" ] && [ -s ${PROJECT_SRC_DIR}/${APPEND_VALUES_FILE} ] ; then
                    python ${BASE}/scripts/validate_schema_keys/validate.py ${SCHEMA_FILE} ${PROJECT_SRC_DIR}/${APPEND_VALUES_FILE}
                    [ $? -eq 0 ] || { echo "error, failed to validate ${PROJECT_SRC_DIR}/${APPEND_VALUES_FILE} with ${SCHEMA_FILE}" ; exit 1 ; }
              fi
          done

  call_e2e:
    uses: ./.github/workflows/call-e2e.yml
    needs: [check_change]
    if: ${{ needs.check_change.outputs.project != '' }}
    with:
      project: ${{ needs.check_change.outputs.project }}
      ref: ${{ github.event.pull_request.head.sha }}
    secrets: inherit

#  nofity_version_changed:
#    runs-on: ubuntu-latest
#    needs: [ check_change ]
#    if: ${{ needs.check_change.outputs.pro_version_changed != '' }}
#    steps:
#      - name: create an issue
#        uses: dacbd/create-issue-action@main
#        with:
#          token: ${{ secrets.GITHUB_TOKEN }}
#          repo: DaoCloud/DaoCloud-docs
#          title: "project ${{ needs.check_change.outputs.pro_version_changed }} update to version "
#          body: my new issue
#
#      - name: Print outputs
#        run: |
#          echo ${{ steps.create-issue.outputs.id }}
#          echo ${{ steps.create-issue.outputs.number }}