From 01496d9332b2fcecc90f35fa9c747c3c32856c55 Mon Sep 17 00:00:00 2001 From: 0xtzData Date: Mon, 25 Mar 2024 17:18:33 +0100 Subject: [PATCH] revert workflows --- .github/workflows/charts.yaml | 48 +++ .github/workflows/cleanup_docker.yaml | 53 +++ .github/workflows/cleanup_helmfile.yaml | 139 ++++++++ .github/workflows/deploy.yaml | 373 ++++++++++++++++++++ .github/workflows/deploy_helmfile.yaml | 135 +++++++ .github/workflows/e2e_account.yaml | 131 +------ .github/workflows/e2e_admin.yaml | 3 + .github/workflows/e2e_comparison.yaml | 7 +- .github/workflows/e2e_new.yaml | 143 ++++++++ .github/workflows/e2e_prod.yaml | 4 + .github/workflows/e2e_prod_performance.yaml | 7 +- .github/workflows/e2e_rollup.yaml | 3 + .github/workflows/e2e_verification.yaml | 5 + .github/workflows/trigger_deploy.yaml | 69 ++++ 14 files changed, 989 insertions(+), 131 deletions(-) create mode 100644 .github/workflows/charts.yaml create mode 100644 .github/workflows/cleanup_docker.yaml create mode 100644 .github/workflows/cleanup_helmfile.yaml create mode 100644 .github/workflows/deploy.yaml create mode 100644 .github/workflows/deploy_helmfile.yaml create mode 100644 .github/workflows/e2e_new.yaml create mode 100644 .github/workflows/trigger_deploy.yaml diff --git a/.github/workflows/charts.yaml b/.github/workflows/charts.yaml new file mode 100644 index 00000000..2eb976ff --- /dev/null +++ b/.github/workflows/charts.yaml @@ -0,0 +1,48 @@ +name: Lint and publish charts + +on: + pull_request: + paths: + - charts/**/* + push: + paths: + - .github/workflows/charts.yaml + - charts/**/* + - scripts/build-repo.sh +jobs: + helm-publish: + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + runs-on: ubuntu-latest + needs: [helm-lint] + steps: + - name: Checkout + uses: actions/checkout@v2.0.0 + with: + fetch-depth: '0' + - name: Download Helm + run: | + curl -L https://get.helm.sh/helm-v3.4.1-linux-amd64.tar.gz | tar -xzf - --strip-component=1 linux-amd64/helm + chmod +x helm + - name: Package charts + run: | + sh ./scripts/build-repo.sh + - name: GitHub Pages action + uses: peaceiris/actions-gh-pages@v3.7.3 + with: + publish_branch: gh-pages + force_orphan: true + publish_dir: ./repo + personal_token: ${{ secrets.GITHUB_TOKEN }} + + helm-lint: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2.0.0 + - name: Download Helm + run: | + curl -L https://get.helm.sh/helm-v3.4.1-linux-amd64.tar.gz | tar -xzf - --strip-component=1 linux-amd64/helm + chmod +x helm + - name: Lint using helm + run: | + ./helm lint charts/* diff --git a/.github/workflows/cleanup_docker.yaml b/.github/workflows/cleanup_docker.yaml new file mode 100644 index 00000000..8387e8a6 --- /dev/null +++ b/.github/workflows/cleanup_docker.yaml @@ -0,0 +1,53 @@ +name: Cleanup docker + +# Triggered when a pull request is merged +on: + workflow_dispatch: + inputs: + dockerImage: + description: 'Docker image to delete' + required: false + type: string + workflow_call: + inputs: + dockerImage: + description: 'Docker image to delete' + required: false + type: string + +jobs: + cleanup: + name: Cleanup + permissions: + id-token: write + contents: read + runs-on: ubuntu-latest + strategy: + fail-fast: true + max-parallel: 1 + matrix: + environment: [Dev] + region: [us-east-1] + # environment: + # name: ${{ matrix.environment }} + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Add inputs to GITHUB_ENV + run: | + echo "DOCKER_IMAGE=${{ inputs.dockerImage }}" >> $GITHUB_ENV + + - name: Delete image + if: "${{ always() && env.DOCKER_IMAGE != ''}}" + uses: bots-house/ghcr-delete-image-action@v1.0.0 + with: + # NOTE: at now only orgs is supported + owner: ${{ env.GITHUB_REPOSITORY_OWNER_PART_SLUG }} + name: ${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG }} + # NOTE: using Personal Access Token + token: ${{ secrets.PAT }} + tag: ${{ env.DOCKER_IMAGE }} diff --git a/.github/workflows/cleanup_helmfile.yaml b/.github/workflows/cleanup_helmfile.yaml new file mode 100644 index 00000000..fa0b729f --- /dev/null +++ b/.github/workflows/cleanup_helmfile.yaml @@ -0,0 +1,139 @@ +name: Cleanup release + +# Triggered when a pull request is merged +on: + workflow_dispatch: + inputs: + dockerImage: + description: 'Docker image to delete' + required: false + type: string + appName: + description: 'Name of application to deploy' + required: true + default: '' + type: string + globalEnv: + description: 'Environment Name' + required: true + default: 'testing' + type: string + helmfileDir: + description: 'Directory to run helmfile' + required: false + default: '.' + type: string + kubeConfigSecret: + description: 'Path to kubeconfig in vault' + required: true + default: '' + type: string + vaultRole: + description: 'Role to authenticate in vault' + required: true + default: '' + type: string + workflow_call: + inputs: + appName: + description: 'Name of application to deploy' + required: true + default: '' + type: string + globalEnv: + description: 'Environment Name' + required: true + default: 'testing' + type: string + helmfileDir: + description: 'Directory to run helmfile' + required: false + default: '.' + type: string + kubeConfigSecret: + description: 'Path to kubeconfig in vault' + required: true + default: '' + type: string + vaultRole: + description: 'Role to authenticate in vault' + required: true + default: '' + type: string + dockerImage: + description: 'Docker image to delete' + required: false + type: string + +jobs: + cleanup: + name: Cleanup + permissions: + id-token: write + contents: read + runs-on: ubuntu-latest + strategy: + fail-fast: true + max-parallel: 1 + matrix: + environment: [Dev] + region: [us-east-1] + # environment: + # name: ${{ matrix.environment }} + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Add inputs to GITHUB_ENV + run: | + echo "APP_NAMESPACE=${{ inputs.appNamespace }}" >> $GITHUB_ENV + echo "DOCKER_IMAGE=${{ inputs.dockerImage }}" >> $GITHUB_ENV + + - name: Configure Kubectl + uses: azure/setup-kubectl@v2.0 + with: + version: 'v1.23.6' + id: install + + # Install helm, helm-secrets, sops + - name: Install tools + shell: bash + run: | + curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null + sudo apt-get install apt-transport-https --yes + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list + sudo apt-get update -y + sudo apt-get install helm gnupg -y + helm plugin install https://github.com/jkroepke/helm-secrets + helm plugin install https://github.com/databus23/helm-diff + cd /tmp + curl -O -L -C - https://github.com/helmfile/helmfile/releases/download/v0.153.1/helmfile_0.153.1_linux_amd64.tar.gz + tar -xzvf helmfile_0.153.1_linux_amd64.tar.gz + sudo mv helmfile /usr/bin/helmfile + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + + - name: Get Vault credentials + id: retrieve-vault-secrets + uses: hashicorp/vault-action@v2.4.1 + with: + url: https://vault.k8s.blockscout.com + role: ${{ inputs.vaultRole }} + path: github-jwt + method: jwt + tlsSkipVerify: false + exportToken: true + secrets: | + ${{ inputs.kubeConfigSecret }} kubeconfig | KUBECONFIG_BASE64 ; + # Delete release + - name: Delete + shell: bash + run: | + mkdir -p ~/.kube + echo $KUBECONFIG_BASE64 | base64 --decode > ~/.kube/config + chmod 600 ~/.kube/config + cd ${{ inputs.helmfileDir }} + helmfile -e ${{ inputs.globalEnv }} -l app=${{ inputs.appName }} destroy diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml new file mode 100644 index 00000000..21a8f712 --- /dev/null +++ b/.github/workflows/deploy.yaml @@ -0,0 +1,373 @@ +name: Deploy blockscout-stack + +# run-name: Deploy to ${{ inputs.appNamespace }} by @${{ github.actor }} +on: + # push: + workflow_dispatch: + inputs: + env_vars: + description: VALUES_DIR=charts/blockscout-stack/values/e2e,APP_NAME=bs-stack,DOCKER_IMAGE=1,APP=app-name + required: false + type: string + globalEnv: + description: 'Network type' + required: false + default: '_default' + type: string + # valuesDir: + # description: 'Directory with values' + # required: true + # default: 'charts/blockscout-stack/values/e2e' + # type: string + # appName: + # description: 'App name' + # required: true + # default: 'bs-stack' + # type: string + appNamespace: + description: 'Namespace to deploy in' + required: false + type: string + blockscoutImage: + description: 'Blockscout image' + required: false + type: string + blockscoutIngressHost: + description: 'Blockscout url' + required: false + type: string + frontendImage: + description: 'Frontend image' + required: false + type: string + frontendIngressHost: + description: 'Frontend url' + required: false + type: string + scVerifierImage: + description: 'Smart-contract-verifier image' + required: false + type: string + scVerifierIngressHost: + description: 'Smart-contract-verifier url' + required: false + type: string + gethIngressHost: + description: 'Geth node url' + required: false + type: string + workflow_call: + inputs: + env_vars: + description: VALUES_DIR=charts/blockscout-stack/values/e2e,APP_NAME=bs-stack,DOCKER_IMAGE=1,APP=app-name + required: false + type: string + globalEnv: + description: 'Network type' + required: false + default: '_default' + type: string + # valuesDir: + # description: 'Directory with values' + # required: true + # default: 'charts/blockscout-stack/values/e2e' + # type: string + # appName: + # description: 'App name' + # required: false + # default: 'bs-stack' + # type: string + appNamespace: + description: 'Namespace to deploy in' + required: false + type: string + blockscoutImage: + description: 'Blockscout image' + required: false + type: string + blockscoutIngressHost: + description: 'Blockscout url' + required: false + type: string + frontendImage: + description: 'Frontend image' + required: false + type: string + frontendIngressHost: + description: 'Frontend url' + required: false + type: string + scVerifierImage: + description: 'Smart-contract-verifier image' + required: false + type: string + scVerifierIngressHost: + description: 'Smart-contract-verifier url' + required: false + type: string + gethIngressHost: + description: 'Geth node url' + required: false + type: string + +env: + K8S_LOCAL_PORT: ${{ secrets.K8S_LOCAL_PORT }} + K8S_HOST: ${{ secrets.K8S_HOST }} + BASTION_HOST: ${{ secrets.BASTION_HOST }} + K8S_PORT: ${{ secrets.K8S_PORT }} + USERNAME: ${{ secrets.USERNAME }} + BASTION_SSH_KEY: ${{secrets.BASTION_SSH_KEY}} + K8S_DOMAIN: test.aws-k8s.blockscout.com + REGION: us-east-1 + +jobs: + deploy: + name: Deploy ${{ inputs.appNamespace }} + permissions: write-all + # permissions: + # id-token: write + # contents: read + runs-on: ubuntu-latest + # strategy: + # fail-fast: true + # max-parallel: 1 + # matrix: + # environment: [Dev] + # region: [us-east-1] + # environment: + # name: Dev + # url: ${{ steps.deploy-args.outputs.app_url }} + #https://${{ inputs.appNamespace }}.${{ env.K8S_DOMAIN }} + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Set environment variables + run: | + env_vars=${{ inputs.env_vars }} + for i in ${env_vars//,/ } + do + echo "$i" >> $GITHUB_ENV + echo "${{ inputs.appNamespace }}.${{ env.K8S_DOMAIN }}" + done + + # - name: Add inputs to GITHUB_ENV + # if: "${{ inputs.valuesDir != '' && inputs.appName != '' }}" + # run: | + # echo "VALUES_DIR=${{ inputs.valuesDir }}" >> $GITHUB_ENV + # echo "APP_NAME=${{ inputs.appName }}" >> $GITHUB_ENV + + - name: Generate namespace name + if: "${{ inputs.appNamespace == '' }}" + id: generate + uses: filipstefansson/uuid-action@v1 + with: + name: ${{ github.sha }} + + - name: Set APP_NAMESPACE + run: | + if ${{ inputs.appNamespace != '' }}; then + echo "APP_NAMESPACE=${{ inputs.appNamespace }}" >> "$GITHUB_ENV" + else + echo "APP_NAMESPACE=bs-${{ steps.generate.outputs.uuid }}" >> "$GITHUB_ENV" + fi + + - name: Configure Kubectl + uses: azure/setup-kubectl@v2.0 + with: + version: 'v1.23.6' + id: install + + # This action will use the GitHub OIDC id-token and request AWS STS to get temporary credentials. + # The temporary credentials are exported as env variables. + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: ${{ env.REGION }} + role-to-assume: ${{ secrets.ROLE_TO_ASSUME }} + + # Install helm, helm-secrets, sops + - name: Install helm and helm-secrets, sops + shell: bash + run: | + curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null + sudo apt-get install apt-transport-https --yes + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list + sudo apt-get update -y + sudo apt-get install helm gnupg -y + helm plugin install https://github.com/jkroepke/helm-secrets + curl -O -L -C - https://github.com/mozilla/sops/releases/download/v3.7.3/sops-v3.7.3.linux + sudo mv sops-v3.7.3.linux /usr/bin/sops + sudo chmod +x /usr/bin/sops + + # The following command will update the kube config file with our cluster information. + # It will use the credentials exported by the previous action. + - name: Update KubeConfig + shell: bash + run: | + aws eks update-kubeconfig --name ${{ secrets.AWS_CLUSTER_NAME }} --region=${{ env.REGION }} + + # Import GPG key + - name: Import GPG key + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + fingerprint: ${{ secrets.GPG_FINGERPRINT }} + passphrase: '' + + - name: Deploy args + id: deploy-args + shell: bash + run: | + echo $DEPLOY_ARGS + if ${{ env.GITHUB_REF_SLUG == 'main' }}; then + DEPLOY_SUFFIX=$GITHUB_REF_SLUG + else + DEPLOY_SUFFIX=$GITHUB_REF_SLUG + fi + echo "DEPLOY_SUFFIX=$DEPLOY_SUFFIX" >> "$GITHUB_ENV" + if ${{ inputs.globalEnv != '' }}; then + DEPLOY_ARGS="$DEPLOY_ARGS--set global.env=${{ inputs.globalEnv }} " + fi + if ${{ env.DOCKER_IMAGE != '' }}; then + DEPLOY_ARGS="$DEPLOY_ARGS--set $APP.image.${{ inputs.globalEnv }}=$DOCKER_IMAGE " + fi + if ${{ inputs.frontendImage != '' }}; then + DEPLOY_ARGS="$DEPLOY_ARGS--set frontend.image._default=${{ inputs.frontendImage }} " + fi + if ${{ inputs.frontendIngressHost != '' }}; then + FRONTEND_INGRESS_URL=${{ inputs.frontendIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN + FRONTEND_URL=https://$FRONTEND_INGRESS_URL + echo "FRONTEND_URL=$FRONTEND_URL" >> "$GITHUB_ENV" + DEPLOY_ARGS="$DEPLOY_ARGS--set frontend.ingress.host._default=$FRONTEND_INGRESS_URL " + DEPLOY_ARGS="$DEPLOY_ARGS--set frontend.environment.NEXT_PUBLIC_APP_HOST._default=$FRONTEND_INGRESS_URL " + fi + if ${{ inputs.blockscoutImage != '' }}; then + DEPLOY_ARGS="$DEPLOY_ARGS--set blockscout.image._default=${{ inputs.blockscoutImage }} " + fi + if ${{ inputs.blockscoutIngressHost != '' }}; then + DEPLOY_ARGS="$DEPLOY_ARGS--set blockscout.environment.ACCOUNT_AUTH0_CALLBACK_URL._default=http://${{ inputs.blockscoutIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN/auth/auth0/callback " + DEPLOY_ARGS="$DEPLOY_ARGS--set blockscout.environment.ACCOUNT_AUTH0_LOGOUT_RETURN_URL._default=http://${{ inputs.blockscoutIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN/auth/logout " + DEPLOY_ARGS="$DEPLOY_ARGS--set frontend.environment.NEXT_PUBLIC_LOGOUT_RETURN_URL._default=http://${{ inputs.blockscoutIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN/auth/logout " + BLOCKSCOUT_URL=https://${{ inputs.blockscoutIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN + DEPLOY_ARGS="$DEPLOY_ARGS--set blockscout.ingress.host._default=${{ inputs.blockscoutIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN " + DEPLOY_ARGS="$DEPLOY_ARGS--set frontend.environment.NEXT_PUBLIC_API_HOST._default=${{ inputs.blockscoutIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN " + echo "BLOCKSCOUT_URL=$BLOCKSCOUT_URL" >> "$GITHUB_ENV" + echo "::set-output name=app_url::$BLOCKSCOUT_URL" + fi + if ${{ inputs.scVerifierImage != '' }}; then + DEPLOY_ARGS="$DEPLOY_ARGS--set scVerifier.image._default=${{ inputs.scVerifierImage }} " + fi + if ${{ inputs.scVerifierIngressHost != '' }}; then + SC_VERIFIER_URL=https://${{ inputs.scVerifierIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN + echo "SC_VERIFIER_URL=$SC_VERIFIER_URL" >> "$GITHUB_ENV" + DEPLOY_ARGS="$DEPLOY_ARGS--set scVerifier.ingress.host._default=${{ inputs.scVerifierIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN " + fi + if ${{ inputs.gethIngressHost != '' }}; then + NETWORK_URL=${{ inputs.gethIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN/http + echo "NETWORK_URL=http://$NETWORK_URL" >> "$GITHUB_ENV" + DEPLOY_ARGS="$DEPLOY_ARGS--set geth.ingress.host._default=${{ inputs.gethIngressHost }}-$DEPLOY_SUFFIX.$K8S_DOMAIN " + fi + echo "$DEPLOY_ARGS" + echo "DEPLOY_ARGS=$DEPLOY_ARGS" >> "$GITHUB_ENV" + + - name: Check GitHub deployment name + id: deployment-name + shell: bash + run: | + echo $DEPLOY_ARGS + if ${{ env.GITHUB_REF_SLUG == 'main' }}; then + DEPLOYMENT_NAME=$GITHUB_REF_SLUG + else + DEPLOYMENT_NAME=review + fi + echo "DEPLOYMENT_NAME=$DEPLOYMENT_NAME" >> "$GITHUB_ENV" + + - uses: chrnorm/deployment-action@v2 + name: Create GitHub deployment + id: deployment + with: + token: '${{ github.token }}' + environment-url: ${{env.BLOCKSCOUT_URL}} + environment: ${{env.DEPLOYMENT_NAME}} + auto-inactive: false + + # Deploy newly built changes to EKS cluster. + - name: Deploy To Cluster + shell: bash + run: | + mkdir ~/.ssh + ssh-keyscan -H $BASTION_HOST >> ~/.ssh/known_hosts + eval `ssh-agent -s` + ssh-add - <<< "$BASTION_SSH_KEY" + sudo echo "127.0.0.1 $K8S_HOST" | sudo tee -a /etc/hosts + sed -i 's/eks.amazonaws.com/eks.amazonaws.com:9443/g' ~/.kube/config + ssh -fN -v -L $K8S_LOCAL_PORT:$K8S_HOST:$K8S_PORT $USERNAME@$BASTION_HOST + helm secrets upgrade --install --repo https://blockscout.github.io/blockscout-ci-cd/ \ + $APP_NAME blockscout-stack \ + -f $VALUES_DIR/values.yaml \ + -f $VALUES_DIR/secrets.yaml \ + $DEPLOY_ARGS \ + -n $APP_NAMESPACE --create-namespace + + # Copy registry secret if private docker repo + - name: Copy private registry secret + # if: github.repository == 'blockscout/frontend' + shell: bash + run: | + kubectl delete secret regcred -n $APP_NAMESPACE --ignore-not-found + kubectl get secret regcred -o yaml | sed "s/namespace: .*/namespace: $APP_NAMESPACE/" | kubectl apply -f - + + - name: Update deployment status (success) + if: success() + uses: chrnorm/deployment-status@v2 + with: + token: '${{ github.token }}' + environment-url: ${{env.BLOCKSCOUT_URL}} + deployment-id: ${{ steps.deployment.outputs.deployment_id }} + state: 'success' + auto-inactive: false + + - name: Update deployment status (failure) + if: failure() + uses: chrnorm/deployment-status@v2 + with: + token: '${{ github.token }}' + environment-url: ${{env.BLOCKSCOUT_URL}} + deployment-id: ${{ steps.deployment.outputs.deployment_id }} + state: 'failure' + + # Show app URLs + - name: Print URLs of blockscout-stack + shell: bash + run: | + echo "Your Blockscout app is here:" > urls.txt + echo "$BLOCKSCOUT_URL" >> urls.txt + echo "Your Frontend app is here:" >> urls.txt + echo "$FRONTEND_URL" >> urls.txt + echo "Your Smart-contract-verifier app is here:" >> urls.txt + echo "$SC_VERIFIER_URL" >> urls.txt + echo "Your geth app is here:" >> urls.txt + echo "$NETWORK_URL" >> urls.txt + cat urls.txt + + # check if instance is ready + - name: Check the deployed service URL + if: "${{ env.BLOCKSCOUT_URL != '' }}" + uses: jtalk/url-health-check-action@v2 + with: + # Check the following URLs one by one sequentially + url: ${{ env.BLOCKSCOUT_URL }} + # + # Follow redirects, or just report success on 3xx status codes + follow-redirect: false # Optional, defaults to "false" + # Fail this action after this many failed attempts + max-attempts: 30 # Optional, defaults to 1 + # Delay between retries + retry-delay: 20s # Optional, only applicable to max-attempts > 1 + # Retry all errors, including 404. This option might trigger curl upgrade. + retry-all: true # Optional, defaults to "false" diff --git a/.github/workflows/deploy_helmfile.yaml b/.github/workflows/deploy_helmfile.yaml new file mode 100644 index 00000000..fc022f9d --- /dev/null +++ b/.github/workflows/deploy_helmfile.yaml @@ -0,0 +1,135 @@ +name: Deploy blockscout-stack + +on: + workflow_call: + inputs: + appName: + description: 'Name of application to deploy' + required: true + default: '' + type: string + globalEnv: + description: 'Environment Name' + required: true + default: 'testing' + type: string + helmfileDir: + description: 'Directory to run helmfile' + required: false + default: '.' + type: string + kubeConfigSecret: + description: 'Path to kubeconfig in vault' + required: true + default: '' + type: string + vaultRole: + description: 'Role to authenticate in vault' + required: true + default: '' + type: string + +jobs: + deploy: + name: Deploy + permissions: write-all + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Configure Kubectl + uses: azure/setup-kubectl@v2.0 + with: + version: 'v1.25.4' + id: install + + # Install helm, helm-secrets, sops + - name: Install tools + shell: bash + run: | + curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null + sudo apt-get install apt-transport-https --yes + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list + sudo apt-get update -y + sudo apt-get install helm gnupg -y + helm plugin install https://github.com/jkroepke/helm-secrets + helm plugin install https://github.com/databus23/helm-diff + cd /tmp + curl -O -L -C - https://github.com/helmfile/helmfile/releases/download/v0.153.1/helmfile_0.153.1_linux_amd64.tar.gz + tar -xzvf helmfile_0.153.1_linux_amd64.tar.gz + sudo mv helmfile /usr/bin/helmfile + sudo wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 + sudo chmod +x /usr/local/bin/yq + + - name: Get Vault credentials + id: retrieve-vault-secrets + uses: hashicorp/vault-action@v2.4.1 + with: + url: https://vault.k8s.blockscout.com + role: ${{ inputs.vaultRole }} + path: github-jwt + method: jwt + tlsSkipVerify: false + exportToken: true + secrets: | + ${{ inputs.kubeConfigSecret }} kubeconfig | KUBECONFIG_BASE64 ; + - name: Check GitHub deployment name + if: ${{ inputs.globalEnv == 'review' }} + id: deployment-name + shell: bash + run: | + mkdir -p ~/.kube + echo $KUBECONFIG_BASE64 | base64 --decode > ~/.kube/config + chmod 600 ~/.kube/config + cd ${{ inputs.helmfileDir }} + helmfile -e ${{ inputs.globalEnv }} -l app=${{ inputs.appName }} write-values --output-file-template "{{ .State.BaseName }}/{{ .Release.Name}}.yaml" + echo "DEPLOYMENT_URL=https://$(cat helmfile/bs-stack.yaml|yq '.frontend.ingress.hostname')" >> "$GITHUB_ENV" + kubectl create namespace ${{ inputs.appName }} --dry-run=client -o yaml | kubectl apply -f - + kubectl annotate namespace ${{ inputs.appName }} field.cattle.io/projectId=${{ secrets.RANCHER_PROJECT_ID }} --overwrite + + - uses: chrnorm/deployment-action@v2 + name: Create GitHub deployment + if: ${{ inputs.globalEnv == 'review' }} + id: deployment + with: + token: '${{ github.token }}' + environment-url: ${{env.DEPLOYMENT_URL}} + environment: review + auto-inactive: false + + # Deploy newly built changes to EKS cluster. + - name: Deploy + shell: bash + run: | + mkdir -p ~/.kube + echo $KUBECONFIG_BASE64 | base64 --decode > ~/.kube/config + chmod 600 ~/.kube/config + cd ${{ inputs.helmfileDir }} + helmfile -e ${{ inputs.globalEnv }} -l app=${{ inputs.appName }} apply --suppress-diff + + - name: Update deployment status (success) + if: ${{ success() && inputs.globalEnv == 'review' }} + uses: chrnorm/deployment-status@v2 + with: + token: '${{ github.token }}' + environment-url: ${{env.DEPLOYMENT_URL}} + deployment-id: ${{ steps.deployment.outputs.deployment_id }} + state: 'success' + auto-inactive: false + + - name: Update deployment status (failure) + if: ${{ failure() && inputs.globalEnv == 'review' }} + uses: chrnorm/deployment-status@v2 + with: + token: '${{ github.token }}' + environment-url: ${{env.DEPLOYMENT_URL}} + deployment-id: ${{ steps.deployment.outputs.deployment_id }} + state: 'failure' + + - name: Add annotation to workflow + if: ${{ success() && inputs.globalEnv == 'review' }} + run: echo "::notice title='Deployment URL'::${{env.DEPLOYMENT_URL}}" \ No newline at end of file diff --git a/.github/workflows/e2e_account.yaml b/.github/workflows/e2e_account.yaml index f981142b..668b7ae6 100644 --- a/.github/workflows/e2e_account.yaml +++ b/.github/workflows/e2e_account.yaml @@ -1,137 +1,10 @@ name: E2E K8s account -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - on: push: workflow_dispatch: -env: - BLOCKSCOUT_URL: ${{ secrets.E2E_BLOCKSCOUT_URL }} - FRONTEND_URL: ${{ secrets.E2E_FRONTEND_URL }} - NETWORK_URL: ${{ secrets.E2E_NETWORK_URL }} - ACCOUNT_USERNAME: ${{ secrets.ACCOUNT_USERNAME }} - ACCOUNT_PASSWORD: ${{ secrets.ACCOUNT_PASSWORD }} - MAILSLURP_API_KEY: ${{ secrets.MAILSLURP_API_KEY }} - MAILSLURP_EMAIL_ID: ${{ secrets.MAILSLURP_EMAIL_ID }} - ENV: test - RESOURCE_MODE: account - PUBLIC: false - PORT: 4000 - PORT_PG: 5432 - PORT_NETWORK_HTTP: 8545 - PORT_NETWORK_WS: 8546 - WALLET: ${{ secrets.WALLET }} - PWDEBUG: 0 - LOAD_AUTH_CTX: 0 - LOAD_CONTRACTS_DATA: 0 - ACCOUNT: 1 - jobs: tests: - name: Tests - permissions: - id-token: write - contents: read - runs-on: ubuntu-20.04 - environment: - name: Tests - steps: - - name: checkout - uses: actions/checkout@v2 - with: - repository: blockscout/blockscout-ci-cd - path: blockscout-ci-cd - - - name: Inject slug/short variables - uses: rlespinasse/github-slug-action@v4 - - - name: Set environment variables - run: | - env_vars=${{ inputs.env_vars }} - for i in ${env_vars//,/ } - do - echo "$i" >> $GITHUB_ENV - echo "${{ inputs.appNamespace }}.${{ env.K8S_DOMAIN }}" - done - - - name: Setup node - uses: actions/setup-node@v3 - with: - node-version: v14.17.0 - - name: Cache node modules - id: cache-npm - uses: actions/cache@v3 - env: - cache-name: cache-node-modules - with: - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - name: Install deps - working-directory: blockscout-ci-cd/tests/e2e - run: npm ci - - name: Build contracts - working-directory: blockscout-ci-cd/tests/contracts - run: | - # gyp build from root - npm config set user 0 - npm ci - npm run build - - name: Cache playwright binaries - uses: actions/cache@v2 - id: playwright-cache - with: - path: | - ~/.cache/ms-playwright - key: ${{ runner.os }}-build-${{ env.cache-name }} - restore-keys: ${{ runner.os }}-build-${{ env.cache-name }} - - name: Install Playwright - run: | - npx playwright install --with-deps - - # check if instance is ready - - name: Check the deployed service URL - uses: jtalk/url-health-check-action@v2 - with: - # Check the following URLs one by one sequentially - url: ${{ env.FRONTEND_URL }}|${{ env.NETWORK_URL }}|${{ env.BLOCKSCOUT_URL }} - # - # Follow redirects, or just report success on 3xx status codes - follow-redirect: false # Optional, defaults to "false" - # Fail this action after this many failed attempts - max-attempts: 30 # Optional, defaults to 1 - # Delay between retries - retry-delay: 10s # Optional, only applicable to max-attempts > 1 - # Retry all errors, including 404. This option might trigger curl upgrade. - retry-all: true # Optional, defaults to "false" - - - name: Run playwright smoke suite - working-directory: blockscout-ci-cd/tests/e2e - run: | - npm run test:smoke:account - - uses: actions/upload-artifact@v3 - if: always() - with: - name: report - path: blockscout-ci-cd/tests/e2e/html-report/index.html - - - uses: actions/upload-artifact@v3 - if: always() - with: - name: tests-results - path: blockscout-ci-cd/tests/e2e/test-results - - uses: actions/upload-artifact@v3 - if: always() - with: - name: state - path: blockscout-ci-cd/tests/e2e/state.json - -defaults: - run: - shell: bash + uses: blockscout/blockscout-ci-cd/.github/workflows/e2e_new.yaml@master + secrets: inherit diff --git a/.github/workflows/e2e_admin.yaml b/.github/workflows/e2e_admin.yaml index 0fe6e137..e8084c35 100644 --- a/.github/workflows/e2e_admin.yaml +++ b/.github/workflows/e2e_admin.yaml @@ -48,6 +48,8 @@ jobs: ${{ runner.os }}- - name: Install deps run: npm ci + - name: Lint + run: npm run lint - name: Build contracts working-directory: ./tests/contracts run: | @@ -64,6 +66,7 @@ jobs: key: ${{ runner.os }}-build-${{ env.cache-name }} restore-keys: ${{ runner.os }}-build-${{ env.cache-name }} - name: Install Playwright + if: steps.playwright-cache.outputs.cache-hit != 'true' run: npx playwright install --with-deps - name: Run playwright run: | diff --git a/.github/workflows/e2e_comparison.yaml b/.github/workflows/e2e_comparison.yaml index 169c8f5e..c9ca3ea1 100644 --- a/.github/workflows/e2e_comparison.yaml +++ b/.github/workflows/e2e_comparison.yaml @@ -3,7 +3,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: - workflow_dispatch: + push: + schedule: + - cron: '0 */3 * * *' env: ENV: prod jobs: @@ -30,6 +32,8 @@ jobs: ${{ runner.os }}- - name: Install deps run: npm ci + - name: Lint + run: npm run lint - name: Build contracts working-directory: ./tests/contracts run: | @@ -46,6 +50,7 @@ jobs: key: ${{ runner.os }}-build-${{ env.cache-name }} restore-keys: ${{ runner.os }}-build-${{ env.cache-name }} - name: Install Playwright + if: steps.playwright-cache.outputs.cache-hit != 'true' run: npx playwright install --with-deps - name: Run headed playwright uses: coactions/setup-xvfb@v1 diff --git a/.github/workflows/e2e_new.yaml b/.github/workflows/e2e_new.yaml new file mode 100644 index 00000000..9582e2cc --- /dev/null +++ b/.github/workflows/e2e_new.yaml @@ -0,0 +1,143 @@ +name: E2E tests account + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + workflow_call: + +env: + BLOCKSCOUT_URL: ${{ secrets.E2E_BLOCKSCOUT_URL }} + FRONTEND_URL: ${{ secrets.E2E_FRONTEND_URL }} + NETWORK_URL: ${{ secrets.E2E_NETWORK_URL }} + ACCOUNT_USERNAME: ${{ secrets.ACCOUNT_USERNAME }} + ACCOUNT_PASSWORD: ${{ secrets.ACCOUNT_PASSWORD }} + MAILSLURP_API_KEY: ${{ secrets.MAILSLURP_API_KEY }} + MAILSLURP_EMAIL_ID: ${{ secrets.MAILSLURP_EMAIL_ID }} + ENV: test + RESOURCE_MODE: account + PUBLIC: false + PORT: 4000 + PORT_PG: 5432 + PORT_NETWORK_HTTP: 8545 + PORT_NETWORK_WS: 8546 + WALLET: ${{ secrets.WALLET }} + PWDEBUG: 0 + LOAD_AUTH_CTX: 0 + LOAD_CONTRACTS_DATA: 0 + ACCOUNT: 1 + +jobs: + e2e_tests: + name: Tests + permissions: + id-token: write + contents: read + runs-on: ubuntu-20.04 + environment: + name: Tests + steps: + - name: checkout + uses: actions/checkout@v2 + with: + repository: blockscout/blockscout-ci-cd + path: blockscout-ci-cd + + - name: Inject slug/short variables + uses: rlespinasse/github-slug-action@v4 + + - name: Set environment variables + run: | + env_vars=${{ inputs.env_vars }} + for i in ${env_vars//,/ } + do + echo "$i" >> $GITHUB_ENV + echo "${{ inputs.appNamespace }}.${{ env.K8S_DOMAIN }}" + done + + - name: Setup node + uses: actions/setup-node@v3 + with: + node-version: v14.17.0 + - name: Cache node modules + id: cache-npm + uses: actions/cache@v3 + env: + cache-name: cache-node-modules + with: + path: ~/.npm + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}- + ${{ runner.os }}-build- + ${{ runner.os }}- + - name: Install deps + working-directory: blockscout-ci-cd/tests/e2e + run: npm ci + - name: Lint + working-directory: blockscout-ci-cd/tests/e2e + run: npm run lint + - name: Build contracts + working-directory: blockscout-ci-cd/tests/contracts + run: | + # gyp build from root + npm config set user 0 + npm ci + npm run build + - name: Cache playwright binaries + uses: actions/cache@v2 + id: playwright-cache + with: + path: | + ~/.cache/ms-playwright + key: ${{ runner.os }}-build-${{ env.cache-name }} + restore-keys: ${{ runner.os }}-build-${{ env.cache-name }} + - name: Install Playwright + working-directory: blockscout-ci-cd/tests/e2e + if: steps.playwright-cache.outputs.cache-hit != 'true' + run: | + # echo "deb http://ftp.us.debian.org/debian buster main non-free" >> /etc/apt/sources.list.d/fonts.list + npx playwright install --with-deps + + # check if instance is ready + - name: Check the deployed service URL + uses: jtalk/url-health-check-action@v2 + with: + # Check the following URLs one by one sequentially + url: ${{ env.FRONTEND_URL }}|${{ env.NETWORK_URL }}|${{ env.BLOCKSCOUT_URL }} + # + # Follow redirects, or just report success on 3xx status codes + follow-redirect: false # Optional, defaults to "false" + # Fail this action after this many failed attempts + max-attempts: 30 # Optional, defaults to 1 + # Delay between retries + retry-delay: 10s # Optional, only applicable to max-attempts > 1 + # Retry all errors, including 404. This option might trigger curl upgrade. + retry-all: true # Optional, defaults to "false" + + - name: Run playwright smoke suite + working-directory: blockscout-ci-cd/tests/e2e + run: | + npm run test:smoke:account + - uses: actions/upload-artifact@v3 + if: always() + with: + name: report + path: blockscout-ci-cd/tests/e2e/html-report/index.html + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: tests-results + path: blockscout-ci-cd/tests/e2e/test-results + - uses: actions/upload-artifact@v3 + if: always() + with: + name: state + path: blockscout-ci-cd/tests/e2e/state.json + +defaults: + run: + shell: bash diff --git a/.github/workflows/e2e_prod.yaml b/.github/workflows/e2e_prod.yaml index b12868d0..6864ff29 100644 --- a/.github/workflows/e2e_prod.yaml +++ b/.github/workflows/e2e_prod.yaml @@ -3,6 +3,7 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true on: + push: schedule: - cron: '*/180 * * * *' env: @@ -38,6 +39,8 @@ jobs: ${{ runner.os }}- - name: Install deps run: npm ci + - name: Lint + run: npm run lint - name: Build contracts working-directory: ./tests/contracts run: | @@ -54,6 +57,7 @@ jobs: key: ${{ runner.os }}-build-${{ env.cache-name }} restore-keys: ${{ runner.os }}-build-${{ env.cache-name }} - name: Install Playwright + if: steps.playwright-cache.outputs.cache-hit != 'true' run: npx playwright install --with-deps - name: Run playwright run: | diff --git a/.github/workflows/e2e_prod_performance.yaml b/.github/workflows/e2e_prod_performance.yaml index c00f4adb..7d74148e 100644 --- a/.github/workflows/e2e_prod_performance.yaml +++ b/.github/workflows/e2e_prod_performance.yaml @@ -1,6 +1,8 @@ name: E2E Prod performance tests on: - workflow_dispatch: + push: + schedule: + - cron: '*/180 * * * *' env: ENV: prod jobs: @@ -34,6 +36,8 @@ jobs: ${{ runner.os }}- - name: Install deps run: npm ci + - name: Lint + run: npm run lint - name: Build contracts working-directory: ./tests/contracts run: | @@ -50,6 +54,7 @@ jobs: key: ${{ runner.os }}-build-${{ env.cache-name }} restore-keys: ${{ runner.os }}-build-${{ env.cache-name }} - name: Install Playwright + if: steps.playwright-cache.outputs.cache-hit != 'true' run: npx playwright install --with-deps - name: Run headed playwright uses: coactions/setup-xvfb@v1 diff --git a/.github/workflows/e2e_rollup.yaml b/.github/workflows/e2e_rollup.yaml index bf2a9d55..1060b82c 100644 --- a/.github/workflows/e2e_rollup.yaml +++ b/.github/workflows/e2e_rollup.yaml @@ -32,6 +32,8 @@ jobs: ${{ runner.os }}- - name: Install deps run: npm ci + - name: Lint + run: npm run lint - name: Build contracts working-directory: ./tests/contracts run: | @@ -48,6 +50,7 @@ jobs: key: ${{ runner.os }}-build-${{ env.cache-name }} restore-keys: ${{ runner.os }}-build-${{ env.cache-name }} - name: Install Playwright + if: steps.playwright-cache.outputs.cache-hit != 'true' run: npx playwright install --with-deps - name: Run playwright run: | diff --git a/.github/workflows/e2e_verification.yaml b/.github/workflows/e2e_verification.yaml index cc68b68b..a09b44c7 100644 --- a/.github/workflows/e2e_verification.yaml +++ b/.github/workflows/e2e_verification.yaml @@ -4,6 +4,7 @@ concurrency: cancel-in-progress: true on: + push: workflow_dispatch: workflow_call: @@ -75,6 +76,9 @@ jobs: - name: Install deps working-directory: blockscout-ci-cd/tests/e2e run: npm ci + - name: Lint + working-directory: blockscout-ci-cd/tests/e2e + run: npm run lint - name: Build contracts working-directory: blockscout-ci-cd/tests/contracts run: | @@ -92,6 +96,7 @@ jobs: restore-keys: ${{ runner.os }}-build-${{ env.cache-name }} - name: Install Playwright working-directory: blockscout-ci-cd/tests/e2e + if: steps.playwright-cache.outputs.cache-hit != 'true' run: | # echo "deb http://ftp.us.debian.org/debian buster main non-free" >> /etc/apt/sources.list.d/fonts.list npx playwright install --with-deps diff --git a/.github/workflows/trigger_deploy.yaml b/.github/workflows/trigger_deploy.yaml new file mode 100644 index 00000000..256b27bd --- /dev/null +++ b/.github/workflows/trigger_deploy.yaml @@ -0,0 +1,69 @@ +name: Trigger deploy + +on: + workflow_call: + inputs: + appName: + description: 'Name of application to deploy' + required: true + default: '' + type: string + appType: + description: 'Application type - service or blockscout' + required: false + default: 'blockscout' + type: string + globalEnv: + description: 'Environment Name' + required: true + default: 'testing' + type: string + +jobs: + deploy: + name: Deploy + permissions: write-all + runs-on: ubuntu-latest + steps: + - name: Get Vault credentials + id: retrieve-vault-secrets + uses: hashicorp/vault-action@v2.4.1 + with: + url: https://vault.k8s.blockscout.com + role: ci-dev + path: github-jwt + method: jwt + tlsSkipVerify: false + exportToken: true + secrets: | + ci/data/dev/github token | WORKFLOW_TRIGGER_TOKEN ; + - name: Set WORKFLOW_ID + run: | + if ${{ inputs.appType == 'service' }}; then + echo "WORKFLOW_ID=40670344" >> "$GITHUB_ENV" + fi + if ${{ inputs.appType == 'blockscout' }}; then + echo "WORKFLOW_ID=40670342" >> "$GITHUB_ENV" + fi + - name: Trigger deploy + uses: actions/github-script@v6 + with: + debug: true + github-token: ${{env.WORKFLOW_TRIGGER_TOKEN}} + script: |- + try { + const result = await github.rest.actions.createWorkflowDispatch({ + owner: 'blockscout', + repo: "deployment-values", + workflow_id: '${{env.WORKFLOW_ID}}', + ref: "main", + inputs: { + instance: '${{ inputs.appName }}', + globalEnv: '${{ inputs.globalEnv }}', + }, + }) + console.log(result); + } catch(error) { + console.error(error); + core.setFailed(error); + }