flux-e2e.yml

---
# Copyright (C) Nicolas Lamirault <nicolas.lamirault@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# SPDX-License-Identifier: Apache-2.0

name: Flux / E2E

on: # yamllint disable-line rule:truthy
  pull_request:
    branches:
    - master
    paths:
    - 'gitops/fluxcd/clusters/**'
    - 'gitops/fluxcd/kubernetes/**'
    # push:
    #   branches:
    #     - master
    #   paths:
    #     - 'gitops/fluxcd/clusters/**'
    #     - 'gitops/fluxcd/kubernetes/**'

############################################################################
# K3S

jobs:
  k3s:
    runs-on: ubuntu-latest

    strategy:
      matrix:
        # https://github.com/k3s-io/k3s/releases
        k8s_version:
        # - v1.18.20+k3s1
        # - v1.19.13+k3s1
        - v1.20.9+k3s1
        # - v1.21.3+k3s1

    steps:
    - name: Monitor Actions
      uses: GitHubSecurityLab/actions-permissions/monitor@v1
      with:
        config: ${{ vars.PERMISSIONS_CONFIG }}

    - name: Checkout
      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

    - name: Step that prints name of pull request's base branch
      run: |
        echo "Pull request's base branch is: ${CURRENT_BRANCH}"
      env:
        CURRENT_BRANCH: ${{ github.head_ref }}
      if: github.event_name == 'pull_request'

    - name: Start k8s locally
      uses: jupyterhub/action-k3s-helm@v4
      with:
        k3s-version: ${{ matrix.k8s_version }}
        helm-version: v3.4.2 # releases:  https://github.com/helm/helm/tags
        traefik-enabled: false

    - name: Verify function of k8s, kubectl, and helm
      run: |
        echo "kubeconfig: $KUBECONFIG"
        kubectl version
        kubectl get pods --all-namespaces

        helm version
        helm list

    - name: Setup Flux CLI
      uses: fluxcd/flux2/action@v2.1.1

    - name: Install Flux in Kubernetes
      run: flux install

    - name: Create Kubernetes secret for Sops
      run: |
        echo ${SOPS_GPG_KEY} | base64 -d
        echo ${SOPS_GPG_KEY} | base64 -d | kubectl create secret generic sops-gpg --namespace=flux-system --from-file=sops.asc=/dev/stdin
      env:
        SOPS_GPG_KEY: ${{ secrets.SOPS_GPG_KEY }}

    - name: Flux
      run: |
        flux create source git flux-system \
          --url=${{ github.event.repository.html_url }} \
          --branch=${CURRENT_BRANCH}
        flux create kustomization flux-system \
          --source=flux-system \
          --path=./gitops/fluxcd/clusters/k3s/cicd/
        sleep 60

        echo "#############################"
        echo "###         Flux          ###"
        echo "#############################"
        flux check
        sleep 30

        echo "#############################"
        echo "###         CRDs          ###"
        echo "#############################"
        kubectl get crds
        sleep 30

        echo "#############################"
        echo "###     Kustomizations    ###"
        echo "#############################"
        flux get kustomizations
        sleep 30

        echo "#############################"
        echo "###      HelmRelease      ###"
        echo "#############################"
        flux get helmreleases -A
        sleep 30

        echo "=========================="
        kubectl describe helmreleases.helm.toolkit.fluxcd.io contour -n ingress-controllers
        echo "=========================="
        kubectl -n ingress-controllers describe pod -l "app.kubernetes.io/component=envoy"
        echo "=========================="

      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        CURRENT_BRANCH: ${{ github.head_ref }}

    - name: Verify reconciliation
      run: |
        echo "--> Pods"
        kubectl get pods -A

        echo "--> Kustomizations"
        flux get kustomizations

        echo "--> HelmRelease"
        flux get helmreleases -A

        echo "--> Flux System"
        kubectl -n flux-system wait kustomization/flux-system-charts --for=condition=ready --timeout=5m

        echo "--> Monitoring"
        kubectl -n flux-system wait kustomization/kube-prometheus-stack --for=condition=ready --timeout=5m
        kubectl -n flux-system wait kustomization/thanos --for=condition=ready --timeout=5m
        kubectl -n flux-system wait kustomization/grafana --for=condition=ready --timeout=5m

        echo "--> Chaos"
        kubectl -n flux-system wait kustomization/chaos-mesh --for=condition=ready --timeout=5m
        kubectl -n flux-system wait kustomization/litmus-chaos --for=condition=ready --timeout=5m

        echo "--> Ingress Controllers"
        # kubectl -n flux-system wait kustomization/ingress-nginx --for=condition=ready --timeout=5m
        kubectl -n flux-system wait kustomization/contour --for=condition=ready --timeout=5m

        # echo "--> Policies"
        kubectl -n flux-system wait kustomization/kyverno --for=condition=ready --timeout=5m
        kubectl -n flux-system wait kustomization/policy-reporter --for=condition=ready --timeout=5m

    - name: Debug failure
      if: failure()
      run: |
        kubectl get crds
        kubectl get pods -A

        kubectl -n flux-system get all
        kubectl -n flux-system logs deploy/source-controller
        kubectl -n flux-system logs deploy/kustomize-controller
        kubectl -n flux-system logs deploy/helm-controller

        kubectl get secret sops-gpg --namespace=flux-system -o yaml

        kubectl get pods -A --field-selector status.phase=Pending

        # kubectl -n monitoring logs prometheus-kube-prometheus-stack-prometheus-0