k8s_e2e.yml

name: K8S E2E Suite

on:
  workflow_dispatch:
  push:
    branches:
      - master
    paths:
      - ".github/workflows/k8s_e2e.yml"
      - ".cargo/**"
      - "benches/**"
      - "lib/**"
      - "proto/**"
      - "scripts/**"
      - "skaffold/**"
      - "src/**"
      - "tests/**"
      - "build.rs"
      - "Cargo.lock"
      - "Cargo.toml"
      - "Makefile"
      - "rust-toolchain"
      - "distribution/**"
  pull_request:

concurrency:
  # For pull requests, cancel running workflows, for master, run all
  #
  # `github.event.number` exists for pull requests, otherwise fall back to SHA
  # for master
  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
  cancel-in-progress: true

env:
  AWS_ACCESS_KEY_ID: "dummy"
  AWS_SECRET_ACCESS_KEY: "dummy"
  CONTAINER_TOOL: "docker"
  RUST_BACKTRACE: full
  RUST_TEST_THREADS: 1
  TEST_LOG: vector=debug
  VERBOSE: true
  DISABLE_MOLD: true
  CI: true
  PROFILE: debug

jobs:
  build-x86_64-unknown-linux-gnu:
    name: Build - x86_64-unknown-linux-gnu
    runs-on: [linux, test-runner]
    if: |
      !github.event.pull_request
        || contains(github.event.pull_request.labels.*.name, 'ci-condition: k8s e2e tests enable')
        || contains(github.event.pull_request.labels.*.name, 'ci-condition: k8s e2e all targets')
    # cargo-deb requires a release build, but we don't need optimizations for tests
    env:
      CARGO_PROFILE_RELEASE_OPT_LEVEL: 0
      CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 256
      CARGO_INCREMENTAL: 0
    steps:
      - uses: actions/checkout@v3
      - run: make ci-sweep
      - uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry
            ~/.cargo/git
          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
      - run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh
      - run: bash scripts/environment/prepare.sh
      - run: echo "::add-matcher::.github/matchers/rust.json"
      - run: make package-deb-x86_64-unknown-linux-gnu
      - uses: actions/upload-artifact@v3
        with:
          name: e2e-test-deb-package
          path: target/artifacts/*

  # Github Actions don't support `matrix` at the job-level `if:` condition.
  # We apply this workaround - compute `matrix` in a preceding job, and assign
  # it's value dynamically at the actual test job.
  # This approach can be advanced further by, for instance, dynamically
  # detecting versions of various components, or reading them from `.meta`.
  # See https://github.community/t/feature-request-and-use-case-example-to-allow-matrix-in-if-s/126067
  compute-k8s-test-plan:
    name: Compute K8s test plan
    runs-on: [linux, test-runner]
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    if: |
      !github.event.pull_request
        || contains(github.event.pull_request.labels.*.name, 'ci-condition: k8s e2e tests enable')
        || contains(github.event.pull_request.labels.*.name, 'ci-condition: k8s e2e all targets')
    steps:
      - uses: actions/github-script@v6.3.3
        id: set-matrix
        with:
          script: |
            // Parameters.
            const minikube_version = [
              "v1.24.0",
            ]

            // Aim to test against oldest supported k8s cloud-provider versions
            // https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html
            // https://cloud.google.com/kubernetes-engine/docs/release-notes
            // https://docs.microsoft.com/en-us/azure/aks/supported-kubernetes-versions?tabs=azure-cli#aks-kubernetes-release-calendar
            const kubernetes_version = [
              { version: "v1.23.3", is_essential: true },
              { version: "v1.22.5", is_essential: true },
              { version: "v1.21.8", is_essential: true },
              { version: "v1.20.14", is_essential: true },
              { version: "v1.19.8" },
            ]
            const container_runtime = [
              "docker",
              "containerd",
              // https://github.com/kubernetes/minikube/issues/12928
              // "crio",
            ]
            const ci_condition_label = 'ci-condition: k8s e2e all targets'

            // Planing.
            const is_in_pull_request = !!context.payload.pull_request;
            const should_test_all_targets = (
              !is_in_pull_request ||
              context.payload.pull_request.labels.some(label => label.name === ci_condition_label)
            )
            const filter_targets = array => array.filter(val => should_test_all_targets || val.is_essential)

            const matrix = {
              minikube_version,
              kubernetes_version: filter_targets(kubernetes_version).map(e => ({
                version: e.version,
                role: e.is_essential ? "essential" : "extra",
              })),
              container_runtime,
            }
            core.setOutput('matrix', matrix)
      - name: Dump matrix context
        env:
          MATRIX_CONTEXT: ${{ toJson(steps.set-matrix.outputs.matrix) }}
        run: echo "$MATRIX_CONTEXT"

  test-e2e-kubernetes:
    name: K8s ${{ matrix.kubernetes_version.version }} / ${{ matrix.container_runtime }} (${{ matrix.kubernetes_version.role }})
    runs-on: [linux, test-runner]
    needs:
      - build-x86_64-unknown-linux-gnu
      - compute-k8s-test-plan
    strategy:
      matrix: ${{ fromJson(needs.compute-k8s-test-plan.outputs.matrix) }}
      fail-fast: false
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - uses: actions/download-artifact@v3
        with:
          name: e2e-test-deb-package
          path: target/artifacts
      - name: Setup Minikube
        run: scripts/ci-setup-minikube.sh
        env:
          KUBERNETES_VERSION: ${{ matrix.kubernetes_version.version }}
          MINIKUBE_VERSION: ${{ matrix.minikube_version }}
          CONTAINER_RUNTIME: ${{ matrix.container_runtime }}
      - run: make test-e2e-kubernetes
        env:
          USE_MINIKUBE_CACHE: "true"
          SKIP_PACKAGE_DEB: "true"
          CARGO_INCREMENTAL: 0

  master-failure:
    name: master-failure
    if: failure() && github.ref == 'refs/heads/master'
    needs:
      - build-x86_64-unknown-linux-gnu
      - compute-k8s-test-plan
      - test-e2e-kubernetes
    runs-on: ubuntu-20.04
    steps:
    - name: Discord notification
      env:
        DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}
      uses: Ilshidur/action-discord@0.3.2
      with:
        args: "Master k8s e2e tests failed: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}>"