deploy-to-eks.yaml

name: Create EKS cluster, deploy CKF and run bundle test
on:
  workflow_dispatch: # This event allows manual triggering from the Github UI
    inputs:
      bundle_version:
        description: 'Comma-separated list of bundle versions e.g. 1.8, 1.9, latest. Make sure that the corresponding K8s version is supported by the cloud.'
        default: '1.8, 1.9, latest'
        required: true
      k8s_version:
        description: 'Kubernetes version to be used for the AKS cluster'
        required: false
      uats_branch:
        description: 'Branch to run the UATs from e.g. main or track/1.9. By default, this is defined by the dependencies.yaml file.'
        required: false
  schedule:
    - cron: "23 0 * * 2"
jobs:
  preprocess-input:
    runs-on: ubuntu-22.04
    outputs:
      processed_bundle_versions: ${{ steps.process_bundle_versions.outputs.bundle_versions }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.x'

      - name: Install CLI tools
        run: |
          sudo snap install yq
          
      - name: Process bundle versions
        id: process_bundle_versions
        run: |
          if [[ "${{ github.event_name }}" == "schedule" ]]; then
              # Use `tr` to remove new lines as a workaround to:
              # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#multiline-strngs
              bundle_versions=$(yq '. | keys' .github/dependencies.yaml -o=json | tr -d '\n')
              echo "bundle_versions=${bundle_versions}"
              echo "bundle_versions=${bundle_versions}" >> $GITHUB_OUTPUT
          else
              python scripts/gh-actions/parse_versions.py "${{ inputs.bundle_version }}"
          fi
        
  deploy-ckf-to-eks:
    needs: preprocess-input
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        bundle_version: ${{ fromJSON(needs.preprocess-input.outputs.processed_bundle_versions) }}
      fail-fast: false
    env:
      PYTHON_VERSION: "3.8"

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Run YAML to Github Output Action
        id: yaml-output
        uses: christian-ci/action-yaml-github-output@v2
        with:
          file_path: ".github/dependencies.yaml"
          main_key: ${{ matrix.bundle_version }}

      - name: Update ENV variables from inputs if available
        run: |
          K8S_VERSION=${{ inputs.k8s_version || env.K8S_VERSION }}
          echo "K8S_VERSION=${K8S_VERSION}" >> $GITHUB_ENV
          UATS_BRANCH=${{ inputs.uats_branch || env.UATS_BRANCH }}
          echo "UATS_BRANCH=${UATS_BRANCH}" >> $GITHUB_ENV

      # Remove once https://github.com/canonical/bundle-kubeflow/issues/761
      # is resolved and applied to uats repository.
      - name: Install python ${{ env.PYTHON_VERSION }}
        run: |
          sudo add-apt-repository ppa:deadsnakes/ppa -y
          sudo apt update -y
          sudo apt install python${{ env.PYTHON_VERSION }} python${{ env.PYTHON_VERSION }}-distutils python${{ env.PYTHON_VERSION }}-venv -y

      - name: Install CLI tools
        run: |
          wget https://bootstrap.pypa.io/get-pip.py
          python${{ env.PYTHON_VERSION }} get-pip.py
          python${{ env.PYTHON_VERSION }} -m pip install tox
          sudo snap install charmcraft --classic
          # We need to install from binary because of this https://bugs.launchpad.net/juju/+bug/2007575
          curl -LO https://launchpad.net/juju/${{ env.JUJU_VERSION }}/${{ env.JUJU_VERSION_WITH_PATCH }}/+download/juju-${{ env.JUJU_VERSION_WITH_PATCH }}-linux-amd64.tar.xz
          tar xf juju-${{ env.JUJU_VERSION_WITH_PATCH }}-linux-amd64.tar.xz
          sudo install -o root -g root -m 0755 juju /usr/local/bin/juju
          juju version

      - name: Configure AWS Credentials
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.BUNDLE_KUBEFLOW_EKS_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.BUNDLE_KUBEFLOW_EKS_AWS_SECRET_ACCESS_KEY }}
        run: |
          aws configure set aws_access_key_id $AWS_ACCESS_KEY_ID
          aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY
          aws configure set default.region eu-central-1
      
      - name: Install kubectl
        run: |
          sudo snap install kubectl --classic --channel=${{ env.K8S_VERSION }}/stable
          mkdir ~/.kube
          kubectl version --client
      
      - name: Install eksctl
        run: |
          sudo apt-get update
          sudo apt-get install -y unzip
          curl --silent --location "https://github.com/weaveworks/eksctl/releases/latest/download/eksctl_$(uname -s)_amd64.tar.gz" | tar xz -C /tmp
          sudo mv /tmp/eksctl /usr/local/bin
          eksctl version
      
      - name: Create cluster
        run: |
          VERSION=${{ matrix.bundle_version }}
          VERSION_WITHOUT_DOT="${VERSION//.}"
          yq e ".metadata.name |= \"kubeflow-test-$VERSION_WITHOUT_DOT\"" -i .github/cluster.yaml
          yq e ".metadata.version |= \"${{ env.K8S_VERSION }}\"" -i .github/cluster.yaml
          eksctl create cluster -f .github/cluster.yaml
          kubectl get nodes
      
      - name: Setup juju
        run: |
          juju add-k8s eks --client
          juju bootstrap eks kubeflow-controller
          juju add-model kubeflow
      
      - name: Test bundle deployment
        run: |
          tox -vve test_bundle_deployment-${{ matrix.bundle_version }} -- --model kubeflow --keep-models -vv -s

      - name: Run Kubeflow UATs
        run: |
          git clone https://github.com/canonical/charmed-kubeflow-uats.git ~/charmed-kubeflow-uats
          cd ~/charmed-kubeflow-uats
          git checkout ${{ env.UATS_BRANCH }}
          tox -e kubeflow-remote

      # On failure, capture debugging resources
      - name: Save debug artifacts
        uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main
        if: failure() || cancelled()

      # On failure, capture debugging resources
      - name: Get juju status
        run: juju status
        if: failure() || cancelled()

      - name: Get juju debug logs
        run: juju debug-log --replay --no-tail
        if: failure() || cancelled()

      - name: Get all kubernetes resources
        run: kubectl get all -A
        if: failure() || cancelled()

      - name: Get logs from pods with status = Pending
        run: kubectl -n kubeflow get pods | tail -n +2 | grep Pending | awk '{print $1}' | xargs -n1 kubectl -n kubeflow logs --all-containers=true --tail 100
        if: failure() || cancelled()

      - name: Get logs from pods with status = Failed
        run: kubectl -n kubeflow get pods | tail -n +2 | grep Failed | awk '{print $1}' | xargs -n1 kubectl -n kubeflow logs --all-containers=true --tail 100
        if: failure() || cancelled()

      - name: Get logs from pods with status = CrashLoopBackOff
        run: kubectl -n kubeflow get pods | tail -n +2 | grep CrashLoopBackOff | awk '{print $1}' | xargs -n1 kubectl -n kubeflow logs --all-containers=true --tail 100
        if: failure() || cancelled()

      - name: Delete EKS cluster
        if: always()
        run: |
          VERSION=${{ matrix.bundle_version }}
          VERSION_WITHOUT_DOT="${VERSION//.}"
          eksctl delete cluster --region eu-central-1 --name=kubeflow-test-$VERSION_WITHOUT_DOT

        # Clean up leftover CloudFormation stack when cluster creation fails unexpectedly
      - name: Delete CloudFormation stack (if present)
        if: always()
        run: |
          VERSION=${{ matrix.bundle_version }}
          VERSION_WITHOUT_DOT="${VERSION//.}"
          aws cloudformation delete-stack --region eu-central-1 --stack-name eksctl-kubeflow-test-$VERSION_WITHOUT_DOT-cluster

  delete-unattached-volumes:
    if: always()
    uses: ./.github/workflows/delete-aws-volumes.yaml
    secrets: inherit
    with:
      region: eu-central-1
    needs: [deploy-ckf-to-eks]