ci.yaml

name: "Run build"
on:
  pull_request:
    # Run when the PR is ready and each time a review is re-requested
    #  (i.e. after feedback has been addressed).
    types: [review_requested, ready_for_review]
  workflow_dispatch:
    inputs:
      image_formats:
        description: |
          Space-separated vendor formats to build.
        required: true
        default: qemu_uefi

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} 
  cancel-in-progress: true

permissions:
  pull-requests: write

jobs:
  packages:
    # Do not run when still in draft mode but a review was requested anyway
    if: github.event.pull_request.draft == false
    name: "Build Flatcar packages"
    runs-on:
      - self-hosted
      - debian
      - build
      - x64
    strategy:
      fail-fast: false
      matrix:
        arch: ["amd64", "arm64"]
    defaults:
      run:
        working-directory: scripts

    steps:
      - name: Prepare machine
        shell: bash
        working-directory: ${{ github.workspace }}
        run: |
          sudo rm /bin/sh
          sudo ln -s /bin/bash /bin/sh
          sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git
          sudo mkdir -p /etc/apt/keyrings
          curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
          echo \
            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
          sudo apt-get update
          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin

      - uses: actions/checkout@v3
        with:
          path: scripts
          fetch-depth: 0

      # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR
      #  instead of its head commit. That commit is not connected to any branch.
      # This causes breakage downstream e.g. when the devcontainer test wants to check out
      #  the ref in the scripts repo that corresponds to this build.
      - name: If this is a PR build, use head commit instead of the merge commit
        if: ${{ github.event.pull_request.head.sha }}
        shell: bash
        run: |
          exec 2>&1
          set -x
          set -euo pipefail

          git checkout ${{ github.event.pull_request.head.sha }}
          git submodule update

      - name: Set environment
        shell: bash
        run: |
          arch="${{ matrix.arch }}"
          echo "arch=${arch}" >> $GITHUB_ENV

          IMAGE_FORMATS="qemu_uefi"
          [ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}"
          echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV

          # Artifact root for images and torcx tarball as seen from within the container
          echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV
          echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV
          mkdir -p artifacts/torcx

          # Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace
          #  this with its IP address.
          echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV

      - name: Build packages
        shell: bash
        run: |
          exec 2>&1
          set -x
          set -euo pipefail

          source ci-automation/ci_automation_common.sh
          source sdk_container/.repo/manifests/version.txt

          version="alpha-$FLATCAR_VERSION_ID"
          check_version_string "$version"
          sdk_version="${FLATCAR_SDK_VERSION}"

          sdk_name="flatcar-sdk-${arch}"
          docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
          docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}"
          sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")"

          container_name="flatcar-ci-build"
          echo "container_name=${container_name}" >> "$GITHUB_ENV"

          # Create version file
          (
            source sdk_lib/sdk_container_common.sh
            create_versionfile "$sdk_version" "$version"
          )

          # Run the packages build. This will create the ci build container
          #  which will be re-used by subsequent build steps.
          ./run_sdk_container -n "${container_name}" -v "${version}" \
            -C "${sdk_image}" \
            ./build_packages --board="${arch}-usr" \
                --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \
                --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}"

          # Create binpkgs tarball for archiving as artifact later
          ./run_sdk_container -n "${container_name}" \
              tar -C "/build/${arch}-usr/var/lib/portage/pkgs/" \
                -cvf binpkgs.tar .

      - name: Extract build logs
        if: always()
        shell: bash
        run: |
          set -euo pipefail
          set -x
          # Copy logs
          ./run_sdk_container -n "${container_name}" \
              tar -cJf ebuild_logs.tar.xz /build/${arch}-usr/var/log/portage \
              /build/${arch}-usr/var/tmp/portage

      - name: Upload build logs
        if: always()
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-build-logs
          path: |
            scripts/ebuild_logs.tar.xz

      - name: Build image
        shell: bash
        run: |
          set -euo pipefail
          set -x

          echo 'channel="developer"' >> $GITHUB_ENV
          channel="developer"

          source ci-automation/ci_automation_common.sh

          official_arg="--noofficial"

          ./run_sdk_container -n  "${container_name}" \
                  ./set_official --board="${arch}-usr" "${official_arg}"
          ./run_sdk_container -n "${container_name}" \
                  ./build_image --board="${arch}-usr" --group="${channel}" \
                                --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \
                                --torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container

      - name: Build VM image(s)
        shell: bash
        run: |
          set -euo pipefail
          set -x

          source ci-automation/ci_automation_common.sh

          images_out="images"

          has_packet=0
          has_pxe=0
          formats="${IMAGE_FORMATS}"
          for format in "${formats}";do
            [[ "${format}" = 'packet' ]] || [[ "${format}" = 'equinix_metal' ]] && has_packet=1
            [[ "${format}" = 'pxe' ]] && has_pxe=1
          done

          [[ ${has_packet} -eq 1 ]] && [[ ${has_pxe} -eq 0 ]] && set -- 'pxe' "${@}"
          if echo "$formats" | tr ' ' '\n' | grep -q '^vmware'; then
            formats=$(echo "$formats" | tr ' ' '\n' | sed '/vmware.*/d')
            formats+=" vmware vmware_insecure vmware_ova vmware_raw"
          fi
          if echo "$formats" | tr ' ' '\n' | grep -q -P '^(ami|aws)'; then
            formats=$(echo "$formats" | tr ' ' '\n' | sed '/ami.*/d' | sed '/aws/d')
            formats+=" ami ami_vmdk"
          fi
          # Keep compatibility with SDK scripts where "equinix_metal" remains unknown.
          formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')

          for format in ${formats}; do
              echo " ###################  VENDOR '${format}' ################### "
              ./run_sdk_container -n "${container_name}" \
                  ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
                      --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \
                      --image_compression_formats=bz2
          done

          # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really)
          #  so we move things around.
          mkdir -p artifacts/images
          (
            cd artifacts/${arch}-usr/latest/
            mv * ../../images/
          )

          # create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons
          #  (such as "docker:20.10.torcx.tgz")
          mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/
          tar -C artifacts/torcx/pkgs/ -cvf torcx.tar .


      - name: Upload binpkgs
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-binpkgs
          path: |
            scripts/binpkgs.tar

      - name: Upload update image (used with kola tests later)
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-test-update
          path: |
            scripts/artifacts/images/flatcar_test_update.gz

      - name: Upload generic image
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-generic-image
          path: |
            scripts/artifacts/images/flatcar_production_image.bin.bz2
            scripts/artifacts/images/flatcar_production_image.grub
            scripts/artifacts/images/flatcar_production_image.shim
            scripts/artifacts/images/flatcar_production_image.vmlinuz
            scripts/artifacts/images/flatcar_production_image*.txt
            scripts/artifacts/images/flatcar_production_image*.json
            scripts/artifacts/images/flatcar_production_image_pcr_policy.zip
            scripts/artifacts/images/flatcar_production_*_efi_*.fd

      - name: Upload developer container
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-devcontainer
          path: |
            scripts/artifacts/images/flatcar_developer_container*

      - name: Upload torcx tarball
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-torcx
          path: |
            scripts/torcx.tar

      # Clean up what we uploaded already so the "vendor images" wildcard
      #  works when uploading artifacts in the next step.
      - name: Remove update, generic and devcontainer images
        shell: bash
        run: |
          set -euo pipefail
          set -x
          rm -f artifacts/images/flatcar_test_update.gz \
                artifacts/images/flatcar_production_image* \
                artifacts/images/flatcar_developer_container* \
                artifacts/images/flatcar_production_update*

      - name: Upload vendor images
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          name: ${{ matrix.arch }}-vm-images
          path: |
            scripts/artifacts/images/*.img.bz2
            scripts/artifacts/images/*.bin.bz2
            scripts/artifacts/images/flatcar_production_*_efi_*.fd
            scripts/artifacts/images/*.txt
            scripts/artifacts/images/flatcar_production_*.sh

  test:
    needs: packages
    name: "Run kola tests"
    uses: ./.github/workflows/run-kola-tests.yaml