test_publish.yml

name: Test, lint and publish

on:
  push:
    branches:
      - main
    tags:
      - "v*"
  pull_request:
    branches:
concurrency:
  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
  cancel-in-progress: true
env:
  DEVCONTAINER_REGISTRY: ghcr.io
  DEVCONTAINER_IMAGE_NAME: ${{ github.repository }}/devcontainer
  DEVCONTAINER_IMAGE_TAG_MAIN: "cache-main"
  CACHE_KEY: main-branch-poetry-cache-ubuntu
  CACHE_PATH: .devcontainer/.poetry_cache

jobs:
  build-devcontainer:
    runs-on: ubuntu-latest
    outputs:
      image_repository: ${{ steps.docker_image.outputs.image_repository }}
      image_tag: ${{ steps.docker_image.outputs.image_tag }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Docker image metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.DEVCONTAINER_REGISTRY }}/${{ env.DEVCONTAINER_IMAGE_NAME }}
          tags: |
            type=ref,event=pr,prefix=cache-pr-,priority=600
            type=ref,event=branch,prefix=cache-,priority=500
            type=ref,event=tag,prefix=cache-,priority=500
          flavor: |
            latest=false
      - name: Extract Docker image name
        id: docker_image
        env:
          IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
        run: |
          IMAGE=$(echo "$IMAGE_TAGS" | cut -d" " -f1)
          IMAGE_REPOSITORY=$(echo "$IMAGE" | cut -d":" -f1)
          IMAGE_TAG=$(echo "$IMAGE" | cut -d":" -f2)
          echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
          echo "image_repository=$IMAGE_REPOSITORY" >> "$GITHUB_OUTPUT"
          echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT"
      - uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Pre-build devcontainer
        uses: devcontainers/ci@v0.3
        continue-on-error: true
        with:
          push: always
          skipContainerUserIdUpdate: false
          imageName: ${{ steps.docker_image.outputs.image_repository }}
          imageTag: ${{ steps.docker_image.outputs.image_tag }}
          cacheFrom: |
            ${{ steps.docker_image.outputs.image_repository }}:${{ steps.docker_image.outputs.image_tag }}
            ${{ steps.docker_image.outputs.image_repository }}:${{ env.DEVCONTAINER_IMAGE_TAG_MAIN }}
  style-checks:
    runs-on: ubuntu-latest
    needs:
      - build-devcontainer
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/cache/restore@v4
        name: Restore cache
        with:
          path: ${{ env.CACHE_PATH }}
          key: ${{ env.CACHE_KEY }}
      - name: Set Git config
        shell: bash
        run: |
          git config --add user.name "Renku Bot"
          git config --add user.email "renku@datascience.ch"
      - name: Style checks
        uses: devcontainers/ci@v0.3
        with:
          runCmd: make style_checks
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
  optional-checks:
    runs-on: ubuntu-latest
    needs:
      - build-devcontainer
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/cache/restore@v4
        name: Restore cache
        with:
          path: ${{ env.CACHE_PATH }}
          key: ${{ env.CACHE_KEY }}
      - name: Set Git config
        shell: bash
        run: |
          git config --add user.name "Renku Bot"
          git config --add user.email "renku@datascience.ch"
      - name: Check Avro Schemas are up to date
        uses: devcontainers/ci@v0.3
        with:
          runCmd: make check_avro
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
  test-main:
    runs-on: ubuntu-latest
    needs:
      - build-devcontainer
      - style-checks
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/cache/restore@v4
        name: Restore cache
        with:
          path: ${{ env.CACHE_PATH }}
          key: ${{ env.CACHE_KEY }}
      - name: Set Git config
        shell: bash
        run: |
          git config --add user.name "Renku Bot"
          git config --add user.email "renku@datascience.ch"
      - name: Test setup
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make test_setup
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
      - name: Main tests
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make main_tests
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
      - name: Combine coverage data
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make collect_coverage
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
      - name: Coveralls Parallel
        uses: coverallsapp/github-action@v2
        with:
          flag-name: run-main-tests
          parallel: true
          path-to-lcov: coverage.lcov
  test-schemathesis:
    runs-on: ubuntu-latest
    needs:
      - build-devcontainer
      - style-checks
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/cache/restore@v4
        name: Restore cache
        with:
          path: ${{ env.CACHE_PATH }}
          key: ${{ env.CACHE_KEY }}
      - name: Set Git config
        shell: bash
        run: |
          git config --add user.name "Renku Bot"
          git config --add user.email "renku@datascience.ch"
      - name: Test setup
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make test_setup
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
      - name: Schemathesis Tests
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make schemathesis_tests
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
          env: |
            HYPOTHESIS_PROFILE=ci
      - name: Combine coverage data
        uses: devcontainers/ci@v0.3
        with:
          runCmd: |
            make collect_coverage
          push: never
          skipContainerUserIdUpdate: false
          cacheFrom: ${{ needs.build-devcontainer.outputs.image_repository }}:${{ needs.build-devcontainer.outputs.image_tag }}
      - name: Coveralls Parallel
        uses: coverallsapp/github-action@v2
        with:
          flag-name: run-schemathesis-tests
          parallel: true
          path-to-lcov: coverage.lcov
  coveralls:
    needs:
      - test-main
      - test-schemathesis
    runs-on: ubuntu-latest
    steps:
      - name: Coveralls Finished
        uses: coverallsapp/github-action@v2
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}
          COVERALLS_SERVICE_NAME: gihub-action
        with:
          parallel-finished: true
          carryforward: "run-main-tests,run-schemathesis-tests"
        continue-on-error: true

  publish:
    runs-on: ubuntu-latest
    needs:
      - test-main
      - test-schemathesis
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Docker meta data services
        id: meta
        uses: docker/metadata-action@v5
        with:
          # list of Docker images to use as base name for tags
          images: |
            renku/renku-data-service
          # generate Docker tags based on the following events/attributes
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=sha
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.RENKU_DOCKER_USERNAME }}
          password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
      - name: Build and push data services
        uses: docker/build-push-action@v4
        with:
          context: .
          file: ./projects/renku_data_service/Dockerfile
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=registry,ref=renku/renku-data-service:buildcache
          cache-to: type=registry,ref=renku/renku-data-service:buildcache,mode=max
      - name: Docker meta background jobs
        id: meta-background-jobs
        uses: docker/metadata-action@v5
        with:
          # list of Docker images to use as base name for tags
          images: |
            renku/data-service-background-jobs
          # generate Docker tags based on the following events/attributes
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=sha
      - name: Build and push background jobs
        uses: docker/build-push-action@v4
        with:
          context: .
          file: ./projects/background_jobs/Dockerfile
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ steps.meta-background-jobs.outputs.tags }}
          labels: ${{ steps.meta-background-jobs.outputs.labels }}
          cache-from: type=registry,ref=renku/data-service-background-jobs:buildcache
          cache-to: type=registry,ref=renku/data-service-background-jobs:buildcache,mode=max
      - name: Docker meta secrets
        id: meta-secrets-storage
        uses: docker/metadata-action@v5
        with:
          # list of Docker images to use as base name for tags
          images: |
            renku/secrets-storage
          # generate Docker tags based on the following events/attributes
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=sha
      - name: Build and push secrets image
        uses: docker/build-push-action@v4
        with:
          context: .
          file: ./projects/secrets_storage/Dockerfile
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ steps.meta-secrets-storage.outputs.tags }}
          labels: ${{ steps.meta-secrets-storage.outputs.labels }}
          cache-from: type=registry,ref=renku/secrets-storage:buildcache
          cache-to: type=registry,ref=renku/secrets-storage:buildcache,mode=max