main.yml

name: CI

# Controls when the action will run. Triggers the workflow on push
# or pull request events, but only for the primary branch.
on:
  push:
    branches: [master]
    paths-ignore:
      - 'site/**'
  pull_request:
    branches: [master]
    paths-ignore:
      - 'site/**'


env:
  CARGO_TERM_COLOR: always
  CARGO_CACHE_VERSION: 0.8.2
  RUSTFLAGS: "-C link-arg=-fuse-ld=lld"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  gnu-binaries:
    runs-on: ubuntu-2404-large

    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
          submodules: true
          lfs: true

      - uses: actions/setup-go@v2
        with:
          go-version: "1.21"

      - uses: dtolnay/rust-toolchain@stable
      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Google Cloud Auth
        uses: google-github-actions/auth@v2
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'

      - name: Set up Google Cloud SDK
        uses: google-github-actions/setup-gcloud@v2
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}

      - uses: supabase/setup-cli@v1
        with:
          version: latest
      - run: supabase start
      - name: SQL tests
        run: ./supabase/run_sql_tests.sh

      - run: make extra-ci-runner-setup
      - run: make print-versions

      - uses: Swatinem/rust-cache@v2
        with:
          prefix-key: "2404"

      - name: Cache/Restore Go dependencies.
        uses: actions/cache@v2
        with:
          path: ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

      - run: make install-tools
      - run: go mod download
      - run: make rust-gnu-test
      - run: make linux-gnu-binaries

      - name: Ensure that we didn't dirty the tree.
        run: git clean --force -d && git diff --exit-code || exit 1

      - name: Upload Binaries
        uses: actions/upload-artifact@v4
        with:
          name: linux-gnu-binaries
          path: |
            .build/package/bin/*
            target/x86_64-unknown-linux-gnu/release/libbindings.a
            crates/bindings/flow_bindings.h
            target/x86_64-unknown-linux-gnu/release/librocksdb-exp/

  musl-binaries:
    runs-on: ubuntu-2404-large

    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
          submodules: true

      - uses: dtolnay/rust-toolchain@stable
        with:
          targets: x86_64-unknown-linux-musl
      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - run: make extra-ci-runner-setup
      - run: make print-versions

      - uses: Swatinem/rust-cache@v2
        with:
          prefix-key: "2404"

      - run: go mod download
      - run: make rust-musl-test
      - run: make linux-musl-binaries

      - name: Upload Binaries
        uses: actions/upload-artifact@v4
        with:
          name: linux-musl-binaries
          path: |
            .build/package/bin/*

  assembly:
    runs-on: ubuntu-2404-large
    needs: [gnu-binaries, musl-binaries]

    services:
      postgres:
        image: postgres:latest
        env:
          POSTGRES_DB: flow
          POSTGRES_USER: flow
          POSTGRES_PASSWORD: flow
        ports:
          - 5432:5432
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
          submodules: true

      - name: Install Protoc
        uses: arduino/setup-protoc@v1
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - uses: actions/setup-go@v2
        with:
          go-version: "1.23"

      - name: Cache/Restore Go dependencies.
        uses: actions/cache@v2
        with:
          path: ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

      - run: make extra-ci-runner-setup
      - run: make print-versions

      - name: Google Cloud Auth
        uses: google-github-actions/auth@v2
        with:
          credentials_json: '${{ secrets.GCP_SA_KEY }}'

      - name: Set up Google Cloud SDK
        uses: google-github-actions/setup-gcloud@v2
        with:
          project_id: ${{ secrets.GCP_PROJECT_ID }}

      - name: Login to GitHub container registry
        run: |
          echo "${{ secrets.GITHUB_TOKEN }}" | \
            docker login --username ${{ github.actor }} --password-stdin ghcr.io

      - name: Download gnu-linux-binaries
        uses: actions/download-artifact@v4
        with:
          name: linux-gnu-binaries
          path: linux-gnu-binaries

      - name: Move linux-gnu-binaries to PKGBIN
        run: |
          mkdir -p .build/package/bin && \
          mv linux-gnu-binaries/.build/package/bin/* .build/package/bin && \
          mkdir -p target/x86_64-unknown-linux-gnu/release && \
          mv -f linux-gnu-binaries/target/x86_64-unknown-linux-gnu/release/* target/x86_64-unknown-linux-gnu/release/ && \
          mv -f linux-gnu-binaries/crates/bindings/flow_bindings.h crates/bindings/

      - name: Download linux-musl-binaries
        uses: actions/download-artifact@v4
        with:
          name: linux-musl-binaries
          path: linux-musl-binaries

      - name: Move linux-musl-binaries to PKGBIN
        run: |
          mkdir -p .build/package/bin && \
          mv linux-musl-binaries/* .build/package/bin

      # Downloaded artifacts lose their prior permission settings
      - name: Adjust binaries permissions
        run: |
          chmod +x .build/package/bin/*

      - name: Add PKGDIR/bin to PATH
        run: echo "$GITHUB_WORKSPACE/.build/package/bin" >> $GITHUB_PATH

      - name: make catalog-test
        run: |
          make catalog-test SKIP_BUILD=true

      - name: make go-test-ci
        run: |
          make go-test-ci SKIP_BUILD=true

      - name: make end-to-end-test
        run: |
          make end-to-end-test SKIP_BUILD=true

      - name: make package (tar only)
        run: |
          make /home/runner/work/flow/flow/.build/package/flow-x86-linux.tar.gz

      - name: Ensure that we didn't dirty the tree.
        run: git clean --force -d && git diff --exit-code || exit 1

      - run: make docker-image

      - run: make docker-push
      - if: ${{ github.ref == 'refs/heads/master' }}
        run: make docker-push-dev

      - if: ${{ github.ref == 'refs/heads/master' }}
        name: Upload release assets
        uses: softprops/action-gh-release@v1
        with:
          body: |
            This is the latest development build, which gets updated on every successful build of the master branch.
          target_commitish: ${{ github.sha }}
          draft: false
          prerelease: true
          tag_name: dev
          generate_release_notes: false
          files: .build/package/flow-x86-linux.tar.gz