build_and_upload.yml

name: Build and Upload

on:
  # for manual release on the v1 release channel
  workflow_dispatch:
    inputs:
      ref:
        description: 'ref to build'
        required: true
        type: string
      release_channel:
        description: 'release channel'
        required: true
        type: choice
        options:
        - "v1"
        - "unstable"

  # for automatic release from main on the unstable release channel
  workflow_call:
    inputs:
      ref:
        description: 'ref to build'
        required: true
        type: string
      release_channel:
        description: 'release channel'
        required: true
        type: string
    secrets:
      RWX_STAGING_CREATE_RELEASE_ACCESS_TOKEN:
        required: true
      RWX_PRODUCTION_CREATE_RELEASE_ACCESS_TOKEN:
        required: true
      AWS_S3_ABQ_RELEASES_DEVELOPMENT_ACCESS_KEY_ID:
        required: true
      AWS_S3_ABQ_RELEASES_DEVELOPMENT_SECRET_ACCESS_KEY:
        required: true
      AWS_S3_ABQ_RELEASES_STAGING_ACCESS_KEY_ID:
        required: true
      AWS_S3_ABQ_RELEASES_STAGING_SECRET_ACCESS_KEY:
        required: true
      AWS_S3_ABQ_RELEASES_PRODUCTION_ACCESS_KEY_ID:
        required: true
      AWS_S3_ABQ_RELEASES_PRODUCTION_SECRET_ACCESS_KEY:
        required: true
      RWX_APPLE_DEVELOPER_ID_APPLICATION_CERT:
        required: true
      RWX_APPLE_APP_STORE_CONNECT_API_KEY:
        required: true

jobs:
  release:
    name: Build, Upload, Release ABQ
    strategy:
      # in the future we'll want to release against different architectures
      matrix:
        include:
          - runs-on: ubuntu-latest
            # todo: deprecate plaform key when we move to RFC 20 style abq uploads
            deprecated-platform: linux_x86-64
            os: linux
            architecture: x86_64
            cross-target: 'x86_64-unknown-linux-musl'
            install-musl-tools: true
          - runs-on: ubuntu-latest
            deprecated-platform: linux_aarch64
            os: linux
            architecture: aarch64
            cross-target: 'aarch64-unknown-linux-musl'
            container: messense/rust-musl-cross:aarch64-musl@sha256:777bd4c61179c38dc213bb8472500584646d28fd4a7c3e0b30b9ef70cb446d58
          - runs-on: macos-11 # use an older version for broader osx support
            deprecated-platform: darwin_x86-64
            os: darwin
            architecture: x86_64
            cross-target: ''
          - runs-on: macos-11 # first OS X to support arm64 -- so the first os for cross compilation
            deprecated-platform: darwin_aarch64
            os: darwin
            architecture: aarch64
            cross-target: 'aarch64-apple-darwin'
    runs-on: ${{ matrix.runs-on }}
    container: ${{ matrix.container }}
    outputs:
      abq_version: ${{ steps.abq_version.outputs.abq_version }}
    env:
      AWS_DEFAULT_REGION: us-east-2
      AWS_DEFAULT_OUTPUT: json
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0 # get whole history for versioning
          ref: ${{inputs.ref}}

      - name: Mark safe directory
        run: |
          # For some reason we need this, even though actions/checkout sets
          # safe.directory by default?
          # Perhaps it is because we work in a container?
          git config --global --add safe.directory "$PWD"
          git describe

      # this may not be unecessary when https://github.com/actions/checkout/issues/882 is resolved
      - name: re-fetch overwritten tag
        run: git fetch --tags --force

      - name: Restore cargo cache
        uses: actions/cache@v3.3.1
        env:
          cache-name: ci
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: cargo-release-${{ matrix.os }}-${{ matrix.architecture }}-${{ hashFiles('rust-toolchain.toml')}}-${{ hashFiles('Cargo.lock') }}
          restore-keys: |
            cargo-release-${{ matrix.os }}-${{ matrix.architecture }}-${{ hashFiles('rust-toolchain.toml')}}-
      - name: Install Rust toolchain
        uses: rwx-research/rust-toolchain@abq
        with:
          toolchain: 1.65.0
          target: ${{ matrix.cross-target }}

      # We don't build a musl ABQ on MacOS
      - name: Install musl libc tools
        if: matrix.install-musl-tools
        run: |
          sudo apt-get install -y musl-tools

      - name: Build release
        if: '!matrix.cross-target'
        run: cargo build --release --all-features

      - name: Build release
        if: matrix.cross-target
        run: cargo build --release --all-features --target="${{ matrix.cross-target }}"

      - name: Prepare release environment
        run: |
          ABQ_VERSION="$(cat build_artifact/abq_version.txt)"
          RELEASE_DIR="abq_${ABQ_VERSION}_${{ matrix.deprecated-platform }}"
          # deprecated, remove these when we fully migrate to RFC-20 style storage
          RELEASE_ARCHIVE="${RELEASE_DIR}.tar.gz"
          # drop `NEW_` when we migrate to RFC-20 style storage
          {
          echo "ABQ_VERSION=$ABQ_VERSION"
          echo "RELEASE_DIR=$RELEASE_DIR"
          echo "RELEASE_ARCHIVE=$RELEASE_ARCHIVE"
          echo "RELEASE_S3_OBJECT=abq/${ABQ_VERSION}/${RELEASE_ARCHIVE}"
          echo "RELEASE_S3_NIGHTLY_OBJECT=abq/nightly/abq_nightly_${{ matrix.deprecated-platform }}"

          echo "NEW_RELEASE_S3_OBJECT=abq/${ABQ_VERSION}/${{ matrix.os }}/${{ matrix.architecture }}/abq"
          echo "NEW_RELEASE_S3_NIGHTLY_OBJECT=abq/nightly/${{ matrix.os }}/${{ matrix.architecture }}/abq"

          echo "NEW_RELEASE_S3_OBJECT_TESTER_HARNESS=abq/${ABQ_VERSION}/${{ matrix.os }}/${{ matrix.architecture }}/abq_tester_harness"
          echo "NEW_RELEASE_S3_NIGHTLY_OBJECT_TESTER_HARNESS=abq/nightly/${{ matrix.os }}/${{ matrix.architecture }}/abq_tester_harness"
          } >> "$GITHUB_ENV"

      - name: Move release to consistent directory
        run: |
          mkdir "${RELEASE_DIR}"
          if [ -z "${{ matrix.cross-target }}" ]; then
            cp target/release/abq "${RELEASE_DIR}/"
            cp target/release/abq_tester_harness "${RELEASE_DIR}/"
          else
            cp target/${{ matrix.cross-target }}/release/abq "${RELEASE_DIR}/"
            cp target/${{ matrix.cross-target }}/release/abq_tester_harness "${RELEASE_DIR}/"
          fi

      - name: Sign & Notarize Release
        if: matrix.os == 'darwin'
        run: |
          (rcodesign --version | grep "$CODESIGN_VERSION")  || cargo install apple-codesign --force --locked --version "$CODESIGN_VERSION"
          echo "$RWX_APPLE_DEVELOPER_ID_APPLICATION_CERT" > rwx-developer-id-application-cert.pem
          # first we sign the binary. This happens locally.
          rcodesign sign --pem-source rwx-developer-id-application-cert.pem --code-signature-flags runtime "$RELEASE_DIR/abq"
          zip -r abq.zip "$RELEASE_DIR"
          echo "$RWX_APPLE_APP_STORE_CONNECT_API_KEY" > rwx-apple-app-store-connect-api-key.json
          submission_id="$(rcodesign notary-submit --wait --api-key-path rwx-apple-app-store-connect-api-key.json abq.zip 2>&1 | tee /dev/fd/2 | grep "created submission ID:" | awk '{print $4}')"
          echo "Submission ID: $submission_id"
          echo "submission_id=$submission_id" >> "$GITHUB_OUTPUT"
        env:
          CODESIGN_VERSION: 0.22.0
          RWX_APPLE_DEVELOPER_ID_APPLICATION_CERT: ${{ secrets.RWX_APPLE_DEVELOPER_ID_APPLICATION_CERT }}
          RWX_APPLE_APP_STORE_CONNECT_API_KEY: ${{ secrets.RWX_APPLE_APP_STORE_CONNECT_API_KEY }}
      - name: ensure sign & notary cleanup happens
        if: matrix.os == 'darwin' && always()
        run: rm -f rwx-apple-app-store-connect-api-key.json rwx-developer-id-application-cert.pem

      # Deprecated. Remove archiving release when we fully migrate to RFC-20 style storage
      - name: (Deprecated) Archive release
        # we do this in its own step so that for darwin, abq is already signed
        run: tar -czvf "${RELEASE_ARCHIVE}" "${RELEASE_DIR}"

      - name: Install tools to container that are baked into default image
        if: matrix.container
        # we need to install aws tools if we use a container. The default images have the tools baked in.
        # The AWS code is approximately copied from the native image, https://github.com/actions/runner-images/blob/4aeccc7b5b5a63058b2d8a9a29072708e1f267bf/images/linux/scripts/installers/aws.sh#L11
        # perhaps we should fork the musl container
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip -qq awscliv2.zip
          ./aws/install -i /usr/local/aws-cli -b /usr/local/bin

          curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o "session-manager-plugin.deb"
          apt install -y ./session-manager-plugin.deb

          # Download & install the latest aws sam cli release
          curl -L "https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip" -o "aws-sam-cli-linux-x86_64.zip"
          mkdir aws-sam
          unzip aws-sam-cli-linux-x86_64.zip -d aws-sam
          ./aws-sam/install

          apt-get update && apt-get install jq -y

      - name: Push release to development S3
        env:
          RELEASE_BUCKET: abq-releases-development
        run: |
          scripts/upload-to-aws.sh ${{ secrets.AWS_S3_ABQ_RELEASES_DEVELOPMENT_ACCESS_KEY_ID }} ${{ secrets.AWS_S3_ABQ_RELEASES_DEVELOPMENT_SECRET_ACCESS_KEY }} development

      - name: Push release to staging S3
        env:
          RELEASE_BUCKET: abq-releases-staging
        run: |
          scripts/upload-to-aws.sh ${{ secrets.AWS_S3_ABQ_RELEASES_STAGING_ACCESS_KEY_ID }} ${{ secrets.AWS_S3_ABQ_RELEASES_STAGING_SECRET_ACCESS_KEY }} staging
      - name: Push release to production S3
        env:
          RELEASE_BUCKET: abq-releases
        run: |
          scripts/upload-to-aws.sh ${{ secrets.AWS_S3_ABQ_RELEASES_PRODUCTION_ACCESS_KEY_ID }} ${{ secrets.AWS_S3_ABQ_RELEASES_PRODUCTION_SECRET_ACCESS_KEY }} production
      - name: export abq version
        id: abq_version
        run: |
          echo "abq_version=$ABQ_VERSION" >> "$GITHUB_OUTPUT"

  notify-staging-api:
    name: Release new abq to staging API
    needs: release
    uses: ./.github/workflows/release_staging.yml
    with:
      version: ${{ needs.release.outputs.abq_version }}
      release_channel: ${{inputs.release_channel}}
    secrets:
      RWX_STAGING_CREATE_RELEASE_ACCESS_TOKEN: ${{ secrets.RWX_STAGING_CREATE_RELEASE_ACCESS_TOKEN }}

  notify-production-api:
    name: Release new abq to production API
    needs: release
    uses: ./.github/workflows/release_production.yml
    with:
      version: ${{ needs.release.outputs.abq_version }}
      release_channel: ${{inputs.release_channel}}
    secrets:
      RWX_PRODUCTION_CREATE_RELEASE_ACCESS_TOKEN: ${{ secrets.RWX_PRODUCTION_CREATE_RELEASE_ACCESS_TOKEN }}