Build and Upload #23
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Upload | |
| on: | |
| # for manual release on the v1 release channel | |
| workflow_dispatch: | |
| inputs: | |
| ref: | |
| description: "ref to build" | |
| required: true | |
| type: string | |
| release_channel: | |
| description: "release channel" | |
| required: true | |
| type: choice | |
| options: | |
| - "v1" | |
| - "unstable" | |
| # for automatic release from main on the unstable release channel | |
| workflow_call: | |
| inputs: | |
| ref: | |
| description: "ref to build" | |
| required: true | |
| type: string | |
| release_channel: | |
| description: "release channel" | |
| required: true | |
| type: string | |
| secrets: | |
| RWX_STAGING_CREATE_RELEASE_ACCESS_TOKEN: | |
| required: true | |
| RWX_PRODUCTION_CREATE_RELEASE_ACCESS_TOKEN: | |
| required: true | |
| AWS_S3_ABQ_RELEASES_DEVELOPMENT_ACCESS_KEY_ID: | |
| required: true | |
| AWS_S3_ABQ_RELEASES_DEVELOPMENT_SECRET_ACCESS_KEY: | |
| required: true | |
| AWS_S3_ABQ_RELEASES_STAGING_ACCESS_KEY_ID: | |
| required: true | |
| AWS_S3_ABQ_RELEASES_STAGING_SECRET_ACCESS_KEY: | |
| required: true | |
| AWS_S3_ABQ_RELEASES_PRODUCTION_ACCESS_KEY_ID: | |
| required: true | |
| AWS_S3_ABQ_RELEASES_PRODUCTION_SECRET_ACCESS_KEY: | |
| required: true | |
| RWX_APPLE_DEVELOPER_ID_APPLICATION_CERT: | |
| required: true | |
| RWX_APPLE_APP_STORE_CONNECT_API_KEY: | |
| required: true | |
| jobs: | |
| release: | |
| name: Build, Upload, Release ABQ | |
| strategy: | |
| # in the future we'll want to release against different architectures | |
| matrix: | |
| include: | |
| - runs-on: ubuntu-latest | |
| # todo: deprecate plaform key when we move to RFC 20 style abq uploads | |
| deprecated-platform: linux_x86-64 | |
| os: linux | |
| architecture: x86_64 | |
| cross-target: "x86_64-unknown-linux-musl" | |
| install-musl-tools: true | |
| - runs-on: ubuntu-latest | |
| deprecated-platform: linux_aarch64 | |
| os: linux | |
| architecture: aarch64 | |
| cross-target: "aarch64-unknown-linux-musl" | |
| container: messense/rust-musl-cross:aarch64-musl@sha256:777bd4c61179c38dc213bb8472500584646d28fd4a7c3e0b30b9ef70cb446d58 | |
| - runs-on: macos-13 # use an older version for broader osx support | |
| deprecated-platform: darwin_x86-64 | |
| os: darwin | |
| architecture: x86_64 | |
| cross-target: "" | |
| - runs-on: macos-13 # first OS X to support arm64 -- so the first os for cross compilation | |
| deprecated-platform: darwin_aarch64 | |
| os: darwin | |
| architecture: aarch64 | |
| cross-target: "aarch64-apple-darwin" | |
| runs-on: ${{ matrix.runs-on }} | |
| container: ${{ matrix.container }} | |
| outputs: | |
| abq_version: ${{ steps.abq_version.outputs.abq_version }} | |
| env: | |
| AWS_DEFAULT_REGION: us-east-2 | |
| AWS_DEFAULT_OUTPUT: json | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 # get whole history for versioning | |
| ref: ${{inputs.ref}} | |
| - name: Mark safe directory | |
| run: | | |
| # For some reason we need this, even though actions/checkout sets | |
| # safe.directory by default? | |
| # Perhaps it is because we work in a container? | |
| git config --global --add safe.directory "$PWD" | |
| git describe | |
| # this may not be unecessary when https://github.com/actions/checkout/issues/882 is resolved | |
| - name: re-fetch overwritten tag | |
| run: git fetch --tags --force | |
| - name: Restore cargo cache | |
| uses: actions/[email protected] | |
| env: | |
| cache-name: ci | |
| with: | |
| path: | | |
| ~/.cargo/bin/ | |
| ~/.cargo/registry/index/ | |
| ~/.cargo/registry/cache/ | |
| ~/.cargo/git/db/ | |
| target/ | |
| key: cargo-release-${{ matrix.os }}-${{ matrix.architecture }}-${{ hashFiles('rust-toolchain.toml')}}-${{ hashFiles('Cargo.lock') }} | |
| restore-keys: | | |
| cargo-release-${{ matrix.os }}-${{ matrix.architecture }}-${{ hashFiles('rust-toolchain.toml')}}- | |
| - name: Install Rust toolchain | |
| uses: rwx-research/rust-toolchain@abq | |
| with: | |
| toolchain: 1.81.0 | |
| target: ${{ matrix.cross-target }} | |
| # We don't build a musl ABQ on MacOS | |
| - name: Install musl libc tools | |
| if: matrix.install-musl-tools | |
| run: | | |
| sudo apt-get install -y musl-tools | |
| - name: Build release | |
| if: "!matrix.cross-target" | |
| run: cargo build --release --all-features | |
| - name: Build release | |
| if: matrix.cross-target | |
| run: cargo build --release --all-features --target="${{ matrix.cross-target }}" | |
| - name: Prepare release environment | |
| run: | | |
| ABQ_VERSION="$(cat build_artifact/abq_version.txt)" | |
| RELEASE_DIR="abq_${ABQ_VERSION}_${{ matrix.deprecated-platform }}" | |
| # deprecated, remove these when we fully migrate to RFC-20 style storage | |
| RELEASE_ARCHIVE="${RELEASE_DIR}.tar.gz" | |
| # drop `NEW_` when we migrate to RFC-20 style storage | |
| { | |
| echo "ABQ_VERSION=$ABQ_VERSION" | |
| echo "RELEASE_DIR=$RELEASE_DIR" | |
| echo "RELEASE_ARCHIVE=$RELEASE_ARCHIVE" | |
| echo "RELEASE_S3_OBJECT=abq/${ABQ_VERSION}/${RELEASE_ARCHIVE}" | |
| echo "RELEASE_S3_NIGHTLY_OBJECT=abq/nightly/abq_nightly_${{ matrix.deprecated-platform }}" | |
| echo "NEW_RELEASE_S3_OBJECT=abq/${ABQ_VERSION}/${{ matrix.os }}/${{ matrix.architecture }}/abq" | |
| echo "NEW_RELEASE_S3_NIGHTLY_OBJECT=abq/nightly/${{ matrix.os }}/${{ matrix.architecture }}/abq" | |
| echo "NEW_RELEASE_S3_OBJECT_TESTER_HARNESS=abq/${ABQ_VERSION}/${{ matrix.os }}/${{ matrix.architecture }}/abq_tester_harness" | |
| echo "NEW_RELEASE_S3_NIGHTLY_OBJECT_TESTER_HARNESS=abq/nightly/${{ matrix.os }}/${{ matrix.architecture }}/abq_tester_harness" | |
| } >> "$GITHUB_ENV" | |
| - name: Move release to consistent directory | |
| run: | | |
| mkdir "${RELEASE_DIR}" | |
| if [ -z "${{ matrix.cross-target }}" ]; then | |
| cp target/release/abq "${RELEASE_DIR}/" | |
| cp target/release/abq_tester_harness "${RELEASE_DIR}/" | |
| else | |
| cp target/${{ matrix.cross-target }}/release/abq "${RELEASE_DIR}/" | |
| cp target/${{ matrix.cross-target }}/release/abq_tester_harness "${RELEASE_DIR}/" | |
| fi | |
| - name: Sign & Notarize Release | |
| if: matrix.os == 'darwin' | |
| run: | | |
| (rcodesign --version | grep "$CODESIGN_VERSION") || cargo install apple-codesign --force --locked --version "$CODESIGN_VERSION" | |
| echo "$RWX_APPLE_DEVELOPER_ID_APPLICATION_CERT" > rwx-developer-id-application-cert.pem | |
| # first we sign the binary. This happens locally. | |
| rcodesign sign --pem-source rwx-developer-id-application-cert.pem --code-signature-flags runtime "$RELEASE_DIR/abq" | |
| zip -r abq.zip "$RELEASE_DIR" | |
| echo "$RWX_APPLE_APP_STORE_CONNECT_API_KEY" > rwx-apple-app-store-connect-api-key.json | |
| submission_id="$(rcodesign notary-submit --wait --api-key-path rwx-apple-app-store-connect-api-key.json abq.zip 2>&1 | tee /dev/fd/2 | grep "created submission ID:" | awk '{print $4}')" | |
| echo "Submission ID: $submission_id" | |
| echo "submission_id=$submission_id" >> "$GITHUB_OUTPUT" | |
| env: | |
| CODESIGN_VERSION: 0.22.0 | |
| RWX_APPLE_DEVELOPER_ID_APPLICATION_CERT: ${{ secrets.RWX_APPLE_DEVELOPER_ID_APPLICATION_CERT }} | |
| RWX_APPLE_APP_STORE_CONNECT_API_KEY: ${{ secrets.RWX_APPLE_APP_STORE_CONNECT_API_KEY }} | |
| - name: ensure sign & notary cleanup happens | |
| if: matrix.os == 'darwin' && always() | |
| run: rm -f rwx-apple-app-store-connect-api-key.json rwx-developer-id-application-cert.pem | |
| # Deprecated. Remove archiving release when we fully migrate to RFC-20 style storage | |
| - name: (Deprecated) Archive release | |
| # we do this in its own step so that for darwin, abq is already signed | |
| run: tar -czvf "${RELEASE_ARCHIVE}" "${RELEASE_DIR}" | |
| - name: Install tools to container that are baked into default image | |
| if: matrix.container | |
| # we need to install aws tools if we use a container. The default images have the tools baked in. | |
| # The AWS code is approximately copied from the native image, https://github.com/actions/runner-images/blob/4aeccc7b5b5a63058b2d8a9a29072708e1f267bf/images/linux/scripts/installers/aws.sh#L11 | |
| # perhaps we should fork the musl container | |
| run: | | |
| curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" | |
| unzip -qq awscliv2.zip | |
| ./aws/install -i /usr/local/aws-cli -b /usr/local/bin | |
| curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_64bit/session-manager-plugin.deb" -o "session-manager-plugin.deb" | |
| apt install -y ./session-manager-plugin.deb | |
| # Download & install the latest aws sam cli release | |
| curl -L "https://github.com/aws/aws-sam-cli/releases/latest/download/aws-sam-cli-linux-x86_64.zip" -o "aws-sam-cli-linux-x86_64.zip" | |
| mkdir aws-sam | |
| unzip aws-sam-cli-linux-x86_64.zip -d aws-sam | |
| ./aws-sam/install | |
| apt-get update && apt-get install jq -y | |
| - name: Push release to development S3 | |
| env: | |
| RELEASE_BUCKET: abq-releases-development | |
| run: | | |
| scripts/upload-to-aws.sh ${{ secrets.AWS_S3_ABQ_RELEASES_DEVELOPMENT_ACCESS_KEY_ID }} ${{ secrets.AWS_S3_ABQ_RELEASES_DEVELOPMENT_SECRET_ACCESS_KEY }} development | |
| - name: Push release to staging S3 | |
| env: | |
| RELEASE_BUCKET: abq-releases-staging | |
| run: | | |
| scripts/upload-to-aws.sh ${{ secrets.AWS_S3_ABQ_RELEASES_STAGING_ACCESS_KEY_ID }} ${{ secrets.AWS_S3_ABQ_RELEASES_STAGING_SECRET_ACCESS_KEY }} staging | |
| - name: Push release to production S3 | |
| env: | |
| RELEASE_BUCKET: abq-releases | |
| run: | | |
| scripts/upload-to-aws.sh ${{ secrets.AWS_S3_ABQ_RELEASES_PRODUCTION_ACCESS_KEY_ID }} ${{ secrets.AWS_S3_ABQ_RELEASES_PRODUCTION_SECRET_ACCESS_KEY }} production | |
| - name: export abq version | |
| id: abq_version | |
| run: | | |
| echo "abq_version=$ABQ_VERSION" >> "$GITHUB_OUTPUT" | |
| notify-staging-api: | |
| name: Release new abq to staging API | |
| needs: release | |
| uses: ./.github/workflows/release_staging.yml | |
| with: | |
| version: ${{ needs.release.outputs.abq_version }} | |
| release_channel: ${{inputs.release_channel}} | |
| secrets: | |
| RWX_STAGING_CREATE_RELEASE_ACCESS_TOKEN: ${{ secrets.RWX_STAGING_CREATE_RELEASE_ACCESS_TOKEN }} | |
| notify-production-api: | |
| name: Release new abq to production API | |
| needs: release | |
| uses: ./.github/workflows/release_production.yml | |
| with: | |
| version: ${{ needs.release.outputs.abq_version }} | |
| release_channel: ${{inputs.release_channel}} | |
| secrets: | |
| RWX_PRODUCTION_CREATE_RELEASE_ACCESS_TOKEN: ${{ secrets.RWX_PRODUCTION_CREATE_RELEASE_ACCESS_TOKEN }} |