From a2faa2211378c0cdece06a3dedc62ba556a606bf Mon Sep 17 00:00:00 2001 From: Siddharth Venkatesan Date: Wed, 31 Jan 2024 13:15:14 -0800 Subject: [PATCH] [ci] refactor cloudwatch metric publishing to avoid needing changes in primary workflows (#2974) --- .github/workflows/codeql-analysis-java.yml | 7 ------ .github/workflows/docker_publish.yml | 7 ------ .github/workflows/docs.yml | 7 ------ .github/workflows/native_jni_s3_paddle.yml | 7 ------ .github/workflows/native_jni_s3_pytorch.yml | 7 ------ .../native_jni_s3_pytorch_android.yml | 7 ------ .github/workflows/nightly_android.yml | 7 ------ .github/workflows/nightly_publish.yml | 7 ------ .github/workflows/publish-job-success.yml | 24 ++++++++++++------- .../workflows/publish_android_packages.yml | 7 ------ .github/workflows/serving_publish.yml | 7 ------ 11 files changed, 15 insertions(+), 79 deletions(-) diff --git a/.github/workflows/codeql-analysis-java.yml b/.github/workflows/codeql-analysis-java.yml index 4b738e9a118..19fe93e1737 100644 --- a/.github/workflows/codeql-analysis-java.yml +++ b/.github/workflows/codeql-analysis-java.yml @@ -73,10 +73,3 @@ jobs: - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 - - publish-success-metric: - needs: [ analyze ] - if: always() - uses: ./.github/workflows/publish-job-success.yml - with: - metric-name: DJL-CodeQL-Failure diff --git a/.github/workflows/docker_publish.yml b/.github/workflows/docker_publish.yml index 5c8da07ee72..a85ea409f4e 100644 --- a/.github/workflows/docker_publish.yml +++ b/.github/workflows/docker_publish.yml @@ -57,10 +57,3 @@ jobs: file: docker/spark/Dockerfile build-args: DJL_VERSION=${DJL_VERSION} tags: deepjavalibrary/djl-spark:${{ env.DJL_VERSION }}-cpu - - publish-success-metric: - needs: [ publish ] - if: always() - uses: ./.github/workflows/publish-job-success.yml - with: - metric-name: DJL-SparkDockerPublish-Failure diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index ae228838506..42b458ebe1d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -65,10 +65,3 @@ jobs: run: | aws s3 sync ../site s3://djl-ai/documentation/nightly --delete aws cloudfront create-invalidation --distribution-id E733IIDCG0G5U --paths "/*" - - publish-success-metric: - needs: [ documentation ] - if: always() - uses: ./.github/workflows/publish-job-success.yml - with: - metric-name: DJL-DocumentationPublish-Failure diff --git a/.github/workflows/native_jni_s3_paddle.yml b/.github/workflows/native_jni_s3_paddle.yml index 6cb89cea353..3a0176ead31 100644 --- a/.github/workflows/native_jni_s3_paddle.yml +++ b/.github/workflows/native_jni_s3_paddle.yml @@ -108,10 +108,3 @@ jobs: PADDLE_VERSION="$(cat gradle.properties | awk -F '=' '/paddlepaddle_version/ {print $2}')" aws s3 sync jnilib s3://djl-ai/publish/paddlepaddle-${PADDLE_VERSION}/jnilib aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/paddlepaddle-${PADDLE_VERSION}/jnilib*" - - publish-success-metric: - needs: [ publish ] - if: always() - uses: ./.github/workflows/publish-job-success.yml - with: - metric-name: DJL-NativeJNIPaddleS3Publish-Failure diff --git a/.github/workflows/native_jni_s3_pytorch.yml b/.github/workflows/native_jni_s3_pytorch.yml index 9a361f7196a..ef4efeb0528 100644 --- a/.github/workflows/native_jni_s3_pytorch.yml +++ b/.github/workflows/native_jni_s3_pytorch.yml @@ -306,10 +306,3 @@ jobs: cd /home/ubuntu/djl_benchmark_script/scripts instance_id=${{ needs.create-aarch64-runner.outputs.aarch64_instance_id }} ./stop_instance.sh $instance_id - - publish-success-metric: - needs: [ build-pytorch-jni-macos, build-pytorch-jni-linux, build-pytorch-jni-precxx11, build-pytorch-jni-windows, build-pytorch-jni-arm64-macos, stop-runners ] - if: always() - uses: ./.github/workflows/publish-job-success.yml - with: - metric-name: DJL-NativeJNIPytorchS3Publish-Failure diff --git a/.github/workflows/native_jni_s3_pytorch_android.yml b/.github/workflows/native_jni_s3_pytorch_android.yml index 60c994010b4..b871f1f3962 100644 --- a/.github/workflows/native_jni_s3_pytorch_android.yml +++ b/.github/workflows/native_jni_s3_pytorch_android.yml @@ -43,10 +43,3 @@ jobs: PYTORCH_VERSION=${PYTORCH_VERSION:-$(cat gradle.properties | awk -F '=' '/pytorch_version/ {print $2}')} aws s3 sync engines/pytorch/pytorch-native/jnilib s3://djl-ai/publish/pytorch/${PYTORCH_VERSION}/jnilib aws cloudfront create-invalidation --distribution-id E371VB8JQ6NRVY --paths "/pytorch/${PYTORCH_VERSION}/jnilib*" - - publish-success-metric: - needs: [ build-pytorch-jni-android ] - if: always() - uses: ./.github/workflows/publish-job-success.yml - with: - metric-name: DJL-NativeJNIPytorchAndroidS3Publish-Failure diff --git a/.github/workflows/nightly_android.yml b/.github/workflows/nightly_android.yml index dc66b56845c..b46528bc29f 100644 --- a/.github/workflows/nightly_android.yml +++ b/.github/workflows/nightly_android.yml @@ -31,10 +31,3 @@ jobs: emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none disable-animations: true script: cd android/core && ./gradlew cAT - - publish-success-metric: - needs: [ build ] - if: always() - uses: ./.github/workflows/publish-job-success.yml - with: - metric-name: DJL-AndroidIntegrationTests-Failure diff --git a/.github/workflows/nightly_publish.yml b/.github/workflows/nightly_publish.yml index 0930eb84404..b034d84c397 100644 --- a/.github/workflows/nightly_publish.yml +++ b/.github/workflows/nightly_publish.yml @@ -255,10 +255,3 @@ jobs: cd /home/ubuntu/djl_benchmark_script/scripts instance_id=${{ needs.create-runners.outputs.gpu_1_instance_id }} ./stop_instance.sh $instance_id - - publish-success-metric: - needs: [ publish, stop-runners ] - if: always() - uses: ./.github/workflows/publish-job-success.yml - with: - metric-name: DJL-NightlyIntegrationTestsPublish-Failure diff --git a/.github/workflows/publish-job-success.yml b/.github/workflows/publish-job-success.yml index 84a6d3030b4..5613cd547fa 100644 --- a/.github/workflows/publish-job-success.yml +++ b/.github/workflows/publish-job-success.yml @@ -1,16 +1,16 @@ name: Publish Job Success Metric to CloudWatch on: - workflow_call: - inputs: - metric-name: - description: "The name of the job to publish a metric for" - type: string - required: true + workflow_run: + workflows: "*" + types: + - completed + branches: + - master jobs: publish-job-success-to-cloudwatch: - if: ${{ github.event_name == 'schedule' }} + if: ${{ github.event.workflow_run.event == "schedule" }} runs-on: [ self-hosted, scheduler ] steps: - name: Configure AWS Credentials @@ -18,10 +18,16 @@ jobs: with: aws-region: us-west-2 - name: Publish Job Success Metric + env: + WORKFLOW_NAME: ${{ github.event.workflow_run.display_title }} + REPO_NAME: ${{ github.event.workflow_run.repository.name }} + CONCLUSION: ${{ github.event.workflow_run.conclusion }} run: | - [[ ${{ job.status }} == "success" ]] + workflow_name=$(echo "$WORKFLOW_NAME" | tr -d ' ') + metric_name="${REPO_NAME}-${workflow_name}-Failure" + [[ "$CONCLUSION" == "success" ]] failedBuild=$? aws cloudwatch put-metric-data --namespace GithubCI \ - --metric-name ${{ inputs.metric-name }} \ + --metric-name "$metric_name" \ --value $failedBuild \ --unit Count diff --git a/.github/workflows/publish_android_packages.yml b/.github/workflows/publish_android_packages.yml index 2c12b0c1b4f..4ae8bcefcb2 100644 --- a/.github/workflows/publish_android_packages.yml +++ b/.github/workflows/publish_android_packages.yml @@ -50,10 +50,3 @@ jobs: ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.ORG_GRADLE_PROJECT_signingPassword }} ORG_GRADLE_PROJECT_ossrhUsername: ${{ secrets.ORG_GRADLE_PROJECT_ossrhUsername }} ORG_GRADLE_PROJECT_ossrhPassword: ${{ secrets.ORG_GRADLE_PROJECT_ossrhPassword }} - - publish-success-metric: - needs: [ release-android ] - if: always() - uses: ./.github/workflows/publish-job-success.yml - with: - metric-name: DJL-AndroidPublish-Failurei diff --git a/.github/workflows/serving_publish.yml b/.github/workflows/serving_publish.yml index 3c77ea0a59d..380666487a7 100644 --- a/.github/workflows/serving_publish.yml +++ b/.github/workflows/serving_publish.yml @@ -105,10 +105,3 @@ jobs: ORG_GRADLE_PROJECT_ossrhUsername: ${{ secrets.ORG_GRADLE_PROJECT_ossrhUsername }} ORG_GRADLE_PROJECT_ossrhPassword: ${{ secrets.ORG_GRADLE_PROJECT_ossrhPassword }} DJL_STAGING: ${{ github.event.inputs.repo-id }} - - publish-success-metric: - needs: [ publish ] - if: always() - uses: ./.github/workflows/publish-job-success.yml - with: - metric-name: DJL-DJLServingPublish-Failure