From 0cf3bb035bf4f3c753f6cc9dc9e4f171887028fb Mon Sep 17 00:00:00 2001 From: Frank Liu Date: Fri, 13 Sep 2024 13:16:32 -0700 Subject: [PATCH] [ci] configure aws creds manually to avoid node20 issues for AL2 --- .github/workflows/native_s3_fasttext.yml | 26 +++++++++-- .github/workflows/native_s3_huggingface.yml | 52 +++++++++++++++++---- 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/.github/workflows/native_s3_fasttext.yml b/.github/workflows/native_s3_fasttext.yml index 0f82a66152e..2aae9e90482 100644 --- a/.github/workflows/native_s3_fasttext.yml +++ b/.github/workflows/native_s3_fasttext.yml @@ -16,7 +16,7 @@ jobs: run: | yum -y update yum -y groupinstall "Development Tools" - yum -y install patch cmake3 + yum -y install patch cmake3 curl jq ln -sf /usr/bin/cmake3 /usr/bin/cmake pip3 install awscli --upgrade - uses: actions/checkout@v3 @@ -37,10 +37,26 @@ jobs: ./gradlew :extensions:fasttext:compileJNI ./gradlew -Pjni :extensions:fasttext:test - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role - aws-region: us-east-2 + run: | + oidc_token=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.amazonaws.com" | jq -r ".value") + echo "::add-mask::$oidc_token" + + read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"$(aws sts assume-role-with-web-identity \ + --region "us-east-2" \ + --role-arn "arn:aws:iam::425969335547:role/djl-ci-publish-role" \ + --role-session-name "build-fasttext-jni-linux" \ + --web-identity-token "$oidc_token" \ + --query "[Credentials.AccessKeyId, Credentials.SecretAccessKey, Credentials.SessionToken]" \ + --output text)" + + echo "::add-mask::$AWS_ACCESS_KEY_ID" + echo "::add-mask::$AWS_SECRET_ACCESS_KEY" + echo "::add-mask::$AWS_SESSION_TOKEN" + + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> "$GITHUB_ENV" + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> "$GITHUB_ENV" + echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> "$GITHUB_ENV" - name: Copy files to S3 with the AWS CLI run: | FASTTEXT_VERSION="$(awk -F '=' '/fasttext/ {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)" diff --git a/.github/workflows/native_s3_huggingface.yml b/.github/workflows/native_s3_huggingface.yml index e635c540399..2ce9c190454 100644 --- a/.github/workflows/native_s3_huggingface.yml +++ b/.github/workflows/native_s3_huggingface.yml @@ -23,7 +23,7 @@ jobs: - name: Install Environment run: | yum -y groupinstall "Development Tools" - yum -y install patch perl-IPC-Cmd cmake3 + yum -y install patch perl-IPC-Cmd cmake3 curl jq yum -y install java-17-amazon-corretto-devel ln -s /usr/bin/cmake3 /usr/bin/cmake curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y @@ -38,10 +38,26 @@ jobs: working-directory: extensions/tokenizers/src/main/python/ run: ./setup.py bdist_wheel - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role - aws-region: us-east-2 + run: | + oidc_token=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.amazonaws.com" | jq -r ".value") + echo "::add-mask::$oidc_token" + + read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"$(aws sts assume-role-with-web-identity \ + --region "us-east-2" \ + --role-arn "arn:aws:iam::425969335547:role/djl-ci-publish-role" \ + --role-session-name "build-tokenizers-jni-linux" \ + --web-identity-token "$oidc_token" \ + --query "[Credentials.AccessKeyId, Credentials.SecretAccessKey, Credentials.SessionToken]" \ + --output text)" + + echo "::add-mask::$AWS_ACCESS_KEY_ID" + echo "::add-mask::$AWS_SECRET_ACCESS_KEY" + echo "::add-mask::$AWS_SESSION_TOKEN" + + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> "$GITHUB_ENV" + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> "$GITHUB_ENV" + echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> "$GITHUB_ENV" - name: Copy files to S3 with the AWS CLI run: | DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml) @@ -166,7 +182,7 @@ jobs: - name: Install Environment run: | yum -y groupinstall "Development Tools" - yum -y install patch perl-IPC-Cmd cmake3 + yum -y install patch perl-IPC-Cmd cmake3 curl jq yum -y install java-17-amazon-corretto-devel ln -s /usr/bin/cmake3 /usr/bin/cmake curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y @@ -178,10 +194,26 @@ jobs: ./gradlew :extensions:tokenizers:compileJNI PYTORCH_PRECXX11=true ./gradlew -Pjni :extensions:tokenizers:test - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: arn:aws:iam::425969335547:role/djl-ci-publish-role - aws-region: us-east-2 + run: | + oidc_token=$(curl -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=sts.amazonaws.com" | jq -r ".value") + echo "::add-mask::$oidc_token" + + read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN <<<"$(aws sts assume-role-with-web-identity \ + --region "us-east-2" \ + --role-arn "arn:aws:iam::425969335547:role/djl-ci-publish-role" \ + --role-session-name "build-tokenizer-jni-aarch64" \ + --web-identity-token "$oidc_token" \ + --query "[Credentials.AccessKeyId, Credentials.SecretAccessKey, Credentials.SessionToken]" \ + --output text)" + + echo "::add-mask::$AWS_ACCESS_KEY_ID" + echo "::add-mask::$AWS_SECRET_ACCESS_KEY" + echo "::add-mask::$AWS_SESSION_TOKEN" + + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> "$GITHUB_ENV" + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> "$GITHUB_ENV" + echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> "$GITHUB_ENV" - name: Copy files to S3 with the AWS CLI run: | DJL_VERSION=$(awk -F '=' '/djl / {gsub(/ ?"/, "", $2); print $2}' gradle/libs.versions.toml)