adp #3276
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: adp | |
| on: | |
| schedule: | |
| - cron: "0 * * * *" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| jobs: | |
| process-reference-files: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # https://stackoverflow.com/questions/58033366/how-to-get-the-current-branch-within-github-actions | |
| - name: Extract Branch Name and Configure Runner | |
| shell: bash | |
| run: | | |
| if [ "${GITHUB_REF##*/}" = "adp_test" ]; then | |
| echo "G_READ_API_KEY=${{ secrets.G_READ_API_KEY }}" >> "$GITHUB_ENV"; | |
| echo "G_WRITE_API_KEY=${{ secrets.G_WRITE_API_KEY }}" >> "$GITHUB_ENV"; | |
| echo "BRANCH_NAME=${GITHUB_REF##*/}" >> "$GITHUB_ENV"; | |
| echo "A_API_KEY=${{ secrets.ADP_TEST_API_KEY }}" >> "$GITHUB_ENV"; | |
| echo "CVE-API-ORG=secretariat-reference" >> "$GITHUB_ENV"; | |
| echo "CVE-API-USER=automated-ingest" >> "$GITHUB_ENV"; | |
| echo "SERVICES_URL=https://cveawg-adp-test.mitre.org/api/cve/" >> "$GITHUB_ENV"; | |
| elif [ "${GITHUB_REF##*/}" = "main" ]; then | |
| echo "G_READ_API_KEY=${{ secrets.G_READ_API_KEY }}" >> "$GITHUB_ENV"; | |
| echo "G_WRITE_API_KEY=${{ secrets.G_WRITE_API_KEY }}" >> "$GITHUB_ENV"; | |
| echo "BRANCH_NAME=${GITHUB_REF##*/}" >> "$GITHUB_ENV"; | |
| echo "A_API_KEY=${{ secrets.MAIN_API_KEY }}" >> "$GITHUB_ENV"; | |
| echo "CVE-API-ORG=CVE" >> "$GITHUB_ENV"; | |
| echo "CVE-API-USER=reference-ingest" >> "$GITHUB_ENV"; | |
| echo "SERVICES_URL=https://cveawg.mitre.org/api/cve/" >> "$GITHUB_ENV"; | |
| fi | |
| - name: check_out_contrib1_repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| repository: "CVEProject/cve-reference-ingest-data" | |
| path: "data-contrib1" | |
| - name: process_new_files_from_contrib1 | |
| run: | | |
| CONTRIBUTOR="contrib1" | |
| DIR_IN_PRIMARY_REPO="CVEProject/cve-reference-ingest-data" | |
| CHECKOUT_PATH="data-$CONTRIBUTOR" | |
| DIRECTORY="references" | |
| cd $CHECKOUT_PATH | |
| curl -s -S -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://raw.githubusercontent.com/CVEProject/cve-reference-ingest/${{ env.BRANCH_NAME }}/actions-bin/create-file-via-api.py > create-file-via-api.py | |
| curl -s -S -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://raw.githubusercontent.com/CVEProject/cve-reference-ingest/${{ env.BRANCH_NAME }}/actions-bin/read-file-via-api.py > read-file-via-api.py | |
| if ! test -e create-file-via-api.py; then | |
| echo "Failed to copy create-file-via-api.py" | |
| exit 1 | |
| fi | |
| if ! test -e read-file-via-api.py; then | |
| echo "Failed to copy read-file-via-api.py" | |
| exit 1 | |
| fi | |
| CURRENT_SHA=$(git rev-parse HEAD) | |
| PREVIOUS_SHA=$(python read-file-via-api.py $DIR_IN_PRIMARY_REPO last_run_sha) | |
| if [[ $PREVIOUS_SHA =~ ^[0-9a-f]{40}$ ]]; then | |
| echo "PREVIOUS_SHA is valid" | |
| else | |
| echo "PREVIOUS_SHA is invalid, issue reading from ${DIR_IN_PRIMARY_REPO}, ${PREVIOUS_SHA}" | |
| exit 1 | |
| fi | |
| if [ -n "$PREVIOUS_SHA" ]; then | |
| git diff --name-only --diff-filter=A $PREVIOUS_SHA $CURRENT_SHA $DIRECTORY | while read REFERENCE_FILE; do | |
| echo "Processing: $REFERENCE_FILE" | |
| if ! python create-file-via-api.py $REFERENCE_FILE $CONTRIBUTOR; then | |
| echo "Unable To process ${REFERENCE_FILE}" | |
| exit 1 | |
| fi | |
| done | |
| fi | |
| - name: check_out_primary_repository | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.BRANCH_NAME }} | |
| fetch-depth: 0 | |
| repository: "CVEProject/cve-reference-ingest" | |
| path: "data-primary" | |
| - name: process_new_files_from_primary | |
| run: | | |
| DIR_IN_PRIMARY_REPO="CVEProject/cve-reference-ingest" | |
| CHECKOUT_PATH="data-primary" | |
| DIRECTORY="references" | |
| pwd | |
| PRIMARY_LAST_RUN_SHA="last_run_shas/$DIR_IN_PRIMARY_REPO/last_run_sha" | |
| cd $CHECKOUT_PATH | |
| CURRENT_SHA=$(git rev-parse HEAD) | |
| PREVIOUS_SHA=$(cat $PRIMARY_LAST_RUN_SHA) | |
| if [[ $PREVIOUS_SHA =~ ^[0-9a-f]{40}$ ]]; then | |
| echo "PREVIOUS_SHA is valid" | |
| else | |
| echo "PREVIOUS_SHA is invalid, issue reading from ${DIR_IN_PRIMARY_REPO}" | |
| exit 1 | |
| fi | |
| git config user.name "GitHub Action" | |
| git config user.email "[email protected]" | |
| git pull | |
| if [ -n "$PREVIOUS_SHA" ]; then | |
| git diff --name-only --diff-filter=A $PREVIOUS_SHA $CURRENT_SHA $DIRECTORY | while read REFERENCE_FILE; do | |
| echo "Processing: $REFERENCE_FILE" | |
| if ! python actions-bin/adp.py $CHECKOUT_PATH $DIRECTORY $REFERENCE_FILE; then | |
| cp $REFERENCE_FILE retry/$REFERENCE_FILE | |
| git add retry/$REFERENCE_FILE | |
| git commit -m "Adding failed file to retry folder" | |
| git push | |
| if [ $? -ne 0 ]; then | |
| # The push to the retry folder failed, log the information so it can be recovered from the logs | |
| echo "git push failed for File: ${REFERENCE_FILE}" | |
| fi | |
| fi | |
| sleep 0.7 | |
| done | |
| fi | |
| echo $CURRENT_SHA > $PRIMARY_LAST_RUN_SHA | |
| git add $PRIMARY_LAST_RUN_SHA | |
| THIS_CONTRIBUTOR="contrib1" | |
| THIS_CONTRIBUTOR_REPO="CVEProject/cve-reference-ingest-data" | |
| CURRENT_SHA_OF_CONTRIBUTOR=$(cd ../data-$THIS_CONTRIBUTOR; git rev-parse HEAD) | |
| CONTRIBUTOR_LAST_RUN_SHA="last_run_shas/$THIS_CONTRIBUTOR_REPO/last_run_sha" | |
| echo $CURRENT_SHA_OF_CONTRIBUTOR > $CONTRIBUTOR_LAST_RUN_SHA | |
| git add $CONTRIBUTOR_LAST_RUN_SHA | |
| if [ -s .rbp ]; then | |
| RBP_BASENAME=$(head -1 .rbp) | |
| RBP_PATHNAME=rbp/$RBP_BASENAME | |
| git add $RBP_PATHNAME | |
| fi | |
| git commit -m "Update last run SHA for primary" | |
| git push | |
| - name: attempt_to_process_retry | |
| if: always() | |
| run: | | |
| DIR_IN_PRIMARY_REPO="CVEProject/cve-reference-ingest" | |
| CHECKOUT_PATH="data-primary" | |
| DIRECTORY="retry" | |
| cd $CHECKOUT_PATH | |
| git config user.name "GitHub Action" | |
| git config user.email "[email protected]" | |
| for file in retry/references/*; do | |
| if [ -f "$file" ]; then | |
| echo "Processing: $file" | |
| if python actions-bin/adp.py $CHECKOUT_PATH $DIRECTORY $file; then | |
| git rm $file | |
| git commit -m "Completed Retry of File: ${file}" | |
| git push | |
| fi | |
| fi | |
| sleep 0.7 | |
| done |