Skip to content

adp

adp #3283

Workflow file for this run

name: adp
on:
schedule:
- cron: "0 * * * *"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
jobs:
process-reference-files:
runs-on: ubuntu-latest
steps:
# https://stackoverflow.com/questions/58033366/how-to-get-the-current-branch-within-github-actions
- name: Extract Branch Name and Configure Runner
shell: bash
run: |
if [ "${GITHUB_REF##*/}" = "adp_test" ]; then
echo "G_READ_API_KEY=${{ secrets.G_READ_API_KEY }}" >> "$GITHUB_ENV";
echo "G_WRITE_API_KEY=${{ secrets.G_WRITE_API_KEY }}" >> "$GITHUB_ENV";
echo "BRANCH_NAME=${GITHUB_REF##*/}" >> "$GITHUB_ENV";
echo "A_API_KEY=${{ secrets.ADP_TEST_API_KEY }}" >> "$GITHUB_ENV";
echo "CVE-API-ORG=secretariat-reference" >> "$GITHUB_ENV";
echo "CVE-API-USER=automated-ingest" >> "$GITHUB_ENV";
echo "SERVICES_URL=https://cveawg-adp-test.mitre.org/api/cve/" >> "$GITHUB_ENV";
elif [ "${GITHUB_REF##*/}" = "main" ]; then
echo "G_READ_API_KEY=${{ secrets.G_READ_API_KEY }}" >> "$GITHUB_ENV";
echo "G_WRITE_API_KEY=${{ secrets.G_WRITE_API_KEY }}" >> "$GITHUB_ENV";
echo "BRANCH_NAME=${GITHUB_REF##*/}" >> "$GITHUB_ENV";
echo "A_API_KEY=${{ secrets.MAIN_API_KEY }}" >> "$GITHUB_ENV";
echo "CVE-API-ORG=CVE" >> "$GITHUB_ENV";
echo "CVE-API-USER=reference-ingest" >> "$GITHUB_ENV";
echo "SERVICES_URL=https://cveawg.mitre.org/api/cve/" >> "$GITHUB_ENV";
fi
- name: check_out_contrib1_repository
uses: actions/checkout@v4
with:
fetch-depth: 0
repository: "CVEProject/cve-reference-ingest-data"
path: "data-contrib1"
- name: process_new_files_from_contrib1
run: |
CONTRIBUTOR="contrib1"
DIR_IN_PRIMARY_REPO="CVEProject/cve-reference-ingest-data"
CHECKOUT_PATH="data-$CONTRIBUTOR"
DIRECTORY="references"
cd $CHECKOUT_PATH
curl -s -S -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://raw.githubusercontent.com/CVEProject/cve-reference-ingest/${{ env.BRANCH_NAME }}/actions-bin/create-file-via-api.py > create-file-via-api.py
curl -s -S -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" https://raw.githubusercontent.com/CVEProject/cve-reference-ingest/${{ env.BRANCH_NAME }}/actions-bin/read-file-via-api.py > read-file-via-api.py
if ! test -e create-file-via-api.py; then
echo "Failed to copy create-file-via-api.py"
exit 1
fi
if ! test -e read-file-via-api.py; then
echo "Failed to copy read-file-via-api.py"
exit 1
fi
CURRENT_SHA=$(git rev-parse HEAD)
PREVIOUS_SHA=$(python read-file-via-api.py $DIR_IN_PRIMARY_REPO last_run_sha)
if [[ $PREVIOUS_SHA =~ ^[0-9a-f]{40}$ ]]; then
echo "PREVIOUS_SHA is valid"
else
echo "PREVIOUS_SHA is invalid, issue reading from ${DIR_IN_PRIMARY_REPO}, ${PREVIOUS_SHA}"
exit 1
fi
if [ -n "$PREVIOUS_SHA" ]; then
git diff --name-only --diff-filter=A $PREVIOUS_SHA $CURRENT_SHA $DIRECTORY | while read REFERENCE_FILE; do
echo "Processing: $REFERENCE_FILE"
if ! python create-file-via-api.py $REFERENCE_FILE $CONTRIBUTOR; then
echo "Unable To process ${REFERENCE_FILE}"
exit 1
fi
done
fi
- name: check_out_primary_repository
uses: actions/checkout@v4
with:
ref: ${{ env.BRANCH_NAME }}
fetch-depth: 0
repository: "CVEProject/cve-reference-ingest"
path: "data-primary"
- name: process_new_files_from_primary
run: |
DIR_IN_PRIMARY_REPO="CVEProject/cve-reference-ingest"
CHECKOUT_PATH="data-primary"
DIRECTORY="references"
pwd
PRIMARY_LAST_RUN_SHA="last_run_shas/$DIR_IN_PRIMARY_REPO/last_run_sha"
cd $CHECKOUT_PATH
CURRENT_SHA=$(git rev-parse HEAD)
PREVIOUS_SHA=$(cat $PRIMARY_LAST_RUN_SHA)
if [[ $PREVIOUS_SHA =~ ^[0-9a-f]{40}$ ]]; then
echo "PREVIOUS_SHA is valid"
else
echo "PREVIOUS_SHA is invalid, issue reading from ${DIR_IN_PRIMARY_REPO}"
exit 1
fi
git config user.name "GitHub Action"
git config user.email "[email protected]"
git pull
if [ -n "$PREVIOUS_SHA" ]; then
git diff --name-only --diff-filter=A $PREVIOUS_SHA $CURRENT_SHA $DIRECTORY | while read REFERENCE_FILE; do
echo "Processing: $REFERENCE_FILE"
if ! python actions-bin/adp.py $CHECKOUT_PATH $DIRECTORY $REFERENCE_FILE; then
cp $REFERENCE_FILE retry/$REFERENCE_FILE
git add retry/$REFERENCE_FILE
git commit -m "Adding failed file to retry folder"
git push
if [ $? -ne 0 ]; then
# The push to the retry folder failed, log the information so it can be recovered from the logs
echo "git push failed for File: ${REFERENCE_FILE}"
fi
fi
sleep 0.7
done
fi
echo $CURRENT_SHA > $PRIMARY_LAST_RUN_SHA
git add $PRIMARY_LAST_RUN_SHA
THIS_CONTRIBUTOR="contrib1"
THIS_CONTRIBUTOR_REPO="CVEProject/cve-reference-ingest-data"
CURRENT_SHA_OF_CONTRIBUTOR=$(cd ../data-$THIS_CONTRIBUTOR; git rev-parse HEAD)
CONTRIBUTOR_LAST_RUN_SHA="last_run_shas/$THIS_CONTRIBUTOR_REPO/last_run_sha"
echo $CURRENT_SHA_OF_CONTRIBUTOR > $CONTRIBUTOR_LAST_RUN_SHA
git add $CONTRIBUTOR_LAST_RUN_SHA
if [ -s .rbp ]; then
RBP_BASENAME=$(head -1 .rbp)
RBP_PATHNAME=rbp/$RBP_BASENAME
git add $RBP_PATHNAME
fi
git commit -m "Update last run SHA for primary"
git push
- name: attempt_to_process_retry
if: always()
run: |
DIR_IN_PRIMARY_REPO="CVEProject/cve-reference-ingest"
CHECKOUT_PATH="data-primary"
DIRECTORY="retry"
cd $CHECKOUT_PATH
git config user.name "GitHub Action"
git config user.email "[email protected]"
for file in retry/references/*; do
if [ -f "$file" ]; then
echo "Processing: $file"
if python actions-bin/adp.py $CHECKOUT_PATH $DIRECTORY $file; then
git rm $file
git commit -m "Completed Retry of File: ${file}"
git push
fi
fi
sleep 0.7
done