Generate and Sync Recent Changes #20788
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Be aware: This workflow should be kept in sync with generate-and-deploy-with-delete.yml and generate-and-deploy.yml | |
| name: Generate and Sync Recent Changes | |
| on: | |
| schedule: | |
| - cron: '5,20,35,50 * * * *' | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| type: choice | |
| description: The environment of the Registry | |
| options: | |
| - Development | |
| - Production | |
| jobs: | |
| generate-and-sync: | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: ${{ inputs.environment || 'Production' }} | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version-file: './src/go.mod' | |
| cache-dependency-path: './src/go.sum' | |
| - name: Setup dependencies | |
| run: sudo apt-get update && sudo apt-get install rclone | |
| - name: Prune modules and providers to only modified in the past 100 commits | |
| run: | | |
| # Copy files changed into pruned directory | |
| git fetch origin | |
| git checkout origin/main | |
| mkdir ./pruned/ | |
| echo "Copy Changed Providers" | |
| git diff --name-only "$(git rev-list -1 --before="$(date -d '-1 hours' '+%Y-%m-%d %H:%M:%S')" HEAD)" | grep ^providers | xargs -I foo echo "mkdir -p ./pruned/\$(dirname foo); cp foo ./pruned/foo; echo foo" | bash | |
| echo "Copy Changed Providers by Keys" | |
| git diff --name-only "$(git rev-list -1 --before="$(date -d '-1 hours' '+%Y-%m-%d %H:%M:%S')" HEAD)" | grep ^keys | sed -e 's,^keys/\([^/]*\)/\([^/]*\).*,providers/\1/\2,' | sort | uniq | xargs -I foo echo "mkdir -p ./pruned/\$(dirname foo); cp -r foo ./pruned/foo; echo foo" | bash | |
| echo "Copy Changed Modules" | |
| git diff --name-only "$(git rev-list -1 --before="$(date -d '-1 hours' '+%Y-%m-%d %H:%M:%S')" HEAD)" | grep ^modules | xargs -I foo echo "mkdir -p ./pruned/\$(dirname foo); cp foo ./pruned/foo; echo foo" | bash | |
| - name: Run Generation script | |
| working-directory: ./src | |
| run: go run ./cmd/generate-v1 --destination ../generated -module-data ../pruned/modules/ -provider-data ../pruned/providers/ | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| - name: Sync Data to R2 | |
| run: rclone copy --checkers=512 --transfers=512 --checksum --no-traverse ./generated R2:${{ secrets.R2_BUCKET_NAME }} | |
| env: | |
| # R2 credentials should be stored as GitHub secrets | |
| RCLONE_CONFIG_R2_TYPE: s3 | |
| RCLONE_CONFIG_R2_PROVIDER: Cloudflare | |
| RCLONE_CONFIG_R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| RCLONE_CONFIG_R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| RCLONE_CONFIG_R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} |