Update Data Model Version #2322
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Data Model Version | |
| on: | |
| schedule: | |
| - cron: "29 * * * *" # every hour on 29th minute, randomly chose | |
| workflow_dispatch: | |
| jobs: | |
| update-data-model-version: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| include: | |
| - branch-name: dev | |
| tag_enabled: false | |
| # - branch-name: dev2 | |
| # tag_enabled: false | |
| # - branch-name: qa | |
| # tag_enabled: false | |
| # - branch-name: qa2 | |
| # tag_enabled: false | |
| # - branch-name: stage | |
| # tag_enabled: true # Enables tag checkout for this branch | |
| # - branch-name: prod | |
| # tag_enabled: true # Enables tag checkout for this branch | |
| steps: | |
| - name: Printing the branch currently working on | |
| run: echo "BRANCH_NAME=${{ matrix.branch-name }}" | |
| - name: Check out the branch | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ matrix.branch-name }} | |
| submodules: true # This ensures submodules are also checked out | |
| submodules_recursive: true # Recursively fetches submodules | |
| fetch-depth: 0 # Fetch all history | |
| - name: Fetch all tags in the main repository | |
| if: matrix.tag_enabled == true | |
| run: | | |
| git fetch --tags | |
| - name: Check if tag_enabled is true and Determine and checkout latest tag for main repository | |
| if: matrix.tag_enabled == true | |
| run: | | |
| latest_tag=$(git tag -l "[0-9]*.[0-9]*.[0-9]*" | sort -V | tail -n 1) # Get latest tag in <major>.<minor>.<patch> format | |
| if [ -n "$latest_tag" ]; then | |
| echo "Checking out latest tag $latest_tag for main repository" | |
| git checkout $latest_tag | |
| else | |
| echo "No valid tag found for the main repository" | |
| fi | |
| - name: Rebase latest tag onto the matrix branch | |
| run: | | |
| git checkout ${{ matrix.branch-name }} # Switch back to the target branch (dev) | |
| if [ "${{ matrix.tag_enabled }}" = "true" ]; then | |
| git rebase $latest_tag | |
| fi | |
| - name: Update submodule to the latest commit | |
| run: | | |
| git submodule update --remote --recursive | |
| git submodule foreach --recursive ' | |
| if [ "${{ matrix.tag_enabled }}" = "true" ]; then | |
| git fetch --tags | |
| latest_tag=$(git tag -l "[0-9]*.[0-9]*.[0-9]*" | sort -V | tail -n 1) # Get the latest tag in <major>.<minor>.<patch> format | |
| if [ -n "$latest_tag" ]; then | |
| echo "Checking out latest tag $latest_tag for $name" | |
| git checkout $latest_tag | |
| else | |
| echo "No valid tag found for $name" | |
| fi | |
| else | |
| branch=$(git config -f $toplevel/.gitmodules submodule.$name.branch) | |
| echo "Fetching submodule branch: $branch" | |
| git fetch origin $branch | |
| fi | |
| git merge --allow-unrelated-histories FETCH_HEAD || echo "No merge needed" | |
| ' | |
| - name: Overwrite all files from the model in the submodule | |
| run: | | |
| # get the latest version from the cache and overwrite all files | |
| # pass if the no version in the cache; meaning never run github action | |
| content=$(cat cache/content.json) | |
| to_be_updated="" | |
| echo "$content" | jq -r 'to_entries[] | "\(.key)"' | while read key; do | |
| # store models in the env | |
| CACHE_DIR="cache/$key" | |
| largest_version_folder=$(ls -1 "$CACHE_DIR" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -n 1) | |
| echo "Largest semantic version folder: $largest_version_folder" | |
| # The path containing files must be a model-desc | |
| model_prefix=$(echo "${key}-model" | tr 'A-Z' 'a-z') | |
| model_path="$model_prefix/model-desc" | |
| target_folder="cache/$key/$largest_version_folder/" | |
| if [ -d "$target_folder" ]; then | |
| echo "copying files.... $target_folder" | |
| cp -r "$model_path/"* "$target_folder" | |
| git add "$target_folder" # Stage changes in the target folder | |
| fi | |
| # no folder has been created => never run => run! | |
| # latest folder has been changed => needs to be updated => run! | |
| if ! git diff --cached --quiet -- "$target_folder" || ([ ! -d "$target_folder" ] && [ -d "$model_path" ]); then | |
| echo "Detected changes: $key" | |
| git diff --cached -- "$target_folder" # Show detailed changes for debugging | |
| to_be_updated=${to_be_updated:+$to_be_updated,}$model_path | |
| else | |
| echo "No changes to commit for $key" | |
| git diff --cached -- "$target_folder" # Show detailed changes for debugging | |
| fi | |
| echo "updated_path=$to_be_updated" >> $GITHUB_ENV | |
| done | |
| - name: Check and create version folders | |
| run: | | |
| content=$(cat cache/content.json) | |
| echo "$content" | jq -r 'to_entries[] | "\(.key) \(.value["current-version"]) \(.value["model-files"]) \(.value["prop-file"])"' | while read key current_version model_files prop_file; do | |
| # uses a prefix path ex) cds(lowercase) + -model, icdc(lowercase) + -model | |
| model_prefix=$(echo "${key}-model" | tr 'A-Z' 'a-z') | |
| model_path="$model_prefix/model-desc" | |
| latest_version="" | |
| if [[ "${{env.updated_path}}" == *"$model_path"* ]]; then | |
| update_version="" | |
| for model_file in $(echo "$model_files" | jq -r '.[]'); do | |
| version_file_path="$model_path/$model_file" | |
| if [ -f "$version_file_path" ] && grep -q '^Version:' "$version_file_path"; then | |
| version=$(grep -i '^Version:' "$version_file_path" | sed 's/[Vv]ersion:[ vV]*//' | tr -cd '0-9.') | |
| if [[ -n "$version" ]]; then | |
| echo "$version_file_path found version: $version" | |
| update_version="$version" | |
| break | |
| else | |
| echo "$version_file_path has no version" | |
| fi | |
| fi | |
| done | |
| if [[ -n "$update_version" ]]; then | |
| echo "$key has the latest version $update_version found" | |
| latest_version=$update_version | |
| else | |
| echo "$key has no version" | |
| fi | |
| fi | |
| # if the model path exits, files are overwrittin already; contents only updated already without version change | |
| if [ -d "$model_path" ]; then | |
| if [ "$latest_version" = "$current_version" ] && [[ "${{env.updated_path}}" == *"$model_path"* ]]; then | |
| echo "Contents updated without any version change" | |
| fi | |
| if [ "$latest_version" != "$current_version" ] && [[ -n "$latest_version" ]]; then | |
| echo "New version is detected" | |
| # Create a new directory if it doesn't exist | |
| new_cache_directory="cache/$key/$latest_version" | |
| echo "new_cache_directory: $new_cache_directory" | |
| if [ ! -d "$new_cache_directory" ]; then | |
| echo "Version $latest_version does NOT exist in cache/$key/. Creating directory..." | |
| mkdir -p "$new_cache_directory" | |
| echo "Directory $new_cache_directory created." | |
| # Copy all files from the previous version into the new folder | |
| prev_version_dir="cache/$key/$current_version/" | |
| if [ -d "$prev_version_dir" ]; then | |
| cp -r "$prev_version_dir/." "$new_cache_directory/" | |
| fi | |
| # Copy(overwrite) all existing contents into the new folder | |
| cp -r "$model_path/"* "$new_cache_directory/" | |
| echo "Copied contents from $model_path to cache/$key/$new_cache_directory." | |
| else | |
| echo "Directory $new_cache_directory already exists." | |
| fi | |
| # Update content.json and handle version mismatch | |
| updated_content=$(echo "$content" | jq --arg key "$key" --arg model_v "$latest_version" ' | |
| .[$key]["versions"] |= (if index($model_v) == null then [$model_v] + . else . end) | | |
| .[$key]["current-version"] = $model_v | |
| ') | |
| echo "$updated_content" > cache/content.json | |
| fi | |
| else | |
| echo "Model file $model_path not found or model file is not properly defined" | |
| fi | |
| done | |
| - name: Ensure branch is created and checked out | |
| run: | | |
| git checkout ${{ matrix.branch-name }} || git checkout -b ${{ matrix.branch-name }} | |
| - name: Configure Git | |
| run: | | |
| git config --global user.name "GitHub Action" | |
| git config --global user.email "[email protected]" | |
| - name: Stage all files includes submodules | |
| run: | | |
| git add --all | |
| - name: Check for any changes before committing | |
| run: | | |
| if git diff --cached --quiet; then | |
| echo "No changes to commit" | |
| echo "skip_commit=true" >> $GITHUB_ENV | |
| else | |
| git commit -m "Detected changes" | |
| echo "skip_commit=false" >> $GITHUB_ENV | |
| fi | |
| - name: Push changes | |
| if: env.skip_commit == 'false' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git | |
| git push origin ${{ matrix.branch-name }} |