Merge branch 'linkedin:main' into ttlRepush

linkedin · Dec 18, 2024 · 91a472b · 91a472b
2 parents bc6967c + 543f65c
commit 91a472b
Show file tree

Hide file tree

Showing 408 changed files with 6,383 additions and 54,201 deletions.
diff --git a/.github/rawWorkflows/gh-ci-parameterized-flow.txt b/.github/rawWorkflows/gh-ci-parameterized-flow.txt
@@ -1,4 +1,5 @@
   $FlowName:
+    name: $FlowName
     strategy:
       fail-fast: false
       matrix:
@@ -8,9 +9,14 @@
      id-token: write
      contents: read
      checks: write
+     pull-requests: write
+     issues: write
     needs: $Dependency
     if: $Conditional
     timeout-minutes: $TimeOut
+    concurrency:
+     group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-jdk${{ matrix.jdk }}-$FlowName
+     cancel-in-progress: ${{ github.event_name == 'pull_request' }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -28,6 +34,8 @@
           git fetch upstream
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4
+        with:
+          add-job-summary: never
       - name: Run Integration Tests
         run: ./gradlew $GradleArguments
       - name: Package Build Artifacts
@@ -47,18 +55,42 @@
         env:
          NODE_OPTIONS: --max-old-space-size=9182
         with:
-         token: ${{ secrets.GITHUB_TOKEN }}
-         name: ${{ github.job }} Test Reports       # Name where it report the test results
-         path: '**/TEST-*.xml'
-         fail-on-error: 'false'
-         max-annotations: '10'
-         list-tests: 'all'
-         list-suites: 'all'
-         reporter: java-junit
+          token: ${{ secrets.GITHUB_TOKEN }}
+          name: ${{ github.job }} Test Reports       # Name where it report the test results
+          path: '**/TEST-*.xml'
+          fail-on-error: 'false'
+          max-annotations: '10'
+          list-tests: 'all'
+          list-suites: 'all'
+          reporter: java-junit
+      - name: Publish Test Report
+        continue-on-error: true
+        env:
+          NODE_OPTIONS: "--max_old_space_size=8192"
+        uses: mikepenz/action-junit-report@v5
+        if: always()
+        with:
+          check_name: ${{ github.job }}-jdk${{ matrix.jdk }} Report
+          comment: false
+          annotate_only: true
+          flaky_summary: true
+          commit: ${{github.event.workflow_run.head_sha}}
+          detailed_summary: true
+          report_paths: '**/build/test-results/test/TEST-*.xml'
       - name: Upload Build Artifacts
         if: success() || failure()
         uses: actions/upload-artifact@v4
         with:
           name: ${{ github.job }}
           path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
           retention-days: 30
+      - name: Upload test results to BuildPulse for flaky test detection
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main' && !cancelled()
+        uses: buildpulse/buildpulse-action@main
+        with:
+          account: 100582612927
+          repository: 100441445875
+          path: |
+            **/TEST-*.xml
+          key: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
+          secret: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
diff --git a/.github/workflows/UnitTests-core.yml b/.github/workflows/UnitTests-core.yml
@@ -22,6 +22,8 @@ jobs:
       contents: read
       checks: write
       id-token: write
+      pull-requests: write
+      issues: write
     steps:
       - uses: actions/checkout@v4
         with:
@@ -39,6 +41,8 @@ jobs:
           git fetch upstream
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4
+        with:
+          add-job-summary: never
       - name: Run Unit Tests with Code Coverage
         run: ./gradlew  -x :internal:venice-avro-compatibility-test:test  ${{ inputs.arg }}
       - name: Package Build Artifacts
@@ -50,12 +54,19 @@ jobs:
           rsync -R --files-from=artifacts.list . ${{ inputs.artifact_suffix }}-artifacts
           tar -zcvf ${{ inputs.artifact_suffix }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ inputs.artifact_suffix }}-artifacts
       - name: Publish Test Report
+        continue-on-error: true
         env:
           NODE_OPTIONS: "--max_old_space_size=8192"
-        uses: mikepenz/action-junit-report@v3
-        if: always() # always run even if the previous step fails
+        uses: mikepenz/action-junit-report@v5
+        if: always()
         with:
-          report_paths: '**/build/test-results/test/TEST-*.xml'
+         check_name: ${{ inputs.artifact_suffix }}-jdk${{ matrix.jdk }} Report
+         comment: false
+         annotate_only: true
+         flaky_summary: true
+         commit: ${{github.event.workflow_run.head_sha}}
+         detailed_summary: true
+         report_paths: '**/build/test-results/test/TEST-*.xml'
       - name: Upload Build Artifacts
         if: success() || failure()
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/VeniceCI-CompatibilityTests.yml b/.github/workflows/VeniceCI-CompatibilityTests.yml
@@ -30,6 +30,8 @@ jobs:
           git fetch upstream
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4
+        with:
+          add-job-summary: never
       - name: Run Avro Compatibility Tests
         run: ./gradlew -DmaxParallelForks=2 --parallel :internal:venice-avro-compatibility-test:test --continue
       - name: Package Build Artifacts
@@ -55,27 +57,16 @@ jobs:
         jdk: [8, 11, 17]
     runs-on: ubuntu-latest
     timeout-minutes: 120
-    outputs:
-      alpini_touched: ${{ steps.check_alpini_files_changed.outputs.alpini }}
     steps:
       - uses: actions/checkout@v4
         with:
           # Checkout as many commits as needed for the diff
           fetch-depth: 2
-      - name: Check if files have changed
-        uses: dorny/paths-filter@v3
-        id: check_alpini_files_changed
-        with:
-          filters: |
-            alpini:
-              - 'internal/alpini/**'
       - uses: actions/checkout@v4
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true'
         with:
           fetch-depth: 0
       - name: Set up JDK
         uses: actions/setup-java@v4
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true'
         with:
           java-version: ${{ matrix.jdk }}
           distribution: 'temurin'
@@ -92,101 +83,29 @@ jobs:
       #     echo "java.security file after modifications: "
       #     cat "$JAVA_HOME/conf/security/java.security"
       - shell: bash
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true'
         run: |
           git remote set-head origin --auto
           git remote add upstream https://github.com/linkedin/venice
           git fetch upstream
       - name: Setup Gradle
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true'
         uses: gradle/actions/setup-gradle@v4
+        with:
+          add-job-summary: never
       - name: Run alpini unit tests
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true'
         run: ./gradlew --continue --no-daemon -DmaxParallelForks=1 alpiniUnitTest
       - name: Package Build Artifacts
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true' && (success() || failure())
-        shell: bash
-        run: |
-          mkdir ${{ github.job }}-artifacts
-          find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
-          rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
-          tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts
-      - name: Upload Build Artifacts
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true' && (success() || failure())
-        uses: actions/upload-artifact@v4
-        with:
-          name: ${{ github.job }}
-          path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
-          retention-days: 30
-
-  AlpiniFunctionalTests:
-    strategy:
-      fail-fast: false
-      matrix:
-        jdk: [17]
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    outputs:
-      alpini_touched: ${{ steps.check_alpini_files_changed.outputs.alpini }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          # Checkout as many commits as needed for the diff
-          fetch-depth: 2
-      - name: Check if files have changed
-        uses: dorny/paths-filter@v3
-        id: check_alpini_files_changed
-        with:
-          filters: |
-            alpini:
-              - 'internal/alpini/**'
-      - uses: actions/checkout@v4
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true'
-        with:
-          fetch-depth: 0
-      - name: Set up JDK
-        uses: actions/setup-java@v4
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true'
-        with:
-          java-version: ${{ matrix.jdk }}
-          distribution: 'temurin'
-          cache: 'gradle'
-      # - name: Allow Deprecated TLS versions for Alpini tests
-      #   run: |
-      #     echo "java.security file before modifications: "
-      #     cat "$JAVA_HOME/conf/security/java.security"
-
-      #     # This is possibly flaky but
-      #     sed -i 's/TLSv1, //g' "$JAVA_HOME/conf/security/java.security" # Allow TLSv1
-      #     sed -i 's/TLSv1.1, //g' "$JAVA_HOME/conf/security/java.security" # Allow TLSv1.1
-
-      #     echo "java.security file after modifications: "
-      #     cat "$JAVA_HOME/conf/security/java.security"
-      - shell: bash
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true'
-        run: |
-          git remote set-head origin --auto
-          git remote add upstream https://github.com/linkedin/venice
-          git fetch upstream
-      - name: Setup Gradle
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true'
-        uses: gradle/actions/setup-gradle@v4
-      - name: Run alpini functional tests
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true'
-        run: ./gradlew --continue --no-daemon -DmaxParallelForks=1 alpiniFunctionalTest
-      - name: Package Build Artifacts
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true' && (success() || failure())
+        if: (success() || failure())
         shell: bash
         run: |
           mkdir ${{ github.job }}-artifacts
           find . -path "**/build/reports/*" -or -path "**/build/test-results/*" > artifacts.list
           rsync -R --files-from=artifacts.list . ${{ github.job }}-artifacts
           tar -zcvf ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz ${{ github.job }}-artifacts
       - name: Upload Build Artifacts
-        if: steps.check_alpini_files_changed.outputs.alpini == 'true' && (success() || failure())
+        if: (success() || failure())
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ github.job }}
+          name: ${{ github.job }}-jdk${{ matrix.jdk }}
           path: ${{ github.job }}-jdk${{ matrix.jdk }}-logs.tar.gz
           retention-days: 30
 
@@ -214,6 +133,8 @@ jobs:
           git fetch upstream
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v4
+        with:
+          add-job-summary: never
       - name: Build with gradle
         run: ./gradlew assemble --continue --no-daemon -DforkEvery=1 -DmaxParallelForks=1
 
@@ -257,7 +178,7 @@ jobs:
     strategy:
       fail-fast: false
     runs-on: ubuntu-latest
-    needs: [AvroCompatibilityTests, AlpiniUnitTests, AlpiniFunctionalTests, PulsarVeniceIntegrationTests]
+    needs: [AvroCompatibilityTests, AlpiniUnitTests, PulsarVeniceIntegrationTests]
     timeout-minutes: 120
     steps:
       - name: AllIsWell