Skip flaky eval integration tests for test registry #16231
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2020 The Chromium Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| name: devtools | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - master | |
| # Declare default permissions as read only. | |
| permissions: read-all | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| jobs: | |
| flutter-prep: | |
| uses: ./.github/workflows/flutter-prep.yaml | |
| main: | |
| name: main | |
| needs: flutter-prep | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: git clone | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| - name: Load Cached Flutter SDK | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 | |
| with: | |
| path: | | |
| ./tool/flutter-sdk | |
| key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} | |
| - name: tool/ci/bots.sh | |
| env: | |
| BOT: main | |
| run: ./tool/ci/bots.sh | |
| dcm: | |
| name: Dart Code Metrics | |
| needs: flutter-prep | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: Clone Flutter DevTools | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| with: | |
| ref: "${{ github.event.pull_request.head.sha }}" | |
| - name: Load Cached Flutter SDK | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 | |
| with: | |
| path: | | |
| ./tool/flutter-sdk | |
| key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} | |
| - name: Run tool/ci/bots.sh | |
| run: ./tool/ci/bots.sh | |
| - name: Install DCM | |
| run: | | |
| sudo apt-get update | |
| wget -qO- https://dcm.dev/pgp-key.public | sudo gpg --dearmor -o /usr/share/keyrings/dcm.gpg | |
| echo 'deb [signed-by=/usr/share/keyrings/dcm.gpg arch=amd64] https://dcm.dev/debian stable main' | sudo tee /etc/apt/sources.list.d/dart_stable.list | |
| sudo apt-get update | |
| sudo apt-get install dcm=1.24.2-1 # To avoid errors add `-1` (build number) to the version | |
| sudo chmod +x /usr/bin/dcm | |
| echo "$(dcm --version)" | |
| - name: Setup Dart SDK | |
| uses: dart-lang/setup-dart@0a8a0fc875eb934c15d08629302413c671d3f672 | |
| - name: Run DCM on root | |
| run: | | |
| dcm analyze packages/devtools_app packages/devtools_app_shared packages/devtools_extensions packages/devtools_shared packages/devtools_test | |
| test-packages: | |
| name: ${{ matrix.package }} test | |
| needs: flutter-prep | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: | |
| - devtools_app_shared | |
| - devtools_extensions | |
| - devtools_shared | |
| steps: | |
| - name: git clone | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| - name: Load Cached Flutter SDK | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 | |
| with: | |
| path: | | |
| ./tool/flutter-sdk | |
| key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} | |
| - name: tool/ci/package_tests.sh | |
| env: | |
| PACKAGE: ${{ matrix.package }} | |
| run: ./tool/ci/package_tests.sh | |
| test-tool: | |
| name: Tool test | |
| needs: flutter-prep | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: git clone | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| - name: Load Cached Flutter SDK | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 | |
| with: | |
| path: | | |
| ./tool/flutter-sdk | |
| key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} | |
| - name: tool/ci/tool_tests.sh | |
| run: ./tool/ci/tool_tests.sh | |
| test: | |
| name: ${{ matrix.bot }} | |
| needs: flutter-prep | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| bot: | |
| - build_ddc | |
| - build_dart2js | |
| - test_ddc | |
| - test_dart2js | |
| steps: | |
| - name: git clone | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| - name: Load Cached Flutter SDK | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 | |
| with: | |
| path: | | |
| ./tool/flutter-sdk | |
| key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} | |
| - name: tool/ci/bots.sh | |
| env: | |
| BOT: ${{ matrix.bot }} | |
| PLATFORM: vm | |
| run: ./tool/ci/bots.sh | |
| macos-test: | |
| needs: flutter-prep | |
| name: macos goldens ${{ matrix.bot }} | |
| runs-on: macos-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| bot: | |
| - test_dart2js | |
| only_golden: | |
| - true | |
| steps: | |
| - name: git clone | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| - name: Load Cached Flutter SDK | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 | |
| with: | |
| path: | | |
| ./tool/flutter-sdk | |
| key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} | |
| - name: tool/ci/bots.sh | |
| env: | |
| BOT: ${{ matrix.bot }} | |
| PLATFORM: vm | |
| ONLY_GOLDEN: ${{ matrix.only_golden }} | |
| run: ./tool/ci/bots.sh | |
| - name: Upload Golden Failure Artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: golden_image_failures.${{ matrix.bot }} | |
| path: packages/devtools_app/test/**/failures/*.png | |
| - name: Notify of Quick Fix | |
| if: failure() | |
| env: | |
| WORKFLOW_ID: ${{ github.run_id }} | |
| run: | | |
| echo "::notice title=To Quickly Fix Goldens:: Run \`dt fix-goldens --run-id=$WORKFLOW_ID\` on your local branch." | |
| devtools-app-integration-test: | |
| name: devtools_app integration-test ${{ matrix.bot }} - ${{ matrix.device }} - shard ${{ matrix.shard }} | |
| needs: flutter-prep | |
| runs-on: macos-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Consider running integration tests in ddc mode, too. | |
| bot: [integration_dart2js] | |
| device: [flutter, flutter-web, dart-cli] | |
| # Option 1/1 will run all tests for a device in a single shard. | |
| # Option 1/2 and 2/2 should be enabled to run tests for a device in 2 shards. | |
| shard: [1/1, 1/2, 2/2, 1/3, 2/3, 3/3] | |
| exclude: | |
| # The 'flutter' device should be ran in three shards. | |
| - device: flutter | |
| shard: 1/1 | |
| - device: flutter | |
| shard: 1/2 | |
| - device: flutter | |
| shard: 2/2 | |
| # The 'flutter-web' device should be ran in two shards. | |
| - device: flutter-web | |
| shard: 1/1 | |
| - device: flutter-web | |
| shard: 1/3 | |
| - device: flutter-web | |
| shard: 2/3 | |
| - device: flutter-web | |
| shard: 3/3 | |
| # The 'dart-cli' device can be ran in a single shard. | |
| - device: dart-cli | |
| shard: 1/2 | |
| - device: dart-cli | |
| shard: 2/2 | |
| - device: dart-cli | |
| shard: 1/3 | |
| - device: dart-cli | |
| shard: 2/3 | |
| - device: dart-cli | |
| shard: 3/3 | |
| steps: | |
| - name: git clone | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| - name: Load Cached Flutter SDK | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 | |
| with: | |
| path: | | |
| ./tool/flutter-sdk | |
| key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} | |
| - name: tool/ci/bots.sh | |
| env: | |
| BOT: ${{ matrix.bot }} | |
| DEVICE: ${{ matrix.device }} | |
| SHARD: ${{ matrix.shard }} | |
| DEVTOOLS_PACKAGE: devtools_app | |
| run: ./tool/ci/bots.sh | |
| - name: Upload Golden Failure Artifacts | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: golden_image_failures.${{ matrix.bot }} | |
| path: packages/devtools_app/integration_test/**/failures/*.png | |
| devtools-extensions-integration-test: | |
| name: devtools_extensions integration-test ${{ matrix.bot }} | |
| needs: flutter-prep | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Consider running integration tests in ddc mode, too. | |
| bot: [integration_dart2js] | |
| steps: | |
| - name: git clone | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| - name: Load Cached Flutter SDK | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 | |
| with: | |
| path: | | |
| ./tool/flutter-sdk | |
| key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} | |
| - name: tool/ci/bots.sh | |
| env: | |
| BOT: ${{ matrix.bot }} | |
| DEVTOOLS_PACKAGE: devtools_extensions | |
| run: ./tool/ci/bots.sh | |
| benchmark-performance: | |
| name: benchmark-performance | |
| needs: flutter-prep | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: git clone | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| - name: Load Cached Flutter SDK | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 | |
| with: | |
| path: | | |
| ./tool/flutter-sdk | |
| key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} | |
| - name: tool/ci/benchmark_performance.sh | |
| run: ./tool/ci/benchmark_performance.sh | |
| benchmark-size: | |
| name: benchmark-size | |
| needs: flutter-prep | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: git clone | |
| uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| - name: Load Cached Flutter SDK | |
| uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 | |
| with: | |
| path: | | |
| ./tool/flutter-sdk | |
| key: flutter-sdk-${{ runner.os }}-${{ needs.flutter-prep.outputs.latest_flutter_candidate }} | |
| - name: tool/ci/benchmark_size.sh | |
| run: ./tool/ci/benchmark_size.sh | |
| # TODO(https://github.com/flutter/devtools/issues/1715): add a windows compatible version of tool/ci/bots.sh | |
| # and run it from this job. | |
| # windows-test: | |
| # name: windows ${{ matrix.bot }} | |
| # runs-on: windows-latest | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # bot: | |
| # - test_dart2js | |
| # steps: | |
| # - name: git clone | |
| # uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| # | |
| # - name: tool/ci/bots.sh | |
| # env: | |
| # BOT: ${{ matrix.bot }} | |
| # PLATFORM: vm | |
| # run: ./tool/ci/bots.sh | |
| # TODO(https://github.com/flutter/devtools/issues/1987): rewrite integration tests. | |
| # integration: | |
| # name: integration ${{ matrix.bot }} | |
| # runs-on: ubuntu-latest | |
| # strategy: | |
| # fail-fast: false | |
| # matrix: | |
| # bot: | |
| # - integration_ddc | |
| # - integration_dart2js | |
| # steps: | |
| # - name: git clone | |
| # uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac | |
| # - name: tool/ci/bots.sh | |
| # env: | |
| # BOT: ${{ matrix.bot }} | |
| # run: ./tool/ci/bots.sh | |
| # TODO(https://github.com/flutter/devtools/issues/2437): | |
| # PLATFORM=chrome is going away. We need to move these tests to run with | |
| # chromedriver. | |
| # - BOT=test_ddc PLATFORM=chrome | |
| # PLATFORM=chrome is going away. We need to move these tests to run with | |
| # chromedriver. | |
| # - BOT=test_dart2js PLATFORM=chrome | |