[monorepo] split out build-only web engine into its own builder

flutter · Dec 13, 2024 · 1606fa2 · 1606fa2
1 parent 93b7c61
commit 1606fa2
Show file tree

Hide file tree

Showing 4 changed files with 143 additions and 56 deletions.
diff --git a/.ci.yaml b/.ci.yaml
@@ -380,11 +380,35 @@ targets:
       config_name: linux_license
       clobber: "true"
 
-  - name: Linux linux_web_engine
+  - name: Linux linux_web_engine_build
     recipe: engine_v2/engine_v2
     timeout: 120
     properties:
       release_build: "true"
+      config_name: linux_web_engine_build
+    # Do not remove(https://github.com/flutter/flutter/issues/144644)
+    # Scheduler will fail to get the platform
+    drone_dimensions:
+      - os=Linux
+    dimensions:
+      # This is needed so that orchestrators that only spawn subbuilds are not
+      # assigned to the large 32 core workers when doing release builds.
+      # For more details see the issue
+      # at https://github.com/flutter/flutter/issues/152186.
+      cores: "8"
+    runIf:
+      - DEPS
+      - .ci.yaml
+      - lib/web_ui/**
+      - web_sdk/**
+      - tools/**
+      - ci/**
+      - flutter_frontend_server/**
+
+  - name: Linux linux_web_engine
+    recipe: engine_v2/engine_v2
+    timeout: 120
+    properties:
       config_name: linux_web_engine
     # Do not remove(https://github.com/flutter/flutter/issues/144644)
     # Scheduler will fail to get the platform

diff --git a/ci/builders/linux_web_engine.json b/ci/builders/linux_web_engine.json
@@ -25,17 +25,6 @@
           "flutter/web_sdk:flutter_web_sdk_archive"
         ]
       },
-      "archives": [
-        {
-          "name": "wasm_release",
-          "base_path": "out/wasm_release/zip_archives/",
-          "type": "gcs",
-          "include_paths": [
-            "out/wasm_release/zip_archives/flutter-web-sdk.zip"
-          ],
-          "realm": "production"
-        }
-      ],
       "generators": {
         "tasks": [
           {

diff --git a/ci/builders/linux_web_engine_build.json b/ci/builders/linux_web_engine_build.json
@@ -0,0 +1,41 @@
+{
+  "_comment": "THIS IS A GENERATED FILE. Do not edit this file directly.",
+  "_comment2": "See `generate_builder_json.dart` for the generator code",
+  "builds": [
+    {
+      "name": "web_build/artifacts",
+      "drone_dimensions": [
+        "device_type=none",
+        "os=Linux",
+        "cores=32"
+      ],
+      "gclient_variables": {
+        "download_android_deps": false,
+        "download_jdk": false,
+        "download_emsdk": true
+      },
+      "gn": [
+        "--web",
+        "--runtime-mode=release",
+        "--no-goma"
+      ],
+      "ninja": {
+        "config": "wasm_release",
+        "targets": [
+          "flutter/web_sdk:flutter_web_sdk_archive"
+        ]
+      },
+      "archives": [
+        {
+          "name": "wasm_release",
+          "base_path": "out/wasm_release/zip_archives/",
+          "type": "gcs",
+          "include_paths": [
+            "out/wasm_release/zip_archives/flutter-web-sdk.zip"
+          ],
+          "realm": "production"
+        }
+      ]
+    }
+  ]
+}
diff --git a/lib/web_ui/dev/generate_builder_json.dart b/lib/web_ui/dev/generate_builder_json.dart
@@ -28,34 +28,66 @@ class GenerateBuilderJsonCommand extends Command<bool> {
     final FeltConfig config = FeltConfig.fromFile(
       path.join(environment.webUiTestDir.path, 'felt_config.yaml')
     );
-    final String configString = generate(config, packageLock);
-    final io.File configFile = io.File(path.join(
+
+    // Generate the config that only builds the engine, but does not run tests.
+    // This allows starting framework as soon as the engine is built, without
+    // waiting for the engine tests. This is also used to skip tests entirely
+    // when running in the merge queue.
+    final String buildConfigString = generateBuildConfig(config, packageLock);
+    final io.File buildConfigFile = io.File(path.join(
+      environment.flutterDirectory.path,
+      'ci',
+      'builders',
+      'linux_web_engine_build.json',
+    ));
+    buildConfigFile.writeAsStringSync('$buildConfigString\n');
+
+    // Generate the config for the full build that includes both the engine
+    // build and tests.
+    final String buildAndTestConfigString = generate(config, packageLock);
+    final io.File buildAndTestConfigFile = io.File(path.join(
       environment.flutterDirectory.path,
       'ci',
       'builders',
       'linux_web_engine.json',
     ));
-    configFile.writeAsStringSync('$configString\n');
+    buildAndTestConfigFile.writeAsStringSync('$buildAndTestConfigString\n');
+
     return true;
   }
 
+  String generateBuildConfig(FeltConfig config, PackageLock packageLock) {
+    final Map<String, dynamic> outputJson = <String, dynamic>{
+      '_comment': 'THIS IS A GENERATED FILE. Do not edit this file directly.',
+      '_comment2': 'See `generate_builder_json.dart` for the generator code',
+      'builds': <dynamic>[
+        _getArtifactBuildStep(name: 'web_build/artifacts', forTesting: false),
+      ],
+    };
+    return const JsonEncoder.withIndent('  ').convert(outputJson);
+  }
+
   String generate(FeltConfig config, PackageLock packageLock) {
     final Map<String, dynamic> outputJson = <String, dynamic>{
       '_comment': 'THIS IS A GENERATED FILE. Do not edit this file directly.',
       '_comment2': 'See `generate_builder_json.dart` for the generator code',
       'builds': <dynamic>[
-        _getArtifactBuildStep(),
-        for (final TestBundle bundle in config.testBundles)
-          _getBundleBuildStep(bundle),
+        _getArtifactBuildStep(name: 'web_tests/artifacts', forTesting: true),
+        if (true)
+          for (final TestBundle bundle in config.testBundles)
+            _getBundleBuildStep(bundle),
       ],
       'tests': _getAllTestSteps(config.testSuites, packageLock)
     };
     return const JsonEncoder.withIndent('  ').convert(outputJson);
   }
 
-  Map<String, dynamic> _getArtifactBuildStep() {
+  Map<String, dynamic> _getArtifactBuildStep({
+    required String name,
+    required bool forTesting,
+  }) {
     return <String, dynamic>{
-      'name': 'web_tests/artifacts',
+      'name': name,
       'drone_dimensions': <String>[
         'device_type=none',
         'os=Linux',
@@ -77,44 +109,45 @@ class GenerateBuilderJsonCommand extends Command<bool> {
           'flutter/web_sdk:flutter_web_sdk_archive'
         ]
       },
-      'archives': <dynamic>[
-        <String, dynamic>{
-          'name': 'wasm_release',
-          'base_path': 'out/wasm_release/zip_archives/',
-          'type': 'gcs',
-          'include_paths': <String>[
-            'out/wasm_release/zip_archives/flutter-web-sdk.zip'
-          ],
-          'realm': 'production',
-        }
-      ],
-      'generators': <String, dynamic>{
-        'tasks': <dynamic>[
+      if (!forTesting)
+        'archives': <dynamic>[
           <String, dynamic>{
-            'name': 'check licenses',
-            'parameters': <String>[
-              'check-licenses'
-            ],
-            'scripts': <String>[ 'flutter/lib/web_ui/dev/felt' ],
-
-          },
-          <String, dynamic>{
-            'name': 'web engine analysis',
-            'parameters': <String>[
-              'analyze'
-            ],
-            'scripts': <String>[ 'flutter/lib/web_ui/dev/felt' ],
-          },
-          <String, dynamic>{
-            'name': 'copy artifacts for web tests',
-            'parameters': <String>[
-              'test',
-              '--copy-artifacts',
+            'name': 'wasm_release',
+            'base_path': 'out/wasm_release/zip_archives/',
+            'type': 'gcs',
+            'include_paths': <String>[
+              'out/wasm_release/zip_archives/flutter-web-sdk.zip'
             ],
-            'scripts': <String>[ 'flutter/lib/web_ui/dev/felt' ],
-          },
-        ]
-      },
+            'realm': 'production',
+          }
+        ],
+      if (forTesting)
+        'generators': <String, dynamic>{
+          'tasks': <dynamic>[
+            <String, dynamic>{
+              'name': 'check licenses',
+              'parameters': <String>[
+                'check-licenses'
+              ],
+              'scripts': <String>[ 'flutter/lib/web_ui/dev/felt' ],
+            },
+            <String, dynamic>{
+              'name': 'web engine analysis',
+              'parameters': <String>[
+                'analyze'
+              ],
+              'scripts': <String>[ 'flutter/lib/web_ui/dev/felt' ],
+            },
+            <String, dynamic>{
+              'name': 'copy artifacts for web tests',
+              'parameters': <String>[
+                'test',
+                '--copy-artifacts',
+              ],
+              'scripts': <String>[ 'flutter/lib/web_ui/dev/felt' ],
+            },
+          ]
+        },
     };
   }