Reland buildbucket v2 migration parts 3&4 (#3697)

Reverts #3686, re-landing the most recent parts of the buildbucket v2 migration.

The one difference relative to the original PRs (see commit 2 in the PR) is that in addition to duplicating all the services and handlers, the original PR duplicated the *cron jobs*. My initial impression of the scheduler code is that there's a non-trivial risk of race conditions in the current design, and so my speculation is that having two cron jobs scheduled at the same time, both trying to run backfill, tripped race conditions that put the databases into bad states.

This is definitely speculative, but given that my understanding is that v2 was successfully handling traffic for at least a day, a race condition rather than a fundamental problem with the implementation of the v2 communication protocol is certainly plausible. My proposal is that we deploy this to test that theory, and to get new data for failure analysis if it doesn't work, but be prepared to roll back and manually adjust the DB again if necessary.

app_dart/bin/gae_server.dart

@@ -7,7 +7,11 @@ import 'dart:io';
 import 'package:appengine/appengine.dart';
 import 'package:cocoon_service/cocoon_service.dart';
 import 'package:cocoon_service/server.dart';
+import 'package:cocoon_service/src/service/build_bucket_v2_client.dart';
 import 'package:cocoon_service/src/service/commit_service.dart';
+import 'package:cocoon_service/src/service/github_checks_service_v2.dart';
+import 'package:cocoon_service/src/service/luci_build_service_v2.dart';
+import 'package:cocoon_service/src/service/scheduler_v2.dart';
 import 'package:gcloud/db.dart';
 
 Future<void> main() async {
@@ -18,15 +22,28 @@ Future<void> main() async {
     final Config config = Config(dbService, cache);
     final AuthenticationProvider authProvider = AuthenticationProvider(config: config);
     final AuthenticationProvider swarmingAuthProvider = SwarmingAuthenticationProvider(config: config);
+
     final BuildBucketClient buildBucketClient = BuildBucketClient(
       accessTokenService: AccessTokenService.defaultProvider(config),
     );
 
+    final BuildBucketV2Client buildBucketV2Client = BuildBucketV2Client(
+      accessTokenService: AccessTokenService.defaultProvider(config),
+    );
+
     /// LUCI service class to communicate with buildBucket service.
     final LuciBuildService luciBuildService = LuciBuildService(
       config: config,
       cache: cache,
       buildBucketClient: buildBucketClient,
+      buildBucketV2Client: buildBucketV2Client,
+      pubsub: const PubSub(),
+    );
+
+    final LuciBuildServiceV2 luciBuildServiceV2 = LuciBuildServiceV2(
+      config: config,
+      cache: cache,
+      buildBucketV2Client: buildBucketV2Client,
       pubsub: const PubSub(),
     );
 
@@ -35,6 +52,10 @@ Future<void> main() async {
       config,
     );
 
+    final GithubChecksServiceV2 githubChecksServiceV2 = GithubChecksServiceV2(
+      config,
+    );
+
     // Gerrit service class to communicate with GoB.
     final GerritService gerritService = GerritService(config: config);
 
@@ -46,6 +67,13 @@ Future<void> main() async {
       luciBuildService: luciBuildService,
     );
 
+    final SchedulerV2 schedulerV2 = SchedulerV2(
+      cache: cache,
+      config: config,
+      githubChecksService: githubChecksServiceV2,
+      luciBuildService: luciBuildServiceV2,
+    );
+
     final BranchService branchService = BranchService(
       config: config,
       gerritService: gerritService,
@@ -59,10 +87,14 @@ Future<void> main() async {
       authProvider: authProvider,
       branchService: branchService,
       buildBucketClient: buildBucketClient,
+      buildBucketV2Client: buildBucketV2Client,
       gerritService: gerritService,
       scheduler: scheduler,
+      schedulerV2: schedulerV2,
       luciBuildService: luciBuildService,
+      luciBuildServiceV2: luciBuildServiceV2,
       githubChecksService: githubChecksService,
+      githubChecksServiceV2: githubChecksServiceV2,
       commitService: commitService,
       swarmingAuthProvider: swarmingAuthProvider,
     );

app_dart/bin/local_server.dart

@@ -8,8 +8,12 @@ import 'package:appengine/appengine.dart';
 import 'package:cocoon_service/cocoon_service.dart';
 import 'package:cocoon_service/server.dart';
 import 'package:cocoon_service/src/model/appengine/cocoon_config.dart';
+import 'package:cocoon_service/src/service/build_bucket_v2_client.dart';
 import 'package:cocoon_service/src/service/commit_service.dart';
 import 'package:cocoon_service/src/service/datastore.dart';
+import 'package:cocoon_service/src/service/github_checks_service_v2.dart';
+import 'package:cocoon_service/src/service/luci_build_service_v2.dart';
+import 'package:cocoon_service/src/service/scheduler_v2.dart';
 import 'package:gcloud/db.dart';
 
 import '../test/src/datastore/fake_datastore.dart';
@@ -25,15 +29,28 @@ Future<void> main() async {
   final Config config = Config(dbService, cache);
   final AuthenticationProvider authProvider = AuthenticationProvider(config: config);
   final AuthenticationProvider swarmingAuthProvider = SwarmingAuthenticationProvider(config: config);
+
   final BuildBucketClient buildBucketClient = BuildBucketClient(
     accessTokenService: AccessTokenService.defaultProvider(config),
   );
 
+  final BuildBucketV2Client buildBucketV2Client = BuildBucketV2Client(
+    accessTokenService: AccessTokenService.defaultProvider(config),
+  );
+
   /// LUCI service class to communicate with buildBucket service.
   final LuciBuildService luciBuildService = LuciBuildService(
     config: config,
     cache: cache,
     buildBucketClient: buildBucketClient,
+    buildBucketV2Client: buildBucketV2Client,
+    pubsub: const PubSub(),
+  );
+
+  final LuciBuildServiceV2 luciBuildServiceV2 = LuciBuildServiceV2(
+    config: config,
+    cache: cache,
+    buildBucketV2Client: buildBucketV2Client,
     pubsub: const PubSub(),
   );
 
@@ -42,6 +59,10 @@ Future<void> main() async {
     config,
   );
 
+  final GithubChecksServiceV2 githubChecksServiceV2 = GithubChecksServiceV2(
+    config,
+  );
+
   // Gerrit service class to communicate with GoB.
   final GerritService gerritService = GerritService(config: config);
 
@@ -53,6 +74,13 @@ Future<void> main() async {
     luciBuildService: luciBuildService,
   );
 
+  final SchedulerV2 schedulerV2 = SchedulerV2(
+    cache: cache,
+    config: config,
+    githubChecksService: githubChecksServiceV2,
+    luciBuildService: luciBuildServiceV2,
+  );
+
   final BranchService branchService = BranchService(
     config: config,
     gerritService: gerritService,
@@ -66,10 +94,14 @@ Future<void> main() async {
     authProvider: authProvider,
     branchService: branchService,
     buildBucketClient: buildBucketClient,
+    buildBucketV2Client: buildBucketV2Client,
     gerritService: gerritService,
     scheduler: scheduler,
+    schedulerV2: schedulerV2,
     luciBuildService: luciBuildService,
+    luciBuildServiceV2: luciBuildServiceV2,
     githubChecksService: githubChecksService,
+    githubChecksServiceV2: githubChecksServiceV2,
     commitService: commitService,
     swarmingAuthProvider: swarmingAuthProvider,
   );

app_dart/lib/server.dart

@@ -6,7 +6,18 @@ import 'dart:io';
 import 'dart:math';
 
 import 'package:cocoon_service/cocoon_service.dart';
+import 'package:cocoon_service/src/request_handlers/postsubmit_luci_subscription_v2.dart';
+import 'package:cocoon_service/src/request_handlers/presubmit_luci_subscription_v2.dart';
+import 'package:cocoon_service/src/request_handlers/reset_prod_task_v2.dart';
+import 'package:cocoon_service/src/request_handlers/reset_try_task_v2.dart';
+import 'package:cocoon_service/src/request_handlers/scheduler/batch_backfiller_v2.dart';
+import 'package:cocoon_service/src/request_handlers/scheduler/scheduler_request_subscription.dart';
+import 'package:cocoon_service/src/request_handlers/vacuum_github_commits_v2.dart';
+import 'package:cocoon_service/src/service/build_bucket_v2_client.dart';
 import 'package:cocoon_service/src/service/commit_service.dart';
+import 'package:cocoon_service/src/service/github_checks_service_v2.dart';
+import 'package:cocoon_service/src/service/luci_build_service_v2.dart';
+import 'package:cocoon_service/src/service/scheduler_v2.dart';
 
 typedef Server = Future<void> Function(HttpRequest);
 
@@ -18,11 +29,15 @@ Server createServer({
   required AuthenticationProvider swarmingAuthProvider,
   required BranchService branchService,
   required BuildBucketClient buildBucketClient,
+  required BuildBucketV2Client buildBucketV2Client,
   required LuciBuildService luciBuildService,
+  required LuciBuildServiceV2 luciBuildServiceV2,
   required GithubChecksService githubChecksService,
+  required GithubChecksServiceV2 githubChecksServiceV2,
   required CommitService commitService,
   required GerritService gerritService,
   required Scheduler scheduler,
+  required SchedulerV2 schedulerV2,
 }) {
   final Map<String, RequestHandler<dynamic>> handlers = <String, RequestHandler<dynamic>>{
     '/api/check_flaky_builders': CheckFlakyBuilders(
@@ -37,7 +52,7 @@ Server createServer({
     '/api/dart-internal-subscription': DartInternalSubscription(
       cache: cache,
       config: config,
-      buildBucketClient: buildBucketClient,
+      buildBucketV2Client: buildBucketV2Client,
     ),
     '/api/file_flaky_issue_and_pr': FileFlakyIssueAndPR(
       config: config,
@@ -65,24 +80,36 @@ Server createServer({
       config: config,
       cache: cache,
       gerritService: gerritService,
-      githubChecksService: githubChecksService,
       scheduler: scheduler,
+      schedulerV2: schedulerV2,
       commitService: commitService,
     ),
     '/api/presubmit-luci-subscription': PresubmitLuciSubscription(
       cache: cache,
       config: config,
-      buildBucketClient: buildBucketClient,
       luciBuildService: luciBuildService,
       githubChecksService: githubChecksService,
       scheduler: scheduler,
     ),
+    '/api/v2/presubmit-luci-subscription': PresubmitLuciSubscriptionV2(
+      cache: cache,
+      config: config,
+      luciBuildService: luciBuildServiceV2,
+      githubChecksService: githubChecksServiceV2,
+      scheduler: schedulerV2,
+    ),
     '/api/postsubmit-luci-subscription': PostsubmitLuciSubscription(
       cache: cache,
       config: config,
       scheduler: scheduler,
       githubChecksService: githubChecksService,
     ),
+    '/api/v2/postsubmit-luci-subscription': PostsubmitLuciSubscriptionV2(
+      cache: cache,
+      config: config,
+      scheduler: schedulerV2,
+      githubChecksService: githubChecksServiceV2,
+    ),
     '/api/push-build-status-to-github': PushBuildStatusToGithub(
       config: config,
       authenticationProvider: authProvider,
@@ -91,26 +118,47 @@ Server createServer({
       config: config,
       authenticationProvider: authProvider,
     ),
+    // I do not believe these recieve a build message.
     '/api/reset-prod-task': ResetProdTask(
       config: config,
       authenticationProvider: authProvider,
       luciBuildService: luciBuildService,
       scheduler: scheduler,
     ),
+    '/api/v2/reset-prod-task': ResetProdTaskV2(
+      config: config,
+      authenticationProvider: authProvider,
+      luciBuildService: luciBuildServiceV2,
+      scheduler: schedulerV2,
+    ),
     '/api/reset-try-task': ResetTryTask(
       config: config,
       authenticationProvider: authProvider,
       scheduler: scheduler,
     ),
+    '/api/v2/reset-try-task': ResetTryTaskV2(
+      config: config,
+      authenticationProvider: authProvider,
+      scheduler: schedulerV2,
+    ),
     '/api/scheduler/batch-backfiller': BatchBackfiller(
       config: config,
       scheduler: scheduler,
     ),
+    '/api/v2/scheduler/batch-backfiller': BatchBackfillerV2(
+      config: config,
+      scheduler: schedulerV2,
+    ),
     '/api/scheduler/batch-request-subscription': SchedulerRequestSubscription(
       cache: cache,
       config: config,
       buildBucketClient: buildBucketClient,
     ),
+    '/api/v2/scheduler/batch-request-subscription': SchedulerRequestSubscriptionV2(
+      cache: cache,
+      config: config,
+      buildBucketClient: buildBucketV2Client,
+    ),
     '/api/scheduler/vacuum-stale-tasks': VacuumStaleTasks(
       config: config,
     ),
@@ -144,6 +192,11 @@ Server createServer({
       authenticationProvider: authProvider,
       scheduler: scheduler,
     ),
+    '/api/v2/vacuum-github-commits': VacuumGithubCommitsV2(
+      config: config,
+      authenticationProvider: authProvider,
+      scheduler: schedulerV2,
+    ),
 
     /// Returns status of the framework tree.
     ///