Add a max concurrency option (Issue #75) (#76)

* feat: Add a max concurrency option to build/test commands fixes issue #75 * feat: Add maxConcurrency option to global config - Also use 'yup' to validate the global config since it comes with Yarn and provides expressive validation
ojkelly · Mar 2, 2021 · 7189eed · 7189eed
1 parent 9858261
commit 7189eed
Show file tree

Hide file tree

Showing 7 changed files with 112 additions and 72 deletions.
diff --git a/.yarn/plugins/@ojkelly/plugin-build.cjs b/.yarn/plugins/@ojkelly/plugin-build.cjs
diff --git a/.yarnbuildrc.yml b/.yarnbuildrc.yml
@@ -4,3 +4,4 @@ folders:
 enableBetaFeatures:
   folderConfiguration: true
   targetedBuilds: true
+maxConcurrency: 8
diff --git a/packages/plugins/plugin-build/src/commands/build/index.ts b/packages/plugins/plugin-build/src/commands/build/index.ts
@@ -9,9 +9,14 @@ import {
 import { PortablePath } from "@yarnpkg/fslib";
 import { Command, Usage } from "clipanion";
 import path from "path";
+import * as yup from "yup";
 
 import { EventEmitter } from "events";
-import { GetPluginConfiguration, YarnBuildConfiguration } from "../../config";
+import {
+  GetPluginConfiguration,
+  maxConcurrencyValidation,
+  YarnBuildConfiguration,
+} from "../../config";
 import RunSupervisor, { RunSupervisorReporterEvents } from "../supervisor";
 
 import { addTargets } from "../supervisor/workspace";
@@ -38,9 +43,16 @@ export default class Build extends BaseCommand {
   @Command.Boolean(`--ignore-cache`)
   ignoreBuildCache = false;
 
+  @Command.String(`-m,--max-concurrency`)
+  maxConcurrency: string | undefined;
+
   @Command.Rest()
   public buildTarget: string[] = [];
 
+  static schema = yup.object().shape({
+    maxConcurrency: maxConcurrencyValidation,
+  });
+
   static usage: Usage = Command.Usage({
     category: `Build commands`,
     description: `build a package and all its dependencies`,
@@ -49,6 +61,8 @@ export default class Build extends BaseCommand {
       will traverse the dependency graph and efficiently ensure, the packages
       are built in the right order.
 
+      \`-c,--build-command\` is the command to be run in each package (if available), defaults to "build"
+
       - If \`-p,--parallel\` and \`-i,--interlaced\` are both set, Yarn
       will print the lines from the output as it receives them.
       Parallel defaults to true.
@@ -57,10 +71,18 @@ export default class Build extends BaseCommand {
       from each process and print the resulting buffers only after their
       source processes have exited. Defaults to false.
 
+      If the \`--verbose\` flag is set, more information will be logged to stdout than normal.
+      \
+      If the \`--dry-run\` flag is set, it will simulate running a build, but not actually run it.
+
       If the \`--json\` flag is set the output will follow a JSON-stream output
       also known as NDJSON (https://github.com/ndjson/ndjson-spec).
 
-      \`-c,--build-command\` is the command to be run in each package (if available), defaults to "build"
+      If the \`--ignore-cache\` flag is set, every package will be built,
+      regardless of whether is has changed or not.
+
+      \`-m,--max-concurrency\` is the maximum number of builds that can run at a time,
+      defaults to the number of logical CPUs on the current machine. Will override the global config option.
     `,
   });
 
@@ -74,7 +96,16 @@ export default class Build extends BaseCommand {
       this.context.plugins
     );
 
-    const pluginConfiguration: YarnBuildConfiguration = await GetPluginConfiguration(configuration);
+    const pluginConfiguration: YarnBuildConfiguration = await GetPluginConfiguration(
+      configuration
+    );
+
+    // Safe to run because the input string is validated by clipanion using the schema property
+    // TODO: Why doesn't the Command validation cast this for us?
+    const maxConcurrency =
+      this.maxConcurrency === undefined
+        ? pluginConfiguration.maxConcurrency
+        : parseInt(this.maxConcurrency);
 
     const report = await StreamReport.start(
       {
@@ -153,6 +184,7 @@ export default class Build extends BaseCommand {
           dryRun: this.dryRun,
           ignoreRunCache: this.ignoreBuildCache,
           verbose: this.verbose,
+          concurrency: maxConcurrency,
         });
 
         await supervisor.setup();

diff --git a/packages/plugins/plugin-build/src/commands/supervisor/index.ts b/packages/plugins/plugin-build/src/commands/supervisor/index.ts
@@ -20,7 +20,7 @@ import fs from "fs";
 import stripAnsi from "strip-ansi";
 import sliceAnsi from "slice-ansi";
 import { Graph, Node } from "./graph";
-import {Hansi} from "./hansi";
+import { Hansi } from "./hansi";
 
 const YARN_RUN_CACHE_FILENAME = "yarn.build.json" as Filename;
 
@@ -102,25 +102,23 @@ class RunSupervisor {
   dryRun = false;
   ignoreRunCache = false;
   verbose = false;
+  concurrency: number;
+  limit: Limit;
   queue: PQueue;
 
   entrypoints: Node[] = [];
 
-  limit: Limit = PLimit(Math.max(1, cpus().length));
-
   runReporter: EventEmitter = new EventEmitter();
   runReport: RunReport = {
     mutex: new Mutex(),
     totalJobs: 0,
-    previousOutput:``,
+    previousOutput: ``,
     successCount: 0,
     failCount: 0,
     workspaces: {},
     done: false,
   };
 
-  concurrency = Math.max(1, cpus().length);
-
   nextUnitOfWork: Promise<void>[] = [];
 
   errorLogFile: fs.WriteStream | undefined;
@@ -137,6 +135,7 @@ class RunSupervisor {
     dryRun,
     ignoreRunCache,
     verbose,
+    concurrency,
   }: {
     project: Project;
     report: StreamReport;
@@ -147,7 +146,10 @@ class RunSupervisor {
     dryRun: boolean;
     ignoreRunCache: boolean;
     verbose: boolean;
+    concurrency?: number | undefined;
   }) {
+    const resolvedConcurrency = concurrency ?? Math.max(1, cpus().length);
+
     this.configuration = configuration;
     this.pluginConfiguration = pluginConfiguration;
     this.project = project;
@@ -157,9 +159,11 @@ class RunSupervisor {
     this.dryRun = dryRun;
     this.ignoreRunCache = ignoreRunCache;
     this.verbose = verbose;
+    this.concurrency = resolvedConcurrency;
+    this.limit = PLimit(resolvedConcurrency);
 
     this.queue = new PQueue({
-      concurrency: this.concurrency, // TODO: make this customisable
+      concurrency: resolvedConcurrency,
       carryoverConcurrencyCount: true,
       timeout: 50000, // TODO: make this customisable
       throwOnTimeout: true,

diff --git a/packages/plugins/plugin-build/src/commands/test/index.ts b/packages/plugins/plugin-build/src/commands/test/index.ts
@@ -9,9 +9,14 @@ import {
 import { PortablePath } from "@yarnpkg/fslib";
 import { Command, Usage } from "clipanion";
 import path from "path";
+import * as yup from "yup";
 
 import { EventEmitter } from "events";
-import { GetPluginConfiguration, YarnBuildConfiguration } from "../../config";
+import {
+  GetPluginConfiguration,
+  maxConcurrencyValidation,
+  YarnBuildConfiguration,
+} from "../../config";
 import RunSupervisor, { RunSupervisorReporterEvents } from "../supervisor";
 
 import { addTargets } from "../supervisor/workspace";
@@ -26,15 +31,34 @@ export default class Test extends BaseCommand {
   @Command.Boolean(`--ignore-cache`)
   ignoreTestCache = false;
 
+  @Command.String(`-m,--max-concurrency`)
+  maxConcurrency: string | undefined;
+
   @Command.Rest()
   public runTarget: string[] = [];
 
+  static schema = yup.object().shape({
+    maxConcurrency: maxConcurrencyValidation,
+  });
+
   static usage: Usage = Command.Usage({
     category: `Test commands`,
     description: `test a package and all its dependencies`,
     details: `
-      Run tests.
+    In a monorepo with internal packages that depend on others, this command
+    will traverse the dependency graph and efficiently ensure, the packages
+    are tested in the right order.
 
+    If the \`--verbose\` flag is set, more information will be logged to stdout than normal.
+    \
+    If the \`--json\` flag is set the output will follow a JSON-stream output
+    also known as NDJSON (https://github.com/ndjson/ndjson-spec).
+
+    If the \`--ignore-cache\` flag is set, every package will be tested,
+    regardless of whether is has changed or not.
+
+    \`-m,--max-concurrency\` is the maximum number of tests that can run at a time,
+    defaults to the number of logical CPUs on the current machine. Will override the global config option.
     `,
   });
 
@@ -48,7 +72,16 @@ export default class Test extends BaseCommand {
       this.context.plugins
     );
 
-    const pluginConfiguration: YarnBuildConfiguration = await GetPluginConfiguration(configuration);
+    const pluginConfiguration: YarnBuildConfiguration = await GetPluginConfiguration(
+      configuration
+    );
+
+    // Safe to run because the input string is validated by clipanion using the schema property
+    // TODO: Why doesn't the Command validation cast this for us?
+    const maxConcurrency =
+      this.maxConcurrency === undefined
+        ? pluginConfiguration.maxConcurrency
+        : parseInt(this.maxConcurrency);
 
     const report = await StreamReport.start(
       {
@@ -127,6 +160,7 @@ export default class Test extends BaseCommand {
           dryRun: false,
           ignoreRunCache: this.ignoreTestCache,
           verbose: this.verbose,
+          concurrency: maxConcurrency,
         });
 
         await supervisor.setup();

diff --git a/packages/plugins/plugin-build/src/config.ts b/packages/plugins/plugin-build/src/config.ts
@@ -1,9 +1,13 @@
 import { Filename, PortablePath, ppath, xfs } from "@yarnpkg/fslib";
 import { parseSyml } from "@yarnpkg/parsers";
 import { Configuration } from "@yarnpkg/core";
+import * as yup from "yup";
 
 const DEFAULT_YARN_BUILD_CONFIGRATION_FILENAME = `.yarnbuildrc.yml` as Filename;
 
+/** The validation pattern used to validate any max concurrency option */
+export const maxConcurrencyValidation = yup.number().integer().moreThan(0);
+
 type YarnBuildConfiguration = {
   // Customising the commands are runtime is not currently supported in Yarn
   // So this part is a WIP
@@ -23,26 +27,13 @@ type YarnBuildConfiguration = {
     // yarn build package/example/lorem-ipsum
     targetedBuilds: boolean;
   };
+  maxConcurrency?: number | undefined;
 };
 
 async function getConfiguration(
   configuration: Configuration
 ): Promise<YarnBuildConfiguration> {
-  let configOnDisk: {
-    commands?: {
-      build?: string;
-      test?: string;
-      dev?: string;
-    };
-    folders?: {
-      input?: string;
-      output?: string;
-    };
-    enableBetaFeatures?: {
-      folderConfiguration?: boolean;
-      targetedBuilds?: boolean;
-    };
-  } = {};
+  let configOnDisk: unknown = {};
 
   // TODO: make this more customisable
   const rcFilename = DEFAULT_YARN_BUILD_CONFIGRATION_FILENAME;
@@ -56,7 +47,7 @@ async function getConfiguration(
     const content = await xfs.readFilePromise(rcPath, `utf8`);
 
     try {
-      configOnDisk = parseSyml(content) as YarnBuildConfiguration;
+      configOnDisk = parseSyml(content);
     } catch (error) {
       let tip = ``;
 
@@ -69,48 +60,24 @@ async function getConfiguration(
     }
   }
 
-  // I thought about more complex ways to load and check the config.
-  // For now, we don't need to overdo it. Later on that might make sense
-  // but not now.
-  return {
-    commands: {
-      build:
-        typeof configOnDisk?.commands?.build === "string"
-          ? configOnDisk.commands.build
-          : "build",
-      test:
-        typeof configOnDisk?.commands?.test === "string"
-          ? configOnDisk.commands.test
-          : "test",
-      dev:
-        typeof configOnDisk?.commands?.dev === "string"
-          ? configOnDisk.commands.dev
-          : "dev",
-    },
-    folders: {
-      input:
-        typeof configOnDisk?.folders?.input === "string"
-          ? configOnDisk.folders.input
-          : ".",
-      output:
-        typeof configOnDisk?.folders?.output === "string"
-          ? configOnDisk.folders.output
-          : "build",
-    },
-    enableBetaFeatures: {
-      folderConfiguration:
-        typeof configOnDisk?.enableBetaFeatures?.folderConfiguration ===
-          "string" &&
-        configOnDisk?.enableBetaFeatures?.folderConfiguration === "false"
-          ? false
-          : true,
-      targetedBuilds:
-        typeof configOnDisk?.enableBetaFeatures?.targetedBuilds === "string" &&
-        configOnDisk?.enableBetaFeatures?.targetedBuilds === "true"
-          ? true
-          : false,
-    },
-  };
+  const configSchema = yup.object().shape({
+    commands: yup.object().shape({
+      build: yup.string().default("build"),
+      test: yup.string().default("test"),
+      dev: yup.string().default("dev"),
+    }),
+    folders: yup.object().shape({
+      input: yup.string().default("."),
+      output: yup.string().default("build"),
+    }),
+    enableBetaFeatures: yup.object().shape({
+      folderConfiguration: yup.boolean().default(true),
+      targetedBuilds: yup.boolean().default(false),
+    }),
+    maxConcurrency: maxConcurrencyValidation,
+  });
+
+  return configSchema.validate(configOnDisk);
 }
 
 async function GetPluginConfiguration(

diff --git a/readme.md b/readme.md
@@ -41,6 +41,8 @@ enableBetaFeatures:
   folderConfiguration: true
   # To enable yarn build path/to/package
   targetedBuilds: true
+# Optional: Limit the number of concurrent builds/tests that can occur at once globally. This can also be set as a command line switch.
+maxConcurrency: 4
 ```
 
 ---