Allow users to use a custom output dir via an env var (#3530)

This allows users to change the Mill output directory (by default, `out`
under the project root) to a directory of their choice, via the
`MILL_OUTPUT_DIR` environment variable.

Fixes #3144

docs/modules/ROOT/pages/Out_Dir.adoc

@@ -111,3 +111,7 @@ This is very useful if Mill is being unexpectedly slow, and you want to find out
 
 `mill-server/*`::
  Each Mill server instance needs to keep some temporary files in one of these directories. Deleting it will also terminate the associated server instance, if it is still running.
+
+== Using another location than the `out/` directory
+
+include::example/depth/out-dir/1-custom-out.adoc[]

example/depth/out-dir/1-custom-out/build.mill

@@ -0,0 +1,28 @@
+// The default location for Mill's output directory is `out/` under the project workspace.
+// A task `printDest` of a module `foo` will have a default scratch space folder
+// `out/foo/printDest.dest/`:
+package build
+
+import mill._
+
+object foo extends Module {
+  def printDest = Task {
+    println(T.dest)
+  }
+}
+
+/** Usage
+> ./mill foo.printDest
+...
+.../out/foo/printDest.dest
+*/
+
+// If you'd rather use another location than `out/`, that lives
+// in a faster or a writable filesystem for example, you can change the output directory
+// via the `MILL_OUTPUT_DIR` environment variable.
+
+/** Usage
+> MILL_OUTPUT_DIR=build-stuff/working-dir ./mill foo.printDest
+...
+.../build-stuff/working-dir/foo/printDest.dest
+*/

example/package.mill

@@ -59,6 +59,7 @@ object `package` extends RootModule with Module {
     object modules extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "modules"))
     object cross extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "cross"))
     object large extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "large"))
+    object `out-dir` extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "out-dir"))
     object sandbox extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "sandbox"))
     object libraries extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "libraries"))
   }

integration/feature/output-directory/resources/build.mill

@@ -0,0 +1,8 @@
+package build
+
+import mill._
+import mill.scalalib._
+
+object `package` extends RootModule with ScalaModule {
+  def scalaVersion = scala.util.Properties.versionNumberString
+}

integration/feature/output-directory/src/OutputDirectoryTests.scala

@@ -0,0 +1,41 @@
+package mill.integration
+
+import mill.main.client.{EnvVars, OutFiles}
+import mill.testkit.UtestIntegrationTestSuite
+import utest._
+
+object OutputDirectoryTests extends UtestIntegrationTestSuite {
+
+  def tests: Tests = Tests {
+    test("Output directory sanity check") - integrationTest { tester =>
+      import tester._
+      eval("__.compile").isSuccess ==> true
+      val defaultOutDir = workspacePath / OutFiles.defaultOut
+      assert(os.isDir(defaultOutDir))
+    }
+
+    test("Output directory elsewhere in workspace") - integrationTest { tester =>
+      import tester._
+      eval(
+        "__.compile",
+        env = millTestSuiteEnv + (EnvVars.MILL_OUTPUT_DIR -> "testing/test-out")
+      ).isSuccess ==> true
+      val expectedOutDir = workspacePath / "testing/test-out"
+      val defaultOutDir = workspacePath / OutFiles.defaultOut
+      assert(os.isDir(expectedOutDir))
+      assert(!os.exists(defaultOutDir))
+    }
+
+    test("Output directory outside workspace") - integrationTest { tester =>
+      import tester._
+      val outDir = os.temp.dir() / "tmp-out"
+      eval(
+        "__.compile",
+        env = millTestSuiteEnv + (EnvVars.MILL_OUTPUT_DIR -> outDir.toString)
+      ).isSuccess ==> true
+      val defaultOutDir = workspacePath / OutFiles.defaultOut
+      assert(os.isDir(outDir))
+      assert(!os.exists(defaultOutDir))
+    }
+  }
+}

main/client/src/mill/main/client/EnvVars.java

@@ -22,6 +22,13 @@ public class EnvVars {
 
     public static final String MILL_JVM_OPTS_PATH = "MILL_JVM_OPTS_PATH";
 
+
+    /**
+     * Output directory where Mill workers' state and Mill tasks output should be
+     * written to
+     */
+    public static final String MILL_OUTPUT_DIR = "MILL_OUTPUT_DIR";
+
     // INTERNAL ENVIRONMENT VARIABLES
     /**
      * Used to pass the Mill workspace root from the client to the server, so

main/client/src/mill/main/client/OutFiles.java

@@ -5,10 +5,19 @@
  * and documentation about what they do
  */
 public class OutFiles {
+
+    final private static String envOutOrNull = System.getenv(EnvVars.MILL_OUTPUT_DIR);
+
+    /**
+     * Default hard-coded value for the Mill `out/` folder path. Unless you know
+     * what you are doing, you should favor using [[out]] instead.
+     */
+    final public static String defaultOut = "out";
+
     /**
      * Path of the Mill `out/` folder
      */
-    final public static String out = "out";
+    final public static String out = envOutOrNull == null ? defaultOut : envOutOrNull;
 
     /**
      * Path of the Mill "meta-build", used to compile the `build.sc` file so we can

runner/src/mill/runner/CodeGen.scala

@@ -15,7 +15,8 @@ object CodeGen {
       allScriptCode: Map[os.Path, String],
       targetDest: os.Path,
       enclosingClasspath: Seq[os.Path],
-      millTopLevelProjectRoot: os.Path
+      millTopLevelProjectRoot: os.Path,
+      output: os.Path
   ): Unit = {
     for (scriptSource <- scriptSources) {
       val scriptPath = scriptSource.path
@@ -94,6 +95,7 @@ object CodeGen {
             projectRoot,
             enclosingClasspath,
             millTopLevelProjectRoot,
+            output,
             scriptPath,
             scriptFolderPath,
             childAliases,
@@ -112,6 +114,7 @@ object CodeGen {
       projectRoot: os.Path,
       enclosingClasspath: Seq[os.Path],
       millTopLevelProjectRoot: os.Path,
+      output: os.Path,
       scriptPath: os.Path,
       scriptFolderPath: os.Path,
       childAliases: String,
@@ -126,7 +129,8 @@ object CodeGen {
       segments,
       scriptFolderPath,
       enclosingClasspath,
-      millTopLevelProjectRoot
+      millTopLevelProjectRoot,
+      output
     )
 
     val instrument = new ObjectDataInstrument(scriptCode)
@@ -183,13 +187,15 @@ object CodeGen {
       segments: Seq[String],
       scriptFolderPath: os.Path,
       enclosingClasspath: Seq[os.Path],
-      millTopLevelProjectRoot: os.Path
+      millTopLevelProjectRoot: os.Path,
+      output: os.Path
   ): String = {
     s"""import _root_.mill.runner.MillBuildRootModule
        |@_root_.scala.annotation.nowarn
        |object MillMiscInfo extends MillBuildRootModule.MillMiscInfo(
        |  ${enclosingClasspath.map(p => literalize(p.toString))},
        |  ${literalize(scriptFolderPath.toString)},
+       |  ${literalize(output.toString)},
        |  ${literalize(millTopLevelProjectRoot.toString)},
        |  _root_.scala.Seq(${segments.map(pprint.Util.literalize(_)).mkString(", ")})
        |)

runner/src/mill/runner/FileImportGraph.scala

@@ -43,7 +43,11 @@ object FileImportGraph {
    * starting from `build.mill`, collecting the information necessary to
    * instantiate the [[MillRootModule]]
    */
-  def parseBuildFiles(topLevelProjectRoot: os.Path, projectRoot: os.Path): FileImportGraph = {
+  def parseBuildFiles(
+      topLevelProjectRoot: os.Path,
+      projectRoot: os.Path,
+      output: os.Path
+  ): FileImportGraph = {
     val seenScripts = mutable.Map.empty[os.Path, String]
     val seenIvy = mutable.Set.empty[String]
     val seenRepo = mutable.ListBuffer.empty[(String, os.Path)]
@@ -193,7 +197,7 @@ object FileImportGraph {
             projectRoot,
             followLinks = true,
             skip = p =>
-              p == projectRoot / out ||
+              p == output ||
                 p == projectRoot / millBuild ||
                 (os.isDir(p) && !os.exists(p / nestedBuildFileName))
           )

runner/src/mill/runner/MillBuildBootstrap.scala

@@ -8,7 +8,7 @@ import mill.eval.Evaluator
 import mill.main.RunScript
 import mill.resolve.SelectMode
 import mill.define.{BaseModule, Discover, Segments}
-import mill.main.client.OutFiles._
+import mill.main.client.OutFiles.{millBuild, millRunnerState}
 
 import java.net.URLClassLoader
 
@@ -30,6 +30,7 @@ import java.net.URLClassLoader
 @internal
 class MillBuildBootstrap(
     projectRoot: os.Path,
+    output: os.Path,
     home: os.Path,
     keepGoing: Boolean,
     imports: Seq[String],
@@ -46,15 +47,15 @@ class MillBuildBootstrap(
 ) {
   import MillBuildBootstrap._
 
-  val millBootClasspath: Seq[os.Path] = prepareMillBootClasspath(projectRoot / out)
+  val millBootClasspath: Seq[os.Path] = prepareMillBootClasspath(output)
   val millBootClasspathPathRefs: Seq[PathRef] = millBootClasspath.map(PathRef(_, quick = true))
 
   def evaluate(): Watching.Result[RunnerState] = CliImports.withValue(imports) {
     val runnerState = evaluateRec(0)
 
     for ((frame, depth) <- runnerState.frames.zipWithIndex) {
       os.write.over(
-        recOut(projectRoot, depth) / millRunnerState,
+        recOut(output, depth) / millRunnerState,
         upickle.default.write(frame.loggedData, indent = 4),
         createFolders = true
       )
@@ -102,7 +103,8 @@ class MillBuildBootstrap(
       } else {
         val parsedScriptFiles = FileImportGraph.parseBuildFiles(
           projectRoot,
-          recRoot(projectRoot, depth) / os.up
+          recRoot(projectRoot, depth) / os.up,
+          output
         )
 
         if (parsedScriptFiles.millImport) evaluateRec(depth + 1)
@@ -111,6 +113,7 @@ class MillBuildBootstrap(
             new MillBuildRootModule.BootstrapModule(
               projectRoot,
               recRoot(projectRoot, depth),
+              output,
               millBootClasspath
             )(
               mill.main.RootModule.Info(
@@ -340,8 +343,8 @@ class MillBuildBootstrap(
     mill.eval.EvaluatorImpl(
       home,
       projectRoot,
-      recOut(projectRoot, depth),
-      recOut(projectRoot, depth),
+      recOut(output, depth),
+      recOut(output, depth),
       rootModule,
       PrefixLogger(logger, "", tickerContext = bootLogPrefix),
       classLoaderSigHash = millClassloaderSigHash,
@@ -422,8 +425,8 @@ object MillBuildBootstrap {
     projectRoot / Seq.fill(depth)(millBuild)
   }
 
-  def recOut(projectRoot: os.Path, depth: Int): os.Path = {
-    projectRoot / out / Seq.fill(depth)(millBuild)
+  def recOut(output: os.Path, depth: Int): os.Path = {
+    output / Seq.fill(depth)(millBuild)
   }
 
 }

runner/src/mill/runner/MillBuildRootModule.scala

@@ -121,7 +121,8 @@ abstract class MillBuildRootModule()(implicit
         parsed.seenScripts,
         T.dest,
         millBuildRootModuleInfo.enclosingClasspath,
-        millBuildRootModuleInfo.topLevelProjectRoot
+        millBuildRootModuleInfo.topLevelProjectRoot,
+        millBuildRootModuleInfo.output
       )
       Result.Success(Seq(PathRef(T.dest)))
     }
@@ -265,12 +266,14 @@ object MillBuildRootModule {
   class BootstrapModule(
       topLevelProjectRoot0: os.Path,
       projectRoot: os.Path,
+      output: os.Path,
       enclosingClasspath: Seq[os.Path]
   )(implicit baseModuleInfo: RootModule.Info) extends MillBuildRootModule()(
         implicitly,
         MillBuildRootModule.Info(
           enclosingClasspath,
           projectRoot,
+          output,
           topLevelProjectRoot0
         )
       ) {
@@ -281,25 +284,29 @@ object MillBuildRootModule {
   case class Info(
       enclosingClasspath: Seq[os.Path],
       projectRoot: os.Path,
+      output: os.Path,
       topLevelProjectRoot: os.Path
   )
 
   def parseBuildFiles(millBuildRootModuleInfo: MillBuildRootModule.Info): FileImportGraph = {
     FileImportGraph.parseBuildFiles(
       millBuildRootModuleInfo.topLevelProjectRoot,
-      millBuildRootModuleInfo.projectRoot / os.up
+      millBuildRootModuleInfo.projectRoot / os.up,
+      millBuildRootModuleInfo.output
     )
   }
 
   class MillMiscInfo(
       enclosingClasspath: Seq[String],
       projectRoot: String,
+      output: String,
       topLevelProjectRoot: String,
       segments: Seq[String]
   ) {
     implicit lazy val millBuildRootModuleInfo: MillBuildRootModule.Info = MillBuildRootModule.Info(
       enclosingClasspath.map(os.Path(_)),
       os.Path(projectRoot),
+      os.Path(output),
       os.Path(topLevelProjectRoot)
     )
     implicit lazy val millBaseModuleInfo: RootModule.Info = RootModule.Info(

runner/src/mill/runner/MillMain.scala

@@ -8,7 +8,7 @@ import mill.java9rtexport.Export
 import mill.api.{MillException, SystemStreams, WorkspaceRoot, internal}
 import mill.bsp.{BspContext, BspServerResult}
 import mill.main.BuildInfo
-import mill.main.client.ServerFiles
+import mill.main.client.{OutFiles, ServerFiles}
 import mill.util.{PromptLogger, PrintLogger}
 
 import java.lang.reflect.InvocationTargetException
@@ -234,6 +234,7 @@ object MillMain {
 
                         new MillBuildBootstrap(
                           projectRoot = WorkspaceRoot.workspaceRoot,
+                          output = os.Path(OutFiles.out, WorkspaceRoot.workspaceRoot),
                           home = config.home,
                           keepGoing = config.keepGoing.value,
                           imports = config.imports,

testkit/src/mill/testkit/IntegrationTester.scala

@@ -91,7 +91,7 @@ object IntegrationTester {
       )
     }
 
-    private val millTestSuiteEnv = Map("MILL_TEST_SUITE" -> this.getClass().toString())
+    def millTestSuiteEnv: Map[String, String] = Map("MILL_TEST_SUITE" -> this.getClass().toString())
 
     /**
      * Helpers to read the `.json` metadata files belonging to a particular task