From 4c3d2cb4ee6e77efa4e79e718bb1e72d8b9b1eba Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 26 Sep 2024 11:36:40 +0200 Subject: [PATCH] Allow users to use a custom output dir via an env var (#3530) This allows users to change the Mill output directory (by default, `out` under the project root) to a directory of their choice, via the `MILL_OUTPUT_DIR` environment variable. Fixes https://github.com/com-lihaoyi/mill/issues/3144 --- docs/modules/ROOT/pages/Out_Dir.adoc | 4 ++ example/depth/out-dir/1-custom-out/build.mill | 28 +++++++++++++ example/package.mill | 1 + .../output-directory/resources/build.mill | 8 ++++ .../src/OutputDirectoryTests.scala | 41 +++++++++++++++++++ main/client/src/mill/main/client/EnvVars.java | 7 ++++ .../client/src/mill/main/client/OutFiles.java | 11 ++++- runner/src/mill/runner/CodeGen.scala | 12 ++++-- runner/src/mill/runner/FileImportGraph.scala | 8 +++- .../src/mill/runner/MillBuildBootstrap.scala | 19 +++++---- .../src/mill/runner/MillBuildRootModule.scala | 11 ++++- runner/src/mill/runner/MillMain.scala | 3 +- .../src/mill/testkit/IntegrationTester.scala | 2 +- .../mill/testkit/IntegrationTesterBase.scala | 21 ++++++++-- 14 files changed, 154 insertions(+), 22 deletions(-) create mode 100644 example/depth/out-dir/1-custom-out/build.mill create mode 100644 integration/feature/output-directory/resources/build.mill create mode 100644 integration/feature/output-directory/src/OutputDirectoryTests.scala diff --git a/docs/modules/ROOT/pages/Out_Dir.adoc b/docs/modules/ROOT/pages/Out_Dir.adoc index 0deb6330950..cd49ed4d57d 100644 --- a/docs/modules/ROOT/pages/Out_Dir.adoc +++ b/docs/modules/ROOT/pages/Out_Dir.adoc @@ -111,3 +111,7 @@ This is very useful if Mill is being unexpectedly slow, and you want to find out `mill-server/*`:: Each Mill server instance needs to keep some temporary files in one of these directories. Deleting it will also terminate the associated server instance, if it is still running. + +== Using another location than the `out/` directory + +include::example/depth/out-dir/1-custom-out.adoc[] diff --git a/example/depth/out-dir/1-custom-out/build.mill b/example/depth/out-dir/1-custom-out/build.mill new file mode 100644 index 00000000000..b36b339830b --- /dev/null +++ b/example/depth/out-dir/1-custom-out/build.mill @@ -0,0 +1,28 @@ +// The default location for Mill's output directory is `out/` under the project workspace. +// A task `printDest` of a module `foo` will have a default scratch space folder +// `out/foo/printDest.dest/`: +package build + +import mill._ + +object foo extends Module { + def printDest = Task { + println(T.dest) + } +} + +/** Usage +> ./mill foo.printDest +... +.../out/foo/printDest.dest +*/ + +// If you'd rather use another location than `out/`, that lives +// in a faster or a writable filesystem for example, you can change the output directory +// via the `MILL_OUTPUT_DIR` environment variable. + +/** Usage +> MILL_OUTPUT_DIR=build-stuff/working-dir ./mill foo.printDest +... +.../build-stuff/working-dir/foo/printDest.dest +*/ diff --git a/example/package.mill b/example/package.mill index dab238c7f8b..3ee18f56f75 100644 --- a/example/package.mill +++ b/example/package.mill @@ -59,6 +59,7 @@ object `package` extends RootModule with Module { object modules extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "modules")) object cross extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "cross")) object large extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "large")) + object `out-dir` extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "out-dir")) object sandbox extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "sandbox")) object libraries extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "libraries")) } diff --git a/integration/feature/output-directory/resources/build.mill b/integration/feature/output-directory/resources/build.mill new file mode 100644 index 00000000000..b5fae4e0f89 --- /dev/null +++ b/integration/feature/output-directory/resources/build.mill @@ -0,0 +1,8 @@ +package build + +import mill._ +import mill.scalalib._ + +object `package` extends RootModule with ScalaModule { + def scalaVersion = scala.util.Properties.versionNumberString +} diff --git a/integration/feature/output-directory/src/OutputDirectoryTests.scala b/integration/feature/output-directory/src/OutputDirectoryTests.scala new file mode 100644 index 00000000000..64d6e09a92b --- /dev/null +++ b/integration/feature/output-directory/src/OutputDirectoryTests.scala @@ -0,0 +1,41 @@ +package mill.integration + +import mill.main.client.{EnvVars, OutFiles} +import mill.testkit.UtestIntegrationTestSuite +import utest._ + +object OutputDirectoryTests extends UtestIntegrationTestSuite { + + def tests: Tests = Tests { + test("Output directory sanity check") - integrationTest { tester => + import tester._ + eval("__.compile").isSuccess ==> true + val defaultOutDir = workspacePath / OutFiles.defaultOut + assert(os.isDir(defaultOutDir)) + } + + test("Output directory elsewhere in workspace") - integrationTest { tester => + import tester._ + eval( + "__.compile", + env = millTestSuiteEnv + (EnvVars.MILL_OUTPUT_DIR -> "testing/test-out") + ).isSuccess ==> true + val expectedOutDir = workspacePath / "testing/test-out" + val defaultOutDir = workspacePath / OutFiles.defaultOut + assert(os.isDir(expectedOutDir)) + assert(!os.exists(defaultOutDir)) + } + + test("Output directory outside workspace") - integrationTest { tester => + import tester._ + val outDir = os.temp.dir() / "tmp-out" + eval( + "__.compile", + env = millTestSuiteEnv + (EnvVars.MILL_OUTPUT_DIR -> outDir.toString) + ).isSuccess ==> true + val defaultOutDir = workspacePath / OutFiles.defaultOut + assert(os.isDir(outDir)) + assert(!os.exists(defaultOutDir)) + } + } +} diff --git a/main/client/src/mill/main/client/EnvVars.java b/main/client/src/mill/main/client/EnvVars.java index 60fd440ee6b..c11b77e9c64 100644 --- a/main/client/src/mill/main/client/EnvVars.java +++ b/main/client/src/mill/main/client/EnvVars.java @@ -22,6 +22,13 @@ public class EnvVars { public static final String MILL_JVM_OPTS_PATH = "MILL_JVM_OPTS_PATH"; + + /** + * Output directory where Mill workers' state and Mill tasks output should be + * written to + */ + public static final String MILL_OUTPUT_DIR = "MILL_OUTPUT_DIR"; + // INTERNAL ENVIRONMENT VARIABLES /** * Used to pass the Mill workspace root from the client to the server, so diff --git a/main/client/src/mill/main/client/OutFiles.java b/main/client/src/mill/main/client/OutFiles.java index ba7dce7bacf..04ffeecb4db 100644 --- a/main/client/src/mill/main/client/OutFiles.java +++ b/main/client/src/mill/main/client/OutFiles.java @@ -5,10 +5,19 @@ * and documentation about what they do */ public class OutFiles { + + final private static String envOutOrNull = System.getenv(EnvVars.MILL_OUTPUT_DIR); + + /** + * Default hard-coded value for the Mill `out/` folder path. Unless you know + * what you are doing, you should favor using [[out]] instead. + */ + final public static String defaultOut = "out"; + /** * Path of the Mill `out/` folder */ - final public static String out = "out"; + final public static String out = envOutOrNull == null ? defaultOut : envOutOrNull; /** * Path of the Mill "meta-build", used to compile the `build.sc` file so we can diff --git a/runner/src/mill/runner/CodeGen.scala b/runner/src/mill/runner/CodeGen.scala index 8baf0736799..485d04caa5a 100644 --- a/runner/src/mill/runner/CodeGen.scala +++ b/runner/src/mill/runner/CodeGen.scala @@ -15,7 +15,8 @@ object CodeGen { allScriptCode: Map[os.Path, String], targetDest: os.Path, enclosingClasspath: Seq[os.Path], - millTopLevelProjectRoot: os.Path + millTopLevelProjectRoot: os.Path, + output: os.Path ): Unit = { for (scriptSource <- scriptSources) { val scriptPath = scriptSource.path @@ -94,6 +95,7 @@ object CodeGen { projectRoot, enclosingClasspath, millTopLevelProjectRoot, + output, scriptPath, scriptFolderPath, childAliases, @@ -112,6 +114,7 @@ object CodeGen { projectRoot: os.Path, enclosingClasspath: Seq[os.Path], millTopLevelProjectRoot: os.Path, + output: os.Path, scriptPath: os.Path, scriptFolderPath: os.Path, childAliases: String, @@ -126,7 +129,8 @@ object CodeGen { segments, scriptFolderPath, enclosingClasspath, - millTopLevelProjectRoot + millTopLevelProjectRoot, + output ) val instrument = new ObjectDataInstrument(scriptCode) @@ -183,13 +187,15 @@ object CodeGen { segments: Seq[String], scriptFolderPath: os.Path, enclosingClasspath: Seq[os.Path], - millTopLevelProjectRoot: os.Path + millTopLevelProjectRoot: os.Path, + output: os.Path ): String = { s"""import _root_.mill.runner.MillBuildRootModule |@_root_.scala.annotation.nowarn |object MillMiscInfo extends MillBuildRootModule.MillMiscInfo( | ${enclosingClasspath.map(p => literalize(p.toString))}, | ${literalize(scriptFolderPath.toString)}, + | ${literalize(output.toString)}, | ${literalize(millTopLevelProjectRoot.toString)}, | _root_.scala.Seq(${segments.map(pprint.Util.literalize(_)).mkString(", ")}) |) diff --git a/runner/src/mill/runner/FileImportGraph.scala b/runner/src/mill/runner/FileImportGraph.scala index 6dcdfa4a1e8..6299844dddc 100644 --- a/runner/src/mill/runner/FileImportGraph.scala +++ b/runner/src/mill/runner/FileImportGraph.scala @@ -43,7 +43,11 @@ object FileImportGraph { * starting from `build.mill`, collecting the information necessary to * instantiate the [[MillRootModule]] */ - def parseBuildFiles(topLevelProjectRoot: os.Path, projectRoot: os.Path): FileImportGraph = { + def parseBuildFiles( + topLevelProjectRoot: os.Path, + projectRoot: os.Path, + output: os.Path + ): FileImportGraph = { val seenScripts = mutable.Map.empty[os.Path, String] val seenIvy = mutable.Set.empty[String] val seenRepo = mutable.ListBuffer.empty[(String, os.Path)] @@ -193,7 +197,7 @@ object FileImportGraph { projectRoot, followLinks = true, skip = p => - p == projectRoot / out || + p == output || p == projectRoot / millBuild || (os.isDir(p) && !os.exists(p / nestedBuildFileName)) ) diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 30e7827e057..ad84d5eb956 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -8,7 +8,7 @@ import mill.eval.Evaluator import mill.main.RunScript import mill.resolve.SelectMode import mill.define.{BaseModule, Discover, Segments} -import mill.main.client.OutFiles._ +import mill.main.client.OutFiles.{millBuild, millRunnerState} import java.net.URLClassLoader @@ -30,6 +30,7 @@ import java.net.URLClassLoader @internal class MillBuildBootstrap( projectRoot: os.Path, + output: os.Path, home: os.Path, keepGoing: Boolean, imports: Seq[String], @@ -46,7 +47,7 @@ class MillBuildBootstrap( ) { import MillBuildBootstrap._ - val millBootClasspath: Seq[os.Path] = prepareMillBootClasspath(projectRoot / out) + val millBootClasspath: Seq[os.Path] = prepareMillBootClasspath(output) val millBootClasspathPathRefs: Seq[PathRef] = millBootClasspath.map(PathRef(_, quick = true)) def evaluate(): Watching.Result[RunnerState] = CliImports.withValue(imports) { @@ -54,7 +55,7 @@ class MillBuildBootstrap( for ((frame, depth) <- runnerState.frames.zipWithIndex) { os.write.over( - recOut(projectRoot, depth) / millRunnerState, + recOut(output, depth) / millRunnerState, upickle.default.write(frame.loggedData, indent = 4), createFolders = true ) @@ -102,7 +103,8 @@ class MillBuildBootstrap( } else { val parsedScriptFiles = FileImportGraph.parseBuildFiles( projectRoot, - recRoot(projectRoot, depth) / os.up + recRoot(projectRoot, depth) / os.up, + output ) if (parsedScriptFiles.millImport) evaluateRec(depth + 1) @@ -111,6 +113,7 @@ class MillBuildBootstrap( new MillBuildRootModule.BootstrapModule( projectRoot, recRoot(projectRoot, depth), + output, millBootClasspath )( mill.main.RootModule.Info( @@ -340,8 +343,8 @@ class MillBuildBootstrap( mill.eval.EvaluatorImpl( home, projectRoot, - recOut(projectRoot, depth), - recOut(projectRoot, depth), + recOut(output, depth), + recOut(output, depth), rootModule, PrefixLogger(logger, "", tickerContext = bootLogPrefix), classLoaderSigHash = millClassloaderSigHash, @@ -422,8 +425,8 @@ object MillBuildBootstrap { projectRoot / Seq.fill(depth)(millBuild) } - def recOut(projectRoot: os.Path, depth: Int): os.Path = { - projectRoot / out / Seq.fill(depth)(millBuild) + def recOut(output: os.Path, depth: Int): os.Path = { + output / Seq.fill(depth)(millBuild) } } diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index 729e6b2f240..3b0891eb53a 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -121,7 +121,8 @@ abstract class MillBuildRootModule()(implicit parsed.seenScripts, T.dest, millBuildRootModuleInfo.enclosingClasspath, - millBuildRootModuleInfo.topLevelProjectRoot + millBuildRootModuleInfo.topLevelProjectRoot, + millBuildRootModuleInfo.output ) Result.Success(Seq(PathRef(T.dest))) } @@ -265,12 +266,14 @@ object MillBuildRootModule { class BootstrapModule( topLevelProjectRoot0: os.Path, projectRoot: os.Path, + output: os.Path, enclosingClasspath: Seq[os.Path] )(implicit baseModuleInfo: RootModule.Info) extends MillBuildRootModule()( implicitly, MillBuildRootModule.Info( enclosingClasspath, projectRoot, + output, topLevelProjectRoot0 ) ) { @@ -281,25 +284,29 @@ object MillBuildRootModule { case class Info( enclosingClasspath: Seq[os.Path], projectRoot: os.Path, + output: os.Path, topLevelProjectRoot: os.Path ) def parseBuildFiles(millBuildRootModuleInfo: MillBuildRootModule.Info): FileImportGraph = { FileImportGraph.parseBuildFiles( millBuildRootModuleInfo.topLevelProjectRoot, - millBuildRootModuleInfo.projectRoot / os.up + millBuildRootModuleInfo.projectRoot / os.up, + millBuildRootModuleInfo.output ) } class MillMiscInfo( enclosingClasspath: Seq[String], projectRoot: String, + output: String, topLevelProjectRoot: String, segments: Seq[String] ) { implicit lazy val millBuildRootModuleInfo: MillBuildRootModule.Info = MillBuildRootModule.Info( enclosingClasspath.map(os.Path(_)), os.Path(projectRoot), + os.Path(output), os.Path(topLevelProjectRoot) ) implicit lazy val millBaseModuleInfo: RootModule.Info = RootModule.Info( diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala index e9b2097ec85..60d104d6a35 100644 --- a/runner/src/mill/runner/MillMain.scala +++ b/runner/src/mill/runner/MillMain.scala @@ -8,7 +8,7 @@ import mill.java9rtexport.Export import mill.api.{MillException, SystemStreams, WorkspaceRoot, internal} import mill.bsp.{BspContext, BspServerResult} import mill.main.BuildInfo -import mill.main.client.ServerFiles +import mill.main.client.{OutFiles, ServerFiles} import mill.util.{PromptLogger, PrintLogger} import java.lang.reflect.InvocationTargetException @@ -234,6 +234,7 @@ object MillMain { new MillBuildBootstrap( projectRoot = WorkspaceRoot.workspaceRoot, + output = os.Path(OutFiles.out, WorkspaceRoot.workspaceRoot), home = config.home, keepGoing = config.keepGoing.value, imports = config.imports, diff --git a/testkit/src/mill/testkit/IntegrationTester.scala b/testkit/src/mill/testkit/IntegrationTester.scala index 309f2b957ac..043b530a5c5 100644 --- a/testkit/src/mill/testkit/IntegrationTester.scala +++ b/testkit/src/mill/testkit/IntegrationTester.scala @@ -91,7 +91,7 @@ object IntegrationTester { ) } - private val millTestSuiteEnv = Map("MILL_TEST_SUITE" -> this.getClass().toString()) + def millTestSuiteEnv: Map[String, String] = Map("MILL_TEST_SUITE" -> this.getClass().toString()) /** * Helpers to read the `.json` metadata files belonging to a particular task diff --git a/testkit/src/mill/testkit/IntegrationTesterBase.scala b/testkit/src/mill/testkit/IntegrationTesterBase.scala index 589728cc2f7..891a0a09845 100644 --- a/testkit/src/mill/testkit/IntegrationTesterBase.scala +++ b/testkit/src/mill/testkit/IntegrationTesterBase.scala @@ -34,21 +34,34 @@ trait IntegrationTesterBase { os.makeDir.all(workspacePath) Retry() { val tmp = os.temp.dir() - if (os.exists(workspacePath / out)) os.move.into(workspacePath / out, tmp) + val outDir = os.Path(out, workspacePath) + if (os.exists(outDir)) os.move.into(outDir, tmp) os.remove.all(tmp) } os.list(workspacePath).foreach(os.remove.all(_)) - os.list(workspaceSourcePath).filter(_.last != out).foreach(os.copy.into(_, workspacePath)) + val outRelPathOpt = os.FilePath(out) match { + case relPath: os.RelPath if relPath.ups == 0 => Some(relPath) + case _ => None + } + os.list(workspaceSourcePath) + .filter( + outRelPathOpt match { + case None => _ => true + case Some(outRelPath) => !_.endsWith(outRelPath) + } + ) + .foreach(os.copy.into(_, workspacePath)) } /** * Remove any ID files to try and force them to exit */ def removeServerIdFile(): Unit = { - if (os.exists(workspacePath / out)) { + val outDir = os.Path(out, workspacePath) + if (os.exists(outDir)) { val serverIdFiles = for { - outPath <- os.list.stream(workspacePath / out) + outPath <- os.list.stream(outDir) if outPath.last.startsWith(millServer) } yield outPath / serverId