diff --git a/share/wake/lib/system/io.wake b/share/wake/lib/system/io.wake index 4aa7f7dcb..7c0972e89 100644 --- a/share/wake/lib/system/io.wake +++ b/share/wake/lib/system/io.wake @@ -166,11 +166,10 @@ def writeRunner (content: String) = | map getPathName | fnInputs - def fileOutputs = - (path, Nil) - | fnOutputs + def cleanable = (path, Nil) + def fileOutputs = fnOutputs cleanable - RunnerOutput fileInputs fileOutputs reality + RunnerOutput fileInputs fileOutputs cleanable reality | Pass makeRunner "write" run @@ -314,11 +313,10 @@ def mkdirRunner: Runner = | map getPathName | fnInputs - def fileOutputs = - (path, Nil) - | fnOutputs + def cleanable = (path, Nil) + def fileOutputs = fnOutputs cleanable - RunnerOutput fileInputs fileOutputs reality + RunnerOutput fileInputs fileOutputs cleanable reality | Pass makeRunner "mkdir" run diff --git a/share/wake/lib/system/job.wake b/share/wake/lib/system/job.wake index ff42e2058..0729e2bc4 100644 --- a/share/wake/lib/system/job.wake +++ b/share/wake/lib/system/job.wake @@ -154,14 +154,13 @@ def runAlways cmd env dir stdin res uusage finputs foutputs vis keep run echo st def final _ = match output Fail e -> primJobFailLaunch job e - Pass (RunnerOutput inputs outputs reality) -> + Pass (RunnerOutput inputs outputs cleanable reality) -> def hashedOutputs = outputs | computeHashes prefix | implode - # TODO: Determine the effect of all_outputs being filtered - primJobFinish job inputs.implode hashedOutputs hashedOutputs reality + primJobFinish job inputs.implode hashedOutputs cleanable.implode reality # Make sure we don't hash files before the job has stopped running def _ = waitJobMerged final job diff --git a/share/wake/lib/system/job_cache_runner.wake b/share/wake/lib/system/job_cache_runner.wake index b96eb904c..111c22697 100644 --- a/share/wake/lib/system/job_cache_runner.wake +++ b/share/wake/lib/system/job_cache_runner.wake @@ -166,7 +166,7 @@ export def mkJobCacheRunner (hashFn: RunnerInput => Result String Error) (wakero def predict = Usage status runtime cputime mem ibytes obytes def _ = primJobVirtual job stdout stderr predict - Pass (RunnerOutput inputs outputs predict) + Pass (RunnerOutput inputs outputs outputs predict) def _ = require True = isDebugOn @@ -176,7 +176,7 @@ export def mkJobCacheRunner (hashFn: RunnerInput => Result String Error) (wakero True # Now we need to run the job - require Pass (RunnerOutput inputs outputs usage) = baseDoIt job (Pass input) + require Pass (RunnerOutput inputs outputs cleanable usage) = baseDoIt job (Pass input) def Usage status runtime cputime mem ibytes obytes = usage def inputsTree = listToTree scmpCanonical inputs @@ -243,6 +243,6 @@ export def mkJobCacheRunner (hashFn: RunnerInput => Result String Error) (wakero job_cache_add jobCacheAddJson - Pass (RunnerOutput (map getPathName vis) outputs usage) + Pass (RunnerOutput (map getPathName vis) outputs cleanable usage) makeRunner "job-cache: {name}" run diff --git a/share/wake/lib/system/remote_cache_runner.wake b/share/wake/lib/system/remote_cache_runner.wake index 7c90677bc..6e1c96533 100644 --- a/share/wake/lib/system/remote_cache_runner.wake +++ b/share/wake/lib/system/remote_cache_runner.wake @@ -216,7 +216,7 @@ export def mkRemoteCacheRunner (rscApi: RemoteCacheApi) (hashFn: RunnerInput => def _ = primJobVirtual job stdout stderr predict - Pass (RunnerOutput inputs outputs predict) + Pass (RunnerOutput inputs outputs outputs predict) def run (job: Job) (input: RunnerInput): Result RunnerOutput Error = def label = input.getRunnerInputLabel diff --git a/share/wake/lib/system/runner.wake b/share/wake/lib/system/runner.wake index 2201ff912..6bd5da0da 100644 --- a/share/wake/lib/system/runner.wake +++ b/share/wake/lib/system/runner.wake @@ -76,9 +76,17 @@ export tuple RunnerInput = # so that it can apply filtering after collecting the files written. export FnOutputs: (List String => List String) +# The result returned by a Runner after running a job export tuple RunnerOutput = + # The filtered list of files actually read by the job. It can be used for more precise reruns export Inputs: List String + # The filtered list of files output by the job. These files are hashed and elevated to Path export Outputs: List String + # The list of files output by the job that should be deleted by wake --clean. May include files + # listed in Outputs as they will be removed later. The files in the list that are not in + # Outputs will not be hashed, published, or elevated to Path but will be deleted by wake --clean + export CleanableOutputs: List String + # The actual resource usage of the job that will be written to the database export Usage: Usage # A Runner describes a way to invoke a Plan to get a Job @@ -121,11 +129,12 @@ export def localRunner: Runner = | map getPathName | fnInputs - # Caller need to fill this in from nothing - def fileOutputs = fnOutputs Nil + # Caller needs to fill this in from nothing + def cleanable = Nil + def fileOutputs = fnOutputs cleanable job.getJobReality - |< RunnerOutput fileInputs fileOutputs + |< RunnerOutput fileInputs fileOutputs cleanable makeRunner "local" run @@ -141,11 +150,12 @@ export def virtualRunner: Runner = | map getPathName | fnInputs - # Caller need to fill this in from nothing - def fileOutputs = fnOutputs Nil + # Caller needs to fill this in from nothing + def cleanable = Nil + def fileOutputs = fnOutputs cleanable job.getJobReality - |< RunnerOutput fileInputs fileOutputs + |< RunnerOutput fileInputs fileOutputs cleanable makeRunner "virtual" run @@ -322,6 +332,6 @@ export def makeJSONRunner ((JSONRunnerPlan rawScript extraArgs extraEnv estimate match usageResult Fail f -> Fail (makeError f) - Pass usage -> Pass (RunnerOutput filteredInputs filteredOutputs usage) + Pass usage -> Pass (RunnerOutput filteredInputs filteredOutputs jsonOutputs usage) makeRunner "json-{script}" run