diff --git a/src/Slurmi/Job.fs b/src/Slurmi/Job.fs index 9dabebd..e95de15 100644 --- a/src/Slurmi/Job.fs +++ b/src/Slurmi/Job.fs @@ -6,6 +6,7 @@ ////// .Split "\n") // Replace("\r","\n")) open DynamicObj open System.Runtime.InteropServices +open Fli type timeUnit = @@ -1734,18 +1735,7 @@ type LongCommand (jobName: string) = job.Remove "mem" - // /// Defer the start of this job until the specified dependencies have been satisfied. - // static member SetDependency - // ([]?Dependency:TypeOfDep*DependencyType[]) = - // (fun (job: LongCommand) -> - // Dependency |> DynObj.setValueOpt job "dependency" - // job - // ) - // static member tryGetDependency (job: LongCommand) = - // job.TryGetValue "dependency" - // static member removeDependency(job: LongCommand) = - // job.Remove "dependency" type EnvironmentSLURM() = let mutable environment : (string * string) list = [] @@ -1824,6 +1814,7 @@ type Job (jobName: string,processList:(string*string list)list)= |> Array.filter (fun x -> x.Key <> "Name") |> Array.map (fun x -> localStr.Append (sprintf "--%s=%s " x.Key (x.Value.ToString()))) |> ignore localStr.ToString() + member this.formatOnlyKey = let localStr = new System.Text.StringBuilder() this.OnlyKey.GetProperties(true) @@ -1854,4 +1845,55 @@ type Job (jobName: string,processList:(string*string list)list)= strLocal.Append this.formatSlurmCalls |> ignore strLocal.AppendLine("EOF") |> ignore strLocal.ToString() - \ No newline at end of file + + member private this.matchOutput x = + match x with + | Some value -> value + | None -> failwith "No output" + member private this.callToTerminalCMD (command:string) = + let processResponse = + cli { + Shell CMD + Command command + } + |> Command.execute + // processResponse.Text + processResponse + member private this.callToTerminalBash (command:string) = + let processResponse = + cli { + Shell BASH + Command command + } + |> Command.execute + // processResponse.Text + processResponse + member this.getResultFromCallCMD (command:string) = + (this.callToTerminalCMD (command)).Text + |> this.matchOutput + + member this.getResultFromCallBash (command:string) = + (this.callToTerminalBash (command)).Text + |> this.matchOutput + + member this.sendToTerminalAndReceiveJobIDBash (job:Job)= + job.OnlyKey |> OnlyKey.SetParsable true |> ignore + + let res = this.getResultFromCallBash (job.formatProcess) + // submit + // get return + job |> Job.SetJobID res |> ignore + // set as Job ID + + + member this.sendToTerminalAndReceiveJobIDCMD (job:Job)= + // job + // set parsable + job.OnlyKey |> OnlyKey.SetParsable true |> ignore + + + let res = this.getResultFromCallCMD (job.formatProcess) + // submit + // get return + job |> Job.SetJobID res |> ignore + // set as Job ID \ No newline at end of file diff --git a/src/Slurmi/Workflow.fs b/src/Slurmi/Workflow.fs index 9aa5199..f8e59ff 100644 --- a/src/Slurmi/Workflow.fs +++ b/src/Slurmi/Workflow.fs @@ -7,40 +7,6 @@ open Graphoscope open System.Collections.Generic module Workflow = - - let callToTerminalCMD (command:string) = - let processResponse = - cli { - Shell CMD - Command command - } - |> Command.execute - // processResponse.Text - processResponse - let callToTerminalBash (command:string) = - let processResponse = - cli { - Shell BASH - Command command - } - |> Command.execute - // processResponse.Text - processResponse - - let matchOutput x = - match x with - | Some value -> value - | None -> failwith "No output" - - let getResultFromCall (command:string) = - (callToTerminalCMD (command)).Text - |> matchOutput - - let getResultFromCallBASH (command:string) = - (callToTerminalBash (command)).Text - |> matchOutput - - type JobWithDep = { @@ -50,146 +16,371 @@ module Workflow = } + type WFGraph = + { + Graph : FGraph + } + static member createGraphFromJobList (input: JobWithDep list)= + let g = FGraph.empty + input + |> List.iter ( + fun jobWithDep -> + //FGraph.addElement jobWithDep.JobInfo.Name jobWithDep.DependingOn jobWithDep.DependingOn |> ignore + jobWithDep.JobInfo.OnlyKey |> OnlyKey.SetParsable true |> ignore + jobWithDep.DependingOn + |> List.iter (fun (job,dep) -> FGraph.addElement job.Name (input|> List.find (fun x -> x.JobInfo.Name = job.Name)) jobWithDep.JobInfo.Name jobWithDep dep g |> ignore)) + + g + static member getNodesWithoutDependencies (graph:FGraph) = + graph.Keys + |> Seq.map (fun x -> ((graph.Item x) |> fun (a,b,c) -> b), graph.Item x |> (FContext.predecessors) |> Seq.length) + |> Seq.filter (fun x -> (snd x = 0 ) ) + |> Seq.map fst + + member this.getPredecessors (graph:FGraph) (jobToSearch:string) = + (graph.Item (jobToSearch)) |> FContext.predecessors + + member this.hasPredecessors (graph:FGraph) (jobToSearch:string) = + if (this.getPredecessors graph jobToSearch |> Seq.length > 0) then true else false + + + member this.getSuccessors (graph:FGraph) (jobToSearch:string) = + (graph.Item (jobToSearch)) |> FContext.successors + + member this.hasSuccessors (graph:FGraph) (jobToSearch:string) = + if (this.getSuccessors graph jobToSearch |> Seq.length > 0) then true else false + + + member this.arePredecessorsWorkedAlready (graph:FGraph)(jobToLook:string) (workedOn:List) = + let predecessors = this.getPredecessors graph jobToLook + let predecessorsWorked = predecessors |> Seq.toArray |> Array.forall (fun x -> workedOn.Contains (fst x)) + predecessorsWorked + + member this.getJob (graph:FGraph) (job:string )= + let name,job,kind = graph.[job] + job + + member this.getKind (graph:FGraph) (job:string )= + let name,job,kind = graph.[job] + kind + + member this.getDependencies (graph:FGraph) (job:string )= + let name,job,kind = graph.[job] + name + + member this.getIdFromJobWithDep (jwd:JobWithDep) = + jwd.JobInfo |> Job.tryGetJobID + + member this.getJobId (graph:FGraph) (job:string)= + let job = this.getJob graph job + let jid = job.JobInfo |> Job.tryGetJobID + jid.Value |> string + + + member this.formatDep (graph:FGraph) (job:string) = + let deps = (this.getDependencies graph job) |> Seq.map (fun x ->(KindOfDependency.toString x.Value)+(this.getJobId graph x.Key)) |> Seq.toArray + let jobInfo = this.getJob graph job + let command = deps |> String.concat (jobInfo.AllOrAny |> TypeOfDep.toString) + //printfn "%A" command + jobInfo.JobInfo.TwoDashes |> LongCommand.SetDependency command |> ignore + + member this.checkForWorkedBash (graph:FGraph)(jobToLook:string) (workedOn:List)= + if (this.arePredecessorsWorkedAlready graph jobToLook workedOn) then + + if workedOn.Contains jobToLook then + //then printfn "already worked on %A "jobToLook + () + else + //printfn "added %A" jobToLook + let jtwo = this.getJob graph jobToLook + + this.formatDep graph jobToLook + jtwo.JobInfo |> Job.SetJobID (jtwo.JobInfo.getResultFromCallBash (jtwo.JobInfo.produceCall)) |> ignore + + workedOn.Add jobToLook + else + //printfn "false" + () + + member this.checkAllForWorked (graph:FGraph) (jobs:string ) (workedOn:List) = + printfn "working on %A rn " jobs + this.checkForWorkedBash graph jobs workedOn + if (this.hasSuccessors graph jobs) then + let successors = this.getSuccessors graph jobs + + successors |> Seq.iter (fun x -> this.checkAllForWorked graph (fst x) workedOn) + else + printfn "no successors" + + + member this.submitAll (graph:FGraph) (workedOn:List)= + let firstNodes = + WFGraph.getNodesWithoutDependencies graph + |> Seq.toArray + firstNodes |> Array.map (fun x -> this.checkAllForWorked graph x.JobInfo.Name workedOn) |> ignore + + type Workflow = + { + Jobs : JobWithDep list + Graph : WFGraph + } + + member private this.callToTerminalCMD (command:string) = + let processResponse = + cli { + Shell CMD + Command command + } + |> Command.execute + processResponse + member private this.callToTerminalBash (command:string) = + let processResponse = + cli { + Shell BASH + Command command + } + |> Command.execute + processResponse + + member private this. matchOutput x = + match x with + | Some value -> value + | None -> failwith "No output" + + member this.getResultFromCallCMD (command:string) = + (this.callToTerminalCMD (command)).Text + |> this.matchOutput + + member this.getResultFromCallBash (command:string) = + (this.callToTerminalBash (command)).Text + |> this.matchOutput + + + member this.sendToTerminalAndReceiveJobIDBash (job:JobWithDep)= + // job + // set parsable + job.JobInfo.OnlyKey |> OnlyKey.SetParsable true |> ignore + + let res = this.getResultFromCallBash (job.JobInfo.formatProcess) + // submit + // get return + job.JobInfo |> Job.SetJobID res |> ignore + // set as Job ID + + + member this.sendToTerminalAndReceiveJobIDCMD (job:JobWithDep)= + // job + // set parsable + job.JobInfo.OnlyKey |> OnlyKey.SetParsable true |> ignore + + let res = this.getResultFromCallCMD (job.JobInfo.formatProcess) + // submit + // get return + job.JobInfo |> Job.SetJobID res |> ignore + // set as Job ID + + let createWorkflow (jobList: JobWithDep list) = + { + Jobs = jobList + //Graph = WFGraph.createGraphFromJobList jobList + Graph = + { + Graph = WFGraph.createGraphFromJobList jobList + } + } - let createGraphFromJobList (input: JobWithDep list)= - let g = FGraph.empty - input - |> List.iter ( - fun jobWithDep -> - //FGraph.addElement jobWithDep.JobInfo.Name jobWithDep.DependingOn jobWithDep.DependingOn |> ignore - jobWithDep.JobInfo.OnlyKey |> OnlyKey.SetParsable true |> ignore - jobWithDep.DependingOn - |> List.iter (fun (job,dep) -> FGraph.addElement job.Name (input|> List.find (fun x -> x.JobInfo.Name = job.Name)) jobWithDep.JobInfo.Name jobWithDep dep g |> ignore)) +//namespace Slurmi + +//open DynamicObj +//open System.Runtime.InteropServices +//open Fli +//open Graphoscope +//open System.Collections.Generic + +//module Workflow = + +// let callToTerminalCMD (command:string) = +// let processResponse = +// cli { +// Shell CMD +// Command command +// } +// |> Command.execute +// // processResponse.Text +// processResponse +// let callToTerminalBash (command:string) = +// let processResponse = +// cli { +// Shell BASH +// Command command +// } +// |> Command.execute +// // processResponse.Text +// processResponse + +// let matchOutput x = +// match x with +// | Some value -> value +// | None -> failwith "No output" + +// let getResultFromCall (command:string) = +// (callToTerminalCMD (command)).Text +// |> matchOutput + +// let getResultFromCallBASH (command:string) = +// (callToTerminalBash (command)).Text +// |> matchOutput + + +// type JobWithDep = + +// { +// JobInfo : Job +// AllOrAny : TypeOfDep +// DependingOn : (Job*KindOfDependency) list + +// } + + +// let createGraphFromJobList (input: JobWithDep list)= +// let g = FGraph.empty +// input +// |> List.iter ( +// fun jobWithDep -> +// //FGraph.addElement jobWithDep.JobInfo.Name jobWithDep.DependingOn jobWithDep.DependingOn |> ignore +// jobWithDep.JobInfo.OnlyKey |> OnlyKey.SetParsable true |> ignore +// jobWithDep.DependingOn +// |> List.iter (fun (job,dep) -> FGraph.addElement job.Name (input|> List.find (fun x -> x.JobInfo.Name = job.Name)) jobWithDep.JobInfo.Name jobWithDep dep g |> ignore)) - g +// g - let sendToTerminalAndReceiveJobID (job:JobWithDep)= - // job - // set parsable - job.JobInfo.OnlyKey |> OnlyKey.SetParsable true |> ignore +// let sendToTerminalAndReceiveJobID (job:JobWithDep)= +// // job +// // set parsable +// job.JobInfo.OnlyKey |> OnlyKey.SetParsable true |> ignore - let res = getResultFromCallBASH (job.JobInfo.formatProcess) - // submit - // get return - job.JobInfo |> Job.SetJobID res |> ignore - // set as Job ID +// let res = getResultFromCallBASH (job.JobInfo.formatProcess) +// // submit +// // get return +// job.JobInfo |> Job.SetJobID res |> ignore +// // set as Job ID - let sendToTerminalAndReceiveJobIDCMD (job:JobWithDep)= - // job - // set parsable - job.JobInfo.OnlyKey |> OnlyKey.SetParsable true |> ignore +// let sendToTerminalAndReceiveJobIDCMD (job:JobWithDep)= +// // job +// // set parsable +// job.JobInfo.OnlyKey |> OnlyKey.SetParsable true |> ignore - let res = getResultFromCall (job.JobInfo.formatProcess) - // submit - // get return - job.JobInfo |> Job.SetJobID res |> ignore - // set as Job ID +// let res = getResultFromCall (job.JobInfo.formatProcess) +// // submit +// // get return +// job.JobInfo |> Job.SetJobID res |> ignore +// // set as Job ID - let getNodesWithoutDependencies (graph:FGraph) = - graph.Keys - |> Seq.map (fun x -> ((graph.Item x) |> fun (a,b,c) -> b), graph.Item x |> (FContext.predecessors) |> Seq.length) - |> Seq.filter (fun x -> (snd x = 0 ) ) - |> Seq.map fst +// let getNodesWithoutDependencies (graph:FGraph) = +// graph.Keys +// |> Seq.map (fun x -> ((graph.Item x) |> fun (a,b,c) -> b), graph.Item x |> (FContext.predecessors) |> Seq.length) +// |> Seq.filter (fun x -> (snd x = 0 ) ) +// |> Seq.map fst - let getPredecessors (graph:FGraph) (jobToSearch:string) = - (graph.Item (jobToSearch)) |> FContext.predecessors +// let getPredecessors (graph:FGraph) (jobToSearch:string) = +// (graph.Item (jobToSearch)) |> FContext.predecessors - let hasPredecessors (graph:FGraph) (jobToSearch:string) = - if (getPredecessors graph jobToSearch |> Seq.length > 0) then true else false +// let hasPredecessors (graph:FGraph) (jobToSearch:string) = +// if (getPredecessors graph jobToSearch |> Seq.length > 0) then true else false - let getSuccessors (graph:FGraph) (jobToSearch:string) = - (graph.Item (jobToSearch)) |> FContext.successors - let hasSuccessors (graph:FGraph) (jobToSearch:string) = - if (getSuccessors graph jobToSearch |> Seq.length > 0) then true else false +// let getSuccessors (graph:FGraph) (jobToSearch:string) = +// (graph.Item (jobToSearch)) |> FContext.successors +// let hasSuccessors (graph:FGraph) (jobToSearch:string) = +// if (getSuccessors graph jobToSearch |> Seq.length > 0) then true else false - let arePredecessorsWorkedAlready (graph:FGraph)(jobToLook:string) (workedOn:List) = - let predecessors = getPredecessors graph jobToLook - let predecessorsWorked = predecessors |> Seq.toArray |> Array.forall (fun x -> workedOn.Contains (fst x)) - predecessorsWorked +// let arePredecessorsWorkedAlready (graph:FGraph)(jobToLook:string) (workedOn:List) = +// let predecessors = getPredecessors graph jobToLook +// let predecessorsWorked = predecessors |> Seq.toArray |> Array.forall (fun x -> workedOn.Contains (fst x)) +// predecessorsWorked - let getJob (graph:FGraph) (job:string )= - let name,job,kind = graph.[job] - job +// let getJob (graph:FGraph) (job:string )= +// let name,job,kind = graph.[job] +// job - let getKind (graph:FGraph) (job:string )= - let name,job,kind = graph.[job] - kind +// let getKind (graph:FGraph) (job:string )= +// let name,job,kind = graph.[job] +// kind - let getDependencies (graph:FGraph) (job:string )= - let name,job,kind = graph.[job] - name +// let getDependencies (graph:FGraph) (job:string )= +// let name,job,kind = graph.[job] +// name - let getIdFromJobWithDep (jwd:JobWithDep) = - jwd.JobInfo |> Job.tryGetJobID +// let getIdFromJobWithDep (jwd:JobWithDep) = +// jwd.JobInfo |> Job.tryGetJobID - let getJobId (graph:FGraph) (job:string)= - let job = getJob graph job - let jid = job.JobInfo |> Job.tryGetJobID - jid.Value |> string +// let getJobId (graph:FGraph) (job:string)= +// let job = getJob graph job +// let jid = job.JobInfo |> Job.tryGetJobID +// jid.Value |> string - let formatDep (graph:FGraph) (job:string) = - let deps = (getDependencies graph job) |> Seq.map (fun x ->(KindOfDependency.toString x.Value)+(getJobId graph x.Key)) |> Seq.toArray - let jobInfo = getJob graph job - let command = deps |> String.concat (jobInfo.AllOrAny |> TypeOfDep.toString) - printfn "%A" command - jobInfo.JobInfo.TwoDashes |> LongCommand.SetDependency command |> ignore +// let formatDep (graph:FGraph) (job:string) = +// let deps = (getDependencies graph job) |> Seq.map (fun x ->(KindOfDependency.toString x.Value)+(getJobId graph x.Key)) |> Seq.toArray +// let jobInfo = getJob graph job +// let command = deps |> String.concat (jobInfo.AllOrAny |> TypeOfDep.toString) +// printfn "%A" command +// jobInfo.JobInfo.TwoDashes |> LongCommand.SetDependency command |> ignore - let checkForWorked (graph:FGraph)(jobToLook:string) (workedOn:List)= - if (arePredecessorsWorkedAlready graph jobToLook workedOn) then +// let checkForWorked (graph:FGraph)(jobToLook:string) (workedOn:List)= +// if (arePredecessorsWorkedAlready graph jobToLook workedOn) then - if workedOn.Contains jobToLook - then printfn "already worked on %A "jobToLook - else - printfn "added %A" jobToLook - let jtwo = getJob graph jobToLook - - formatDep graph jobToLook - jtwo.JobInfo |> Job.SetJobID (getResultFromCallBASH (jtwo.JobInfo.produceCall)) |> ignore +// if workedOn.Contains jobToLook +// then printfn "already worked on %A "jobToLook +// else +// printfn "added %A" jobToLook +// let jtwo = getJob graph jobToLook + +// formatDep graph jobToLook +// jtwo.JobInfo |> Job.SetJobID (getResultFromCallBASH (jtwo.JobInfo.produceCall)) |> ignore - workedOn.Add jobToLook - else - printfn "false" - - let rec checkAllForWorked (graph:FGraph) (jobs:string ) (workedOn:List) = - printfn "working on %A rn " jobs - checkForWorked graph jobs workedOn - if (hasSuccessors graph jobs) then - let successors = getSuccessors graph jobs +// workedOn.Add jobToLook +// else +// printfn "false" + +// let rec checkAllForWorked (graph:FGraph) (jobs:string ) (workedOn:List) = +// printfn "working on %A rn " jobs +// checkForWorked graph jobs workedOn +// if (hasSuccessors graph jobs) then +// let successors = getSuccessors graph jobs - successors |> Seq.iter (fun x -> checkAllForWorked graph (fst x) workedOn) - else - printfn "no successors" +// successors |> Seq.iter (fun x -> checkAllForWorked graph (fst x) workedOn) +// else +// printfn "no successors" - let formatCallWithDep (job:JobWithDep)= - let jobInfo = job.JobInfo - let localStr = new System.Text.StringBuilder() - localStr.Append ("#!/bin/bash \n") |> ignore - localStr.Append ("#SBATCH ") |> ignore - localStr.AppendFormat (sprintf "-J %s " jobInfo.Name) |> ignore - localStr.Append (jobInfo.formatOneDash) |> ignore - localStr.Append (jobInfo.formatTwoDashes) |> ignore - localStr.Append (jobInfo.formatOnlyKey) |> ignore - localStr.Append ("\n") |> ignore - if jobInfo |> Job.tryGetEnvironment |> Option.isSome then - localStr.Append (jobInfo.formatEnv) |> ignore - localStr.Append jobInfo.formatProcess |> ignore - localStr.ToString() +// let formatCallWithDep (job:JobWithDep)= +// let jobInfo = job.JobInfo +// let localStr = new System.Text.StringBuilder() +// localStr.Append ("#!/bin/bash \n") |> ignore +// localStr.Append ("#SBATCH ") |> ignore +// localStr.AppendFormat (sprintf "-J %s " jobInfo.Name) |> ignore +// localStr.Append (jobInfo.formatOneDash) |> ignore +// localStr.Append (jobInfo.formatTwoDashes) |> ignore +// localStr.Append (jobInfo.formatOnlyKey) |> ignore +// localStr.Append ("\n") |> ignore +// if jobInfo |> Job.tryGetEnvironment |> Option.isSome then +// localStr.Append (jobInfo.formatEnv) |> ignore +// localStr.Append jobInfo.formatProcess |> ignore +// localStr.ToString() - let submitAll (graph:FGraph) (workedOn:List)= - let firstNodes = - getNodesWithoutDependencies graph - |> Seq.toArray - firstNodes |> Array.map (fun x -> checkAllForWorked graph x.JobInfo.Name workedOn) |> ignore \ No newline at end of file +// let submitAll (graph:FGraph) (workedOn:List)= +// let firstNodes = +// getNodesWithoutDependencies graph +// |> Seq.toArray +// firstNodes |> Array.map (fun x -> checkAllForWorked graph x.JobInfo.Name workedOn) |> ignore