Skip to content

Commit

Permalink
Refactoring of Job Types
Browse files Browse the repository at this point in the history
  • Loading branch information
@zieglerSe
zieglerSe committed Oct 4, 2023
1 parent 00aaa5b commit 8c9ebfd
Show file tree
Hide file tree
Showing 6 changed files with 3,111 additions and 1,293 deletions.
146 changes: 74 additions & 72 deletions src/Slurmi/Dependency.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,62 +3,63 @@ open DynamicObj
open System.Runtime.InteropServices
open Fli

//let checkStringIntOpt (a:string*int option)=
// match (snd a) with
// |Some value -> sprintf "%s+%i" (fst a) value
// |None -> sprintf "%s" (fst a)
type TypeOfDep =
| All
| Any

static member toString (dep:TypeOfDep)=
match dep with
| All -> ","
| Any -> "?"
type DependencyType =
/// After the specified jobs start or are cancelled and 'time' in minutes from job start or cancellation happens, this job can begin execution. If no 'time' is given then there is no delay after start or cancellation.
| After of (string*int option) list
/// This job can begin execution after the specified jobs have terminated. This is the default dependency type.
| Afterany of string list
/// This job can begin execution after the specified jobs have terminated and any associated burst buffer stage out operations have completed.
| Afterburstbuffer of string list
/// A task of this job array can begin execution after the corresponding task ID in the specified job has completed successfully (ran to completion with an exit code of zero).
| Aftercorr of string list
/// This job can begin execution after the specified jobs have terminated in some failed state (non-zero exit code, node failure, timed out, etc).
| Afternotok of string list
/// This job can begin execution after the specified jobs have successfully executed (ran to completion with an exit code of zero)
| Afterok of string list
/// This job can begin execution after any previously launched jobs sharing the same job name and user have terminated. In other words, only one job by that name and owned by that user can be running or suspended at any point in time. In a federation, a singleton dependency must be fulfilled on all clusters unless DependencyParameters=disable_remote_singleton is used in slurm.conf.
| Singleton of string list

static member toString (dep:DependencyType) =
match dep with
| Afterany (dep) -> sprintf "afterany:%s" (dep |> String.concat ":")
| Afterburstbuffer (dep) -> sprintf "afterburstbuffer:%s" (dep |> String.concat ":")
| Aftercorr (dep) -> sprintf "aftercorr:%s" (dep |> String.concat ":")
| Afternotok (dep) -> sprintf "afternotok:%s" (dep |> String.concat ":")
| Afterok (dep) -> sprintf "afterok:%s" (dep |> String.concat ":")
| Singleton (dep) -> sprintf "singleton"
| After (dep) ->
let depTimes =
dep
|> List.map (
fun x ->
match (snd x) with
|Some value -> sprintf "%s+%i" (fst x) value
|None -> sprintf "%s" (fst x))
////let checkStringIntOpt (a:string*int option)=
//// match (snd a) with
//// |Some value -> sprintf "%s+%i" (fst a) value
//// |None -> sprintf "%s" (fst a)
//type TypeOfDep =
// | All
// | Any

// static member toString (dep:TypeOfDep)=
// match dep with
// | All -> ","
// | Any -> "?"

//type DependencyType =
// /// After the specified jobs start or are cancelled and 'time' in minutes from job start or cancellation happens, this job can begin execution. If no 'time' is given then there is no delay after start or cancellation.
// | After of (string*int option) list
// /// This job can begin execution after the specified jobs have terminated. This is the default dependency type.
// | Afterany of string list
// /// This job can begin execution after the specified jobs have terminated and any associated burst buffer stage out operations have completed.
// | Afterburstbuffer of string list
// /// A task of this job array can begin execution after the corresponding task ID in the specified job has completed successfully (ran to completion with an exit code of zero).
// | Aftercorr of string list
// /// This job can begin execution after the specified jobs have terminated in some failed state (non-zero exit code, node failure, timed out, etc).
// | Afternotok of string list
// /// This job can begin execution after the specified jobs have successfully executed (ran to completion with an exit code of zero)
// | Afterok of string list
// /// This job can begin execution after any previously launched jobs sharing the same job name and user have terminated. In other words, only one job by that name and owned by that user can be running or suspended at any point in time. In a federation, a singleton dependency must be fulfilled on all clusters unless DependencyParameters=disable_remote_singleton is used in slurm.conf.
// | Singleton of string list

// static member toString (dep:DependencyType) =
// match dep with
// | Afterany (dep) -> sprintf "afterany:%s" (dep |> String.concat ":")
// | Afterburstbuffer (dep) -> sprintf "afterburstbuffer:%s" (dep |> String.concat ":")
// | Aftercorr (dep) -> sprintf "aftercorr:%s" (dep |> String.concat ":")
// | Afternotok (dep) -> sprintf "afternotok:%s" (dep |> String.concat ":")
// | Afterok (dep) -> sprintf "afterok:%s" (dep |> String.concat ":")
// | Singleton (dep) -> sprintf "singleton"
// | After (dep) ->
// let depTimes =
// dep
// |> List.map (
// fun x ->
// match (snd x) with
// |Some value -> sprintf "%s+%i" (fst x) value
// |None -> sprintf "%s" (fst x))


|> String.concat (":")
sprintf "after:%s" depTimes
// |> String.concat (":")
// sprintf "after:%s" depTimes

static member concat (typeOfDep: TypeOfDep)(depL: DependencyType[]) =
let sep =
TypeOfDep.toString typeOfDep
depL
|> Array.map (fun x -> DependencyType.toString x)
|> String.concat sep

// static member concat (typeOfDep: TypeOfDep)(depL: DependencyType[]) =
// let sep =
// TypeOfDep.toString typeOfDep
// depL
// |> Array.map (fun x -> DependencyType.toString x)
// |> String.concat sep



Expand All @@ -67,30 +68,31 @@ type DependencyType =



//let depTypeToString (dep:DependencyType) =
// match dep with
// | Afterany (dep) -> sprintf "afterany:%s" (dep |> String.concat ":")
// | Afterburstbuffer (dep) -> sprintf "afterburstbuffer:%s" (dep |> String.concat ":")
// | Aftercorr (dep) -> sprintf "aftercorr:%s" (dep |> String.concat ":")
// | Afternotok (dep) -> sprintf "afternotok:%s" (dep |> String.concat ":")
// | Afterok (dep) -> sprintf "afterok:%s" (dep |> String.concat ":")
// | Singleton (dep) -> sprintf "singleton"
// | After (dep) ->
// let depTimes =
// dep
// |> List.map (
// fun x ->
// checkStringIntOpt x)

////let depTypeToString (dep:DependencyType) =
//// match dep with
//// | Afterany (dep) -> sprintf "afterany:%s" (dep |> String.concat ":")
//// | Afterburstbuffer (dep) -> sprintf "afterburstbuffer:%s" (dep |> String.concat ":")
//// | Aftercorr (dep) -> sprintf "aftercorr:%s" (dep |> String.concat ":")
//// | Afternotok (dep) -> sprintf "afternotok:%s" (dep |> String.concat ":")
//// | Afterok (dep) -> sprintf "afterok:%s" (dep |> String.concat ":")
//// | Singleton (dep) -> sprintf "singleton"
//// | After (dep) ->
//// let depTimes =
//// dep
//// |> List.map (
//// fun x ->
//// checkStringIntOpt x)


// |> String.concat (":")
// sprintf "after:%s" depTimes
//// |> String.concat (":")
//// sprintf "after:%s" depTimes


//let internal concatDependencies (seperator:string) (depL: DependencyType[])=
// depL
// |> Array.map (fun x -> depTypeToString x)
// |> String.concat seperator
////let internal concatDependencies (seperator:string) (depL: DependencyType[])=
//// depL
//// |> Array.map (fun x -> depTypeToString x)
//// |> String.concat seperator



Expand Down
12 changes: 6 additions & 6 deletions src/Slurmi/Environment.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@



type EnvironmentSLURM() =
let mutable environment : (string * string) list = []
//type EnvironmentSLURM() =
// let mutable environment : (string * string) list = []

member this.AddCommandAndArgument (command: string) (argument: string) =
environment <- (command, argument) :: environment
// member this.AddCommandAndArgument (command: string) (argument: string) =
// environment <- (command, argument) :: environment

member this.GetEnvironment() =
environment
// member this.GetEnvironment() =
// environment
100 changes: 50 additions & 50 deletions src/Slurmi/HelperFunctions.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,67 +5,67 @@ open System.Runtime.InteropServices
open Fli


module helperFunctions =
//module helperFunctions =

let findDependencyName (name:string) (jobArray:Job array )=
let id =
jobArray
|> Array.findIndex (fun x -> x.Name = name)
let name = $"$Job{id}"
name
// let findDependencyName (name:string) (jobArray:Job array )=
// let id =
// jobArray
// |> Array.findIndex (fun x -> x.Name = name)
// let name = $"$Job{id}"
// name


let getDependency (job:Job)=
(match (job |> Job.tryGetDependency) with
| (Some value) -> "--dependency="+(DependencyType.concat (fst (value:?>(TypeOfDep*DependencyType[]))) (snd (value:?>(TypeOfDep*DependencyType[]))))
| (None) -> "")
// let getDependency (job:Job)=
// (match (job |> Job.LongCommand.tryGetDependency) with
// | (Some value) -> "--dependency="+(DependencyType.concat (fst (value:?>(TypeOfDep*DependencyType[]))) (snd (value:?>(TypeOfDep*DependencyType[]))))
// | (None) -> "")

let createNamesForWF (jobs:Job array)=
jobs
|> Array.mapi (fun i x -> ($"Job{i}",x))
// let createNamesForWF (jobs:Job array)=
// jobs
// |> Array.mapi (fun i x -> ($"Job{i}",x))

let formatWF (wfJob:(string*Job)) =
let jId = fst wfJob
let job = snd wfJob
let parsable =
(match (job |> Job.tryGetParsable) with
| (Some value) -> (sprintf "--parsable")
| (None) -> "")
let dep: string =
getDependency job
let fullString =
$"{jId}=$(sbatch {parsable} {dep} {job.Name})"
fullString
// let formatWF (wfJob:(string*Job)) =
// let jId = fst wfJob
// let job = snd wfJob
// let parsable =
// (match (job |> Job.tryGetParsable) with
// | (Some value) -> (sprintf "--parsable")
// | (None) -> "")
// let dep: string =
// getDependency job
// let fullString =
// $"{jId}=$(sbatch {parsable} {dep} {job.Name})"
// fullString

let createWorkflowFromJobArray (jobs:Job array) =
let jobNames = createNamesForWF jobs
// let createWorkflowFromJobArray (jobs:Job array) =
// let jobNames = createNamesForWF jobs

let parsableJobs = jobNames |> Array.map (fun x -> fst x,x |> snd |> Job.SetParsable true )
// let parsableJobs = jobNames |> Array.map (fun x -> fst x,x |> snd |> Job.SetParsable true )

let jobString = parsableJobs |> Array.map (fun x -> formatWF x)
jobString
// let jobString = parsableJobs |> Array.map (fun x -> formatWF x)
// jobString

let createWorkflowFromWFType (wf:Workflow) =
let jobs = wf.Jobs
let jobNames = createNamesForWF jobs
// let createWorkflowFromWFType (wf:Workflow) =
// let jobs = wf.Jobs
// let jobNames = createNamesForWF jobs

let parsableJobs = jobNames |> Array.map (fun x -> fst x,x |> snd |> Job.SetParsable true )
// let parsableJobs = jobNames |> Array.map (fun x -> fst x,x |> snd |> Job.SetParsable true )

let jobString = parsableJobs |> Array.map (fun x -> formatWF x)
// let jobString = parsableJobs |> Array.map (fun x -> formatWF x)

let textToDisplay =
[|
"#!/bin/bash"
;
(match (wf |> Workflow.tryGetTime) with
| (Some value) -> (sprintf "#SBATCH --time=%s" (Job.formatTime (value:?>(int*int*int*int))))
| (None) -> "")
;
(match (wf |> Workflow.tryGetPartition) with
| (Some value) -> (sprintf "#SBATCH -p %s" (value:?>string))
| (None) -> "")
// let textToDisplay =
// [|
// "#!/bin/bash"
// ;
// (match (wf |> Workflow.tryGetTime) with
// | (Some value) -> (sprintf "#SBATCH --time=%s" (Job.formatTime (value:?>(int*int*int*int))))
// | (None) -> "")
// ;
// (match (wf |> Workflow.tryGetPartition) with
// | (Some value) -> (sprintf "#SBATCH -p %s" (value:?>string))
// | (None) -> "")

|]
|> Array.filter (fun x -> x <> "" || x <> " ")
// |]
// |> Array.filter (fun x -> x <> "" || x <> " ")

jobString |> Array.append textToDisplay
// jobString |> Array.append textToDisplay
Loading

0 comments on commit 8c9ebfd

Please sign in to comment.