From 3de78a862cc1863b97ed6294979ec609b5b8b0e3 Mon Sep 17 00:00:00 2001 From: Mohammed Diaa Date: Wed, 29 Nov 2023 19:07:13 +0200 Subject: [PATCH 1/3] Support new machine and fleet types --- cmd/get/get.go | 3 --- cmd/output/output.go | 36 ++++++++++++++++++++++++++++++++---- types/list.go | 8 +++++--- types/user.go | 19 ++++++++++--------- util/util.go | 24 +++++++++++++++++++++--- 5 files changed, 68 insertions(+), 22 deletions(-) diff --git a/cmd/get/get.go b/cmd/get/get.go index 7f7bc2f..d9d4bd4 100644 --- a/cmd/get/get.go +++ b/cmd/get/get.go @@ -85,9 +85,6 @@ var GetCmd = &cobra.Command{ status = strings.ToLower(runs[0].Status) } out += fmt.Sprintf(fmtStr, "Status:", status) - availableBees := execute.GetAvailableMachines() - out += fmt.Sprintf(fmtStr, "Max machines:", execute.FormatMachines(version.MaxMachines, true)+ - " (currently available: "+execute.FormatMachines(availableBees, true)+")") out += fmt.Sprintf(fmtStr, "Created:", workflow.CreatedDate.In(time.Local).Format(time.RFC1123)+ " ("+util.FormatDuration(time.Since(workflow.CreatedDate))+" ago)") diff --git a/cmd/output/output.go b/cmd/output/output.go index 3b7df2c..1bdb7db 100644 --- a/cmd/output/output.go +++ b/cmd/output/output.go @@ -139,7 +139,7 @@ The YAML config file should be formatted like: runs = append(runs, runs...) } - version := GetWorkflowVersionByID(*runs[0].WorkflowVersionInfo) + version := GetWorkflowVersionByID(*runs[0].WorkflowVersionInfo, uuid.Nil) if version == nil { return } @@ -175,7 +175,7 @@ func DownloadRunOutput(run *types.Run, nodes map[string]NodeInfo, files []string } if version == nil { - version = GetWorkflowVersionByID(*run.WorkflowVersionInfo) + version = GetWorkflowVersionByID(*run.WorkflowVersionInfo, uuid.Nil) } subJobs := getSubJobs(*run.ID) @@ -544,8 +544,8 @@ func GetRuns(workflowID uuid.UUID, pageSize int) []types.Run { return runs.Results } -func GetWorkflowVersionByID(id uuid.UUID) *types.WorkflowVersionDetailed { - resp := request.Trickest.Get().DoF("library/workflow-version/%s/", id) +func GetWorkflowVersionByID(versionID, fleetID uuid.UUID) *types.WorkflowVersionDetailed { + resp := request.Trickest.Get().DoF("library/workflow-version/%s/", versionID) if resp == nil { fmt.Println("Error: Couldn't get workflow version!") return nil @@ -562,9 +562,37 @@ func GetWorkflowVersionByID(id uuid.UUID) *types.WorkflowVersionDetailed { return nil } + if fleetID != uuid.Nil { + maxMachines, err := GetWorkflowVersionMaxMachines(versionID, fleetID) + if err != nil { + fmt.Printf("Error getting maximum machines: %v", err) + return nil + } + workflowVersion.MaxMachines = maxMachines + + } return &workflowVersion } +func GetWorkflowVersionMaxMachines(version, fleet uuid.UUID) (types.Machines, error) { + resp := request.Trickest.Get().DoF("library/workflow-version/%s/max-machines/?fleet=%s", version, fleet) + if resp == nil { + return types.Machines{}, fmt.Errorf("couldn't get workflow version's maximum machines") + } + + if resp.Status() != http.StatusOK { + request.ProcessUnexpectedResponse(resp) + } + + var machines types.Machines + err := json.Unmarshal(resp.Body(), &machines) + if err != nil { + return types.Machines{}, fmt.Errorf("couldn't unmarshal workflow versions's maximum machines: %v", err) + } + + return machines, nil +} + func getChildrenSubJobsCount(subJobID uuid.UUID) int { urlReq := "subjob/" + subJobID.String() + "/children/" urlReq += "?page_size=" + strconv.Itoa(math.MaxInt) diff --git a/types/list.go b/types/list.go index 1827c92..a7dce6c 100644 --- a/types/list.go +++ b/types/list.go @@ -127,9 +127,11 @@ type Run struct { } type Machines struct { - Small *int `json:"small,omitempty"` - Medium *int `json:"medium,omitempty"` - Large *int `json:"large,omitempty"` + Small *int `json:"small,omitempty"` + Medium *int `json:"medium,omitempty"` + Large *int `json:"large,omitempty"` + Default *int `json:"default,omitempty"` + SelfHosted *int `json:"self_hosted,omitempty"` } type SubJobs struct { diff --git a/types/user.go b/types/user.go index 0a5dc66..20669d4 100644 --- a/types/user.go +++ b/types/user.go @@ -54,14 +54,12 @@ type Fleets struct { } type Fleet struct { - ID uuid.UUID `json:"id"` - Name string `json:"name"` - Vault uuid.UUID `json:"vault"` - Cluster string `json:"cluster"` - State string `json:"state"` - CreatedDate time.Time `json:"created_date"` - ModifiedDate time.Time `json:"modified_date"` - Machines []struct { + ID uuid.UUID `json:"id"` + Name string `json:"name"` + Vault uuid.UUID `json:"vault"` + Cluster string `json:"cluster"` + State string `json:"state"` + Machines []struct { Name string `json:"name"` Description string `json:"description"` Mem string `json:"mem"` @@ -70,6 +68,9 @@ type Fleet struct { Running int `json:"running"` Up int `json:"up"` Down int `json:"down"` - Error int `json:"error"` } `json:"machines"` + CreatedDate time.Time `json:"created_date"` + ModifiedDate time.Time `json:"modified_date"` + Type string `json:"type"` + Default bool `json:"default"` } diff --git a/util/util.go b/util/util.go index b4cdd0d..5463554 100644 --- a/util/util.go +++ b/util/util.go @@ -106,7 +106,7 @@ func GetMe() *types.User { return &user } -func GetFleetInfo() *types.Fleet { +func GetFleetInfo(fleetName string) *types.Fleet { resp := request.Trickest.Get().DoF("fleet/?vault=%s", GetVault()) if resp == nil || resp.Status() != http.StatusOK { request.ProcessUnexpectedResponse(resp) @@ -119,11 +119,29 @@ func GetFleetInfo() *types.Fleet { return nil } - if len(fleets.Results) == 0 { + if fleets.Count == 0 { fmt.Println("Error: Couldn't find any active fleets") + return nil + } else if fleets.Count == 1 { + if fleetName == "" || fleets.Results[0].Name == fleetName { + return &fleets.Results[0] + } else { + fmt.Println("Error: Couldn't find a fleet with the specified name") + } + } else { + for _, fleet := range fleets.Results { + if fleet.Name == fleetName { + return &fleet + } + } } - return &fleets.Results[0] + if fleetName == "" { + fmt.Println("Error: You have multiple fleets. Use the --fleet flag to specify which one to use") + } else { + fmt.Println("Error: Couldn't find a fleet with the specified name") + } + return nil } func GetSpaces(name string) []types.Space { From 534c2b7a384348bbf2b0bee5a6e18aeda3d9078a Mon Sep 17 00:00:00 2001 From: Mohammed Diaa Date: Wed, 29 Nov 2023 19:08:05 +0200 Subject: [PATCH 2/3] Support multiple fleets and new machine config format - Add `--fleet` flag to select a specific fleet - Add fleet auto-selection if one fleet is available - Support running a workflow on self-hosted or default machines --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 06980fd..ec33b1e 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,9 @@ trickest execute --workflow --space --confi | --set-name | string | / | Sets the new workflow name and will copy the workflow to space and project supplied | | --ci | boolean | false | Enable CI mode (in-progress executions will be stopped when the CLI is forcefully stopped - if not set, you will be asked for confirmation) | | --create-project | boolean | false | If the project doesn't exist, create one using the project flag as its name (or workflow/tool name if project flag is not set) | -| --machines | string | / | Specify the number of machines (format: small-medium-large). Examples: 1-1-1, 0-0-3 | +| --machines | string | / | Specify the number of machines. Use one value for default/self-hosted machines (--machines 3) or three values for small-medium-large (--machines 1-1-1) | +| --fleet | string | / | The name of the fleet to use to execute the workflow + #### Provide parameters using **config.yaml** file From fc7da71efc59f81c27d9df6c529eb2715decfc4a Mon Sep 17 00:00:00 2001 From: Mohammed Diaa Date: Wed, 29 Nov 2023 19:10:56 +0200 Subject: [PATCH 3/3] Process different machine configuration formats and handle fleet selection --- cmd/execute/config.go | 3 +- cmd/execute/execute.go | 66 ++++++++++++++-------- cmd/execute/helpers.go | 125 +++++++++++++++++++++++------------------ cmd/execute/watch.go | 2 +- 4 files changed, 114 insertions(+), 82 deletions(-) diff --git a/cmd/execute/config.go b/cmd/execute/config.go index 6100ee4..b1e2838 100644 --- a/cmd/execute/config.go +++ b/cmd/execute/config.go @@ -620,8 +620,7 @@ func readConfigMachines(config *map[string]interface{}, isTool bool, maximumMach if maxMachines { return maximumMachines } else { - execMachines = maximumMachines - setMachinesToMinimum(execMachines) + *execMachines = setMachinesToMinimum(*maximumMachines) } } } diff --git a/cmd/execute/execute.go b/cmd/execute/execute.go index 9a24623..087c039 100644 --- a/cmd/execute/execute.go +++ b/cmd/execute/execute.go @@ -42,6 +42,7 @@ var ( outputNodesFlag string ci bool createProject bool + fleetName string ) // ExecuteCmd represents the execute command @@ -67,7 +68,7 @@ var ExecuteCmd = &cobra.Command{ } } - fleet = util.GetFleetInfo() + fleet = util.GetFleetInfo(fleetName) if fleet == nil { return } @@ -87,35 +88,51 @@ var ExecuteCmd = &cobra.Command{ allNodes, roots = CreateTrees(version, false) if maxMachines { executionMachines = version.MaxMachines - } + } else if machineConfiguration != "" { + machines := &types.Machines{} - if machineConfiguration != "" { - pattern := `^\d+-\d+-\d+$` - regex := regexp.MustCompile(pattern) - if regex.MatchString(machineConfiguration) { - parts := strings.Split(machineConfiguration, "-") + // Managed enterprise fleet, 3 types of machines, small-medium-large + if len(fleet.Machines) == 3 { + pattern := `^\d+-\d+-\d+$` + regex := regexp.MustCompile(pattern) - machines := &types.Machines{} + if regex.MatchString(machineConfiguration) { + parts := strings.Split(machineConfiguration, "-") - if small, err := strconv.Atoi(parts[0]); err == nil && small != 0 { - machines.Small = &small - } + if small, err := strconv.Atoi(parts[0]); err == nil && small != 0 { + machines.Small = &small + } - if medium, err := strconv.Atoi(parts[1]); err == nil && medium != 0 { - machines.Medium = &medium - } + if medium, err := strconv.Atoi(parts[1]); err == nil && medium != 0 { + machines.Medium = &medium + } - if large, err := strconv.Atoi(parts[2]); err == nil && large != 0 { - machines.Large = &large + if large, err := strconv.Atoi(parts[2]); err == nil && large != 0 { + machines.Large = &large + } + executionMachines = *machines + } else { + fmt.Printf("Invalid machine configuration \"%s\".\n", machineConfiguration) + fmt.Println("Please use the format: small-medium-large (e.g., 0-0-3)") + os.Exit(1) } - - executionMachines = *machines - } else { - fmt.Printf("Invalid machine configuration \"%s\".\n", machineConfiguration) - fmt.Println("Please use the format: small-medium-large (e.g., 0-0-3)") - os.Exit(0) + defaultOrSelfHosted, err := strconv.Atoi(machineConfiguration) + if err != nil { + fmt.Printf("Invalid machine configuration \"%s\".\n", machineConfiguration) + os.Exit(1) + } + + if fleet.Type == "MANAGED" { + machines.Default = &defaultOrSelfHosted + } else if fleet.Type == "HOSTED" { + machines.SelfHosted = &defaultOrSelfHosted + } } + executionMachines = *machines + + } else { + executionMachines = setMachinesToMinimum(version.MaxMachines) } outputNodes := make([]string, 0) @@ -130,7 +147,7 @@ var ExecuteCmd = &cobra.Command{ os.Exit(0) } - createRun(version.ID, fleet.ID, watch, &executionMachines, outputNodes, outputsDirectory) + createRun(version.ID, fleet.ID, watch, outputNodes, outputsDirectory) }, } @@ -141,12 +158,13 @@ func init() { ExecuteCmd.Flags().BoolVar(&showParams, "show-params", false, "Show parameters in the workflow tree") // ExecuteCmd.Flags().StringVar(&workflowYAML, "file", "", "Workflow YAML file to execute") ExecuteCmd.Flags().BoolVar(&maxMachines, "max", false, "Use maximum number of machines for workflow execution") - ExecuteCmd.Flags().StringVar(&machineConfiguration, "machines", "", "Specify the number of machines (format: small-medium-large). Examples: 1-1-1, 0-0-3") + ExecuteCmd.Flags().StringVar(&machineConfiguration, "machines", "", "Specify the number of machines. Use one value for default/self-hosted machines (--machines 3) or three values for small-medium-large (--machines 1-1-1)") ExecuteCmd.Flags().BoolVar(&downloadAllNodes, "output-all", false, "Download all outputs when the execution is finished") ExecuteCmd.Flags().StringVar(&outputNodesFlag, "output", "", "A comma separated list of nodes which outputs should be downloaded when the execution is finished") ExecuteCmd.Flags().StringVar(&outputsDirectory, "output-dir", "", "Path to directory which should be used to store outputs") ExecuteCmd.Flags().BoolVar(&ci, "ci", false, "Run in CI mode (in-progreess executions will be stopped when the CLI is forcefully stopped - if not set, you will be asked for confirmation)") ExecuteCmd.Flags().BoolVar(&createProject, "create-project", false, "If the project doesn't exist, create it using the project flag as its name (or workflow name if not set)") + ExecuteCmd.Flags().StringVar(&fleetName, "fleet", "", "The name of the fleet to use to execute the workflow") } func readWorkflowYAMLandCreateVersion(fileName string, workflowName string, objectPath string) *types.WorkflowVersionDetailed { diff --git a/cmd/execute/helpers.go b/cmd/execute/helpers.go index c18821f..0413809 100644 --- a/cmd/execute/helpers.go +++ b/cmd/execute/helpers.go @@ -98,11 +98,10 @@ func getScripts(pageSize int, search string, name string) []types.Script { return scripts.Results } -func createRun(versionID, fleetID uuid.UUID, watch bool, machines *types.Machines, outputNodes []string, outputsDir string) { +func createRun(versionID, fleetID uuid.UUID, watch bool, outputNodes []string, outputsDir string) { run := types.CreateRun{ VersionID: versionID, - Vault: fleet.Vault, Machines: executionMachines, Fleet: &fleetID, } @@ -119,18 +118,6 @@ func createRun(versionID, fleetID uuid.UUID, watch bool, machines *types.Machine os.Exit(0) } - if resp.Status() != http.StatusCreated { - run.Fleet = nil - - data, err := json.Marshal(run) - if err != nil { - fmt.Println("Error encoding create run request!") - os.Exit(0) - } - - resp = request.Trickest.Post().Body(data).DoF("execution/") - } - if resp.Status() != http.StatusCreated { request.ProcessUnexpectedResponse(resp) } @@ -151,9 +138,9 @@ func createRun(versionID, fleetID uuid.UUID, watch bool, machines *types.Machine if watch { WatchRun(createRunResp.ID, outputsDir, nodesToDownload, nil, false, &executionMachines, showParams) } else { - availableMachines := GetAvailableMachines() + availableMachines := GetAvailableMachines(fleetName) fmt.Println("Run successfully created! ID: " + createRunResp.ID.String()) - fmt.Print("Machines:\n" + FormatMachines(*machines, false)) + fmt.Print("Machines:\n" + FormatMachines(executionMachines, false)) fmt.Print("\nAvailable:\n" + FormatMachines(availableMachines, false)) } } @@ -193,7 +180,8 @@ func createNewVersion(version *types.WorkflowVersionDetailed) *types.WorkflowVer return nil } - newVersion := output.GetWorkflowVersionByID(newVersionInfo.ID) + fleet := util.GetFleetInfo(fleetName) + newVersion := output.GetWorkflowVersionByID(newVersionInfo.ID, fleet.ID) return newVersion } @@ -396,8 +384,8 @@ func processInvalidInputType(newPNode, existingPNode types.PrimitiveNode) { os.Exit(0) } -func GetAvailableMachines() types.Machines { - hiveInfo := util.GetFleetInfo() +func GetAvailableMachines(fleetName string) types.Machines { + hiveInfo := util.GetFleetInfo(fleetName) availableMachines := types.Machines{} for _, machine := range hiveInfo.Machines { if machine.Name == "small" { @@ -412,6 +400,14 @@ func GetAvailableMachines() types.Machines { available := machine.Total - machine.Running availableMachines.Large = &available } + if machine.Name == "default" { + available := machine.Total - machine.Running + availableMachines.Default = &available + } + if machine.Name == "self_hosted" { + available := machine.Total - machine.Running + availableMachines.SelfHosted = &available + } } return availableMachines } @@ -477,7 +473,7 @@ func stopRun(runID uuid.UUID) { } } -func setMachinesToMinimum(machines *types.Machines) { +func setMachinesToMinimum(machines types.Machines) types.Machines { if machines.Small != nil { *machines.Small = 1 } @@ -487,50 +483,50 @@ func setMachinesToMinimum(machines *types.Machines) { if machines.Large != nil { *machines.Large = 1 } + if machines.Default != nil { + *machines.Default = 1 + } + if machines.SelfHosted != nil { + *machines.SelfHosted = 1 + } + + return machines } func FormatMachines(machines types.Machines, inline bool) string { - var small, medium, large string - if machines.Small != nil { - small = "small: " + strconv.Itoa(*machines.Small) - } - if machines.Medium != nil { - medium = "medium: " + strconv.Itoa(*machines.Medium) - } - if machines.Large != nil { - large = "large: " + strconv.Itoa(*machines.Large) - } + smallMachines := formatSize("small", machines.Small) + mediumMachines := formatSize("medium", machines.Medium) + largeMachines := formatSize("large", machines.Large) + selfHostedMachines := formatSize("self hosted", machines.SelfHosted) + defaultMachines := formatSize("default", machines.Default) - out := "" + var out string if inline { - if small != "" { - out = small - } - if medium != "" { - if small != "" { - out += ", " - } - out += medium - } - if large != "" { - if small != "" || medium != "" { - out += ", " - } - out += large - } + out = joinNonEmptyValues(", ", smallMachines, mediumMachines, largeMachines, selfHostedMachines, defaultMachines) } else { - if small != "" { - out = " " + small + "\n " - } - if medium != "" { - out += medium + "\n " - } - if large != "" { - out += large + "\n" + out = joinNonEmptyValues("\n ", " "+smallMachines, mediumMachines, largeMachines, selfHostedMachines, defaultMachines) + } + return out +} + +func formatSize(sizeName string, size *int) string { + if size != nil { + return sizeName + ": " + strconv.Itoa(*size) + } + return "" +} + +func joinNonEmptyValues(separator string, values ...string) string { + var nonEmptyValues []string + + for _, value := range values { + if value != "" { + nonEmptyValues = append(nonEmptyValues, value) } } - return out + result := strings.Join(nonEmptyValues, separator) + return result } func getNodeNameFromConnectionID(id string) string { @@ -596,6 +592,25 @@ func uploadFilesIfNeeded(primitiveNodes map[string]*types.PrimitiveNode) { } func maxMachinesTypeCompatible(machines, maxMachines types.Machines) bool { + if machines.Default != nil && *machines.Default > *maxMachines.Default { + return false + } + if machines.SelfHosted != nil && *machines.SelfHosted > *maxMachines.SelfHosted { + return false + } + + if machines.Small != nil && *machines.Small > *maxMachines.Small { + return false + } + + if machines.Medium != nil && *machines.Medium > *maxMachines.Medium { + return false + } + + if machines.Large != nil && *machines.Large > *maxMachines.Large { + return false + } + if (machines.Small != nil && maxMachines.Small == nil) || (machines.Medium != nil && maxMachines.Medium == nil) || (machines.Large != nil && maxMachines.Large == nil) { diff --git a/cmd/execute/watch.go b/cmd/execute/watch.go index 35c4150..3783ff0 100644 --- a/cmd/execute/watch.go +++ b/cmd/execute/watch.go @@ -71,7 +71,7 @@ func WatchRun(runID uuid.UUID, downloadPath string, nodesToDownload map[string]o out := "" out += fmt.Sprintf(fmtStr, "Name:", run.WorkflowName) out += fmt.Sprintf(fmtStr, "Status:", strings.ToLower(run.Status)) - availableMachines := GetAvailableMachines() + availableMachines := GetAvailableMachines(fleetName) out += fmt.Sprintf(fmtStr, "Machines:", FormatMachines(*machines, true)+ " (currently available: "+FormatMachines(availableMachines, true)+")") out += fmt.Sprintf(fmtStr, "Created:", run.CreatedDate.In(time.Local).Format(time.RFC1123)+