From 6d4b92fd96490514f66dee3d8b2d764de6165a3b Mon Sep 17 00:00:00 2001 From: Prasad Ghangal Date: Mon, 23 Jan 2023 13:09:59 +0530 Subject: [PATCH 01/10] Add proposal for phase tracking Signed-off-by: Prasad Ghangal --- design/phase-level-progress-tracking.md | 143 ++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 design/phase-level-progress-tracking.md diff --git a/design/phase-level-progress-tracking.md b/design/phase-level-progress-tracking.md new file mode 100644 index 0000000000..0315348cc3 --- /dev/null +++ b/design/phase-level-progress-tracking.md @@ -0,0 +1,143 @@ +# Problem Statement + +Kanister actions are triggered by creating ActionSet resource. The status field in ActionSet CR represents the status of Kanister operation. The action completion percentage is set by calculating the weights given on phases ([https://github.com/kanisterio/kanister/blob/master/design/progress-tracking.md](https://github.com/kanisterio/kanister/blob/master/design/progress-tracking.md) ). + +This goal of this proposal is + +- Have progress tracking per phase +- Improve action progress tracking mechanism by removing hard coded weights + +## High Level Design + +### Changes in ActionSet status field + +In order to inform progress of each phase execution while performing action, the progress field can be added to status.phase + +``` +// ActionStatus is updated as we execute phases. +type ActionStatus struct { + // Name is the action we'll perform. For example: `backup` or `restore`. + Name string `json:"name"` + // Object refers to the thing we'll perform this action on. + Object ObjectReference `json:"object"` + // Blueprint with instructions on how to execute this action. + Blueprint string `json:"blueprint"` + // Phases are sub-actions an are executed sequentially. + Phases []Phase `json:"phases,omitempty"` + // Artifacts created by this phase. + Artifacts map[string]Artifact `json:"artifacts,omitempty"` + // DeferPhase is the phase that is executed at the end of an action + // irrespective of the status of other phases in the action + DeferPhase Phase `json:"deferPhase,omitempty"` +} + + // Phase is subcomponent of an action. + type Phase struct { + Name string `json:"name"` + State State `json:"state"` + Output map[string]interface{} `json:"output,omitempty"` ++ Progress PhaseProgress `json:"progress,omitempty"` <----- + } + ++type PhaseProgress struct { ++ ProgressPercent int64 ++ LastTransitionTime *metav1.Time ++ SizeUploadedB int64 ++ EstimatedTimeSeconds int64 ++ EstimatedUploadSizeB int64 ++} +``` + +### Action progress tracking + +Action can consists of multiple phases. These all phases can take consume different duration for completion based on the operations it performs. All phases for the action can be given equal weight-age and to calculate action progress completion %, average of all phase completion %ge can be calculated + +``` +% completion of action = sum(% Phase completion of all the phases)/ no. of phases +``` + +### Phase progress tracking + +Each function is designed to perform different operations in different ways. Since the operation execution is specific to function, each function can be different ways to calculate and provide info about the progress. E.g datamover functions can leverage kopia stats to track progress. So it makes sense to delegate progress tracking to function implementation. + +Each Kanister function implements Progress interface. The Kanister function implementation calculates and returns the progress stats. + +**Progress interface** + +``` +type Progress interface { + Stats() (*FuncProgress, error) +} + +type FuncProgress struct { + ProgressPercent int64 + SizeUploadedB int64 + EstimatedTimeSeconds int64 + EstimatedUploadSizeB int64 +} +``` + +**Kanister function interface** + +``` +type Func interface { ++ Progress + RequiredArgs() []string + Arguments() []string + Exec(context.Context, param.TemplateParams, map[string]interface{}) (map[string]interface{}, error) + } +``` + +The PhaseProgress.ProgressPercent would be the of progress the functions it consists of. + +### Kanister functions progress tracking + +On high level we can divide the Kanister functions into two groups + +### Datamover Kanister functions + +Kanister data mover functions (like BackupData, CopyVolumeData, RestoreData, etc) use Kopia to snapshot filesystem and move data to/from objects. + +While snapshotting, kopia provides the info about progress on the terminal + +``` +$ kopia --log-level=error --config-file=/tmp/kopia-repository.config --log-dir=/tmp/kopia-log snapshot --progress-update-interval=5s /data + / 1 hashing, 118 hashed (546.9 MB), 0 cached (0 B), uploaded 0 B, estimated 4.5 GB (12.1%) 35s left + * 0 hashing, 136 hashed (1.4 GB), 0 cached (0 B), uploaded 244 B, estimated 4.5 GB (30.1%) 30s left +``` + +The command output can be parsed to get the progress metadata. + +### Non-datamover Kanister functions + +TBD + +### Updating actionset status + +[TrackActionProgress](https://github.com/kanisterio/kanister/blob/master/pkg/progress/action.go#L47) function is responsible for periodically updating progress for the action in ActionSet resource status. + +Refactor TrackActionsProgress to remove weight based progress calculation to use Progress interface implemented by functions. + +Pass additional parameter Phase to the TrackActionsProgress which can be used to get progress of the function in execution. + +``` ++ go func() { ++ // progress update is computed on a best-effort basis. ++ // if it exits with error, we will just log it. ++ if err := progress.TrackActionsProgress(ctx, c.crClient, as.GetName(), as.GetNamespace(), p); err != nil { ++ log.Error().WithError(err) ++ } ++ }() + output, err = p.Exec(ctx, *bp, action.Name, *tp) + +``` + +Implement Progress() function on Phase type to return Progress of the Kanister function + +``` ++func (p *Phase) Progress() (crv1alpha1.PhaseProgress, error) { ++ return p.f.Stats() ++} +``` + +Once we have information about Progress of current function, the Phase ProgressPercent and Action ProgressPercent in the ActionSet status can be updated by taking out the average. From 62f101cda65489cc07244c74920861e934901489 Mon Sep 17 00:00:00 2001 From: Prasad Ghangal Date: Thu, 9 Feb 2023 18:39:28 +0530 Subject: [PATCH 02/10] Update design/phase-level-progress-tracking.md Co-authored-by: Ankit Jain --- design/phase-level-progress-tracking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/phase-level-progress-tracking.md b/design/phase-level-progress-tracking.md index 0315348cc3..f6f1957769 100644 --- a/design/phase-level-progress-tracking.md +++ b/design/phase-level-progress-tracking.md @@ -2,7 +2,7 @@ Kanister actions are triggered by creating ActionSet resource. The status field in ActionSet CR represents the status of Kanister operation. The action completion percentage is set by calculating the weights given on phases ([https://github.com/kanisterio/kanister/blob/master/design/progress-tracking.md](https://github.com/kanisterio/kanister/blob/master/design/progress-tracking.md) ). -This goal of this proposal is +The goal of this proposal is - Have progress tracking per phase - Improve action progress tracking mechanism by removing hard coded weights From 3e6586dfa30022c76021c8da17f7ded2ad69bff2 Mon Sep 17 00:00:00 2001 From: Prasad Ghangal Date: Mon, 20 Feb 2023 23:30:02 +0530 Subject: [PATCH 03/10] Add actionset status example Signed-off-by: Prasad Ghangal --- design/phase-level-progress-tracking.md | 42 ++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/design/phase-level-progress-tracking.md b/design/phase-level-progress-tracking.md index f6f1957769..b782d5c52b 100644 --- a/design/phase-level-progress-tracking.md +++ b/design/phase-level-progress-tracking.md @@ -90,6 +90,46 @@ type Func interface { The PhaseProgress.ProgressPercent would be the of progress the functions it consists of. +The following is the example ActionSet status of BP action consists of 3 phases. Out of which first phase is completed and second one is running + +``` +status: + actions: + - artifacts: + mysqlCloudDump: + keyValue: + s3path: '{{ .Phases.dumpToObjectStore.Output.s3path }}' + blueprint: mysql-blueprint + name: backup + object: + kind: statefulset + name: mysql + namespace: mysql + phases: + - name: updatePermissions + state: completed + progress: + progressPercent: "100.00" + lastTransitionTime: "2023-02-20T12:448:55Z" + - name: dumpToObjectStore + state: running + progress: + progressPercent: "30.00" + lastTransitionTime: "2023-02-20T12:49:55Z" + sizeUploaded: "50000" + estimatedTimeSeconds: "120" + estimatedUploadSize: "100000000" + - name: cleanup + state: pending + error: + message: "" + progress: + lastTransitionTime: "2023-02-20T12:49:55Z" + percentCompleted: "43.33" + runningPhase: dumpToObjectStore + state: running +``` + ### Kanister functions progress tracking On high level we can divide the Kanister functions into two groups @@ -140,4 +180,4 @@ Implement Progress() function on Phase type to return Progress of the Kanister f +} ``` -Once we have information about Progress of current function, the Phase ProgressPercent and Action ProgressPercent in the ActionSet status can be updated by taking out the average. +Once we have information about Progress of current function, the Phase ProgressPercent and Action ProgressPercent in the ActionSet status can be updated by computing the average. From 2ea378ad90e226c5fb5457c179dca97e6ba12b6c Mon Sep 17 00:00:00 2001 From: Prasad Ghangal Date: Wed, 22 Feb 2023 12:44:33 +0530 Subject: [PATCH 04/10] Add progress check proposal for non-datamover funcs Signed-off-by: Prasad Ghangal --- design/phase-level-progress-tracking.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/design/phase-level-progress-tracking.md b/design/phase-level-progress-tracking.md index b782d5c52b..d91e6ad822 100644 --- a/design/phase-level-progress-tracking.md +++ b/design/phase-level-progress-tracking.md @@ -150,7 +150,15 @@ The command output can be parsed to get the progress metadata. ### Non-datamover Kanister functions -TBD +Non-datamover Kanister functions like `KubeTask`, `KubeExec`, `KubeOps`, `ScaleWorkload`, etc allow users to perform operations like executing scripts on a Pod or managing K8s resources. The duration it takes to execute these functions depends on different factors like the type of operations, function arguments, and types of commands listed in BP in the case of KubeExec or KubeTask functions. We can roughly divide these function execution into 3 steps. + +- Prerequisites - which include preparing Pod specs, creating the pod and waiting for it to be ready. +- Execution - This is the step where the function performs operations. +- Cleanup - Operations like deleting pods or cleaning up resources + +Since there is no standard way to check the progress of these functions, we can divide the progress equally into 3 giving each step equal weightage i.e 33.33%. E.g once the prerequisite step is completed, the progress can be set to 33.33%. And 66.66% once the specified operations are completed. + +A few functions like `ScaleWorkload` may not have any prerequisite step. In that case, the progress can be set to 0 till the operation completes. ### Updating actionset status From f77bc463dc62e9e0b041046e84c526d0380df2e9 Mon Sep 17 00:00:00 2001 From: Prasad Ghangal Date: Wed, 22 Feb 2023 14:32:32 +0530 Subject: [PATCH 05/10] Update design/phase-level-progress-tracking.md --- design/phase-level-progress-tracking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/phase-level-progress-tracking.md b/design/phase-level-progress-tracking.md index d91e6ad822..384c260ada 100644 --- a/design/phase-level-progress-tracking.md +++ b/design/phase-level-progress-tracking.md @@ -152,7 +152,7 @@ The command output can be parsed to get the progress metadata. Non-datamover Kanister functions like `KubeTask`, `KubeExec`, `KubeOps`, `ScaleWorkload`, etc allow users to perform operations like executing scripts on a Pod or managing K8s resources. The duration it takes to execute these functions depends on different factors like the type of operations, function arguments, and types of commands listed in BP in the case of KubeExec or KubeTask functions. We can roughly divide these function execution into 3 steps. -- Prerequisites - which include preparing Pod specs, creating the pod and waiting for it to be ready. +- Prerequisites - which include steps to perform before running actual operations like setting up env, preparing Pod specs, creating the K8s resources and waiting for them to be ready. - Execution - This is the step where the function performs operations. - Cleanup - Operations like deleting pods or cleaning up resources From 00fc58dce7ddc90ac69c0b387bf90652f1e502ba Mon Sep 17 00:00:00 2001 From: Prasad Ghangal Date: Tue, 12 Sep 2023 20:41:51 +0530 Subject: [PATCH 06/10] Update design/phase-level-progress-tracking.md Co-authored-by: Pavan Navarathna <6504783+pavannd1@users.noreply.github.com> --- design/phase-level-progress-tracking.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/design/phase-level-progress-tracking.md b/design/phase-level-progress-tracking.md index 384c260ada..802d69030b 100644 --- a/design/phase-level-progress-tracking.md +++ b/design/phase-level-progress-tracking.md @@ -1,6 +1,9 @@ # Problem Statement -Kanister actions are triggered by creating ActionSet resource. The status field in ActionSet CR represents the status of Kanister operation. The action completion percentage is set by calculating the weights given on phases ([https://github.com/kanisterio/kanister/blob/master/design/progress-tracking.md](https://github.com/kanisterio/kanister/blob/master/design/progress-tracking.md) ). +Kanister actions are triggered by creating an ActionSet resource. The status +field in the ActionSet CR represents the status of a Kanister operation. The +action completion percentage is set by calculating the weights given on phases +([https://github.com/kanisterio/kanister/blob/master/design/progress-tracking.md](https://github.com/kanisterio/kanister/blob/master/design/progress-tracking.md)). The goal of this proposal is From 2a882b185cc567e94d756be08c361d84ca4e8d34 Mon Sep 17 00:00:00 2001 From: Prasad Ghangal Date: Tue, 12 Sep 2023 20:42:09 +0530 Subject: [PATCH 07/10] Update design/phase-level-progress-tracking.md Co-authored-by: Pavan Navarathna <6504783+pavannd1@users.noreply.github.com> --- design/phase-level-progress-tracking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/phase-level-progress-tracking.md b/design/phase-level-progress-tracking.md index 802d69030b..801aa6b5fc 100644 --- a/design/phase-level-progress-tracking.md +++ b/design/phase-level-progress-tracking.md @@ -5,7 +5,7 @@ field in the ActionSet CR represents the status of a Kanister operation. The action completion percentage is set by calculating the weights given on phases ([https://github.com/kanisterio/kanister/blob/master/design/progress-tracking.md](https://github.com/kanisterio/kanister/blob/master/design/progress-tracking.md)). -The goal of this proposal is +The goal of this proposal is to - Have progress tracking per phase - Improve action progress tracking mechanism by removing hard coded weights From 160efcef0cdd9cb325ff67801a4888bce0149632 Mon Sep 17 00:00:00 2001 From: Prasad Ghangal Date: Tue, 12 Sep 2023 20:42:19 +0530 Subject: [PATCH 08/10] Update design/phase-level-progress-tracking.md Co-authored-by: Pavan Navarathna <6504783+pavannd1@users.noreply.github.com> --- design/phase-level-progress-tracking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/phase-level-progress-tracking.md b/design/phase-level-progress-tracking.md index 801aa6b5fc..9d1e26705f 100644 --- a/design/phase-level-progress-tracking.md +++ b/design/phase-level-progress-tracking.md @@ -8,7 +8,7 @@ action completion percentage is set by calculating the weights given on phases The goal of this proposal is to - Have progress tracking per phase -- Improve action progress tracking mechanism by removing hard coded weights +- Improve action progress tracking mechanism by removing hard-coded weights ## High Level Design From 4b0640a77abb864f3eafc30eb812c68c0efbc738 Mon Sep 17 00:00:00 2001 From: Prasad Ghangal Date: Tue, 12 Sep 2023 20:42:29 +0530 Subject: [PATCH 09/10] Update design/phase-level-progress-tracking.md Co-authored-by: Pavan Navarathna <6504783+pavannd1@users.noreply.github.com> --- design/phase-level-progress-tracking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/phase-level-progress-tracking.md b/design/phase-level-progress-tracking.md index 9d1e26705f..60bec61482 100644 --- a/design/phase-level-progress-tracking.md +++ b/design/phase-level-progress-tracking.md @@ -12,7 +12,7 @@ The goal of this proposal is to ## High Level Design -### Changes in ActionSet status field +### Changes in ActionSet `status` Field In order to inform progress of each phase execution while performing action, the progress field can be added to status.phase From 9b81eeb0f2850b47a70742fb372b25b9312d2c3f Mon Sep 17 00:00:00 2001 From: Prasad Ghangal Date: Tue, 12 Sep 2023 20:46:08 +0530 Subject: [PATCH 10/10] Apply suggestions from code review Co-authored-by: Pavan Navarathna <6504783+pavannd1@users.noreply.github.com> --- design/phase-level-progress-tracking.md | 88 +++++++++++++++++-------- 1 file changed, 60 insertions(+), 28 deletions(-) diff --git a/design/phase-level-progress-tracking.md b/design/phase-level-progress-tracking.md index 60bec61482..ba21e22e5e 100644 --- a/design/phase-level-progress-tracking.md +++ b/design/phase-level-progress-tracking.md @@ -14,7 +14,8 @@ The goal of this proposal is to ### Changes in ActionSet `status` Field -In order to inform progress of each phase execution while performing action, the progress field can be added to status.phase +In order to show the progress of each phase execution while performing an +action, a progress field can be added to the `status.phase`. ``` // ActionStatus is updated as we execute phases. @@ -51,19 +52,27 @@ type ActionStatus struct { +} ``` -### Action progress tracking +### Action Progress Tracking -Action can consists of multiple phases. These all phases can take consume different duration for completion based on the operations it performs. All phases for the action can be given equal weight-age and to calculate action progress completion %, average of all phase completion %ge can be calculated +An Action consists of multiple phases. The phases consume different duration +for completion based on the operations they perform. All the phases in an +action can be given equal weightage, and to calculate the action completion +progress %, the average of all phase completion % can be calculated. ``` -% completion of action = sum(% Phase completion of all the phases)/ no. of phases +% completion of action = sum(% completion of all the phases)/ no. of phases ``` -### Phase progress tracking +### Phase Progress Tracking -Each function is designed to perform different operations in different ways. Since the operation execution is specific to function, each function can be different ways to calculate and provide info about the progress. E.g datamover functions can leverage kopia stats to track progress. So it makes sense to delegate progress tracking to function implementation. +Each Kanister function performs its operations in different ways. Since the +operation execution is specific to a function, each function has a specific way +to calculate and provide info about the progress. For example, datamover +functions can leverage Kopia stats to track progress. So, the progress tracking +can be delegated to the function implementation. -Each Kanister function implements Progress interface. The Kanister function implementation calculates and returns the progress stats. +Each Kanister function calculates and returns the progress stats by +implementing the `Progress` interface. **Progress interface** @@ -91,9 +100,10 @@ type Func interface { } ``` -The PhaseProgress.ProgressPercent would be the of progress the functions it consists of. +The `PhaseProgress.ProgressPercent` is the progress of its Kanister function. -The following is the example ActionSet status of BP action consists of 3 phases. Out of which first phase is completed and second one is running +The following ActionSet example shows the status of an action with 3 phases. +The first phase is complete and the second one is running. ``` status: @@ -133,15 +143,17 @@ status: state: running ``` -### Kanister functions progress tracking +### Kanister Functions Progress Tracking -On high level we can divide the Kanister functions into two groups +At a high level, we can divide the Kanister functions into two groups. -### Datamover Kanister functions +#### Datamover Kanister Functions -Kanister data mover functions (like BackupData, CopyVolumeData, RestoreData, etc) use Kopia to snapshot filesystem and move data to/from objects. +Kanister data mover functions (like `BackupData`, `CopyVolumeData`, +`RestoreData`, etc.) use Kopia to snapshot the filesystem and move data to/from +external storage. -While snapshotting, kopia provides the info about progress on the terminal +While snapshotting, Kopia provides the info about progress on stdout. ``` $ kopia --log-level=error --config-file=/tmp/kopia-repository.config --log-dir=/tmp/kopia-log snapshot --progress-update-interval=5s /data @@ -151,25 +163,42 @@ $ kopia --log-level=error --config-file=/tmp/kopia-repository.config --log-dir=/ The command output can be parsed to get the progress metadata. -### Non-datamover Kanister functions +#### Non-datamover Kanister Functions -Non-datamover Kanister functions like `KubeTask`, `KubeExec`, `KubeOps`, `ScaleWorkload`, etc allow users to perform operations like executing scripts on a Pod or managing K8s resources. The duration it takes to execute these functions depends on different factors like the type of operations, function arguments, and types of commands listed in BP in the case of KubeExec or KubeTask functions. We can roughly divide these function execution into 3 steps. +Non-datamover Kanister functions like `KubeTask`, `KubeExec`, `KubeOps`, +`ScaleWorkload`, etc., allow users to perform operations like executing scripts +on a Pod or managing K8s resources. The duration it takes to execute these +functions depends on different factors like the type of operations, the +commands defined, and the function arguments. We can roughly divide the +function execution into 3 steps. -- Prerequisites - which include steps to perform before running actual operations like setting up env, preparing Pod specs, creating the K8s resources and waiting for them to be ready. -- Execution - This is the step where the function performs operations. -- Cleanup - Operations like deleting pods or cleaning up resources +- Prerequisites - includes steps to perform before running actual operations, + e.g., setting up env, preparing Pod specs, creating the K8s resources, and + waiting for them to be ready. +- Execution - this is the step where the function performs its operations. +- Cleanup - operations like deleting pods or cleaning up resources. -Since there is no standard way to check the progress of these functions, we can divide the progress equally into 3 giving each step equal weightage i.e 33.33%. E.g once the prerequisite step is completed, the progress can be set to 33.33%. And 66.66% once the specified operations are completed. +Since there is no standard way to check the progress of these functions, we can +divide the progress equally into 3 giving each step equal weightage, i.e., +33.33%. For example, once the prerequisite step is completed, the progress can +be set to 33.33%, and 66.66% once the specified operations are completed. -A few functions like `ScaleWorkload` may not have any prerequisite step. In that case, the progress can be set to 0 till the operation completes. +A few functions like `ScaleWorkload` may not have any prerequisite step. In +that case, the progress can be set to 0 till the operation completes. -### Updating actionset status +### Updating ActionSet `status` -[TrackActionProgress](https://github.com/kanisterio/kanister/blob/master/pkg/progress/action.go#L47) function is responsible for periodically updating progress for the action in ActionSet resource status. +The following changes are required to update the ActionSet `status`. -Refactor TrackActionsProgress to remove weight based progress calculation to use Progress interface implemented by functions. - -Pass additional parameter Phase to the TrackActionsProgress which can be used to get progress of the function in execution. +- [TrackActionProgress](https://github.com/kanisterio/kanister/blob/master/pkg/progress/action.go#L47) + function is currently responsible for periodically updating the progress of an + action in the ActionSet resource `status`. + + Refactor `TrackActionsProgress` to remove the weight-based progress calculation + and use the `Progress` interface implemented by functions. + + Pass an additional `Phase` parameter to the `TrackActionsProgress`` that can be + used to get the progress of the Kanister function being executed. ``` + go func() { @@ -183,7 +212,8 @@ Pass additional parameter Phase to the TrackActionsProgress which can be used to ``` -Implement Progress() function on Phase type to return Progress of the Kanister function +- Implement `Progress()` function on the `Phase`` type to return the progress + of the Kanister function. ``` +func (p *Phase) Progress() (crv1alpha1.PhaseProgress, error) { @@ -191,4 +221,6 @@ Implement Progress() function on Phase type to return Progress of the Kanister f +} ``` -Once we have information about Progress of current function, the Phase ProgressPercent and Action ProgressPercent in the ActionSet status can be updated by computing the average. +- Use the progress of the current function to compute the average values and + update the `Phase.ProgressPercent` and `Action.ProgressPercent` in the + ActionSet `status`.