Enable retrying failed/cancelled workflow runs

DDMAL · Jul 5, 2023 · 3406029 · 3406029
1 parent 47aa7d2
commit 3406029
Show file tree

Hide file tree

Showing 5 changed files with 44 additions and 3 deletions.
diff --git a/rodan-client/code/src/js/Controllers/ControllerWorkflowRun.js b/rodan-client/code/src/js/Controllers/ControllerWorkflowRun.js
@@ -25,6 +25,7 @@ export default class ControllerWorkflowRun extends BaseController
         Radio.channel('rodan').on(RODAN_EVENTS.EVENT__WORKFLOWRUN_DELETED, options => this._handleEventWorkflowRunDeleteResponse(options));
         Radio.channel('rodan').on(RODAN_EVENTS.EVENT__WORKFLOWRUN_SAVED, options => this._handleEventWorkflowRunSaveResponse(options));
         Radio.channel('rodan').on(RODAN_EVENTS.EVENT__WORKFLOWRUN_STARTED, options => this._handleEventWorkflowRunStartResponse(options));
+        Radio.channel('rodan').on(RODAN_EVENTS.EVENT__WORKFLOWRUN_RETRIED, options => this._handleEventWorkflowRunRetryResponse(options));
 
         // Requests.
         Radio.channel('rodan').on(RODAN_EVENTS.EVENT__WORKFLOWRUN_SELECTED_COLLECTION, options => this._handleEventCollectionSelected(options), this);
@@ -33,6 +34,7 @@ export default class ControllerWorkflowRun extends BaseController
         Radio.channel('rodan').reply(RODAN_EVENTS.REQUEST__WORKFLOWRUN_DELETE, options => this._handleRequestWorkflowRunDelete(options), this);
         Radio.channel('rodan').reply(RODAN_EVENTS.REQUEST__WORKFLOWRUN_SAVE, options => this._handleRequestWorkflowRunSave(options), this);
         Radio.channel('rodan').reply(RODAN_EVENTS.REQUEST__WORKFLOWRUN_START, options => this._handleRequestWorkflowRunStart(options), this);
+        Radio.channel('rodan').reply(RODAN_EVENTS.REQUEST__WORKFLOWRUN_RETRY, options => this._handleRequestWorkflowRunRetry(options), this);        
     }
 
 ///////////////////////////////////////////////////////////////////////////////////////
@@ -166,6 +168,13 @@ export default class ControllerWorkflowRun extends BaseController
         });
     }
 
+    _handleEventWorkflowRunRetryResponse(options)
+    {
+        Radio.channel('rodan').request(RODAN_EVENTS.REQUEST__MODAL_HIDE);
+        const project = Radio.channel('rodan').request(RODAN_EVENTS.REQUEST__PROJECT_GET_ACTIVE);
+        Radio.channel('rodan').trigger(RODAN_EVENTS.EVENT__WORKFLOWRUN_SELECTED_COLLECTION, {project: project});
+    }
+
     /**
      * Handle request create WorkflowRun.
      */
@@ -214,6 +223,17 @@ export default class ControllerWorkflowRun extends BaseController
                                  {patch: true, success: (model) => Radio.channel('rodan').trigger(RODAN_EVENTS.EVENT__WORKFLOWRUN_STARTED, {workflowrun: model})});
     }
 
+    /**
+     * Handle request retry WorkflowRun.
+     */
+    _handleRequestWorkflowRunRetry(options)
+    {
+        Radio.channel('rodan').request(RODAN_EVENTS.REQUEST__MODAL_SHOW_IMPORTANT, {title: 'Restarting Workflow Run', content: 'Please wait...'});
+        options.workflowrun.set({status: 31});
+        options.workflowrun.save(options.workflowrun.changed,
+                                    {patch: true, success: (model) => Radio.channel('rodan').trigger(RODAN_EVENTS.EVENT__WORKFLOWRUN_RETRIED, {workflowrun: model})});
+    }
+
 ///////////////////////////////////////////////////////////////////////////////////////
 // PRIVATE METHODS
 ///////////////////////////////////////////////////////////////////////////////////////

diff --git a/rodan-client/code/src/js/Shared/RODAN_EVENTS.js b/rodan-client/code/src/js/Shared/RODAN_EVENTS.js
@@ -512,6 +512,8 @@ class RODAN_EVENTS
         this.EVENT__WORKFLOWRUN_SAVED = 'EVENT__WORKFLOWRUN_SAVED';
         /** Triggered when WorkflowRun started. Sends {workflowrun: WorkflowRun}. */
         this.EVENT__WORKFLOWRUN_STARTED = 'EVENT__WORKFLOWRUN_STARTED';
+        /** Triggered when WorkflowRun retried. Sends {workflowrun: WorkflowRun}. */
+        this.EVENT__WORKFLOWRUN_RETRIED = 'EVENT__WORKFLOWRUN_RETRIED';
         /** Triggered when the user selects an individual WorkflowRun. Sends {workflow: WorkflowRun}. */
         this.EVENT__WORKFLOWRUN_SELECTED = 'EVENT__WORKFLOWRUN_SELECTED';
         /** Triggered when the user selects to see all available WorkflowRuns. Sends {project: Project (Project associated with WorkflowRunCollection)}. */
@@ -524,6 +526,8 @@ class RODAN_EVENTS
         this.REQUEST__WORKFLOWRUN_SAVE = 'REQUEST__WORKFLOWRUN_SAVE';
         /** Request a WorkflowRun be started. Takes {model: WorkflowRun}. */
         this.REQUEST__WORKFLOWRUN_START = 'REQUEST__WORKFLOWRUN_START';
+        /** Request a failed or cancelled WorkflowRun be retried. Takes {model: WorkflowRun}. */
+        this.REQUEST__WORKFLOWRUN_RETRY = 'REQUEST__WORKFLOWRUN_RETRY';
 
         ///////////////////////////////////////////////////////////////////////////////////////
         // VERSION COMPATIBILITY CHECKS

diff --git a/...t/code/src/js/Views/Master/Main/WorkflowRun/Individual/LayoutViewIndividualWorkflowRun.js b/...t/code/src/js/Views/Master/Main/WorkflowRun/Individual/LayoutViewIndividualWorkflowRun.js
@@ -115,6 +115,14 @@ export default class LayoutViewIndividualWorkflowRun extends Marionette.View
     {
         Radio.channel('rodan').request(RODAN_EVENTS.REQUEST__WORKFLOWRUN_DELETE, {workflowrun: this.model});
     }
+
+    /**
+     * Handle retry button.
+     */
+    _handleButtonRetry()
+    {
+        Radio.channel('rodan').request(RODAN_EVENTS.REQUEST__WORKFLOWRUN_RETRY, {workflowrun: this.model});
+    }
 }
 LayoutViewIndividualWorkflowRun.prototype.modelEvents = {
     'all': 'render'
@@ -124,14 +132,15 @@ LayoutViewIndividualWorkflowRun.prototype.ui = {
     buttonShowRunJobs: '#button-runjobs_show',
     buttonSave: '#button-save_workflowrun',
     buttonDelete: '#button-delete_workflowrun',
+    buttonRetry: '#button-retry_workflowrun',
     textName: '#text-workflowrun_name',
     textDescription: '#text-workflowrun_description'
 };
 LayoutViewIndividualWorkflowRun.prototype.events = {
     'click @ui.buttonShowResources': '_showResources',
     'click @ui.buttonShowRunJobs': '_showRunJobs',
     'click @ui.buttonSave': '_handleButtonSave',
-    'click @ui.buttonDelete': '_handleButtonDelete'
-
+    'click @ui.buttonDelete': '_handleButtonDelete',
+    'click @ui.buttonRetry': '_handleButtonRetry',
 };
 LayoutViewIndividualWorkflowRun.prototype.template = _.template($('#template-main_workflowrun_individual').text());
diff --git a/...plates/Views/Master/Main/WorkflowRun/Individual/template-main_workflowrun_individual.html b/...plates/Views/Master/Main/WorkflowRun/Individual/template-main_workflowrun_individual.html
@@ -10,6 +10,9 @@ <h3>Workflow Run</h3>
         Updated on: <%= _.formatFromUTC(updated) %><br/>
         URL: <%= url %><br>
         Status: <%= statusText %><br/>
+        <% if (status === -1 || status === 9) { %>
+            <button class="btn btn-xs btn-warning" id="button-retry_workflowrun">Retry</button>
+        <% } %>
         <button class="btn btn-xs btn-warning" id="button-save_workflowrun">Save</button>
         <button class="btn btn-xs btn-danger" id="button-delete_workflowrun">Delete</button>
     </div>

diff --git a/rodan-main/code/rodan/jobs/base.py b/rodan-main/code/rodan/jobs/base.py
@@ -1011,7 +1011,12 @@ def on_failure(self, exc, task_id, args, kwargs, einfo):
         wfrun_id = RunJob.objects.filter(pk=runjob_id).values_list(
             "workflow_run__uuid", flat=True
         )[0]
-        WorkflowRun.objects.filter(uuid=wfrun_id).update(status=task_status.FAILED)
+
+        # Update status for workflow and pending RunJobs
+        workflow_run = WorkflowRun.objects.get(uuid=wfrun_id)
+        RunJob.objects.filter(workflow_run=workflow_run, status=task_status.SCHEDULED).update(status=task_status.CANCELLED)
+        workflow_run.status = task_status.FAILED
+        workflow_run.save(update_fields=["status"])
 
         # Send an email to owner of WorkflowRun
         workflowrun = WorkflowRun.objects.get(uuid=wfrun_id)