Raise ClientError if UsageError occurs in bundle content upload (#1007)

* Raise ClientError if UsageError occurs in bundle content upload * Propagate upload errors to worker run state
codalab · Dec 11, 2018 · a2cac9c · a2cac9c
1 parent e768632
commit a2cac9c
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 1 deletion.
diff --git a/codalab/rest/bundles.py b/codalab/rest/bundles.py
@@ -656,6 +656,16 @@ def _update_bundle_contents_blob(uuid):
 
         local.upload_manager.update_metadata_and_save(bundle, enforce_disk_quota=True)
 
+    except UsageError as err:
+        # This is a user error (most likely disk quota overuser) so raise a client HTTP error
+        if local.upload_manager.has_contents(bundle):
+            local.upload_manager.cleanup_existing_contents(bundle)
+        msg = "Upload failed: %s" % err
+        local.model.update_bundle(
+            bundle, {'state': State.FAILED, 'metadata': {'failure_message': msg}}
+        )
+        abort(httplib.BAD_REQUEST, msg)
+
     except Exception as e:
         # Upload failed: cleanup, update state if desired, and return HTTP error
         if local.upload_manager.has_contents(bundle):

diff --git a/worker/codalabworker/local_run/local_run_state.py b/worker/codalabworker/local_run/local_run_state.py
@@ -120,7 +120,7 @@ def __init__(
         self.docker_network_internal_name = docker_network_internal_name
         self.docker_runtime = docker_runtime
         # bundle_uuid -> {'thread': Thread, 'run_status': str}
-        self.uploading = ThreadDict(fields={'run_status': 'Upload started'})
+        self.uploading = ThreadDict(fields={'run_status': 'Upload started', 'success': False})
         # bundle_uuid -> {'thread': Thread, 'disk_utilization': int, 'running': bool}
         self.disk_utilization = ThreadDict(
             fields={'disk_utilization': 0, 'running': True, 'lock': None}
@@ -440,6 +440,7 @@ def progress_callback(bytes_uploaded):
                     return True
 
                 self.upload_bundle_callback(bundle_uuid, run_state.bundle_path, progress_callback)
+                self.uploading[bundle_uuid]['success'] = True
             except Exception as e:
                 self.uploading[bundle_uuid]['run_status'] = "Error while uploading: %s" % e
                 traceback.print_exc()
@@ -449,6 +450,15 @@ def progress_callback(bytes_uploaded):
 
         if self.uploading[bundle_uuid].is_alive():
             return run_state._replace(run_status=self.uploading[bundle_uuid]['run_status'])
+        elif not self.uploading[bundle_uuid]['success']:
+            # upload failed
+            failure_message = run_state.info.get('failure_message', None)
+            if failure_message:
+                run_state.info['failure_message'] = (
+                    failure_message + '. ' + self.uploading[bundle_uuid]['run_status']
+                )
+            else:
+                run_state.info['failure_message'] = self.uploading[bundle_uuid]['run_status']
 
         self.uploading.remove(bundle_uuid)
         return self.finalize_run(run_state)