From a2cac9c79ac77e093747a42e2767f558b16bf90a Mon Sep 17 00:00:00 2001 From: Kerem Goksel Date: Mon, 10 Dec 2018 23:07:39 -0800 Subject: [PATCH] Raise ClientError if UsageError occurs in bundle content upload (#1007) * Raise ClientError if UsageError occurs in bundle content upload * Propagate upload errors to worker run state --- codalab/rest/bundles.py | 10 ++++++++++ worker/codalabworker/local_run/local_run_state.py | 12 +++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/codalab/rest/bundles.py b/codalab/rest/bundles.py index 1d642d444..bc502facf 100644 --- a/codalab/rest/bundles.py +++ b/codalab/rest/bundles.py @@ -656,6 +656,16 @@ def _update_bundle_contents_blob(uuid): local.upload_manager.update_metadata_and_save(bundle, enforce_disk_quota=True) + except UsageError as err: + # This is a user error (most likely disk quota overuser) so raise a client HTTP error + if local.upload_manager.has_contents(bundle): + local.upload_manager.cleanup_existing_contents(bundle) + msg = "Upload failed: %s" % err + local.model.update_bundle( + bundle, {'state': State.FAILED, 'metadata': {'failure_message': msg}} + ) + abort(httplib.BAD_REQUEST, msg) + except Exception as e: # Upload failed: cleanup, update state if desired, and return HTTP error if local.upload_manager.has_contents(bundle): diff --git a/worker/codalabworker/local_run/local_run_state.py b/worker/codalabworker/local_run/local_run_state.py index 421c61399..f87fb385c 100644 --- a/worker/codalabworker/local_run/local_run_state.py +++ b/worker/codalabworker/local_run/local_run_state.py @@ -120,7 +120,7 @@ def __init__( self.docker_network_internal_name = docker_network_internal_name self.docker_runtime = docker_runtime # bundle_uuid -> {'thread': Thread, 'run_status': str} - self.uploading = ThreadDict(fields={'run_status': 'Upload started'}) + self.uploading = ThreadDict(fields={'run_status': 'Upload started', 'success': False}) # bundle_uuid -> {'thread': Thread, 'disk_utilization': int, 'running': bool} self.disk_utilization = ThreadDict( fields={'disk_utilization': 0, 'running': True, 'lock': None} @@ -440,6 +440,7 @@ def progress_callback(bytes_uploaded): return True self.upload_bundle_callback(bundle_uuid, run_state.bundle_path, progress_callback) + self.uploading[bundle_uuid]['success'] = True except Exception as e: self.uploading[bundle_uuid]['run_status'] = "Error while uploading: %s" % e traceback.print_exc() @@ -449,6 +450,15 @@ def progress_callback(bytes_uploaded): if self.uploading[bundle_uuid].is_alive(): return run_state._replace(run_status=self.uploading[bundle_uuid]['run_status']) + elif not self.uploading[bundle_uuid]['success']: + # upload failed + failure_message = run_state.info.get('failure_message', None) + if failure_message: + run_state.info['failure_message'] = ( + failure_message + '. ' + self.uploading[bundle_uuid]['run_status'] + ) + else: + run_state.info['failure_message'] = self.uploading[bundle_uuid]['run_status'] self.uploading.remove(bundle_uuid) return self.finalize_run(run_state)