add 404 error handling to get_job_status

nmdc_automation/workflow_automation/watch_nmdc.py

@@ -319,7 +319,10 @@ def cycle(self):
             logger.info(f"Found {len(unclaimed_jobs)} unclaimed jobs.")
             self.claim_jobs(unclaimed_jobs)
 
+
+        logger.info(f"Checking for finished jobs.")
         successful_jobs, failed_jobs = self.job_manager.get_finished_jobs()
+        logger.debug(f"Found {len(successful_jobs)} successful jobs and {len(failed_jobs)} failed jobs.")
         for job in successful_jobs:
             job_database = self.job_manager.process_successful_job(job)
             # sanity checks

nmdc_automation/workflow_automation/wfutils.py

@@ -189,6 +189,18 @@ def get_job_status(self) -> str:
         if not self.job_id:
             return "Unknown"
         status_url = f"{self.service_url}/{self.job_id}/status"
+        # There can be a delay between submitting a job and it
+        # being available in Cromwell so handle 404 errors
+        try:
+            response = requests.get(status_url)
+            response.raise_for_status()
+            return response.json().get("status", "Unknown")
+        except requests.exceptions.HTTPError as e:
+            if e.response.status_code == 404:
+                return "Unknown"
+            raise e
+
+
         response = requests.get(status_url)
         response.raise_for_status()
         return response.json().get("status", "Unknown")