moved finished_metadata_hook to crawlmanager

scrapinghub · Sep 13, 2024 · 77ed19f · 77ed19f
1 parent 5059008
commit 77ed19f
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/shub_workflow/crawl.py b/shub_workflow/crawl.py
@@ -14,6 +14,7 @@
 from shub_workflow.script import (
     JobKey,
     JobDict,
+    JobMeta,
     SpiderName,
     Outcome,
     BaseLoopScriptAsyncMixin,
@@ -148,6 +149,7 @@ def check_running_jobs(self) -> Dict[JobKey, Outcome]:
         while running_job_keys:
             jobkey = running_job_keys.pop()
             if (outcome := self.is_finished(jobkey)) is not None:
+                self.finished_metadata_hook(jobkey, self.get_job_metadata(jobkey))
                 spider, job_args_override = self._running_job_keys.pop(jobkey)
                 if outcome in self.failed_outcomes:
                     _LOG.warning(f"Job {jobkey} finished with outcome {outcome}.")
@@ -161,6 +163,12 @@ def check_running_jobs(self) -> Dict[JobKey, Outcome]:
 
         return outcomes
 
+    def finished_metadata_hook(self, jobkey: JobKey, metadata: JobMeta):
+        """
+        allow to add some reaction on each finished job, based solely on its metadata.
+        Use self.get_metadata_key(metadata, <key>) in order to get metadata with handled retries.
+        """
+
     def bad_outcome_hook(self, spider: SpiderName, outcome: Outcome, job_args_override: JobParams, jobkey: JobKey):
         if self.get_close_reason() is None:
             self.set_close_reason(outcome)

diff --git a/shub_workflow/script.py b/shub_workflow/script.py
@@ -508,16 +508,9 @@ def is_finished(self, jobkey: JobKey) -> Optional[Outcome]:
         """
         metadata = self.get_job_metadata(jobkey)
         if self.get_metadata_key(metadata, "state") == "finished":
-            self.finished_metadata_hook(jobkey, metadata)
             return self.get_metadata_key(metadata, "close_reason")
         return None
 
-    def finished_metadata_hook(self, jobkey: JobKey, metadata: JobMeta):
-        """
-        allow to add some reaction on each finished job, based solely on its metadata.
-        Use self.get_metadata_key(metadata, <key>) in order to get metadata with handled retries.
-        """
-
     @dash_retry_decorator
     def finish(self, jobkey: Optional[JobKey] = None, close_reason: Optional[str] = None):
         close_reason = close_reason or "finished"