exclude paths which do not exist

openedx-unsupported · May 31, 2018 · fe100b0 · fe100b0
1 parent 01f6b2b
commit fe100b0
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/edx/analytics/tasks/common/spark.py b/edx/analytics/tasks/common/spark.py
@@ -184,7 +184,7 @@ def get_event_log_dataframe(self, spark, *args, **kwargs):
             pattern=self.pattern,
             date_pattern=self.date_pattern,
         ).output()
-        self.path_targets = [task.path for task in path_targets]
+        self.path_targets = [task.path for task in path_targets if task.exists()]
         dataframe = spark.read.format('json').load(self.path_targets, schema=self.get_log_schema())
         dataframe = dataframe.filter(dataframe['time'].isNotNull()) \
             .withColumn('event_date', date_format(to_date(dataframe['time']), 'yyyy-MM-dd'))