Debug failing straggler test (#580)

cubed-dev · Sep 27, 2024 · d24f83b · d24f83b
1 parent 5b604ef
commit d24f83b
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 2 deletions.
diff --git a/cubed/runtime/backup.py b/cubed/runtime/backup.py
@@ -29,4 +29,10 @@ def should_launch_backup(
         [end_times[task] - start_times[task] for task in end_times]
     )
     duration = now - start_times[task]
-    return duration > completed_durations[n] * slow_factor
+    result = duration > completed_durations[n] * slow_factor
+    if result:
+        print(
+            "should_launch_backup is True, with duration=%s, completed_durations=%s, n=%s, completed_durations[n]=%s, slow_factor=%s"
+            % (duration, completed_durations, n, completed_durations[n], slow_factor)
+        )
+    return result
diff --git a/cubed/runtime/executors/asyncio.py b/cubed/runtime/executors/asyncio.py
@@ -81,8 +81,8 @@ async def async_map_unordered(
                     task, now, start_times, end_times
                 ):
                     # launch backup task
-                    print("Launching backup task")
                     i = tasks[task]
+                    print(f"Launching backup task for input {i} at time {now}")
                     i, new_task = create_backup_futures_func([i], **kwargs)[0]
                     tasks[new_task] = i
                     start_times[new_task] = time.monotonic()