Merge pull request #31 from NASA-IMPACT/gcorradini/feature

Feature: Keep Flink Dashboards for Failed Runners and Dump Task Manager Logs

.github/workflows/job-runner.yaml

@@ -252,13 +252,19 @@ jobs:
           kubectl get pod | grep -v taskmanager | grep ${{ needs.run-job.outputs.job_name }} | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} > /tmp/jobmanager.log
           cat /tmp/jobmanager.log
           echo "##################### TASK MANAGER ######################"
-          # depending on the `inputs.parallism` we can more than one taskmanager so grab the first one
-          # TODO: loop through them all and dump the logs
-          kubectl get pod | grep taskmanager | grep ${{ needs.run-job.outputs.job_name }} | cut -d' ' -f1 | head -n1 | xargs -I{} kubectl logs pod/{} -c flink-main-container > /tmp/taskmanager.log
-          cat /tmp/taskmanager.log
+          # depending on the `inputs.parallism` we can have more than one taskmanager so only dump logs for the first three
+          iterations=$(expr "${{ github.event.inputs.parallelism }}" + 0)  # cast to integer
+          for (( i = 1; i < iterations; i++ )); do
+              kubectl get pod | grep ${{ needs.run-job.outputs.job_name }} | grep task-manager-$i | cut -d' ' -f1 | head -n1 | xargs -I{} kubectl logs pod/{} -c flink-main-container > /tmp/taskmanager.log
+              cat /tmp/taskmanager.log
+              # break the loop if iteration count exceeds 3
+              if (( i >= 3 )); then
+                  break
+              fi
+          done
 
-          # delete the flinkdeployment so we don't have old failures hanging around
-          kubectl get flinkdeployment --no-headers | grep ${{ needs.run-job.outputs.job_name }} | cut -d' ' -f1 | xargs -I{} kubectl delete flinkdeployment/{}
+          # NOTE: we actually want the failed flink deployments to sick around b/c we might want to inspect the flink dashboard
+          # kubectl get flinkdeployment --no-headers | grep ${{ needs.run-job.outputs.job_name }} | cut -d' ' -f1 | xargs -I{} kubectl delete flinkdeployment/{}
 
           #################################################################
           # provide feedback about OOM errors where we've seen them before