Print summary of failed cases and exit it with an error if a failed c…

…ases exists

Continue docker run so report of all failed tests gets produced.

.github/workflows/ci_test_docker.yml

@@ -28,4 +28,4 @@ jobs:
         run: |
           mkdir $HOME/output
           chmod a+rw $HOME/output
-          docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d --stop_on_error
+          docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d

scm/src/run_scm.py

@@ -3,6 +3,7 @@
 import argparse
 import f90nml
 import logging
+import numpy as np
 import os
 import re
 import shutil
@@ -765,6 +766,30 @@ def copy_outdir(exp_dir):
         shutil.rmtree(home_output_dir)
     shutil.copytree(exp_dir, home_output_dir)
 
+def print_error_report(error_logs, total_count):
+    case_l = len(max(error_logs[:,0], key=len))
+    suite_l = len(max(error_logs[:,1], key=len))
+    namelist_l = len(max(error_logs[:,2], key=len))
+    status_l = len(max(error_logs[:,3], key=len))
+    # error_log contains header, subtracting 1 from error
+    error_count = error_logs.shape[0] - 1
+    passing_count = total_count - error_count
+    header_printed = False
+    column_width = (case_l + suite_l + namelist_l + status_l + 13)
+
+    # print formatted asummary
+    print("Failure Summary:")
+    print("-" * column_width)
+    for error_log in error_logs:
+        case_s, suite, namelist, status = error_log
+        print(f"| {case_s:<{case_l}} | {suite:<{suite_l}} | {namelist:<{namelist_l}} | {status:<{status_l}} |")
+        if not header_printed:
+            print("-" * column_width)
+            header_printed = True
+    print("-" * column_width)
+    print(f"[{error_count}/{total_count}] failed cases, [{passing_count}/{total_count}] passing cases")
+
+
 def main():
     (file, case, sdf, namelist, tracers, use_gdb, runtime, runtime_mult, docker, \
      verbose, levels, npz_type, vert_coord_file, case_data_dir, n_itt_out,       \
@@ -822,6 +847,8 @@ def main():
         if (tracers  != None): run_list[0]["tracers"]  = tracers
 
     # Loop through all input "run dictionaires"
+    error_logs = [["Failed Case", "Suite", "Namelist", "Status"]]
+    failed_case = False
     irun = 0
     for run in run_list:
 
@@ -894,13 +921,23 @@ def main():
             logging.info('Process "(case={0}, suite={1}, namelist={2}" completed successfully'. \
                          format(run["case"], run["suite"], active_suite.namelist))
         else:
-            logging.warning('Process "(case={0}, suite={1}, namelist={2}" exited with code {3}'. \
-                            format( run["case"], run["suite"], active_suite.namelist, status))
+            failed_case = True
+            error_str = 'Process "(case={0}, suite={1}, namelist={2}" exited with code {3}'. \
+                        format( run["case"], run["suite"], active_suite.namelist, status)
+            logging.warning(error_str)
+            error_logs = np.append(error_logs,
+                                   [[run["case"], run["suite"], active_suite.namelist, status]],
+                                   axis=0)
         #
         if time_elapsed:
             logging.info('    Elapsed time: {0}s'.format(time_elapsed))
         if docker:
             copy_outdir(exp_dir)
 
+    if (failed_case):
+        print_error_report(error_logs, len(run_list))
+        sys.exit(1)
+
+
 if __name__ == '__main__':
     main()