Merge pull request #257 from studavekar/test-retry-count

Accept --retry-count argument that retries on test failure

mbed_greentea/mbed_greentea_cli.py

@@ -331,6 +331,12 @@ def main():
                     action="store_true",
                     help='Prints console outputs for failed tests')
 
+    parser.add_option('', '--retry-count',
+                      dest='retry_count',
+                      default=1,
+                      type=int,
+                      help='retry count for individual test failure. By default, there is no retry')
+
     parser.add_option('', '--report-memory-metrics-csv',
                     dest='report_memory_metrics_csv_file_name',
                     help='You can log test suite memory metrics in the form of a CSV file')
@@ -469,6 +475,7 @@ def run_test_thread(test_result_queue, test_queue, opts, mut, build, build_path,
                                          global_resource_mgr=opts.global_resource_mgr,
                                          num_sync_packtes=opts.num_sync_packtes,
                                          tags=opts.tags,
+                                         retry_count=opts.retry_count,
                                          verbose=verbose)
 
         # Some error in htrun, abort test execution

mbed_greentea/mbed_test_api.py

@@ -103,6 +103,42 @@ def get_test_result(output):
     return TEST_RESULT_TIMEOUT
 
 
+def run_command(cmd):
+    """! Runs command and prints proc stdout on screen
+    @paran cmd List with command line to execute e.g. ['ls', '-l]
+    @return Value returned by subprocess.Popen, if failed return None
+    """
+    try:
+        p = Popen(cmd,
+                  stdout=PIPE,
+                  stderr=STDOUT)
+    except OSError as e:
+        gt_logger.gt_log_err("run_host_test.run_command(%s) failed!" % str(cmd))
+        gt_logger.gt_log_tab(str(e))
+        return None
+    return p
+
+def run_htrun(cmd, verbose):
+    # detect overflow when running tests
+    htrun_output = str()
+    # run_command will return None if process can't be opened (Issue #134)
+    p = run_command(cmd)
+    if not p:
+        # int value > 0 notifies caller that starting of host test process failed
+        return RUN_HOST_TEST_POPEN_ERROR
+
+    for line in iter(p.stdout.readline, b''):
+        htrun_output += line
+        # When dumping output to file both \r and \n will be a new line
+        # To avoid this "extra new-line" we only use \n at the end
+        if verbose:
+            sys.stdout.write(line.rstrip() + '\n')
+            sys.stdout.flush()
+
+    # Check if process was terminated by signal
+    returncode = p.wait()
+    return returncode, htrun_output
+
 def run_host_test(image_path,
                   disk,
                   port,
@@ -121,6 +157,7 @@ def run_host_test(image_path,
                   enum_host_tests_path=None,
                   global_resource_mgr=None,
                   num_sync_packtes=None,
+                  retry_count=1,
                   tags=None,
                   run_app=None):
     """! This function runs host test supervisor (executes mbedhtrun) and checks output from host test process.
@@ -149,21 +186,6 @@ def run_host_test(image_path,
             Retrun int < 0 if something went wrong during mbedhtrun execution.
     """
 
-    def run_command(cmd):
-        """! Runs command and prints proc stdout on screen
-        @paran cmd List with command line to execute e.g. ['ls', '-l]
-        @return Value returned by subprocess.Popen, if failed return None
-        """
-        try:
-            p = Popen(cmd,
-                      stdout=PIPE,
-                      stderr=STDOUT)
-        except OSError as e:
-            gt_logger.gt_log_err("run_host_test.run_command(%s) failed!"% str(cmd))
-            gt_logger.gt_log_tab(str(e))
-            return None
-        return p
-
     def get_binary_host_tests_dir(binary_path, level=2):
         """! Checks if in binary test group has host_tests directory
         @param binary_path Path to binary in test specification
@@ -277,34 +299,22 @@ def get_binary_host_tests_dir(binary_path, level=2):
     if tags:
         cmd += ["--tag-filters", tags]
 
-    gt_logger.gt_log_tab("calling mbedhtrun: %s"% " ".join(cmd), print_text=verbose)
+    gt_logger.gt_log_tab("calling mbedhtrun: %s" % " ".join(cmd), print_text=verbose)
     gt_logger.gt_log("mbed-host-test-runner: started")
 
-    htrun_output = str()
-    start_time = time()
-
-    # run_command will return None if process can't be opened (Issue #134)
-    p = run_command(cmd)
-    if not p:
-        # int value > 0 notifies caller that starting of host test process failed
-        return RUN_HOST_TEST_POPEN_ERROR
-
-    for line in iter(p.stdout.readline, b''):
-        htrun_output += line
-        # When dumping output to file both \r and \n will be a new line
-        # To avoid this "extra new-line" we only use \n at the end
-        if verbose:
-            sys.stdout.write(line.rstrip() + '\n')
-            sys.stdout.flush()
-
-    # Check if process was terminated by signal
-    returncode = p.wait()
-    if returncode < 0:
-        return returncode
+    for retry in range(1, 1 + retry_count):
+        start_time = time()
+        returncode, htrun_output = run_htrun(cmd, verbose)
+        end_time = time()
+        if returncode < 0:
+            return returncode
+        elif returncode == 0:
+            break
+        gt_logger.gt_log("retry mbedhtrun {}/{}".format(retry, retry_count))
+    else:
+        gt_logger.gt_log("{} failed after {} count".format(cmd, retry_count))
 
-    end_time = time()
     testcase_duration = end_time - start_time   # Test case duration from reset to {end}
-
     htrun_output = get_printable_string(htrun_output)
     result = get_test_result(htrun_output)
     result_test_cases = get_testcase_result(htrun_output)