fix recursive finding of paths

pipestat/backends/filebackend.py

@@ -216,6 +216,18 @@ def link(self, output_dir: Optional[str] = None) -> str:
         This function creates a link structure such that results are organized by type.
         """
 
+        def get_all_paths(parent_key, result_identifier_value):
+            """If the result identifier is a complex object which contains nested paths"""
+
+            key_value_pairs = []
+
+            for k, v in result_identifier_value.items():
+                if isinstance(v, dict):
+                    key_value_pairs.extend(get_all_paths(k, v))
+                elif k == "path":
+                    key_value_pairs.append((parent_key, v))
+            return key_value_pairs
+
         linkdir = output_dir or os.path.abspath(os.path.dirname(self.results_file_path))
         unique_result_identifiers = []
 
@@ -226,8 +238,9 @@ def link(self, output_dir: Optional[str] = None) -> str:
             print(result_identifiers)
             for k, v in result_identifiers.items():
                 if type(v) == dict:
-                    if "path" in v.keys():
-                        file = os.path.basename(v["path"])
+                    all_paths = get_all_paths(k, v)
+                    for path in all_paths:
+                        file = os.path.basename(path[1])
                         file_name, file_extension = os.path.splitext(file)
                         if k not in unique_result_identifiers:
                             sub_dir_for_type = os.path.join(linkdir, k)
@@ -236,13 +249,12 @@ def link(self, output_dir: Optional[str] = None) -> str:
                                 os.mkdir(sub_dir_for_type)
                             except:
                                 pass
-
                         for subdir in unique_result_identifiers:
                             if k == subdir[0]:
                                 target_dir = subdir[1]
-                        linkname = os.path.join(target_dir, record + file)
+                        linkname = os.path.join(target_dir, record + "_" + path[0] + "_" + file)
                         # src = os.path.join(root, file)
-                        src = os.path.abspath(v["path"])
+                        src = os.path.abspath(path[1])
                         src_rel = os.path.relpath(src, os.path.dirname(linkname))
                         force_symlink(src_rel, linkname)
 

tests/test_pipestat.py

@@ -1065,14 +1065,10 @@ def test_linking(
         backend,
     ):
         # paths to images and files
-        path_file_1 = get_data_file_path(
-            "test_file_links/results/project_dir_example_1/ex1.txt"
-        )  # os.path.abspath("tests/data/test_file_links/results/project_dir_example_1/ex1.txt")
+        path_file_1 = get_data_file_path("test_file_links/results/project_dir_example_1/ex1.txt")
         path_file_2 = get_data_file_path("test_file_links/results/project_dir_example_1/ex2.txt")
         path_image_1 = get_data_file_path("test_file_links/results/project_dir_example_1/ex3.png")
         path_image_2 = get_data_file_path("test_file_links/results/project_dir_example_1/ex4.png")
-        # path_file_3 = get_data_file_path("test_file_links/results/project_dir_example_2/sub_project_dir_example_2/ex7.txt")
-        # Make absolute for the test to run successfully.
 
         values_sample = [
             {"sample1": {"number_of_things": 100}},
@@ -1140,8 +1136,11 @@ def test_linking(
             except Exception:
                 assert False
 
-            linkdir = os.path.join(linkdir, "output_file")
-            for root, dirs, files in os.walk(linkdir):
-                assert "sample1ex1.txt" in files
+            # Test simple
+            for root, dirs, files in os.walk(os.path.join(linkdir, "output_file")):
+                assert "sample1_output_file_ex1.txt" in files
+            # Test complex types
+            for root, dirs, files in os.walk(os.path.join(linkdir, "output_file_in_object")):
+                assert "sample2_example_property_1_ex1.txt" in files
 
             print("done")