diff --git a/pipestat/backends/abstract.py b/pipestat/backends/abstract.py index 9554820b..f69869fb 100644 --- a/pipestat/backends/abstract.py +++ b/pipestat/backends/abstract.py @@ -91,11 +91,49 @@ def get_records( def get_status(self, record_identifier: str) -> Optional[str]: _LOGGER.warning("Not implemented yet for this backend") - def link(self, output_dir: Optional[str] = None) -> str: - """ - This function creates a link structure such that results are organized by type. - """ - _LOGGER.warning("Not implemented yet for this backend") + def link(self, output_dir) -> str: + def get_all_paths(parent_key, result_identifier_value): + """If the result identifier is a complex object which contains nested paths""" + + key_value_pairs = [] + + for k, v in result_identifier_value.items(): + if isinstance(v, dict): + key_value_pairs.extend(get_all_paths(k, v)) + elif k == "path": + key_value_pairs.append((parent_key, v)) + return key_value_pairs + + linkdir = output_dir + + unique_result_identifiers = [] + + all_records = self.get_records() + + for record in all_records["records"]: + result_identifiers = self.retrieve(record_identifier=record) + print(result_identifiers) + for k, v in result_identifiers.items(): + if type(v) == dict: + all_paths = get_all_paths(k, v) + for path in all_paths: + file = os.path.basename(path[1]) + if k not in unique_result_identifiers: + sub_dir_for_type = os.path.join(linkdir, k) + unique_result_identifiers.append((k, sub_dir_for_type)) + try: + os.mkdir(sub_dir_for_type) + except: + pass + for subdir in unique_result_identifiers: + if k == subdir[0]: + target_dir = subdir[1] + linkname = os.path.join(target_dir, record + "_" + path[0] + "_" + file) + src = os.path.abspath(path[1]) + src_rel = os.path.relpath(src, os.path.dirname(linkname)) + force_symlink(src_rel, linkname) + + return linkdir def clear_status( self, record_identifier: str = None, flag_names: List[str] = None diff --git a/pipestat/backends/filebackend.py b/pipestat/backends/filebackend.py index a4ffd18b..7ed49e64 100644 --- a/pipestat/backends/filebackend.py +++ b/pipestat/backends/filebackend.py @@ -211,57 +211,6 @@ def get_status_flag_path(self, status_identifier: str, record_identifier=None) - self.status_file_dir, f"{self.pipeline_name}_{r_id}_{status_identifier}.flag" ) - def link(self, output_dir: Optional[str] = None) -> str: - """ - This function creates a link structure such that results are organized by type. - """ - - def get_all_paths(parent_key, result_identifier_value): - """If the result identifier is a complex object which contains nested paths""" - - key_value_pairs = [] - - for k, v in result_identifier_value.items(): - if isinstance(v, dict): - key_value_pairs.extend(get_all_paths(k, v)) - elif k == "path": - key_value_pairs.append((parent_key, v)) - return key_value_pairs - - linkdir = output_dir or os.path.abspath(os.path.dirname(self.results_file_path)) - unique_result_identifiers = [] - - all_records = self.get_records() - - for record in all_records["records"]: - result_identifiers = self.retrieve(record_identifier=record) - print(result_identifiers) - for k, v in result_identifiers.items(): - if type(v) == dict: - all_paths = get_all_paths(k, v) - for path in all_paths: - file = os.path.basename(path[1]) - file_name, file_extension = os.path.splitext(file) - if k not in unique_result_identifiers: - sub_dir_for_type = os.path.join(linkdir, k) - unique_result_identifiers.append((k, sub_dir_for_type)) - try: - os.mkdir(sub_dir_for_type) - except: - pass - for subdir in unique_result_identifiers: - if k == subdir[0]: - target_dir = subdir[1] - linkname = os.path.join(target_dir, record + "_" + path[0] + "_" + file) - # src = os.path.join(root, file) - src = os.path.abspath(path[1]) - src_rel = os.path.relpath(src, os.path.dirname(linkname)) - force_symlink(src_rel, linkname) - - print(all_records) - - return linkdir - def list_results( self, restrict_to: Optional[List[str]] = None, diff --git a/pipestat/pipestat.py b/pipestat/pipestat.py index 68db4d01..64b222b4 100644 --- a/pipestat/pipestat.py +++ b/pipestat/pipestat.py @@ -500,7 +500,7 @@ def set_status( self.backend.set_status(status_identifier, r_id) @require_backend - def link(self, output_dir: Optional[str] = None) -> str: + def link(self, output_dir) -> str: """ This function creates a link structure such that results are organized by type. """ diff --git a/tests/test_pipestat.py b/tests/test_pipestat.py index c0157ca9..ea762b15 100644 --- a/tests/test_pipestat.py +++ b/tests/test_pipestat.py @@ -1056,7 +1056,7 @@ def test_basics( class TestFileTypeLinking: - @pytest.mark.parametrize("backend", ["file"]) + @pytest.mark.parametrize("backend", ["file", "db"]) def test_linking( self, config_file_path,