From 1d4a21a8522251903485cd68a7ef614859faa36f Mon Sep 17 00:00:00 2001 From: Ziga Luksic Date: Thu, 7 Sep 2023 12:02:16 +0200 Subject: [PATCH 1/2] Remove timestamps in sample merging --- eogrow/pipelines/merge_samples.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/eogrow/pipelines/merge_samples.py b/eogrow/pipelines/merge_samples.py index 33e88fa1..a406385d 100644 --- a/eogrow/pipelines/merge_samples.py +++ b/eogrow/pipelines/merge_samples.py @@ -12,7 +12,7 @@ from eolearn.core.utils.fs import get_full_path from ..core.pipeline import Pipeline -from ..types import Feature, FeatureSpec +from ..types import Feature from ..utils.validators import ensure_storage_key_presence LOGGER = logging.getLogger(__name__) @@ -34,7 +34,6 @@ class Schema(Pipeline.Schema): features_to_merge: List[Feature] = Field( description="Dictionary of all features for which samples are to be merged." ) - include_timestamp: bool = Field(False, description="Whether to also prepare an array of merged timestamps.") id_filename: Optional[str] = Field( description=( "Filename of array holding patch ID of concatenated features. The patch ID is the index of the patch in" @@ -69,12 +68,10 @@ def run_procedure(self) -> tuple[list[str], list[str]]: def build_workflow(self) -> EOWorkflow: """Creates a workflow that outputs the requested features""" - features_to_load: list[FeatureSpec] = [FeatureType.TIMESTAMPS] if self.config.include_timestamp else [] - features_to_load.extend(self.config.features_to_merge) load_task = LoadTask( self.storage.get_folder(self.config.input_folder_key), filesystem=self.storage.filesystem, - features=features_to_load, + features=self.config.features_to_merge, ) output_task = OutputTask(name=self._OUTPUT_NAME) return EOWorkflow(linearly_connect_tasks(load_task, output_task)) @@ -100,14 +97,6 @@ def merge_and_save_features(self, patches: list[EOPatch]) -> None: if patch_sample_nums is None: raise ValueError("Need at least one feature to merge.") - if self.config.include_timestamp: - arrays = [] - for patch, sample_num in zip(patches, patch_sample_nums): - arrays.append(np.tile(np.array(patch.timestamps), (sample_num, 1))) - patch.timestamps = [] - - self._save_array(np.concatenate(arrays, axis=0), "TIMESTAMPS") - if self.config.id_filename: LOGGER.info("Started merging EOPatch ids") patch_id_arrays = [ From f0f1d92ca8a52f5dcf35fa87251fdafdee5ec863 Mon Sep 17 00:00:00 2001 From: Ziga Luksic Date: Thu, 7 Sep 2023 12:47:46 +0200 Subject: [PATCH 2/2] adjust tests --- .../merge_samples/merge_features_samples.json | 1 - .../merge_samples/merge_reference_samples.json | 1 - tests/test_stats/merge_samples/merge_features_samples.json | 7 ------- .../test_stats/merge_samples/merge_reference_samples.json | 7 ------- 4 files changed, 16 deletions(-) diff --git a/tests/test_config_files/merge_samples/merge_features_samples.json b/tests/test_config_files/merge_samples/merge_features_samples.json index 4001806c..6aafecb3 100644 --- a/tests/test_config_files/merge_samples/merge_features_samples.json +++ b/tests/test_config_files/merge_samples/merge_features_samples.json @@ -4,7 +4,6 @@ "input_folder_key": "features_sampled", "output_folder_key": "training_data", "features_to_merge": [["data", "FEATURES"]], - "include_timestamp": true, "id_filename": "PATCH_IDS", "suffix": "", "workers": 2 diff --git a/tests/test_config_files/merge_samples/merge_reference_samples.json b/tests/test_config_files/merge_samples/merge_reference_samples.json index 1d898437..4173ac4a 100644 --- a/tests/test_config_files/merge_samples/merge_reference_samples.json +++ b/tests/test_config_files/merge_samples/merge_reference_samples.json @@ -4,7 +4,6 @@ "input_folder_key": "data_sampled", "output_folder_key": "training_data", "features_to_merge": [["mask_timeless", "LULC_ID"]], - "include_timestamp": true, "id_filename": "PATCH_IDS", "suffix": "", "workers": 2 diff --git a/tests/test_stats/merge_samples/merge_features_samples.json b/tests/test_stats/merge_samples/merge_features_samples.json index 57ce3edd..13417767 100644 --- a/tests/test_stats/merge_samples/merge_features_samples.json +++ b/tests/test_stats/merge_samples/merge_features_samples.json @@ -83,12 +83,5 @@ "value": 1 } ] - }, - "TIMESTAMPS.npy": { - "array_shape": [ - 1400, - 12 - ], - "dtype": "object" } } diff --git a/tests/test_stats/merge_samples/merge_reference_samples.json b/tests/test_stats/merge_samples/merge_reference_samples.json index 5283d69e..8e9f090a 100644 --- a/tests/test_stats/merge_samples/merge_reference_samples.json +++ b/tests/test_stats/merge_samples/merge_reference_samples.json @@ -126,12 +126,5 @@ "value": 1 } ] - }, - "TIMESTAMPS.npy": { - "array_shape": [ - 1400, - 35 - ], - "dtype": "object" } }