From a620e7041bbae9c2ff98fe012be03995446586b3 Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Thu, 12 Oct 2023 20:36:29 -0600 Subject: [PATCH 01/11] added support for mkv files --- elodie/media/video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elodie/media/video.py b/elodie/media/video.py index a53d238a..204e37d2 100644 --- a/elodie/media/video.py +++ b/elodie/media/video.py @@ -28,7 +28,7 @@ class Video(Media): __name__ = 'Video' #: Valid extensions for video files. - extensions = ('avi', 'm4v', 'mov', 'mp4', 'mpg', 'mpeg', '3gp', 'mts') + extensions = ('avi', 'm4v', 'mov', 'mp4', 'mpg', 'mpeg', '3gp', 'mts', 'mkv') def __init__(self, source=None): super(Video, self).__init__(source) From 651c373c0733840e29e5ed80bd6f0edb8075639e Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Thu, 12 Oct 2023 21:13:09 -0600 Subject: [PATCH 02/11] updated and verified code to change result message there is now a new type of result, duplicate files these are files that already exist in the destination folder and are not imported if not importing duplicates I have thoroughly verified that my changes do not cause any other part of the code to break or act in an unexpected manner. I have also removed some code that could never get used: elodie/localstorage.py: checksum() could never return None, code was not accessible due to earlier return because of this: elodie/filesystem.py: process_checksum() did not need to check if checksum was None after calculating it this is what allowed my new code to use None as a flag meaning that the file being imported and skipped is a duplicate file --- elodie/filesystem.py | 3 --- elodie/localstorage.py | 1 - elodie/result.py | 20 +++++++++++++++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/elodie/filesystem.py b/elodie/filesystem.py index 92d7807e..4b96a409 100644 --- a/elodie/filesystem.py +++ b/elodie/filesystem.py @@ -487,9 +487,6 @@ def parse_mask_for_location(self, mask, location_parts, place_name): def process_checksum(self, _file, allow_duplicate): db = Db() checksum = db.checksum(_file) - if(checksum is None): - log.info('Could not get checksum for %s.' % _file) - return None # If duplicates are not allowed then we check if we've seen this file # before via checksum. We also check that the file exists at the diff --git a/elodie/localstorage.py b/elodie/localstorage.py index 17ac26cc..7c8b05d3 100644 --- a/elodie/localstorage.py +++ b/elodie/localstorage.py @@ -127,7 +127,6 @@ def checksum(self, file_path, blocksize=65536): hasher.update(buf) buf = f.read(blocksize) return hasher.hexdigest() - return None def get_hash(self, key): """Get the hash value for a given key. diff --git a/elodie/result.py b/elodie/result.py index 3fa78519..e72e7d9f 100644 --- a/elodie/result.py +++ b/elodie/result.py @@ -8,17 +8,24 @@ def __init__(self): self.success = 0 self.error = 0 self.error_items = [] + self.duplicate = 0 + self.duplicate_items = [] def append(self, row): id, status = row - if status: + # status can only be True, False, or None + if status is True: self.success += 1 + elif status is None: # status is only ever None if file checksum matched an existing file checksum and is therefore a duplicate file + self.duplicate += 1 + self.duplicate_items.append(id) else: self.error += 1 self.error_items.append(id) def write(self): + print("\n") if self.error > 0: error_headers = ["File"] error_result = [] @@ -29,10 +36,21 @@ def write(self): print(tabulate(error_result, headers=error_headers)) print("\n") + if self.duplicate > 0: + duplicate_headers = ["File"] + duplicate_result = [] + for id in self.duplicate_items: + duplicate_result.append([id]) + + print("****** DUPLICATE (NOT IMPORTED) DETAILS ******") + print(tabulate(duplicate_result, headers=duplicate_headers)) + print("\n") + headers = ["Metric", "Count"] result = [ ["Success", self.success], ["Error", self.error], + ["Duplicate, not imported", self.duplicate] ] print("****** SUMMARY ******") From 6953b7a90883bd49998b42959d84057fab76764b Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Thu, 12 Oct 2023 22:02:19 -0600 Subject: [PATCH 03/11] fix for at least one test case --- elodie/result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elodie/result.py b/elodie/result.py index e72e7d9f..8006df03 100644 --- a/elodie/result.py +++ b/elodie/result.py @@ -15,7 +15,7 @@ def append(self, row): id, status = row # status can only be True, False, or None - if status is True: + if status: self.success += 1 elif status is None: # status is only ever None if file checksum matched an existing file checksum and is therefore a duplicate file self.duplicate += 1 From 996bc6a36e6276bf5f101fe3900350977bb345b5 Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Thu, 12 Oct 2023 22:02:45 -0600 Subject: [PATCH 04/11] updated test cases --- elodie/tests/elodie_test.py | 45 +++++++++++++++++++++++-------------- elodie/tests/result_test.py | 27 ++++++++++++---------- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/elodie/tests/elodie_test.py b/elodie/tests/elodie_test.py index a01bc303..b9c5c238 100644 --- a/elodie/tests/elodie_test.py +++ b/elodie/tests/elodie_test.py @@ -300,8 +300,9 @@ def test_import_file_with_single_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--exclude-regex', origin_valid[0:5], '--allow-duplicates', origin_valid]) - assert 'Success 0' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 0' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output def test_import_file_with_multiple_exclude(): temporary_folder, folder = helper.create_working_folder() @@ -313,8 +314,9 @@ def test_import_file_with_multiple_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--exclude-regex', 'does not exist in path', '--exclude-regex', origin_valid[0:5], '--allow-duplicates', origin_valid]) - assert 'Success 0' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 0' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output def test_import_file_with_non_matching_exclude(): temporary_folder, folder = helper.create_working_folder() @@ -326,8 +328,10 @@ def test_import_file_with_non_matching_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--exclude-regex', 'does not exist in path', '--allow-duplicates', origin_valid]) - assert 'Success 1' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 1' in result.output, result.output # FIXME this is not the case :( + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output + def test_import_directory_with_matching_exclude(): temporary_folder, folder = helper.create_working_folder() @@ -339,8 +343,9 @@ def test_import_directory_with_matching_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--source', folder, '--exclude-regex', folder[1:5], '--allow-duplicates']) - assert 'Success 0' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 0' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output def test_import_directory_with_non_matching_exclude(): temporary_folder, folder = helper.create_working_folder() @@ -352,8 +357,10 @@ def test_import_directory_with_non_matching_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--source', folder, '--exclude-regex', 'non-matching', '--allow-duplicates']) - assert 'Success 1' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 1' in result.output, result.output # FIXME this is not the case :( + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output + @mock.patch('elodie.config.config_file', '%s/config.ini-import-file-with-single-config-exclude' % gettempdir()) def test_import_file_with_single_config_exclude(): @@ -379,8 +386,9 @@ def test_import_file_with_single_config_exclude(): if hasattr(load_config, 'config'): del load_config.config - assert 'Success 0' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 0' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output @mock.patch('elodie.config.config_file', '%s/config.ini-import-file-with-multiple-config-exclude' % gettempdir()) def test_import_file_with_multiple_config_exclude(): @@ -407,8 +415,9 @@ def test_import_file_with_multiple_config_exclude(): if hasattr(load_config, 'config'): del load_config.config - assert 'Success 0' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 0' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output def test_update_location_on_audio(): temporary_folder, folder = helper.create_working_folder() @@ -705,8 +714,9 @@ def test_verify_ok(): shutil.rmtree(folder) - assert 'Success 1' in result.output, result.output - assert 'Error 0' in result.output, result.output + assert 'Success 1' in result.output, result.output + assert 'Error 0' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output def test_verify_error(): temporary_folder, folder = helper.create_working_folder() @@ -725,7 +735,8 @@ def test_verify_error(): shutil.rmtree(folder) assert origin in result.output, result.output - assert 'Error 1' in result.output, result.output + assert 'Error 1' in result.output, result.output + assert 'Duplicate, not imported 0' in result.output, result.output @mock.patch('elodie.config.config_file', '%s/config.ini-cli-batch-plugin-googlephotos' % gettempdir()) def test_cli_batch_plugin_googlephotos(): diff --git a/elodie/tests/result_test.py b/elodie/tests/result_test.py index a373fd6a..990a19ce 100644 --- a/elodie/tests/result_test.py +++ b/elodie/tests/result_test.py @@ -30,10 +30,11 @@ def call_result_and_assert(result, expected): def test_add_multiple_rows_with_success(): expected = """****** SUMMARY ****** -Metric Count --------- ------- -Success 2 -Error 0""" +Metric Count +----------------------- ------- +Success 2 +Error 0 +Duplicate, not imported 0""" # FIXME this is not the case :( result = Result() result.append(('id1', '/some/path/1')) result.append(('id2', '/some/path/2')) @@ -48,10 +49,11 @@ def test_add_multiple_rows_with_failure(): ****** SUMMARY ****** -Metric Count --------- ------- -Success 0 -Error 2""" +Metric Count +----------------------- ------- +Success 0 +Error 2 +Duplicate, not imported 0""" result = Result() result.append(('id1', False)) result.append(('id2', False)) @@ -65,10 +67,11 @@ def test_add_multiple_rows_with_failure_and_success(): ****** SUMMARY ****** -Metric Count --------- ------- -Success 1 -Error 1""" +Metric Count +----------------------- ------- +Success 1 +Error 1 +Duplicate, not imported 0""" # FIXME this is not the case :( result = Result() result.append(('id1', False)) result.append(('id2', '/some/path')) From 549ae5e32ce759b91a86d26ce6cc9eadcdafb060 Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Thu, 12 Oct 2023 22:03:12 -0600 Subject: [PATCH 05/11] these tests should be fixed with the previous fix --- elodie/tests/result_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elodie/tests/result_test.py b/elodie/tests/result_test.py index 990a19ce..265714cc 100644 --- a/elodie/tests/result_test.py +++ b/elodie/tests/result_test.py @@ -34,7 +34,7 @@ def test_add_multiple_rows_with_success(): ----------------------- ------- Success 2 Error 0 -Duplicate, not imported 0""" # FIXME this is not the case :( +Duplicate, not imported 0""" result = Result() result.append(('id1', '/some/path/1')) result.append(('id2', '/some/path/2')) @@ -71,7 +71,7 @@ def test_add_multiple_rows_with_failure_and_success(): ----------------------- ------- Success 1 Error 1 -Duplicate, not imported 0""" # FIXME this is not the case :( +Duplicate, not imported 0""" result = Result() result.append(('id1', False)) result.append(('id2', '/some/path')) From fd2d264fd3c27dee355c773834f75d97ad5a2c63 Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Thu, 12 Oct 2023 22:05:15 -0600 Subject: [PATCH 06/11] these last two tests also pass with that fix --- elodie/tests/elodie_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elodie/tests/elodie_test.py b/elodie/tests/elodie_test.py index b9c5c238..333a9416 100644 --- a/elodie/tests/elodie_test.py +++ b/elodie/tests/elodie_test.py @@ -328,7 +328,7 @@ def test_import_file_with_non_matching_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--exclude-regex', 'does not exist in path', '--allow-duplicates', origin_valid]) - assert 'Success 1' in result.output, result.output # FIXME this is not the case :( + assert 'Success 1' in result.output, result.output assert 'Error 0' in result.output, result.output assert 'Duplicate, not imported 0' in result.output, result.output @@ -357,7 +357,7 @@ def test_import_directory_with_non_matching_exclude(): runner = CliRunner() result = runner.invoke(elodie._import, ['--destination', folder_destination, '--source', folder, '--exclude-regex', 'non-matching', '--allow-duplicates']) - assert 'Success 1' in result.output, result.output # FIXME this is not the case :( + assert 'Success 1' in result.output, result.output assert 'Error 0' in result.output, result.output assert 'Duplicate, not imported 0' in result.output, result.output From 432ae0b23b15f8993f6e9442bdac779b22c788e2 Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Thu, 12 Oct 2023 22:12:49 -0600 Subject: [PATCH 07/11] commit to force tests to rerun --- elodie/tests/result_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elodie/tests/result_test.py b/elodie/tests/result_test.py index 265714cc..0be68ef5 100644 --- a/elodie/tests/result_test.py +++ b/elodie/tests/result_test.py @@ -76,3 +76,4 @@ def test_add_multiple_rows_with_failure_and_success(): result.append(('id1', False)) result.append(('id2', '/some/path')) call_result_and_assert(result, expected) + \ No newline at end of file From 2d59391b082514521be80aafe5a85aa4c62bac23 Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Thu, 12 Oct 2023 22:13:03 -0600 Subject: [PATCH 08/11] removing space added in previous commit --- elodie/tests/result_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elodie/tests/result_test.py b/elodie/tests/result_test.py index 0be68ef5..265714cc 100644 --- a/elodie/tests/result_test.py +++ b/elodie/tests/result_test.py @@ -76,4 +76,3 @@ def test_add_multiple_rows_with_failure_and_success(): result.append(('id1', False)) result.append(('id2', '/some/path')) call_result_and_assert(result, expected) - \ No newline at end of file From 6eef0362fac9af502dbc4c55605088a5e2487e7a Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Fri, 13 Oct 2023 09:54:24 -0600 Subject: [PATCH 09/11] exiftool also supports webm, adding to recognized video file extensions --- elodie/media/video.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elodie/media/video.py b/elodie/media/video.py index 204e37d2..c9cecb08 100644 --- a/elodie/media/video.py +++ b/elodie/media/video.py @@ -28,7 +28,7 @@ class Video(Media): __name__ = 'Video' #: Valid extensions for video files. - extensions = ('avi', 'm4v', 'mov', 'mp4', 'mpg', 'mpeg', '3gp', 'mts', 'mkv') + extensions = ('avi', 'm4v', 'mov', 'mp4', 'mpg', 'mpeg', '3gp', 'mts', 'mkv', 'webm') def __init__(self, source=None): super(Video, self).__init__(source) From 0b6a682ff9ce55114908b7c3a5c59e71c65d9943 Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Fri, 13 Oct 2023 10:00:04 -0600 Subject: [PATCH 10/11] added a few audio formats that exiftool supports --- elodie/media/audio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elodie/media/audio.py b/elodie/media/audio.py index 84095125..55131a06 100644 --- a/elodie/media/audio.py +++ b/elodie/media/audio.py @@ -20,7 +20,7 @@ class Audio(Video): __name__ = 'Audio' #: Valid extensions for audio files. - extensions = ('m4a',) + extensions = ('m4a', 'mp3', 'opus', 'ogg', 'flac') def __init__(self, source=None): super(Audio, self).__init__(source) From 6e946d6287c19bfb1b578a7ea4583cbcdc54097c Mon Sep 17 00:00:00 2001 From: Cedric Boucher Date: Fri, 13 Oct 2023 10:00:20 -0600 Subject: [PATCH 11/11] added webp and svg to recognized photo types as they are supported by exiftool --- elodie/media/photo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elodie/media/photo.py b/elodie/media/photo.py index 07f3e047..2afcfa1b 100644 --- a/elodie/media/photo.py +++ b/elodie/media/photo.py @@ -28,7 +28,7 @@ class Photo(Media): __name__ = 'Photo' #: Valid extensions for photo files. - extensions = ('arw', 'cr2', 'dng', 'gif', 'heic', 'jpeg', 'jpg', 'nef', 'png', 'rw2') + extensions = ('arw', 'cr2', 'dng', 'gif', 'heic', 'jpeg', 'jpg', 'nef', 'png', 'rw2', 'webp', 'svg') def __init__(self, source=None): super(Photo, self).__init__(source)