Change this forks "Scanner" tag to something different

README.md

@@ -8,6 +8,15 @@ Tubeup - a multi-VOD service to Archive.org uploader
 
 It was designed by the [Bibliotheca Anonoma](https://github.com/bibanon/bibanon/wiki) to archive single videos, playlists (see warning below about more than video uploads) or accounts to the Internet Archive.
 
+## Changes specific to this fork
+- Clean-up IP addresses contained by Youtube-generated info files before IA upload
+- Do not abort after a failure but try the next item + saner timeout values
+- Accept arbitrary yt-dlp options (⚠️)
+- Can upload existing resource (⚠️ under certain strict condition, see --help and a52031c)
+- Can upload existing resource based on a local JSON info file
+- More efficient at proceessing a large number of files/URL
+- Broken testsuite (⚠️)
+
 ## Prerequisites
 
 This script strongly recommends Linux or some sort of POSIX system (such as macOS), preferably from a rented VPS and not your personal machine or phone.
@@ -31,7 +40,7 @@ For Debian/Ubuntu:
    At a minimum Python 3.8 and up is required (latest Python preferred).
 
 ```
-   python3 -m pip install -U pip tubeup
+   python3 -m pip install -U pip git+https://github.com/drzraf/tubeup
 ```
 
 3. If you don't already have an Internet Archive account, [register for one](https://archive.org/account/login.createaccount.php) to give the script upload privileges.

setup.py

@@ -26,6 +26,7 @@
     },
     install_requires=[
         'internetarchive',
+        'urllib3',
         'docopt==0.6.2',
         'yt-dlp',
     ]

tests/test_tubeup.py

@@ -590,3 +590,83 @@ def test_archive_urls(self):
                  'scanner': SCANNER})]
 
             self.assertEqual(expected_result, result)
+
+    def test_archive_deletion(self):
+        root_path = os.path.join(current_path, 'test_tubeup_rootdir')
+        # Clean up before test
+        shutil.rmtree(root_path, ignore_errors=True)
+
+        tu = TubeUp(dir_path=root_path,
+                    ia_config_path=get_testfile_path('ia_config_for_test.ini'))
+
+        videobasename = os.path.join(
+            current_path, 'test_tubeup_rootdir', 'downloads',
+            'KdsN9YhkDrY')
+
+        copy_testfiles_to_tubeup_rootdir_test()
+        dest = os.path.join(root_path, 'downloads', '*')
+        files_before_upload = glob.glob(dest)
+
+        vid_info = {'mediatype': 'movies',
+                    'creator': 'RelaxingWorld',
+                    'channel': 'http://www.youtube.com/channel/UCWpsozCMdAnfI16rZHQ9XDg',
+                    'collection': 'opensource_movies',
+                    'title': 'Epic Ramadan - Video Background HD1080p',
+                    'description': ('If you enjoy my work, please consider Subscribe to my NEW '
+                                    'channel for more videos: <br>'
+                                    'https://www.youtube.com/MusicForRelaxation?sub_confirmation=1 <br>'
+                                    '▷ If you use this video, please put credits to my channel '
+                                    'in description: <br>'
+                                    'Source from RelaxingWorld: https://goo.gl/HsW75m<br>'
+                                    '<br>'
+                                    '▷ Also, do not forget to Subscribe to my channel. Thanks!'),
+                    'date': '2016-06-25',
+                    'year': '2016',
+                    'subject': ('Youtube;video;Film & Animation;Video Background;'
+                                'Footage;Animation;Cinema;Royalty Free Videos;'
+                                'Stock Video Footage;Video Backdrops;'
+                                'Amazing Nature;youtube;HD;1080p;Creative Commons Videos;'
+                                'relaxing music;Ramadan;'),
+                    'originalurl': 'https://www.youtube.com/watch?v=KdsN9YhkDrY',
+                    'licenseurl': '',
+                    'scanner': SCANNER}
+
+        with requests_mock.Mocker() as m:
+            # Mock the request to s3.us.archive.org, so it will responds
+            # a custom json. `internetarchive` library sends GET request to
+            # that url to check that we don't violate the upload limit.
+            m.get('https://s3.us.archive.org',
+                  content=b'{"over_limit": 0}',
+                  headers={'content-type': 'application/json'})
+
+            m.get('https://archive.org/metadata/youtube-KdsN9YhkDrY',
+                  content=b'{}',
+                  headers={'content-type': 'application/json'})
+
+            # Mock the PUT requests for internetarchive urls that defined
+            # in mock_upload_response_by_videobasename(), so this test
+            # doesn't perform upload to the real archive.org server.
+            mock_upload_response_by_videobasename(
+                m, 'youtube-KdsN9YhkDrY', videobasename)
+
+            # First upload, this actually get uploaded...
+            result = list(tu.archive_urls(
+                ['https://www.youtube.com/watch?v=KdsN9YhkDrY'], use_upload_archive=True))
+
+            # ... and returns a remote IA item name
+            expected_result = [('youtube-KdsN9YhkDrY', vid_info)]
+            self.assertEqual(expected_result, result)
+
+            # ... and no file got deleted
+            files_after_upload = glob.glob(dest)
+            self.assertListEqual(files_before_upload, files_after_upload)
+            # ... and a upload-archive file was created
+            self.assertTrue(os.path.exists(os.path.join(root_path, '.iauparchive')))
+
+            # Second upload, nothing was actually uploaded...
+            result = list(tu.archive_urls(
+                ['https://www.youtube.com/watch?v=KdsN9YhkDrY'], use_upload_archive=True))
+
+            # ... and no remote IA item name is returned
+            expected_result = [(None, vid_info)]
+            self.assertEqual(expected_result, result)

tests/test_utils.py

@@ -1,6 +1,13 @@
 import unittest
 import os
-from tubeup.utils import sanitize_identifier, check_is_file_empty
+import json
+from tubeup.utils import sanitize_identifier, check_is_file_empty, strip_ip_from_meta
+
+current_path = os.path.dirname(os.path.realpath(__file__))
+
+
+def get_testfile_path(name):
+    return os.path.join(current_path, 'test_tubeup_files', name)
 
 
 class UtilsTest(unittest.TestCase):
@@ -48,3 +55,14 @@ def test_check_is_file_empty_when_file_doesnt_exist(self):
                 FileNotFoundError,
                 r"^Path 'file_that_doesnt_exist.txt' doesn't exist$"):
             check_is_file_empty('file_that_doesnt_exist.txt')
+
+    def test_strip_ip_from_meta(self):
+        with open(get_testfile_path(
+                'Mountain_3_-_Video_Background_HD_1080p-6iRV8liah8A.'
+                'info.json')
+        ) as f:
+            vid_meta = json.load(f)
+            mod, new_meta = strip_ip_from_meta(vid_meta)
+            self.assertTrue(mod)
+            self.assertNotEqual(f.read(), json.dumps(new_meta))
+            self.assertNotRegex(json.dumps(new_meta), r'36\.73\.93\.234')