From 953a03ff5952fd2035e816b5245595896e42f2f8 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Thu, 23 May 2024 19:11:02 -0400 Subject: [PATCH 01/33] Add torbox as a debrid option --- .env.template | 8 +- README.md | 11 +- blackhole.py | 501 +++++++++++------------------------ delete_non_linked_folders.py | 4 +- docker-compose.yml | 24 +- import_torrent_folder.py | 4 +- repair.py | 9 +- shared/debrid.py | 392 +++++++++++++++++++++++++++ shared/shared.py | 12 +- 9 files changed, 601 insertions(+), 364 deletions(-) create mode 100644 shared/debrid.py diff --git a/.env.template b/.env.template index a22a867..3b354f7 100644 --- a/.env.template +++ b/.env.template @@ -22,8 +22,15 @@ RADARR_ROOT_FOLDER= TAUTULLI_HOST= TAUTULLI_API_KEY= +REALDEBRID_ENABLED=true REALDEBRID_HOST="https://api.real-debrid.com/rest/1.0/" REALDEBRID_API_KEY= +REALDEBRID_MOUNT_TORRENTS_PATH= + +TORBOX_ENABLED=false +TORBOX_HOST="https://api.torbox.app/v1/api/" +TORBOX_API_KEY= +TORBOX_MOUNT_TORRENTS_PATH= TRAKT_API_KEY= @@ -31,7 +38,6 @@ WATCHLIST_PLEX_PRODUCT="Plex Request Authentication" WATCHLIST_PLEX_VERSION="1.0.0" WATCHLIST_PLEX_CLIENT_IDENTIFIER="576101fc-b425-4685-91cb-5d3c1671fd2b" -BLACKHOLE_RD_MOUNT_TORRENTS_PATH= BLACKHOLE_BASE_WATCH_PATH="./blackhole" BLACKHOLE_RADARR_PATH="Movies" BLACKHOLE_SONARR_PATH="TV Shows" diff --git a/README.md b/README.md index db4aa61..6a7b191 100644 --- a/README.md +++ b/README.md @@ -56,9 +56,17 @@ - `TAUTULLI_HOST`: The host address of your Tautulli instance. - `TAUTULLI_API_KEY`: The API key for accessing Tautulli. - - **RealDebrid** - Blackhole: + - **RealDebrid** - Blackhole, Repair: + - `REALDEBRID_ENABLED`: Set to `true` to enable RealDebrid services. - `REALDEBRID_HOST`: The host address for the RealDebrid API. - `REALDEBRID_API_KEY`: The API key for accessing RealDebrid services. + - `REALDEBRID_MOUNT_TORRENTS_PATH`: The path to the RealDebrid mount torrents folder. + + - **TorBox** - Blackhole, Repair: + - `TORBOX_ENABLED`: Set to `true` to enable TorBox services. + - `TORBOX_HOST`: The host address for the TorBox API. + - `TORBOX_API_KEY`: The API key for accessing TorBox services. + - `TORBOX_MOUNT_TORRENTS_PATH`: The path to the TorBox mount torrents folder. - **Trakt** - Reclaim Space: - `TRAKT_API_KEY`: The API key for integrating with Trakt. @@ -69,7 +77,6 @@ - `WATCHLIST_PLEX_CLIENT_IDENTIFIER`: A unique identifier for the Plex client. - **Blackhole** - Blackhole: - - `BLACKHOLE_RD_MOUNT_TORRENTS_PATH`: The path to the RealDebrid mounted torrents. - `BLACKHOLE_BASE_WATCH_PATH`: The base path for watched folders by the blackhole mechanism. Can be relative or absolute. - `BLACKHOLE_RADARR_PATH`: The path where torrent files will be dropped into by Radarr, relative to the base path. - `BLACKHOLE_SONARR_PATH`: The path where torrent files will be dropped into by Sonarr, relative to the base path. diff --git a/blackhole.py b/blackhole.py index 6b70f78..814120e 100644 --- a/blackhole.py +++ b/blackhole.py @@ -1,79 +1,43 @@ import shutil import time import traceback -import hashlib import os import sys import re import requests import asyncio -import bencode3 from datetime import datetime # import urllib -from werkzeug.utils import cached_property -from abc import ABC, abstractmethod from shared.discord import discordError, discordUpdate -from shared.shared import realdebrid, blackhole, plex, mediaExtensions, checkRequiredEnvs +from shared.shared import realdebrid, torbox, blackhole, plex, checkRequiredEnvs from shared.arr import Arr, Radarr, Sonarr - -rdHost = realdebrid['host'] -authToken = realdebrid['apiKey'] +from shared.debrid import TorrentBase, RealDebridTorrent, RealDebridMagnet, TorboxTorrent, TorboxMagnet _print = print def print(*values: object): _print(f"[{datetime.now()}]", *values) - -def validateRealdebridHost(): - url = f"{realdebrid['host']}/time" - try: - response = requests.get(url) - return response.status_code == 200 - except Exception as e: - return False - -def validateRealdebridApiKey(): - url = f"{realdebrid['host']}/user?auth_token={authToken}" - try: - response = requests.get(url) - - if response.status_code == 401: - return False, "Invalid or expired API key." - elif response.status_code == 403: - return False, "Permission denied, account locked." - except Exception as e: - return False - - return True - -def validateMountTorrentsPath(): - path = blackhole['rdMountTorrentsPath'] - if os.path.exists(path) and any(os.path.isdir(os.path.join(path, child)) for child in os.listdir(path)): - return True - else: - return False, "Path does not exist or has no children." - requiredEnvs = { - 'RealDebrid host': (realdebrid['host'], validateRealdebridHost), - 'RealDebrid API key': (realdebrid['apiKey'], validateRealdebridApiKey, True), - 'Blackhole RealDebrid mount torrents path': (blackhole['rdMountTorrentsPath'], validateMountTorrentsPath), 'Blackhole base watch path': (blackhole['baseWatchPath'],), 'Blackhole Radarr path': (blackhole['radarrPath'],), - 'Blackhole Sonarr path': (blackhole['sonarrPath'],) + 'Blackhole Sonarr path': (blackhole['sonarrPath'],), + 'Blackhole fail if not cached': (blackhole['failIfNotCached'],), + 'Blackhole RD mount refresh seconds': (blackhole['rdMountRefreshSeconds'],), + 'Blackhole wait for torrent timeout': (blackhole['waitForTorrentTimeout'],), + 'Blackhole history page size': (blackhole['historyPageSize'],) } checkRequiredEnvs(requiredEnvs) class TorrentFileInfo(): class FileInfo(): - def __init__(self, filename, filenameWithoutExt, filePath, filePathProcessing, folderPathCompleted, folderPathMountTorrent) -> None: + def __init__(self, filename, filenameWithoutExt, filePath, filePathProcessing, folderPathCompleted) -> None: self.filename = filename self.filenameWithoutExt = filenameWithoutExt self.filePath = filePath self.filePathProcessing = filePathProcessing self.folderPathCompleted = folderPathCompleted - self.folderPathMountTorrent = folderPathMountTorrent class TorrentInfo(): def __init__(self, isTorrentOrMagnet, isDotTorrentFile) -> None: @@ -89,171 +53,9 @@ def __init__(self, filename, isRadarr) -> None: filePath = os.path.join(baseBath, filename) filePathProcessing = os.path.join(baseBath, 'processing', filename) folderPathCompleted = os.path.join(baseBath, 'completed', filenameWithoutExt) - folderPathMountTorrent = os.path.join(blackhole['rdMountTorrentsPath'], filenameWithoutExt) - self.fileInfo = self.FileInfo(filename, filenameWithoutExt, filePath, filePathProcessing, folderPathCompleted, folderPathMountTorrent) + self.fileInfo = self.FileInfo(filename, filenameWithoutExt, filePath, filePathProcessing, folderPathCompleted) self.torrentInfo = self.TorrentInfo(isTorrentOrMagnet, isDotTorrentFile) - - -class TorrentBase(ABC): - def __init__(self, f, file, fail, failIfNotCached, onlyLargestFile) -> None: - super().__init__() - self.f = f - self.file = file - self.fail = fail - self.failIfNotCached = failIfNotCached - self.onlyLargestFile = onlyLargestFile - self.id = None - self._info = None - self._instantAvailability = None - self._hash = None - self.incompatibleHashSize = False - - def print(self, *values: object): - print(f"[{self.file.fileInfo.filenameWithoutExt}]", *values) - - @cached_property - def fileData(self): - fileData = self.f.read() - self.f.seek(0) - - return fileData - - - def submitTorrent(self): - if self.failIfNotCached: - instantAvailability = self.getInstantAvailability() - self.print('instantAvailability:', not not instantAvailability) - if not instantAvailability: - self.fail(self) - return False - - availableHost = self.getAvailableHost() - self.addTorrent(availableHost) - return True - - @abstractmethod - def getHash(self): - pass - - @abstractmethod - def addTorrent(self, host): - pass - - def getInstantAvailability(self, refresh=False): - if refresh or not self._instantAvailability: - torrentHash = self.getHash() - self.print('hash:', torrentHash) - - if len(torrentHash) != 40: - self.incompatibleHashSize = True - return True - - instantAvailabilityRequest = requests.get(f"{rdHost}torrents/instantAvailability/{torrentHash}?auth_token={authToken}") - instantAvailabilities = instantAvailabilityRequest.json() - self.print('instantAvailabilities:', instantAvailabilities) - instantAvailabilityHosters = next(iter(instantAvailabilities.values())) - if not instantAvailabilityHosters: return - - self._instantAvailability = next(iter(instantAvailabilityHosters.values())) - - return self._instantAvailability - - def getAvailableHost(self): - availableHostsRequest = requests.get(f"{rdHost}torrents/availableHosts?auth_token={authToken}") - availableHosts = availableHostsRequest.json() - - return availableHosts[0]['host'] - - def getInfo(self, refresh=False): - self._enforceId() - - if refresh or not self._info: - infoRequest = requests.get(f"{rdHost}torrents/info/{self.id}?auth_token={authToken}") - self._info = infoRequest.json() - - return self._info - - def selectFiles(self): - self._enforceId() - - info = self.getInfo() - self.print('files:', info['files']) - mediaFiles = [file for file in info['files'] if os.path.splitext(file['path'])[1].lower() in mediaExtensions] - - if not mediaFiles: - self.print('no media files found') - return False - - mediaFileIds = {str(file['id']) for file in mediaFiles} - self.print('required fileIds:', mediaFileIds) - - largestMediaFile = max(mediaFiles, key=lambda file: file['bytes']) - largestMediaFileId = str(largestMediaFile['id']) - self.print('only largest file:', self.onlyLargestFile) - self.print('largest file:', largestMediaFile) - - if self.failIfNotCached and not self.incompatibleHashSize: - targetFileIds = {largestMediaFileId} if self.onlyLargestFile else mediaFileIds - if not any(set(fileGroup.keys()) == targetFileIds for fileGroup in self._instantAvailability): - extraFilesGroup = next((fileGroup for fileGroup in self._instantAvailability if largestMediaFileId in fileGroup.keys()), None) - if self.onlyLargestFile and extraFilesGroup: - self.print('extra files required for cache:', extraFilesGroup) - discordUpdate('Extra files required for cache:', extraFilesGroup) - return False - - if self.onlyLargestFile and len(mediaFiles) > 1: - discordUpdate('largest file:', largestMediaFile['path']) - - files = {'files': [largestMediaFileId] if self.onlyLargestFile else ','.join(mediaFileIds)} - selectFilesRequest = requests.post(f"{rdHost}torrents/selectFiles/{self.id}?auth_token={authToken}", data=files) - - return True - - def delete(self): - self._enforceId() - - deleteRequest = requests.delete(f"{rdHost}torrents/delete/{self.id}?auth_token={authToken}") - - - def _enforceId(self): - if not self.id: raise Exception("Id is required. Must be aquired via sucessfully running submitTorrent() first.") - - -class Torrent(TorrentBase): - def getHash(self): - - if not self._hash: - self._hash = hashlib.sha1(bencode3.bencode(bencode3.bdecode(self.fileData)['info'])).hexdigest() - - return self._hash - - def addTorrent(self, host): - addTorrentRequest = requests.put(f"{rdHost}torrents/addTorrent?host={host}&auth_token={authToken}", data=self.f) - addTorrentResponse = addTorrentRequest.json() - self.print('torrent info:', addTorrentResponse) - - self.id = addTorrentResponse['id'] - return self.id - - -class Magnet(TorrentBase): - def getHash(self): - - if not self._hash: - # Consider changing when I'm more familiar with hashes - self._hash = re.search('xt=urn:btih:(.+?)(?:&|$)', self.fileData).group(1) - - return self._hash - - def addTorrent(self, host): - addMagnetRequest = requests.post(f"{rdHost}torrents/addMagnet?host={host}&auth_token={authToken}", data={'magnet': self.fileData}) - addMagnetResponse = addMagnetRequest.json() - self.print('magnet info:', addMagnetResponse) - - self.id = addMagnetResponse['id'] - - return self.id def getPath(isRadarr, create=False): baseWatchPath = blackhole['baseWatchPath'] @@ -319,6 +121,127 @@ def print(*values: object): import signal +async def processTorrent(torrent: TorrentBase, file: TorrentFileInfo, arr: Arr) -> bool: + if not torrent.submitTorrent(): + return False + + count = 0 + while True: + count += 1 + info = torrent.getInfo(refresh=True) + status = info['status'] + + print('status:', status) + + if status == torrent.STATUS_WAITING_FILES_SELECTION: + if not torrent.selectFiles(): + torrent.delete() + return False + elif status == torrent.STATUS_DOWNLOADING: + # Send progress to arr + progress = info['progress'] + print(progress) + if torrent.incompatibleHashSize and torrent.failIfNotCached: + print("Non-cached incompatible hash sized torrent") + torrent.delete() + return False + await asyncio.sleep(1) + elif status == torrent.STATUS_ERROR: + return False + elif status == torrent.STATUS_COMPLETED: + existsCount = 0 + print('Waiting for folders to refresh...') + + filename = info.get('filename') + originalFilename = info.get('original_filename') + + folderPathMountFilenameTorrent = os.path.join(torrent.mountTorrentsPath, filename) + folderPathMountOriginalFilenameTorrent = os.path.join(torrent.mountTorrentsPath, originalFilename) + folderPathMountOriginalFilenameWithoutExtTorrent = os.path.join(torrent.mountTorrentsPath, os.path.splitext(originalFilename)[0]) + + while True: + existsCount += 1 + + if os.path.exists(folderPathMountFilenameTorrent) and os.listdir(folderPathMountFilenameTorrent): + folderPathMountTorrent = folderPathMountFilenameTorrent + elif os.path.exists(folderPathMountOriginalFilenameTorrent) and os.listdir(folderPathMountOriginalFilenameTorrent): + folderPathMountTorrent = folderPathMountOriginalFilenameTorrent + elif (originalFilename.endswith(('.mkv', '.mp4')) and + os.path.exists(folderPathMountOriginalFilenameWithoutExtTorrent) and os.listdir(folderPathMountOriginalFilenameWithoutExtTorrent)): + folderPathMountTorrent = folderPathMountOriginalFilenameWithoutExtTorrent + else: + folderPathMountTorrent = None + + if folderPathMountTorrent: + multiSeasonRegex1 = r'(?<=[\W_][Ss]eason[\W_])[\d][\W_][\d]{1,2}(?=[\W_])' + multiSeasonRegex2 = r'(?<=[\W_][Ss])[\d]{2}[\W_][Ss]?[\d]{2}(?=[\W_])' + multiSeasonRegexCombined = f'{multiSeasonRegex1}|{multiSeasonRegex2}' + + multiSeasonMatch = re.search(multiSeasonRegexCombined, file.fileInfo.filenameWithoutExt) + + for root, dirs, files in os.walk(folderPathMountTorrent): + relRoot = os.path.relpath(root, folderPathMountTorrent) + for filename in files: + # Check if the file is accessible + # if not await is_accessible(os.path.join(root, filename)): + # print(f"Timeout reached when accessing file: {filename}") + # discordError(f"Timeout reached when accessing file", filename) + # Uncomment the following line to fail the entire torrent if the timeout on any of its files are reached + # fail(torrent) + # return + + if multiSeasonMatch: + seasonMatch = re.search(r'S([\d]{2})E[\d]{2}', filename) + + if seasonMatch: + season = seasonMatch.group(1) + seasonShort = season[1:] if season[0] == '0' else season + + seasonFolderPathCompleted = re.sub(multiSeasonRegex1, seasonShort, file.fileInfo.folderPathCompleted) + seasonFolderPathCompleted = re.sub(multiSeasonRegex2, season, seasonFolderPathCompleted) + + os.makedirs(os.path.join(seasonFolderPathCompleted, relRoot), exist_ok=True) + os.symlink(os.path.join(root, filename), os.path.join(seasonFolderPathCompleted, relRoot, filename)) + print('Season Recursive:', f"{os.path.join(seasonFolderPathCompleted, relRoot, filename)} -> {os.path.join(root, filename)}") + # refreshEndpoint = f"{plex['serverHost']}/library/sections/{plex['serverMovieLibraryId'] if isRadarr else plex['serverTvShowLibraryId']}/refresh?path={urllib.parse.quote_plus(os.path.join(seasonFolderPathCompleted, relRoot))}&X-Plex-Token={plex['serverApiKey']}" + # cancelRefreshRequest = requests.delete(refreshEndpoint, headers={'Accept': 'application/json'}) + # refreshRequest = requests.get(refreshEndpoint, headers={'Accept': 'application/json'}) + + continue + + + os.makedirs(os.path.join(file.fileInfo.folderPathCompleted, relRoot), exist_ok=True) + os.symlink(os.path.join(root, filename), os.path.join(file.fileInfo.folderPathCompleted, relRoot, filename)) + print('Recursive:', f"{os.path.join(file.fileInfo.folderPathCompleted, relRoot, filename)} -> {os.path.join(root, filename)}") + # refreshEndpoint = f"{plex['serverHost']}/library/sections/{plex['serverMovieLibraryId'] if isRadarr else plex['serverTvShowLibraryId']}/refresh?path={urllib.parse.quote_plus(os.path.join(file.fileInfo.folderPathCompleted, relRoot))}&X-Plex-Token={plex['serverApiKey']}" + # cancelRefreshRequest = requests.delete(refreshEndpoint, headers={'Accept': 'application/json'}) + # refreshRequest = requests.get(refreshEndpoint, headers={'Accept': 'application/json'}) + + print('Refreshed') + discordUpdate(f"Sucessfully processed {file.fileInfo.filenameWithoutExt}", f"Now available for immediate consumption! existsCount: {existsCount}") + + # refreshEndpoint = f"{plex['serverHost']}/library/sections/{plex['serverMovieLibraryId'] if isRadarr else plex['serverTvShowLibraryId']}/refresh?X-Plex-Token={plex['serverApiKey']}" + # cancelRefreshRequest = requests.delete(refreshEndpoint, headers={'Accept': 'application/json'}) + # refreshRequest = requests.get(refreshEndpoint, headers={'Accept': 'application/json'}) + await refreshArr(arr) + + # await asyncio.get_running_loop().run_in_executor(None, copyFiles, file, folderPathMountTorrent, arr) + return True + + if existsCount >= blackhole['rdMountRefreshSeconds'] + 1: + print(f"Torrent folder not found in filesystem: {file.fileInfo.filenameWithoutExt}") + discordError("Torrent folder not found in filesystem", file.fileInfo.filenameWithoutExt) + + return False + + await asyncio.sleep(1) + + if torrent.failIfNotCached and count >= blackhole['waitForTorrentTimeout']: + print(f"Torrent timeout: {file.fileInfo.filenameWithoutExt} - {status}") + discordError("Torrent timeout", f"{file.fileInfo.filenameWithoutExt} - {status}") + + return False + async def processFile(file: TorrentFileInfo, arr: Arr, isRadarr): try: _print = globals()['print'] @@ -346,148 +269,24 @@ async def is_accessible(path, timeout=10): executor.shutdown(wait=False) with open(file.fileInfo.filePathProcessing, 'rb' if file.torrentInfo.isDotTorrentFile else 'r') as f: - def fail(torrent: TorrentBase, arr: Arr=arr): - print(f"Failing") - - history = arr.getHistory(blackhole['historyPageSize'])['records'] - items = (item for item in history if item['data'].get('torrentInfoHash', '').casefold() == torrent.getHash().casefold() or cleanFileName(item['sourceTitle'].casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()) - - if not items: - raise Exception("No history items found to cancel") - - for item in items: - # TODO: See if we can fail without blacklisting as cached items constantly changes - arr.failHistoryItem(item['id']) - print(f"Failed") + torrentConstructors = [] + if realdebrid['enabled']: + torrentConstructors.append(RealDebridTorrent if file.torrentInfo.isDotTorrentFile else RealDebridMagnet) + if torbox['enabled']: + torrentConstructors.append(TorboxTorrent if file.torrentInfo.isDotTorrentFile else TorboxMagnet) onlyLargestFile = isRadarr or bool(re.search(r'S[\d]{2}E[\d]{2}', file.fileInfo.filename)) - if file.torrentInfo.isDotTorrentFile: - torrent = Torrent(f, file, fail, blackhole['failIfNotCached'], onlyLargestFile) + if not blackhole['failIfNotCached']: + await asyncio.gather(*(processTorrent(constructor(f, file, blackhole['failIfNotCached'], onlyLargestFile), file, arr) for constructor in torrentConstructors)) else: - torrent = Magnet(f, file, fail, blackhole['failIfNotCached'], onlyLargestFile) - - if torrent.submitTorrent(): - count = 0 - while True: - count += 1 - info = torrent.getInfo(refresh=True) - status = info['status'] - - print('status:', status) - - if status == 'waiting_files_selection': - if not torrent.selectFiles(): - torrent.delete() - fail(torrent) - break - elif status == 'magnet_conversion' or status == 'queued' or status == 'downloading' or status == 'compressing' or status == 'uploading': - # Send progress to arr - progress = info['progress'] - print(progress) - if torrent.incompatibleHashSize and torrent.failIfNotCached: - print("Non-cached incompatible hash sized torrent") - torrent.delete() - fail(torrent) - break - await asyncio.sleep(1) - elif status == 'magnet_error' or status == 'error' or status == 'dead' or status == 'virus': - fail(torrent) - break - elif status == 'downloaded': - existsCount = 0 - print('Waiting for folders to refresh...') - - filename = info.get('filename') - originalFilename = info.get('original_filename') - - folderPathMountFilenameTorrent = os.path.join(blackhole['rdMountTorrentsPath'], filename) - folderPathMountOriginalFilenameTorrent = os.path.join(blackhole['rdMountTorrentsPath'], originalFilename) - folderPathMountOriginalFilenameWithoutExtTorrent = os.path.join(blackhole['rdMountTorrentsPath'], os.path.splitext(originalFilename)[0]) - - while existsCount <= blackhole['waitForTorrentTimeout']: - existsCount += 1 - - if os.path.exists(folderPathMountFilenameTorrent) and os.listdir(folderPathMountFilenameTorrent): - folderPathMountTorrent = folderPathMountFilenameTorrent - elif os.path.exists(folderPathMountOriginalFilenameTorrent) and os.listdir(folderPathMountOriginalFilenameTorrent): - folderPathMountTorrent = folderPathMountOriginalFilenameTorrent - elif (originalFilename.endswith(('.mkv', '.mp4')) and - os.path.exists(folderPathMountOriginalFilenameWithoutExtTorrent) and os.listdir(folderPathMountOriginalFilenameWithoutExtTorrent)): - folderPathMountTorrent = folderPathMountOriginalFilenameWithoutExtTorrent - else: - folderPathMountTorrent = None - - if folderPathMountTorrent: - multiSeasonRegex1 = r'(?<=[\W_][Ss]eason[\W_])[\d][\W_][\d]{1,2}(?=[\W_])' - multiSeasonRegex2 = r'(?<=[\W_][Ss])[\d]{2}[\W_][Ss]?[\d]{2}(?=[\W_])' - multiSeasonRegexCombined = f'{multiSeasonRegex1}|{multiSeasonRegex2}' - - multiSeasonMatch = re.search(multiSeasonRegexCombined, file.fileInfo.filenameWithoutExt) - - for root, dirs, files in os.walk(folderPathMountTorrent): - relRoot = os.path.relpath(root, folderPathMountTorrent) - for filename in files: - # Check if the file is accessible - # if not await is_accessible(os.path.join(root, filename)): - # print(f"Timeout reached when accessing file: {filename}") - # discordError(f"Timeout reached when accessing file", filename) - # Uncomment the following line to fail the entire torrent if the timeout on any of its files are reached - # fail(torrent) - # return - - if multiSeasonMatch: - seasonMatch = re.search(r'S([\d]{2})E[\d]{2}', filename) - - if seasonMatch: - season = seasonMatch.group(1) - seasonShort = season[1:] if season[0] == '0' else season - - seasonFolderPathCompleted = re.sub(multiSeasonRegex1, seasonShort, file.fileInfo.folderPathCompleted) - seasonFolderPathCompleted = re.sub(multiSeasonRegex2, season, seasonFolderPathCompleted) - - os.makedirs(os.path.join(seasonFolderPathCompleted, relRoot), exist_ok=True) - os.symlink(os.path.join(root, filename), os.path.join(seasonFolderPathCompleted, relRoot, filename)) - print('Season Recursive:', f"{os.path.join(seasonFolderPathCompleted, relRoot, filename)} -> {os.path.join(root, filename)}") - # refreshEndpoint = f"{plex['serverHost']}/library/sections/{plex['serverMovieLibraryId'] if isRadarr else plex['serverTvShowLibraryId']}/refresh?path={urllib.parse.quote_plus(os.path.join(seasonFolderPathCompleted, relRoot))}&X-Plex-Token={plex['serverApiKey']}" - # cancelRefreshRequest = requests.delete(refreshEndpoint, headers={'Accept': 'application/json'}) - # refreshRequest = requests.get(refreshEndpoint, headers={'Accept': 'application/json'}) - - continue - - - os.makedirs(os.path.join(file.fileInfo.folderPathCompleted, relRoot), exist_ok=True) - os.symlink(os.path.join(root, filename), os.path.join(file.fileInfo.folderPathCompleted, relRoot, filename)) - print('Recursive:', f"{os.path.join(file.fileInfo.folderPathCompleted, relRoot, filename)} -> {os.path.join(root, filename)}") - # refreshEndpoint = f"{plex['serverHost']}/library/sections/{plex['serverMovieLibraryId'] if isRadarr else plex['serverTvShowLibraryId']}/refresh?path={urllib.parse.quote_plus(os.path.join(file.fileInfo.folderPathCompleted, relRoot))}&X-Plex-Token={plex['serverApiKey']}" - # cancelRefreshRequest = requests.delete(refreshEndpoint, headers={'Accept': 'application/json'}) - # refreshRequest = requests.get(refreshEndpoint, headers={'Accept': 'application/json'}) - - print('Refreshed') - discordUpdate(f"Sucessfully processed {file.fileInfo.filenameWithoutExt}", f"Now available for immediate consumption! existsCount: {existsCount}") - - # refreshEndpoint = f"{plex['serverHost']}/library/sections/{plex['serverMovieLibraryId'] if isRadarr else plex['serverTvShowLibraryId']}/refresh?X-Plex-Token={plex['serverApiKey']}" - # cancelRefreshRequest = requests.delete(refreshEndpoint, headers={'Accept': 'application/json'}) - # refreshRequest = requests.get(refreshEndpoint, headers={'Accept': 'application/json'}) - await refreshArr(arr) - - # await asyncio.get_running_loop().run_in_executor(None, copyFiles, file, folderPathMountTorrent, arr) - break - - if existsCount == blackhole['rdMountRefreshSeconds'] + 1: - print(f"Torrent folder not found in filesystem: {file.fileInfo.filenameWithoutExt}") - discordError("Torrent folder not found in filesystem", file.fileInfo.filenameWithoutExt) + for i, constructor in enumerate(torrentConstructors): + isLast = (i == len(torrentConstructors) - 1) + torrent = constructor(f, file, blackhole['failIfNotCached'], onlyLargestFile) - await asyncio.sleep(1) + if await processTorrent(torrent, file, arr): break - - if torrent.failIfNotCached: - if count == 21: - print('infoCount > 20') - discordError(f"{file.fileInfo.filenameWithoutExt} info attempt count > 20", status) - elif count == blackhole['waitForTorrentTimeout']: - print('infoCount == 60 - Failing') - fail(torrent) - break + elif isLast: + fail(torrent, arr) os.remove(file.fileInfo.filePathProcessing) except: @@ -498,6 +297,20 @@ def fail(torrent: TorrentBase, arr: Arr=arr): discordError(f"Error processing {file.fileInfo.filenameWithoutExt}", e) +def fail(torrent: TorrentBase, arr: Arr): + print(f"Failing") + + history = arr.getHistory(blackhole['historyPageSize'])['records'] + items = (item for item in history if item['data'].get('torrentInfoHash', '').casefold() == torrent.getHash().casefold() or cleanFileName(item['sourceTitle'].casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()) + + if not items: + raise Exception("No history items found to cancel") + + for item in items: + # TODO: See if we can fail without blacklisting as cached items constantly changes + arr.failHistoryItem(item['id']) + print(f"Failed") + def getFiles(isRadarr): print('getFiles') files = (TorrentFileInfo(filename, isRadarr) for filename in os.listdir(getPath(isRadarr)) if filename not in ['processing', 'completed']) diff --git a/delete_non_linked_folders.py b/delete_non_linked_folders.py index cdeef85..d9f5ebb 100644 --- a/delete_non_linked_folders.py +++ b/delete_non_linked_folders.py @@ -2,7 +2,7 @@ import argparse import shutil import traceback -from shared.shared import blackhole +from shared.shared import realdebrid def find_non_linked_files(src_folder, dst_folder, dry_run=False, no_confirm=False, only_delete_files=False): # Get the list of links in the dst_folder @@ -52,7 +52,7 @@ def find_non_linked_files(src_folder, dst_folder, dry_run=False, no_confirm=Fals if __name__ == '__main__': parser = argparse.ArgumentParser(description='Find and delete non-linked file directories.') parser.add_argument('dst_folder', type=str, help='Destination folder to check for non-linked files. WARNING: This folder must encompass ALL folders where symlinks may live otherwise folders will unintentionally be deleted') - parser.add_argument('--src-folder', type=str, default=blackhole.get('rdMountTorrentsPath'), help='Source folder to check for non-linked files') + parser.add_argument('--src-folder', type=str, default=realdebrid['mountTorrentsPath'], help='Source folder to check for non-linked files') parser.add_argument('--dry-run', action='store_true', help='print non-linked file directories without deleting') parser.add_argument('--no-confirm', action='store_true', help='delete non-linked file directories without confirmation') parser.add_argument('--only-delete-files', action='store_true', help='delete only the files in the non-linked directories') diff --git a/docker-compose.yml b/docker-compose.yml index d323952..054c2af 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,8 @@ services: environment: - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} volumes: - - ${BLACKHOLE_RD_MOUNT_TORRENTS_PATH}:${BLACKHOLE_RD_MOUNT_TORRENTS_PATH} + - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} + - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH}:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH}:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} profiles: [blackhole, blackhole_all, all] @@ -40,7 +41,8 @@ services: # - RADARR_API_KEY=${RADARR_API_KEY_4K} # - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} # volumes: - # - ${BLACKHOLE_RD_MOUNT_TORRENTS_PATH}:${BLACKHOLE_RD_MOUNT_TORRENTS_PATH} + # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} + # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} 4k:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} 4k:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} # profiles: [blackhole_4k, blackhole_all, all] @@ -55,7 +57,8 @@ services: # - RADARR_API_KEY=${RADARR_API_KEY_ANIME} # - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} # volumes: - # - ${BLACKHOLE_RD_MOUNT_TORRENTS_PATH}:${BLACKHOLE_RD_MOUNT_TORRENTS_PATH} + # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} + # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} anime:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} anime:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} # profiles: [blackhole_anime, blackhole_all, all] @@ -70,7 +73,8 @@ services: # - RADARR_API_KEY=${RADARR_API_KEY_MUX} # - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} # volumes: - # - ${BLACKHOLE_RD_MOUNT_TORRENTS_PATH}:${BLACKHOLE_RD_MOUNT_TORRENTS_PATH} + # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} + # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} mux:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} mux:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} # profiles: [blackhole_mux, blackhole_all, all] @@ -79,7 +83,8 @@ services: <<: *repair container_name: repair_service volumes: - - ${BLACKHOLE_RD_MOUNT_TORRENTS_PATH}:${BLACKHOLE_RD_MOUNT_TORRENTS_PATH} + - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} + - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} - ${SONARR_ROOT_FOLDER}:${SONARR_ROOT_FOLDER} - ${RADARR_ROOT_FOLDER}:${RADARR_ROOT_FOLDER} profiles: [repair, repair_all, all] @@ -93,7 +98,8 @@ services: # - RADARR_HOST=${RADARR_HOST_4K} # - RADARR_API_KEY=${RADARR_API_KEY_4K} # volumes: - # - ${BLACKHOLE_RD_MOUNT_TORRENTS_PATH}:${BLACKHOLE_RD_MOUNT_TORRENTS_PATH} + # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} + # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} # - ${SONARR_ROOT_FOLDER_4K}:${SONARR_ROOT_FOLDER} # - ${RADARR_ROOT_FOLDER_4K}:${RADARR_ROOT_FOLDER} # profiles: [repair_4k, repair_all, all] @@ -107,7 +113,8 @@ services: # - RADARR_HOST=${RADARR_HOST_ANIME} # - RADARR_API_KEY=${RADARR_API_KEY_ANIME} # volumes: - # - ${BLACKHOLE_RD_MOUNT_TORRENTS_PATH}:${BLACKHOLE_RD_MOUNT_TORRENTS_PATH} + # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} + # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} # - ${SONARR_ROOT_FOLDER_ANIME}:${SONARR_ROOT_FOLDER} # - ${RADARR_ROOT_FOLDER_ANIME}:${RADARR_ROOT_FOLDER} # profiles: [repair_anime, repair_all, all] @@ -121,7 +128,8 @@ services: # - RADARR_HOST=${RADARR_HOST_MUX} # - RADARR_API_KEY=${RADARR_API_KEY_MUX} # volumes: - # - ${BLACKHOLE_RD_MOUNT_TORRENTS_PATH}:${BLACKHOLE_RD_MOUNT_TORRENTS_PATH} + # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} + # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} # - ${SONARR_ROOT_FOLDER_MUX}:${SONARR_ROOT_FOLDER} # - ${RADARR_ROOT_FOLDER_MUX}:${RADARR_ROOT_FOLDER} # profiles: [repair_mux, repair_all, all] diff --git a/import_torrent_folder.py b/import_torrent_folder.py index 4cea576..a882296 100644 --- a/import_torrent_folder.py +++ b/import_torrent_folder.py @@ -1,9 +1,9 @@ import os import re import argparse -from shared.shared import blackhole +from shared.shared import blackhole, realdebrid -parentDirectory = blackhole['rdMountTorrentsPath'] +parentDirectory = realdebrid['mountTorrentsPath'] def get_completed_parent_directory(args): if args.symlink_directory: diff --git a/repair.py b/repair.py index ee80ca4..2d8aa42 100644 --- a/repair.py +++ b/repair.py @@ -3,7 +3,7 @@ import time from shared.arr import Sonarr, Radarr from shared.discord import discordUpdate -from shared.shared import repair, blackhole, intersperse +from shared.shared import repair, realdebrid, torbox, intersperse def parse_interval(interval_str): """Parse a smart interval string (e.g., '1w2d3h4m5s') into seconds.""" @@ -68,11 +68,14 @@ def main(): for childFile in childFiles: fullPath = childFile.path + destinationPath = os.readlink(fullPath) realPath = os.path.realpath(fullPath) realPaths.append(realPath) - if os.path.islink(fullPath) and realPath.startswith(blackhole['rdMountTorrentsPath']) and not os.path.exists(realPath): - brokenSymlinks.append(realPath) + if os.path.islink(fullPath): + if ((realdebrid['enabled'] and destinationPath.startswith(realdebrid['mountTorrentsPath']) and not os.path.destinationPath(realPath)) or + (torbox['enabled'] and destinationPath.startswith(torbox['mountTorrentsPath']) and not os.path.destinationPath(realPath))): + brokenSymlinks.append(realPath) # If not full season just repair individual episodes? if brokenSymlinks: diff --git a/shared/debrid.py b/shared/debrid.py new file mode 100644 index 0000000..359816e --- /dev/null +++ b/shared/debrid.py @@ -0,0 +1,392 @@ +import os +import re +import hashlib +import bencode3 +import requests +from abc import ABC, abstractmethod +from urllib.parse import urljoin +from shared.discord import discordUpdate +from shared.shared import realdebrid, torbox, mediaExtensions, checkRequiredEnvs +from werkzeug.utils import cached_property + +def validateDebridEnabled(): + if not realdebrid['enabled'] and not torbox['enabled']: + return False, "At least one of RealDebrid or Torbox must be enabled." + return True + +def validateRealdebridHost(): + url = urljoin(realdebrid['host'], "time") + try: + response = requests.get(url) + return response.status_code == 200 + except Exception as e: + return False + +def validateRealdebridApiKey(): + url = urljoin(realdebrid['host'], "user") + headers = {'Authorization': f'Bearer {realdebrid["apiKey"]}'} + try: + response = requests.get(url, headers=headers) + + if response.status_code == 401: + return False, "Invalid or expired API key." + elif response.status_code == 403: + return False, "Permission denied, account locked." + except Exception as e: + return False + + return True + +def validateRealdebridMountTorrentsPath(): + path = realdebrid['mountTorrentsPath'] + if os.path.exists(path) and any(os.path.isdir(os.path.join(path, child)) for child in os.listdir(path)): + return True + else: + return False, "Path does not exist or has no children." + +def validateTorboxHost(): + url = urljoin(torbox['host'], "stats") + try: + response = requests.get(url) + return response.status_code == 200 + except Exception as e: + return False + +def validateTorboxApiKey(): + url = urljoin(torbox['host'], "user/me") + headers = {'Authorization': f'Bearer {torbox["apiKey"]}'} + try: + response = requests.get(url, headers=headers) + + if response.status_code == 401: + return False, "Invalid or expired API key." + elif response.status_code == 403: + return False, "Permission denied, account locked." + except Exception as e: + return False + + return True + +def validateTorboxMountTorrentsPath(): + path = torbox['mountTorrentsPath'] + if os.path.exists(path) and any(os.path.isdir(os.path.join(path, child)) for child in os.listdir(path)): + return True + else: + return False, "Path does not exist or has no children." + +requiredEnvs = { + 'RealDebrid/TorBox enabled': (True, validateDebridEnabled), +} + +if realdebrid['enabled']: + requiredEnvs.update({ + 'RealDebrid host': (realdebrid['host'], validateRealdebridHost), + 'RealDebrid API key': (realdebrid['apiKey'], validateRealdebridApiKey, True), + 'RealDebrid mount torrents path': (realdebrid['mountTorrentsPath'], validateRealdebridMountTorrentsPath) + }) + +if torbox['enabled']: + requiredEnvs.update({ + 'Torbox host': (torbox['host'], validateTorboxHost), + 'Torbox API key': (torbox['apiKey'], validateTorboxApiKey, True), + 'Torbox mount torrents path': (torbox['mountTorrentsPath'], validateTorboxMountTorrentsPath) + }) + +checkRequiredEnvs(requiredEnvs) + +class TorrentBase(ABC): + STATUS_WAITING_FILES_SELECTION = 'waiting_files_selection' + STATUS_DOWNLOADING = 'downloading' + STATUS_COMPLETED = 'completed' + STATUS_ERROR = 'error' + + def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: + super().__init__() + self.f = f + self.file = file + self.failIfNotCached = failIfNotCached + self.onlyLargestFile = onlyLargestFile + self.incompatibleHashSize = False + self.id = None + self._info = None + self._hash = None + self._instantAvailability = None + + def print(self, *values: object): + print(f"[{self.__class__.__name__} - {self.file.fileInfo.filenameWithoutExt}]", *values) + + @cached_property + def fileData(self): + fileData = self.f.read() + self.f.seek(0) + return fileData + + @abstractmethod + def submitTorrent(self): + pass + + @abstractmethod + def getHash(self): + pass + + @abstractmethod + def addTorrent(self): + pass + + @abstractmethod + def getInfo(self, refresh=False): + pass + + @abstractmethod + def selectFiles(self): + pass + + @abstractmethod + def delete(self): + pass + + def _addTorrentFile(self): + pass + + def _addMagnetFile(self): + pass + + def _enforceId(self): + if not self.id: + raise Exception("Id is required. Must be acquired via successfully running submitTorrent() first.") + +class RealDebrid(TorrentBase): + def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: + super().__init__(f, file, failIfNotCached, onlyLargestFile) + self.headers = {'Authorization': f'Bearer {realdebrid["apiKey"]}'} + self.mountTorrentsPath = realdebrid["mountTorrentsPath"] + + def submitTorrent(self): + if self.failIfNotCached: + instantAvailability = self._getInstantAvailability() + self.print('instantAvailability:', not not instantAvailability) + if not instantAvailability: + return False + + self.addTorrent() + return True + + def _getInstantAvailability(self, refresh=False): + if refresh or not self._instantAvailability: + torrentHash = self.getHash() + self.print('hash:', torrentHash) + + if len(torrentHash) != 40: + self.incompatibleHashSize = True + return True + + instantAvailabilityRequest = requests.get(urljoin(realdebrid['host'], f"torrents/instantAvailability/{torrentHash}"), headers=self.headers) + instantAvailabilities = instantAvailabilityRequest.json() + self.print('instantAvailabilities:', instantAvailabilities) + instantAvailabilityHosters = next(iter(instantAvailabilities.values())) + if not instantAvailabilityHosters: return + + self._instantAvailability = next(iter(instantAvailabilityHosters.values())) + + return self._instantAvailability + + def _getAvailableHost(self): + availableHostsRequest = requests.get(urljoin(realdebrid['host'], "torrents/availableHosts"), headers=self.headers) + availableHosts = availableHostsRequest.json() + return availableHosts[0]['host'] + + def getInfo(self, refresh=False): + self._enforceId() + + if refresh or not self._info: + infoRequest = requests.get(urljoin(realdebrid['host'], f"torrents/info/{self.id}"), headers=self.headers) + info = infoRequest.json() + info['status'] = self._normalize_status(info['status']) + self._info = info + + return self._info + + def selectFiles(self): + self._enforceId() + + info = self.getInfo() + self.print('files:', info['files']) + mediaFiles = [file for file in info['files'] if os.path.splitext(file['path'])[1].lower() in mediaExtensions] + + if not mediaFiles: + self.print('no media files found') + return False + + mediaFileIds = {str(file['id']) for file in mediaFiles} + self.print('required fileIds:', mediaFileIds) + + largestMediaFile = max(mediaFiles, key=lambda file: file['bytes']) + largestMediaFileId = str(largestMediaFile['id']) + self.print('only largest file:', self.onlyLargestFile) + self.print('largest file:', largestMediaFile) + + if self.failIfNotCached and not self.incompatibleHashSize: + targetFileIds = {largestMediaFileId} if self.onlyLargestFile else mediaFileIds + if not any(set(fileGroup.keys()) == targetFileIds for fileGroup in self._instantAvailability): + extraFilesGroup = next((fileGroup for fileGroup in self._instantAvailability if largestMediaFileId in fileGroup.keys()), None) + if self.onlyLargestFile and extraFilesGroup: + self.print('extra files required for cache:', extraFilesGroup) + discordUpdate('Extra files required for cache:', extraFilesGroup) + return False + + if self.onlyLargestFile and len(mediaFiles) > 1: + discordUpdate('largest file:', largestMediaFile['path']) + + files = {'files': [largestMediaFileId] if self.onlyLargestFile else ','.join(mediaFileIds)} + selectFilesRequest = requests.post(urljoin(realdebrid['host'], f"torrents/selectFiles/{self.id}"), headers=self.headers, data=files) + + return True + + def delete(self): + self._enforceId() + + deleteRequest = requests.delete(urljoin(realdebrid['host'], f"torrents/delete/{self.id}"), headers=self.headers) + + def _addFile(self, endpoint, data): + host = self._getAvailableHost() + + request = requests.post(urljoin(realdebrid['host'], endpoint), params={'host': host}, headers=self.headers, data=data) + response = request.json() + + self.print('response info:', response) + self.id = response['id'] + + return self.id + + def _addTorrentFile(self): + return self._addFile("torrents/addTorrent", self.f) + + def _addMagnetFile(self): + return self._addFile("torrents/addMagnet", {'magnet': self.fileData}) + + def _normalize_status(self, status): + if status in ['waiting_files_selection']: + return self.STATUS_WAITING_FILES_SELECTION + elif status in ['magnet_conversion', 'queued', 'downloading', 'compressing', 'uploading']: + return self.STATUS_DOWNLOADING + elif status == 'downloaded': + return self.STATUS_COMPLETED + elif status in ['magnet_error', 'error', 'dead', 'virus']: + return self.STATUS_ERROR + return status + +class Torbox(TorrentBase): + def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: + super().__init__(f, file, failIfNotCached, onlyLargestFile) + self.headers = {'Authorization': f'Bearer {torbox["apiKey"]}'} + self.mountTorrentsPath = torbox["mountTorrentsPath"] + + def submitTorrent(self): + if self.failIfNotCached: + instantAvailability = self._getInstantAvailability() + self.print('instantAvailability:', not not instantAvailability) + if not instantAvailability: + self.fail(self) + return False + + self.addTorrent() + return True + + def _getInstantAvailability(self, refresh=False): + if refresh or not self._instantAvailability: + torrentHash = self.getHash() + self.print('hash:', torrentHash) + + instantAvailabilityRequest = requests.get( + urljoin(torbox['host'], "torrents/checkcached"), + headers=self.headers, + params={'hash': torrentHash, 'format': 'object'} + ) + instantAvailabilities = instantAvailabilityRequest.json() + self.print('instantAvailabilities:', instantAvailabilities) + self._instantAvailability = instantAvailabilities['data'] + + return self._instantAvailability + + def getInfo(self, refresh=False): + self._enforceId() + + if refresh or not self._info: + infoRequest = requests.get(urljoin(torbox['host'], "torrents/mylist"), headers=self.headers) + torrents = infoRequest.json()['data'] + for torrent in torrents: + if torrent['id'] == self.id: + torrent['status'] = self._normalize_status(torrent['download_state'], torrent['download_finished']) + self._info = torrent + break + + return self._info + + def selectFiles(self): + pass + + def delete(self): + self._enforceId() + + deleteRequest = requests.delete(urljoin(torbox['host'], "torrents/controltorrent"), headers=self.headers, data={'torrent_id': self.id, 'operation': "Delete"}) + + def _addFile(self, endpoint, data=None, files=None): + request = requests.post(urljoin(torbox['host'], endpoint), headers=self.headers, data=data, files=files) + + response = request.json() + self.print('response info:', response) + self.id = response['data']['torrent_id'] + + return self.id + + def _addTorrentFile(self): + nametorrent = self.f.name.split('/')[-1] + files = {'file': (nametorrent, self.f, 'application/x-bittorrent')} + return self._addFile("/torrents/createtorrent", files=files) + + def _addMagnetFile(self): + return self._addFile("/torrents/createtorrent", data={'magnet': self.fileData}) + + def _normalize_status(self, status, download_finished): + if download_finished: + return self.STATUS_COMPLETED + elif status in ['paused', 'downloading', 'uploading']: + return self.STATUS_DOWNLOADING + elif status in ['error', 'stalled (no seeds)']: + return self.STATUS_ERROR + return status + +class Torrent(TorrentBase): + def getHash(self): + + if not self._hash: + self._hash = hashlib.sha1(bencode3.bencode(bencode3.bdecode(self.fileData)['info'])).hexdigest() + + return self._hash + + def addTorrent(self): + self._addTorrentFile() + +class Magnet(TorrentBase): + def getHash(self): + + if not self._hash: + # Consider changing when I'm more familiar with hashes + self._hash = re.search('xt=urn:btih:(.+?)(?:&|$)', self.fileData).group(1) + + return self._hash + + def addTorrent(self): + self._addMagnetFile() + +class RealDebridTorrent(RealDebrid, Torrent): + pass + +class RealDebridMagnet(RealDebrid, Magnet): + pass + +class TorboxTorrent(Torbox, Torrent): + pass + +class TorboxMagnet(Torbox, Magnet): + pass \ No newline at end of file diff --git a/shared/shared.py b/shared/shared.py index c97cdeb..9cee2b6 100644 --- a/shared/shared.py +++ b/shared/shared.py @@ -20,7 +20,6 @@ def stringEnvParser(value): } blackhole = { - 'rdMountTorrentsPath': env.string('BLACKHOLE_RD_MOUNT_TORRENTS_PATH', default=None), 'baseWatchPath': env.string('BLACKHOLE_BASE_WATCH_PATH', default=None), 'radarrPath': env.string('BLACKHOLE_RADARR_PATH', default=None), 'sonarrPath': env.string('BLACKHOLE_SONARR_PATH', default=None), @@ -65,8 +64,17 @@ def stringEnvParser(value): } realdebrid = { + 'enabled': env.bool('REALDEBRID_ENABLED', default=True), 'host': env.string('REALDEBRID_HOST', default=None), - 'apiKey': env.string('REALDEBRID_API_KEY', default=None) + 'apiKey': env.string('REALDEBRID_API_KEY', default=None), + 'mountTorrentsPath': env.string('REALDEBRID_MOUNT_TORRENTS_PATH', env.string('BLACKHOLE_RD_MOUNT_TORRENTS_PATH', default=None)) +} + +torbox = { + 'enabled': env.bool('TORBOX_ENABLED', default=None), + 'host': env.string('TORBOX_HOST', default=None), + 'apiKey': env.string('TORBOX_API_KEY', default=None), + 'mountTorrentsPath': env.string('TORBOX_MOUNT_TORRENTS_PATH', default=None) } trakt = { From 2af3e0d8b8dc7ad1dfe42a2ef08b4bce625e90d2 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Thu, 23 May 2024 19:53:03 -0400 Subject: [PATCH 02/33] Fixes --- shared/debrid.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/shared/debrid.py b/shared/debrid.py index 359816e..0b8aee3 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -286,7 +286,6 @@ def submitTorrent(self): instantAvailability = self._getInstantAvailability() self.print('instantAvailability:', not not instantAvailability) if not instantAvailability: - self.fail(self) return False self.addTorrent() @@ -304,8 +303,8 @@ def _getInstantAvailability(self, refresh=False): ) instantAvailabilities = instantAvailabilityRequest.json() self.print('instantAvailabilities:', instantAvailabilities) - self._instantAvailability = instantAvailabilities['data'] - + self._instantAvailability = instantAvailabilities['data']['data'] if 'data' in instantAvailabilities and 'data' in instantAvailabilities['data'] and instantAvailabilities['data']['data'] is not False else None + return self._instantAvailability def getInfo(self, refresh=False): From ab9e5d62e81008d3b2f58687f6003a6146f0e3d7 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Thu, 23 May 2024 19:55:19 -0400 Subject: [PATCH 03/33] Missed one --- shared/debrid.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/debrid.py b/shared/debrid.py index 0b8aee3..df9a67b 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -341,10 +341,10 @@ def _addFile(self, endpoint, data=None, files=None): def _addTorrentFile(self): nametorrent = self.f.name.split('/')[-1] files = {'file': (nametorrent, self.f, 'application/x-bittorrent')} - return self._addFile("/torrents/createtorrent", files=files) + return self._addFile("torrents/createtorrent", files=files) def _addMagnetFile(self): - return self._addFile("/torrents/createtorrent", data={'magnet': self.fileData}) + return self._addFile("torrents/createtorrent", data={'magnet': self.fileData}) def _normalize_status(self, status, download_finished): if download_finished: From aa1996a17926926bc50bcb3df1eb64f9d0017ca1 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Thu, 23 May 2024 20:23:54 -0400 Subject: [PATCH 04/33] Add more statuses --- shared/debrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/debrid.py b/shared/debrid.py index df9a67b..9353c57 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -349,7 +349,7 @@ def _addMagnetFile(self): def _normalize_status(self, status, download_finished): if download_finished: return self.STATUS_COMPLETED - elif status in ['paused', 'downloading', 'uploading']: + elif status in ['paused', 'downloading', 'uploading', 'checkingResumeData']: return self.STATUS_DOWNLOADING elif status in ['error', 'stalled (no seeds)']: return self.STATUS_ERROR From 8d341fc392a4ebac0ed6ca588f33641ee8fee50e Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Thu, 23 May 2024 20:29:51 -0400 Subject: [PATCH 05/33] Really added the statues this time --- shared/debrid.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/shared/debrid.py b/shared/debrid.py index 9353c57..f165266 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -349,9 +349,14 @@ def _addMagnetFile(self): def _normalize_status(self, status, download_finished): if download_finished: return self.STATUS_COMPLETED - elif status in ['paused', 'downloading', 'uploading', 'checkingResumeData']: + elif status in [ + 'paused', 'downloading', 'uploading', 'checkingResumeData', 'metaDL', + 'pausedUP', 'queuedUP', 'checkingUP', 'forcedUP', + 'allocating', 'downloading', 'metaDL', 'pausedDL', 'queuedDL', + 'checkingDL', 'forcedDL', 'checkingResumeData', 'moving' + ]: return self.STATUS_DOWNLOADING - elif status in ['error', 'stalled (no seeds)']: + elif status in ['error', 'stalledUP', 'stalledDL', 'stalled (no seeds)', 'missingFiles']: return self.STATUS_ERROR return status From d56c72097b5d9be5f562139e543f5903295d52b6 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Thu, 23 May 2024 20:35:01 -0400 Subject: [PATCH 06/33] Update print --- shared/debrid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shared/debrid.py b/shared/debrid.py index f165266..bc9749d 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -5,6 +5,7 @@ import requests from abc import ABC, abstractmethod from urllib.parse import urljoin +from datetime import datetime from shared.discord import discordUpdate from shared.shared import realdebrid, torbox, mediaExtensions, checkRequiredEnvs from werkzeug.utils import cached_property @@ -113,7 +114,7 @@ def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: self._instantAvailability = None def print(self, *values: object): - print(f"[{self.__class__.__name__} - {self.file.fileInfo.filenameWithoutExt}]", *values) + print(f"[{datetime.now()}][{self.__class__.__name__}][{self.file.fileInfo.filenameWithoutExt}]", *values) @cached_property def fileData(self): From 925cc74d4868c4e4d43810f963111a541c364428 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 10:30:53 -0400 Subject: [PATCH 07/33] Fix repair and limit refreshArr loop to only run once at a time --- blackhole.py | 20 +++++++++++++++++--- repair.py | 5 +++-- shared/debrid.py | 8 ++++---- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/blackhole.py b/blackhole.py index 814120e..9f16016 100644 --- a/blackhole.py +++ b/blackhole.py @@ -81,11 +81,25 @@ def cleanFileName(name): return result.strip() +refreshingTask = None + async def refreshArr(arr: Arr, count=60): # TODO: Change to refresh until found/imported - for _ in range(count): - arr.refreshMonitoredDownloads() - await asyncio.sleep(1) + async def refresh(): + for _ in range(count): + arr.refreshMonitoredDownloads() + await asyncio.sleep(1) + + global refreshingTask + if refreshingTask and not refreshingTask.done(): + print("Refresh already in progress, restarting...") + refreshingTask.cancel() + + refreshingTask = asyncio.createTask(refresh()) + try: + await refreshingTask + except asyncio.CancelledError: + pass def copyFiles(file: TorrentFileInfo, folderPathMountTorrent, arr: Arr): # Consider removing this and always streaming diff --git a/repair.py b/repair.py index 2d8aa42..75d4f0a 100644 --- a/repair.py +++ b/repair.py @@ -1,6 +1,7 @@ import os import argparse import time +import shared.debrid # Run validation from shared.arr import Sonarr, Radarr from shared.discord import discordUpdate from shared.shared import repair, realdebrid, torbox, intersperse @@ -73,8 +74,8 @@ def main(): realPaths.append(realPath) if os.path.islink(fullPath): - if ((realdebrid['enabled'] and destinationPath.startswith(realdebrid['mountTorrentsPath']) and not os.path.destinationPath(realPath)) or - (torbox['enabled'] and destinationPath.startswith(torbox['mountTorrentsPath']) and not os.path.destinationPath(realPath))): + if ((realdebrid['enabled'] and destinationPath.startswith(realdebrid['mountTorrentsPath']) and not os.path.exists(destinationPath)) or + (torbox['enabled'] and destinationPath.startswith(torbox['mountTorrentsPath']) and not os.path.exists(realPath))): brokenSymlinks.append(realPath) # If not full season just repair individual episodes? diff --git a/shared/debrid.py b/shared/debrid.py index bc9749d..edbf231 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -351,10 +351,10 @@ def _normalize_status(self, status, download_finished): if download_finished: return self.STATUS_COMPLETED elif status in [ - 'paused', 'downloading', 'uploading', 'checkingResumeData', 'metaDL', - 'pausedUP', 'queuedUP', 'checkingUP', 'forcedUP', - 'allocating', 'downloading', 'metaDL', 'pausedDL', 'queuedDL', - 'checkingDL', 'forcedDL', 'checkingResumeData', 'moving' + 'completed', 'cached', 'paused', 'downloading', 'uploading', + 'checkingResumeData', 'metaDL', 'pausedUP', 'queuedUP', 'checkingUP', + 'forcedUP', 'allocating', 'downloading', 'metaDL', 'pausedDL', + 'queuedDL', 'checkingDL', 'forcedDL', 'checkingResumeData', 'moving' ]: return self.STATUS_DOWNLOADING elif status in ['error', 'stalledUP', 'stalledDL', 'stalled (no seeds)', 'missingFiles']: From 975aeacbf207fee07230e6617126813521f019e3 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 12:29:13 -0400 Subject: [PATCH 08/33] Update torrent info regularly for the first 5 minutes --- blackhole.py | 2 +- shared/debrid.py | 35 ++++++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/blackhole.py b/blackhole.py index 9f16016..a444876 100644 --- a/blackhole.py +++ b/blackhole.py @@ -142,7 +142,7 @@ async def processTorrent(torrent: TorrentBase, file: TorrentFileInfo, arr: Arr) count = 0 while True: count += 1 - info = torrent.getInfo(refresh=True) + info = await torrent.getInfo(refresh=True) status = info['status'] print('status:', status) diff --git a/shared/debrid.py b/shared/debrid.py index edbf231..8aba97c 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -1,3 +1,4 @@ +import asyncio import os import re import hashlib @@ -135,7 +136,7 @@ def addTorrent(self): pass @abstractmethod - def getInfo(self, refresh=False): + async def getInfo(self, refresh=False): pass @abstractmethod @@ -196,7 +197,7 @@ def _getAvailableHost(self): availableHosts = availableHostsRequest.json() return availableHosts[0]['host'] - def getInfo(self, refresh=False): + async def getInfo(self, refresh=False): self._enforceId() if refresh or not self._info: @@ -282,6 +283,10 @@ def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: self.headers = {'Authorization': f'Bearer {torbox["apiKey"]}'} self.mountTorrentsPath = torbox["mountTorrentsPath"] + userInfoRequest = requests.get(urljoin(torbox['host'], "user/me"), headers=self.headers) + userInfo = userInfoRequest.json() + self.authId = userInfo['data']['auth_id'] + def submitTorrent(self): if self.failIfNotCached: instantAvailability = self._getInstantAvailability() @@ -290,6 +295,7 @@ def submitTorrent(self): return False self.addTorrent() + self.submittedTime = datetime return True def _getInstantAvailability(self, refresh=False): @@ -308,18 +314,25 @@ def _getInstantAvailability(self, refresh=False): return self._instantAvailability - def getInfo(self, refresh=False): + async def getInfo(self, refresh=False): self._enforceId() if refresh or not self._info: - infoRequest = requests.get(urljoin(torbox['host'], "torrents/mylist"), headers=self.headers) - torrents = infoRequest.json()['data'] - for torrent in torrents: - if torrent['id'] == self.id: - torrent['status'] = self._normalize_status(torrent['download_state'], torrent['download_finished']) - self._info = torrent - break - + if (datetime.now() - self.submittedTime).total_seconds() < 300: + inactiveCheckUrl = f"https://relay.torbox.app/v1/inactivecheck/torrent/{self.authId}/{self.id}" + requests.get(inactiveCheckUrl) + + for _ in range(10): + infoRequest = requests.get(urljoin(torbox['host'], "torrents/mylist"), headers=self.headers) + torrents = infoRequest.json()['data'] + + for torrent in torrents: + if torrent['id'] == self.id: + torrent['status'] = self._normalize_status(torrent['download_state'], torrent['download_finished']) + self._info = torrent + return self._info + + await asyncio.sleep(1) return self._info def selectFiles(self): From 12d84b457b03ee732dcbe688984b36c5a9695d4b Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 12:36:49 -0400 Subject: [PATCH 09/33] Fixes --- blackhole.py | 2 +- shared/debrid.py | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/blackhole.py b/blackhole.py index a444876..ba9a599 100644 --- a/blackhole.py +++ b/blackhole.py @@ -148,7 +148,7 @@ async def processTorrent(torrent: TorrentBase, file: TorrentFileInfo, arr: Arr) print('status:', status) if status == torrent.STATUS_WAITING_FILES_SELECTION: - if not torrent.selectFiles(): + if not await torrent.selectFiles(): torrent.delete() return False elif status == torrent.STATUS_DOWNLOADING: diff --git a/shared/debrid.py b/shared/debrid.py index 8aba97c..cccbcab 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -140,7 +140,7 @@ async def getInfo(self, refresh=False): pass @abstractmethod - def selectFiles(self): + async def selectFiles(self): pass @abstractmethod @@ -208,10 +208,10 @@ async def getInfo(self, refresh=False): return self._info - def selectFiles(self): + async def selectFiles(self): self._enforceId() - info = self.getInfo() + info = await self.getInfo() self.print('files:', info['files']) mediaFiles = [file for file in info['files'] if os.path.splitext(file['path'])[1].lower() in mediaExtensions] @@ -282,6 +282,8 @@ def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: super().__init__(f, file, failIfNotCached, onlyLargestFile) self.headers = {'Authorization': f'Bearer {torbox["apiKey"]}'} self.mountTorrentsPath = torbox["mountTorrentsPath"] + self.submittedTime = None + self.lastInactiveCheck = None userInfoRequest = requests.get(urljoin(torbox['host'], "user/me"), headers=self.headers) userInfo = userInfoRequest.json() @@ -318,9 +320,12 @@ async def getInfo(self, refresh=False): self._enforceId() if refresh or not self._info: - if (datetime.now() - self.submittedTime).total_seconds() < 300: - inactiveCheckUrl = f"https://relay.torbox.app/v1/inactivecheck/torrent/{self.authId}/{self.id}" - requests.get(inactiveCheckUrl) + currentTime = datetime.now() + if (currentTime - self.submittedTime).total_seconds() < 300: + if not self.lastInactiveCheck or (currentTime - self.lastInactiveCheck).total_seconds() > 5: + inactiveCheckUrl = f"https://relay.torbox.app/v1/inactivecheck/torrent/{self.authId}/{self.id}" + requests.get(inactiveCheckUrl) + self.lastInactiveCheck = currentTime for _ in range(10): infoRequest = requests.get(urljoin(torbox['host'], "torrents/mylist"), headers=self.headers) @@ -335,7 +340,7 @@ async def getInfo(self, refresh=False): await asyncio.sleep(1) return self._info - def selectFiles(self): + async def selectFiles(self): pass def delete(self): From d8f7028b580c2c59c53024cffd3898b122ddf47a Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 14:41:46 -0400 Subject: [PATCH 10/33] Fix --- shared/debrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/debrid.py b/shared/debrid.py index cccbcab..781a6a5 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -297,7 +297,7 @@ def submitTorrent(self): return False self.addTorrent() - self.submittedTime = datetime + self.submittedTime = datetime.now() return True def _getInstantAvailability(self, refresh=False): From 528393c700709a2ebf8252411687ef23ab5ce14b Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 14:47:08 -0400 Subject: [PATCH 11/33] Print changes --- blackhole.py | 2 +- shared/debrid.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/blackhole.py b/blackhole.py index ba9a599..1e51ab8 100644 --- a/blackhole.py +++ b/blackhole.py @@ -154,7 +154,7 @@ async def processTorrent(torrent: TorrentBase, file: TorrentFileInfo, arr: Arr) elif status == torrent.STATUS_DOWNLOADING: # Send progress to arr progress = info['progress'] - print(progress) + print(f"Progress: {progress:.2f}%") if torrent.incompatibleHashSize and torrent.failIfNotCached: print("Non-cached incompatible hash sized torrent") torrent.delete() diff --git a/shared/debrid.py b/shared/debrid.py index 781a6a5..7f07129 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -115,7 +115,7 @@ def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: self._instantAvailability = None def print(self, *values: object): - print(f"[{datetime.now()}][{self.__class__.__name__}][{self.file.fileInfo.filenameWithoutExt}]", *values) + print(f"[{datetime.now()}] [{self.__class__.__name__}] [{self.file.fileInfo.filenameWithoutExt}]", *values) @cached_property def fileData(self): From 846d81c54da2c37d37cfdcf645eb1712564e8482 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 14:55:25 -0400 Subject: [PATCH 12/33] More print --- blackhole.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/blackhole.py b/blackhole.py index 1e51ab8..9027735 100644 --- a/blackhole.py +++ b/blackhole.py @@ -104,7 +104,7 @@ async def refresh(): def copyFiles(file: TorrentFileInfo, folderPathMountTorrent, arr: Arr): # Consider removing this and always streaming try: - _print = globals()['print'] + _print = print def print(*values: object): _print(f"[{file.fileInfo.filenameWithoutExt}]", *values) @@ -136,6 +136,11 @@ def print(*values: object): import signal async def processTorrent(torrent: TorrentBase, file: TorrentFileInfo, arr: Arr) -> bool: + _print = print + + def print(*values: object): + _print(f"[{torrent.__class__.__name__}] [{file.fileInfo.filenameWithoutExt}]", *values) + if not torrent.submitTorrent(): return False @@ -258,7 +263,7 @@ async def processTorrent(torrent: TorrentBase, file: TorrentFileInfo, arr: Arr) async def processFile(file: TorrentFileInfo, arr: Arr, isRadarr): try: - _print = globals()['print'] + _print = print def print(*values: object): _print(f"[{file.fileInfo.filenameWithoutExt}]", *values) @@ -312,6 +317,12 @@ async def is_accessible(path, timeout=10): discordError(f"Error processing {file.fileInfo.filenameWithoutExt}", e) def fail(torrent: TorrentBase, arr: Arr): + _print = print + + def print(*values: object): + _print(f"[{torrent.__class__.__name__}] [{torrent.file.fileInfo.filenameWithoutExt}]", *values) + + print(f"Failing") history = arr.getHistory(blackhole['historyPageSize'])['records'] From da35f1968314eb422a4387ce40b5d75ea6230f1f Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 14:57:55 -0400 Subject: [PATCH 13/33] More print fixes --- blackhole.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/blackhole.py b/blackhole.py index 9027735..f992e92 100644 --- a/blackhole.py +++ b/blackhole.py @@ -104,7 +104,7 @@ async def refresh(): def copyFiles(file: TorrentFileInfo, folderPathMountTorrent, arr: Arr): # Consider removing this and always streaming try: - _print = print + _print = globals()['print'] def print(*values: object): _print(f"[{file.fileInfo.filenameWithoutExt}]", *values) @@ -136,7 +136,7 @@ def print(*values: object): import signal async def processTorrent(torrent: TorrentBase, file: TorrentFileInfo, arr: Arr) -> bool: - _print = print + _print = globals()['print'] def print(*values: object): _print(f"[{torrent.__class__.__name__}] [{file.fileInfo.filenameWithoutExt}]", *values) @@ -263,7 +263,7 @@ def print(*values: object): async def processFile(file: TorrentFileInfo, arr: Arr, isRadarr): try: - _print = print + _print = globals()['print'] def print(*values: object): _print(f"[{file.fileInfo.filenameWithoutExt}]", *values) @@ -317,7 +317,7 @@ async def is_accessible(path, timeout=10): discordError(f"Error processing {file.fileInfo.filenameWithoutExt}", e) def fail(torrent: TorrentBase, arr: Arr): - _print = print + _print = globals()['print'] def print(*values: object): _print(f"[{torrent.__class__.__name__}] [{torrent.file.fileInfo.filenameWithoutExt}]", *values) From 2b4da8a261e6e1980de92f0ed2f90b1a33a71a6f Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 18:56:57 -0400 Subject: [PATCH 14/33] Fix torrent folder path detection --- blackhole.py | 20 ++------------------ shared/debrid.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/blackhole.py b/blackhole.py index f992e92..aeced67 100644 --- a/blackhole.py +++ b/blackhole.py @@ -171,26 +171,10 @@ def print(*values: object): existsCount = 0 print('Waiting for folders to refresh...') - filename = info.get('filename') - originalFilename = info.get('original_filename') - - folderPathMountFilenameTorrent = os.path.join(torrent.mountTorrentsPath, filename) - folderPathMountOriginalFilenameTorrent = os.path.join(torrent.mountTorrentsPath, originalFilename) - folderPathMountOriginalFilenameWithoutExtTorrent = os.path.join(torrent.mountTorrentsPath, os.path.splitext(originalFilename)[0]) - while True: existsCount += 1 - - if os.path.exists(folderPathMountFilenameTorrent) and os.listdir(folderPathMountFilenameTorrent): - folderPathMountTorrent = folderPathMountFilenameTorrent - elif os.path.exists(folderPathMountOriginalFilenameTorrent) and os.listdir(folderPathMountOriginalFilenameTorrent): - folderPathMountTorrent = folderPathMountOriginalFilenameTorrent - elif (originalFilename.endswith(('.mkv', '.mp4')) and - os.path.exists(folderPathMountOriginalFilenameWithoutExtTorrent) and os.listdir(folderPathMountOriginalFilenameWithoutExtTorrent)): - folderPathMountTorrent = folderPathMountOriginalFilenameWithoutExtTorrent - else: - folderPathMountTorrent = None - + + folderPathMountTorrent = await torrent.getTorrentPath() if folderPathMountTorrent: multiSeasonRegex1 = r'(?<=[\W_][Ss]eason[\W_])[\d][\W_][\d]{1,2}(?=[\W_])' multiSeasonRegex2 = r'(?<=[\W_][Ss])[\d]{2}[\W_][Ss]?[\d]{2}(?=[\W_])' diff --git a/shared/debrid.py b/shared/debrid.py index 7f07129..f9bd8eb 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -147,9 +147,15 @@ async def selectFiles(self): def delete(self): pass + @abstractmethod + async def getTorrentPath(self): + pass + + @abstractmethod def _addTorrentFile(self): pass + @abstractmethod def _addMagnetFile(self): pass @@ -249,6 +255,26 @@ def delete(self): deleteRequest = requests.delete(urljoin(realdebrid['host'], f"torrents/delete/{self.id}"), headers=self.headers) + async def getTorrentPath(self): + filename = await self.getInfo()['filename'] + originalFilename = await self.getInfo()['original_filename'] + + folderPathMountFilenameTorrent = os.path.join(self.mountTorrentsPath, filename) + folderPathMountOriginalFilenameTorrent = os.path.join(self.mountTorrentsPath, originalFilename) + folderPathMountOriginalFilenameWithoutExtTorrent = os.path.join(self.mountTorrentsPath, os.path.splitext(originalFilename)[0]) + + if os.path.exists(folderPathMountFilenameTorrent) and os.listdir(folderPathMountFilenameTorrent): + folderPathMountTorrent = folderPathMountFilenameTorrent + elif os.path.exists(folderPathMountOriginalFilenameTorrent) and os.listdir(folderPathMountOriginalFilenameTorrent): + folderPathMountTorrent = folderPathMountOriginalFilenameTorrent + elif (originalFilename.endswith(('.mkv', '.mp4')) and + os.path.exists(folderPathMountOriginalFilenameWithoutExtTorrent) and os.listdir(folderPathMountOriginalFilenameWithoutExtTorrent)): + folderPathMountTorrent = folderPathMountOriginalFilenameWithoutExtTorrent + else: + folderPathMountTorrent = None + + return folderPathMountTorrent + def _addFile(self, endpoint, data): host = self._getAvailableHost() @@ -348,6 +374,18 @@ def delete(self): deleteRequest = requests.delete(urljoin(torbox['host'], "torrents/controltorrent"), headers=self.headers, data={'torrent_id': self.id, 'operation': "Delete"}) + async def getTorrentPath(self): + filename = await self.getInfo()['name'] + + folderPathMountFilenameTorrent = os.path.join(self.mountTorrentsPath, filename) + + if os.path.exists(folderPathMountFilenameTorrent) and os.listdir(folderPathMountFilenameTorrent): + folderPathMountTorrent = folderPathMountFilenameTorrent + else: + folderPathMountTorrent = None + + return folderPathMountTorrent + def _addFile(self, endpoint, data=None, files=None): request = requests.post(urljoin(torbox['host'], endpoint), headers=self.headers, data=data, files=files) From 277475a5f0a0a8e89f4c9d0c24ad81d104713570 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 18:59:33 -0400 Subject: [PATCH 15/33] Fix --- shared/debrid.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shared/debrid.py b/shared/debrid.py index f9bd8eb..4d4ab10 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -256,8 +256,8 @@ def delete(self): deleteRequest = requests.delete(urljoin(realdebrid['host'], f"torrents/delete/{self.id}"), headers=self.headers) async def getTorrentPath(self): - filename = await self.getInfo()['filename'] - originalFilename = await self.getInfo()['original_filename'] + filename = (await self.getInfo())['filename'] + originalFilename = (await self.getInfo())['original_filename'] folderPathMountFilenameTorrent = os.path.join(self.mountTorrentsPath, filename) folderPathMountOriginalFilenameTorrent = os.path.join(self.mountTorrentsPath, originalFilename) @@ -375,7 +375,7 @@ def delete(self): deleteRequest = requests.delete(urljoin(torbox['host'], "torrents/controltorrent"), headers=self.headers, data={'torrent_id': self.id, 'operation': "Delete"}) async def getTorrentPath(self): - filename = await self.getInfo()['name'] + filename = (await self.getInfo())['name'] folderPathMountFilenameTorrent = os.path.join(self.mountTorrentsPath, filename) From b32b96e93d8406970a33e90405c7720fe34916c6 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 19:02:22 -0400 Subject: [PATCH 16/33] Upp how long it checks for info --- shared/debrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/debrid.py b/shared/debrid.py index 4d4ab10..baaacfa 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -353,7 +353,7 @@ async def getInfo(self, refresh=False): requests.get(inactiveCheckUrl) self.lastInactiveCheck = currentTime - for _ in range(10): + for _ in range(20): infoRequest = requests.get(urljoin(torbox['host'], "torrents/mylist"), headers=self.headers) torrents = infoRequest.json()['data'] From 9a4e8d83015cc06e6e09405c4222835dbbb70f47 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 19:12:16 -0400 Subject: [PATCH 17/33] Fixes for refresh and error --- blackhole.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/blackhole.py b/blackhole.py index aeced67..7dcf490 100644 --- a/blackhole.py +++ b/blackhole.py @@ -95,7 +95,7 @@ async def refresh(): print("Refresh already in progress, restarting...") refreshingTask.cancel() - refreshingTask = asyncio.createTask(refresh()) + refreshingTask = asyncio.create_task(refresh()) try: await refreshingTask except asyncio.CancelledError: @@ -280,7 +280,12 @@ async def is_accessible(path, timeout=10): onlyLargestFile = isRadarr or bool(re.search(r'S[\d]{2}E[\d]{2}', file.fileInfo.filename)) if not blackhole['failIfNotCached']: - await asyncio.gather(*(processTorrent(constructor(f, file, blackhole['failIfNotCached'], onlyLargestFile), file, arr) for constructor in torrentConstructors)) + torrents = [constructor(f, file, blackhole['failIfNotCached'], onlyLargestFile) for constructor in torrentConstructors] + results = await asyncio.gather(*(processTorrent(torrent, file, arr) for torrent in torrents)) + + if not any(results): + for torrent in torrents: + fail(torrent, arr) else: for i, constructor in enumerate(torrentConstructors): isLast = (i == len(torrentConstructors) - 1) From cebdd9963b59f8987cf4f8df8396cac3d6d140f3 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 19:33:25 -0400 Subject: [PATCH 18/33] Fixes for filedata --- blackhole.py | 8 +++++--- shared/debrid.py | 17 ++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/blackhole.py b/blackhole.py index 7dcf490..139162b 100644 --- a/blackhole.py +++ b/blackhole.py @@ -272,6 +272,9 @@ async def is_accessible(path, timeout=10): executor.shutdown(wait=False) with open(file.fileInfo.filePathProcessing, 'rb' if file.torrentInfo.isDotTorrentFile else 'r') as f: + fileData = f.read() + f.seek(0) + torrentConstructors = [] if realdebrid['enabled']: torrentConstructors.append(RealDebridTorrent if file.torrentInfo.isDotTorrentFile else RealDebridMagnet) @@ -280,7 +283,7 @@ async def is_accessible(path, timeout=10): onlyLargestFile = isRadarr or bool(re.search(r'S[\d]{2}E[\d]{2}', file.fileInfo.filename)) if not blackhole['failIfNotCached']: - torrents = [constructor(f, file, blackhole['failIfNotCached'], onlyLargestFile) for constructor in torrentConstructors] + torrents = [constructor(f, fileData, file, blackhole['failIfNotCached'], onlyLargestFile) for constructor in torrentConstructors] results = await asyncio.gather(*(processTorrent(torrent, file, arr) for torrent in torrents)) if not any(results): @@ -289,7 +292,7 @@ async def is_accessible(path, timeout=10): else: for i, constructor in enumerate(torrentConstructors): isLast = (i == len(torrentConstructors) - 1) - torrent = constructor(f, file, blackhole['failIfNotCached'], onlyLargestFile) + torrent = constructor(f, fileData, file, blackhole['failIfNotCached'], onlyLargestFile) if await processTorrent(torrent, file, arr): break @@ -311,7 +314,6 @@ def fail(torrent: TorrentBase, arr: Arr): def print(*values: object): _print(f"[{torrent.__class__.__name__}] [{torrent.file.fileInfo.filenameWithoutExt}]", *values) - print(f"Failing") history = arr.getHistory(blackhole['historyPageSize'])['records'] diff --git a/shared/debrid.py b/shared/debrid.py index baaacfa..4cf8f51 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -102,9 +102,10 @@ class TorrentBase(ABC): STATUS_COMPLETED = 'completed' STATUS_ERROR = 'error' - def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: + def __init__(self, f, fileData, file, failIfNotCached, onlyLargestFile) -> None: super().__init__() self.f = f + self.fileData = fileData self.file = file self.failIfNotCached = failIfNotCached self.onlyLargestFile = onlyLargestFile @@ -117,12 +118,6 @@ def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: def print(self, *values: object): print(f"[{datetime.now()}] [{self.__class__.__name__}] [{self.file.fileInfo.filenameWithoutExt}]", *values) - @cached_property - def fileData(self): - fileData = self.f.read() - self.f.seek(0) - return fileData - @abstractmethod def submitTorrent(self): pass @@ -164,8 +159,8 @@ def _enforceId(self): raise Exception("Id is required. Must be acquired via successfully running submitTorrent() first.") class RealDebrid(TorrentBase): - def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: - super().__init__(f, file, failIfNotCached, onlyLargestFile) + def __init__(self, f, fileData, file, failIfNotCached, onlyLargestFile) -> None: + super().__init__(f, fileData, file, failIfNotCached, onlyLargestFile) self.headers = {'Authorization': f'Bearer {realdebrid["apiKey"]}'} self.mountTorrentsPath = realdebrid["mountTorrentsPath"] @@ -304,8 +299,8 @@ def _normalize_status(self, status): return status class Torbox(TorrentBase): - def __init__(self, f, file, failIfNotCached, onlyLargestFile) -> None: - super().__init__(f, file, failIfNotCached, onlyLargestFile) + def __init__(self, f, fileData, file, failIfNotCached, onlyLargestFile) -> None: + super().__init__(f, fileData, file, failIfNotCached, onlyLargestFile) self.headers = {'Authorization': f'Bearer {torbox["apiKey"]}'} self.mountTorrentsPath = torbox["mountTorrentsPath"] self.submittedTime = None From d3b8b583133e24c4ab56c3b5c650cbcd1f9a83ae Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Fri, 24 May 2024 19:55:43 -0400 Subject: [PATCH 19/33] Up info count more --- shared/debrid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/debrid.py b/shared/debrid.py index 4cf8f51..9c4f1b1 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -348,7 +348,7 @@ async def getInfo(self, refresh=False): requests.get(inactiveCheckUrl) self.lastInactiveCheck = currentTime - for _ in range(20): + for _ in range(60): infoRequest = requests.get(urljoin(torbox['host'], "torrents/mylist"), headers=self.headers) torrents = infoRequest.json()['data'] From d4371128b2d43921d5c78a43d06f58b40c2412ce Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 11:47:48 -0400 Subject: [PATCH 20/33] Add logging and retries for request --- blackhole.py | 3 ++ shared/debrid.py | 137 +++++++++++++++++++++++++++++++++++------------ shared/shared.py | 39 ++++++++++++++ 3 files changed, 144 insertions(+), 35 deletions(-) diff --git a/blackhole.py b/blackhole.py index 139162b..313681d 100644 --- a/blackhole.py +++ b/blackhole.py @@ -148,6 +148,9 @@ def print(*values: object): while True: count += 1 info = await torrent.getInfo(refresh=True) + if not info: + return False + status = info['status'] print('status:', status) diff --git a/shared/debrid.py b/shared/debrid.py index 9c4f1b1..376acfd 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -8,8 +8,7 @@ from urllib.parse import urljoin from datetime import datetime from shared.discord import discordUpdate -from shared.shared import realdebrid, torbox, mediaExtensions, checkRequiredEnvs -from werkzeug.utils import cached_property +from shared.shared import realdebrid, torbox, mediaExtensions, checkRequiredEnvs, retryRequest def validateDebridEnabled(): if not realdebrid['enabled'] and not torbox['enabled']: @@ -171,8 +170,7 @@ def submitTorrent(self): if not instantAvailability: return False - self.addTorrent() - return True + return not not self.addTorrent() def _getInstantAvailability(self, refresh=False): if refresh or not self._instantAvailability: @@ -183,7 +181,13 @@ def _getInstantAvailability(self, refresh=False): self.incompatibleHashSize = True return True - instantAvailabilityRequest = requests.get(urljoin(realdebrid['host'], f"torrents/instantAvailability/{torrentHash}"), headers=self.headers) + instantAvailabilityRequest = retryRequest( + lambda: requests.get(urljoin(realdebrid['host'], f"torrents/instantAvailability/{torrentHash}"), headers=self.headers), + print=self.print + ) + if instantAvailabilityRequest is None: + return None + instantAvailabilities = instantAvailabilityRequest.json() self.print('instantAvailabilities:', instantAvailabilities) instantAvailabilityHosters = next(iter(instantAvailabilities.values())) @@ -194,7 +198,13 @@ def _getInstantAvailability(self, refresh=False): return self._instantAvailability def _getAvailableHost(self): - availableHostsRequest = requests.get(urljoin(realdebrid['host'], "torrents/availableHosts"), headers=self.headers) + availableHostsRequest = retryRequest( + lambda: requests.get(urljoin(realdebrid['host'], "torrents/availableHosts"), headers=self.headers), + print=self.print + ) + if availableHostsRequest is None: + return None + availableHosts = availableHostsRequest.json() return availableHosts[0]['host'] @@ -202,10 +212,16 @@ async def getInfo(self, refresh=False): self._enforceId() if refresh or not self._info: - infoRequest = requests.get(urljoin(realdebrid['host'], f"torrents/info/{self.id}"), headers=self.headers) - info = infoRequest.json() - info['status'] = self._normalize_status(info['status']) - self._info = info + infoRequest = retryRequest( + lambda: requests.get(urljoin(realdebrid['host'], f"torrents/info/{self.id}"), headers=self.headers), + print=self.print + ) + if infoRequest is None: + self._info = None + else: + info = infoRequest.json() + info['status'] = self._normalize_status(info['status']) + self._info = info return self._info @@ -213,6 +229,9 @@ async def selectFiles(self): self._enforceId() info = await self.getInfo() + if info is None: + return False + self.print('files:', info['files']) mediaFiles = [file for file in info['files'] if os.path.splitext(file['path'])[1].lower() in mediaExtensions] @@ -241,14 +260,24 @@ async def selectFiles(self): discordUpdate('largest file:', largestMediaFile['path']) files = {'files': [largestMediaFileId] if self.onlyLargestFile else ','.join(mediaFileIds)} - selectFilesRequest = requests.post(urljoin(realdebrid['host'], f"torrents/selectFiles/{self.id}"), headers=self.headers, data=files) + selectFilesRequest = retryRequest( + lambda: requests.post(urljoin(realdebrid['host'], f"torrents/selectFiles/{self.id}"), headers=self.headers, data=files), + print=self.print + ) + if selectFilesRequest is None: + return False return True def delete(self): self._enforceId() - deleteRequest = requests.delete(urljoin(realdebrid['host'], f"torrents/delete/{self.id}"), headers=self.headers) + deleteRequest = retryRequest( + lambda: requests.delete(urljoin(realdebrid['host'], f"torrents/delete/{self.id}"), headers=self.headers), + print=self.print + ) + return not not deleteRequest + async def getTorrentPath(self): filename = (await self.getInfo())['filename'] @@ -272,10 +301,17 @@ async def getTorrentPath(self): def _addFile(self, endpoint, data): host = self._getAvailableHost() + if host is None: + return None - request = requests.post(urljoin(realdebrid['host'], endpoint), params={'host': host}, headers=self.headers, data=data) - response = request.json() + request = retryRequest( + lambda: requests.post(urljoin(realdebrid['host'], endpoint), params={'host': host}, headers=self.headers, data=data), + print=self.print + ) + if request is None: + return None + response = request.json() self.print('response info:', response) self.id = response['id'] @@ -306,9 +342,13 @@ def __init__(self, f, fileData, file, failIfNotCached, onlyLargestFile) -> None: self.submittedTime = None self.lastInactiveCheck = None - userInfoRequest = requests.get(urljoin(torbox['host'], "user/me"), headers=self.headers) - userInfo = userInfoRequest.json() - self.authId = userInfo['data']['auth_id'] + userInfoRequest = retryRequest( + lambda: requests.get(urljoin(torbox['host'], "user/me"), headers=self.headers), + print=self.print + ) + if userInfoRequest is not None: + userInfo = userInfoRequest.json() + self.authId = userInfo['data']['auth_id'] def submitTorrent(self): if self.failIfNotCached: @@ -316,21 +356,28 @@ def submitTorrent(self): self.print('instantAvailability:', not not instantAvailability) if not instantAvailability: return False - - self.addTorrent() - self.submittedTime = datetime.now() - return True - + + if self.addTorrent(): + self.submittedTime = datetime.now() + return True + return False + def _getInstantAvailability(self, refresh=False): if refresh or not self._instantAvailability: torrentHash = self.getHash() self.print('hash:', torrentHash) - instantAvailabilityRequest = requests.get( - urljoin(torbox['host'], "torrents/checkcached"), - headers=self.headers, - params={'hash': torrentHash, 'format': 'object'} + instantAvailabilityRequest = retryRequest( + lambda: requests.get( + urljoin(torbox['host'], "torrents/checkcached"), + headers=self.headers, + params={'hash': torrentHash, 'format': 'object'} + ), + print=self.print ) + if instantAvailabilityRequest is None: + return None + instantAvailabilities = instantAvailabilityRequest.json() self.print('instantAvailabilities:', instantAvailabilities) self._instantAvailability = instantAvailabilities['data']['data'] if 'data' in instantAvailabilities and 'data' in instantAvailabilities['data'] and instantAvailabilities['data']['data'] is not False else None @@ -341,15 +388,26 @@ async def getInfo(self, refresh=False): self._enforceId() if refresh or not self._info: + if not self.authId: + return None + currentTime = datetime.now() if (currentTime - self.submittedTime).total_seconds() < 300: if not self.lastInactiveCheck or (currentTime - self.lastInactiveCheck).total_seconds() > 5: inactiveCheckUrl = f"https://relay.torbox.app/v1/inactivecheck/torrent/{self.authId}/{self.id}" - requests.get(inactiveCheckUrl) + retryRequest( + lambda: requests.get(inactiveCheckUrl), + print=self.print + ) self.lastInactiveCheck = currentTime - for _ in range(60): - infoRequest = requests.get(urljoin(torbox['host'], "torrents/mylist"), headers=self.headers) + infoRequest = retryRequest( + lambda: requests.get(urljoin(torbox['host'], "torrents/mylist"), headers=self.headers), + print=self.print + ) + if infoRequest is None: + return None + torrents = infoRequest.json()['data'] for torrent in torrents: @@ -367,7 +425,11 @@ async def selectFiles(self): def delete(self): self._enforceId() - deleteRequest = requests.delete(urljoin(torbox['host'], "torrents/controltorrent"), headers=self.headers, data={'torrent_id': self.id, 'operation': "Delete"}) + deleteRequest = retryRequest( + lambda: requests.delete(urljoin(torbox['host'], "torrents/controltorrent"), headers=self.headers, data={'torrent_id': self.id, 'operation': "Delete"}), + print=self.print + ) + return not not deleteRequest async def getTorrentPath(self): filename = (await self.getInfo())['name'] @@ -382,7 +444,12 @@ async def getTorrentPath(self): return folderPathMountTorrent def _addFile(self, endpoint, data=None, files=None): - request = requests.post(urljoin(torbox['host'], endpoint), headers=self.headers, data=data, files=files) + request = retryRequest( + lambda: requests.post(urljoin(torbox['host'], endpoint), headers=self.headers, data=data, files=files), + print=self.print + ) + if request is None: + return None response = request.json() self.print('response info:', response) @@ -421,7 +488,7 @@ def getHash(self): return self._hash def addTorrent(self): - self._addTorrentFile() + return self._addTorrentFile() class Magnet(TorrentBase): def getHash(self): @@ -433,9 +500,8 @@ def getHash(self): return self._hash def addTorrent(self): - self._addMagnetFile() + return self._addMagnetFile() -class RealDebridTorrent(RealDebrid, Torrent): pass class RealDebridMagnet(RealDebrid, Magnet): @@ -445,4 +511,5 @@ class TorboxTorrent(Torbox, Torrent): pass class TorboxMagnet(Torbox, Magnet): - pass \ No newline at end of file + pass + diff --git a/shared/shared.py b/shared/shared.py index 9cee2b6..9407b9c 100644 --- a/shared/shared.py +++ b/shared/shared.py @@ -1,5 +1,8 @@ import os import re +import time +import requests +from typing import Callable, Optional from environs import Env env = Env() @@ -198,3 +201,39 @@ def checkRequiredEnvs(requiredEnvs): else: previousSuccess = True +def retryRequest( + requestFunc: Callable[[], requests.Response], + print: Callable[..., None] = print, + retries: int = 1, + delay: int = 1 +) -> Optional[requests.Response]: + """ + Retry a request if the response status code is not in the 200 range. + + :param requestFunc: A callable that returns an HTTP response. + :param print: Optional print function for logging. + :param retries: The number of times to retry the request after the initial attempt. + :param delay: The delay between retries in seconds. + :return: The response object or None if all attempts fail. + """ + attempts = retries + 1 # Total attempts including the initial one + for attempt in range(attempts): + try: + response = requestFunc() + if 200 <= response.status_code < 300: + return response + else: + print(f"URL: {response.url}") + print(f"Status code: {response.status_code}") + print(f"Attempt {attempt + 1} failed") + if attempt < retries: + print(f"Retrying in {delay} seconds...") + time.sleep(delay) + except requests.RequestException as e: + print(f"URL: {response.url if 'response' in locals() else 'unknown'}") + print(f"Attempt {attempt + 1} encountered an error: {e}") + if attempt < retries: + print(f"Retrying in {delay} seconds...") + time.sleep(delay) + + return None \ No newline at end of file From d557b1d36535c961c95a742b2804ae458dcef498 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 11:53:52 -0400 Subject: [PATCH 21/33] Fix --- shared/debrid.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/shared/debrid.py b/shared/debrid.py index 376acfd..06ca8bb 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -502,6 +502,8 @@ def getHash(self): def addTorrent(self): return self._addMagnetFile() + +class RealDebridTorrent(RealDebrid, Torrent): pass class RealDebridMagnet(RealDebrid, Magnet): @@ -512,4 +514,3 @@ class TorboxTorrent(Torbox, Torrent): class TorboxMagnet(Torbox, Magnet): pass - From b64d3cbbf1d6d84f5dcc0f4c4bbe9fa22a2a8ed3 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 12:36:41 -0400 Subject: [PATCH 22/33] Fixes --- blackhole.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/blackhole.py b/blackhole.py index 313681d..63d1736 100644 --- a/blackhole.py +++ b/blackhole.py @@ -318,9 +318,10 @@ def print(*values: object): _print(f"[{torrent.__class__.__name__}] [{torrent.file.fileInfo.filenameWithoutExt}]", *values) print(f"Failing") - + + torrentHash = torrent.getHash() history = arr.getHistory(blackhole['historyPageSize'])['records'] - items = (item for item in history if item['data'].get('torrentInfoHash', '').casefold() == torrent.getHash().casefold() or cleanFileName(item['sourceTitle'].casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()) + items = [item for item in history if item['data'].get('torrentInfoHash', '').casefold() == torrentHash.casefold() or cleanFileName(item['sourceTitle'].casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()] if not items: raise Exception("No history items found to cancel") From 60284df30c5e8270f4c84ae34f23f313e604a7ae Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 12:51:22 -0400 Subject: [PATCH 23/33] Fix for unwritten files --- blackhole.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/blackhole.py b/blackhole.py index 63d1736..cf3357f 100644 --- a/blackhole.py +++ b/blackhole.py @@ -274,6 +274,9 @@ async def is_accessible(path, timeout=10): finally: executor.shutdown(wait=False) + await asyncio.sleep(1) # Wait before processing the file in case it isn't fully written yet. + os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) + with open(file.fileInfo.filePathProcessing, 'rb' if file.torrentInfo.isDotTorrentFile else 'r') as f: fileData = f.read() f.seek(0) @@ -353,8 +356,6 @@ async def on_created(isRadarr): while firstGo or not all(future.done() for future in futures): files = getFiles(isRadarr) if files: - for file in files: - os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) futures.append(asyncio.gather(*(processFile(file, arr, isRadarr) for file in files))) elif firstGo: print('No torrent files found') From 6c28bde722a938672738885daf33a0e456c409fc Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 12:53:18 -0400 Subject: [PATCH 24/33] Fix for fix --- blackhole.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/blackhole.py b/blackhole.py index cf3357f..c1adf03 100644 --- a/blackhole.py +++ b/blackhole.py @@ -255,6 +255,9 @@ async def processFile(file: TorrentFileInfo, arr: Arr, isRadarr): def print(*values: object): _print(f"[{file.fileInfo.filenameWithoutExt}]", *values) + await asyncio.sleep(1) # Wait before processing the file in case it isn't fully written yet. + os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) + from concurrent.futures import ThreadPoolExecutor def read_file(path): @@ -274,9 +277,6 @@ async def is_accessible(path, timeout=10): finally: executor.shutdown(wait=False) - await asyncio.sleep(1) # Wait before processing the file in case it isn't fully written yet. - os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) - with open(file.fileInfo.filePathProcessing, 'rb' if file.torrentInfo.isDotTorrentFile else 'r') as f: fileData = f.read() f.seek(0) From 4bffeeb9c21d7830bc7da4c269eac2b5a48a79d3 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 12:58:18 -0400 Subject: [PATCH 25/33] Fix more? --- blackhole.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/blackhole.py b/blackhole.py index c1adf03..21cb591 100644 --- a/blackhole.py +++ b/blackhole.py @@ -255,9 +255,6 @@ async def processFile(file: TorrentFileInfo, arr: Arr, isRadarr): def print(*values: object): _print(f"[{file.fileInfo.filenameWithoutExt}]", *values) - await asyncio.sleep(1) # Wait before processing the file in case it isn't fully written yet. - os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) - from concurrent.futures import ThreadPoolExecutor def read_file(path): @@ -277,6 +274,9 @@ async def is_accessible(path, timeout=10): finally: executor.shutdown(wait=False) + time.sleep(1) # Wait before processing the file in case it isn't fully written yet. + os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) + with open(file.fileInfo.filePathProcessing, 'rb' if file.torrentInfo.isDotTorrentFile else 'r') as f: fileData = f.read() f.seek(0) From 3a356d0334254bc001eef333bb14fab60a1f9877 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 13:29:05 -0400 Subject: [PATCH 26/33] Fix realdebrid torrent adding --- shared/debrid.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/shared/debrid.py b/shared/debrid.py index 06ca8bb..5b7b0cf 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -299,13 +299,13 @@ async def getTorrentPath(self): return folderPathMountTorrent - def _addFile(self, endpoint, data): + def _addFile(self, request, endpoint, data): host = self._getAvailableHost() if host is None: return None request = retryRequest( - lambda: requests.post(urljoin(realdebrid['host'], endpoint), params={'host': host}, headers=self.headers, data=data), + lambda: request(urljoin(realdebrid['host'], endpoint), params={'host': host}, headers=self.headers, data=data), print=self.print ) if request is None: @@ -318,10 +318,10 @@ def _addFile(self, endpoint, data): return self.id def _addTorrentFile(self): - return self._addFile("torrents/addTorrent", self.f) + return self._addFile(requests.put, "torrents/addTorrent", self.f) def _addMagnetFile(self): - return self._addFile("torrents/addMagnet", {'magnet': self.fileData}) + return self._addFile(requests.post, "torrents/addMagnet", {'magnet': self.fileData}) def _normalize_status(self, status): if status in ['waiting_files_selection']: @@ -443,9 +443,9 @@ async def getTorrentPath(self): return folderPathMountTorrent - def _addFile(self, endpoint, data=None, files=None): + def _addFile(self, data=None, files=None): request = retryRequest( - lambda: requests.post(urljoin(torbox['host'], endpoint), headers=self.headers, data=data, files=files), + lambda: requests.post(urljoin(torbox['host'], "torrents/createtorrent"), headers=self.headers, data=data, files=files), print=self.print ) if request is None: @@ -460,10 +460,10 @@ def _addFile(self, endpoint, data=None, files=None): def _addTorrentFile(self): nametorrent = self.f.name.split('/')[-1] files = {'file': (nametorrent, self.f, 'application/x-bittorrent')} - return self._addFile("torrents/createtorrent", files=files) + return self._addFile(files=files) def _addMagnetFile(self): - return self._addFile("torrents/createtorrent", data={'magnet': self.fileData}) + return self._addFile(data={'magnet': self.fileData}) def _normalize_status(self, status, download_finished): if download_finished: From 49acde187b66db065624babe712636e7f6fbcb9d Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 13:53:26 -0400 Subject: [PATCH 27/33] Fixes --- blackhole.py | 2 +- shared/shared.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/blackhole.py b/blackhole.py index 21cb591..2c7d74d 100644 --- a/blackhole.py +++ b/blackhole.py @@ -274,7 +274,7 @@ async def is_accessible(path, timeout=10): finally: executor.shutdown(wait=False) - time.sleep(1) # Wait before processing the file in case it isn't fully written yet. + time.sleep(.1) # Wait before processing the file in case it isn't fully written yet. os.renames(file.fileInfo.filePath, file.fileInfo.filePathProcessing) with open(file.fileInfo.filePathProcessing, 'rb' if file.torrentInfo.isDotTorrentFile else 'r') as f: diff --git a/shared/shared.py b/shared/shared.py index 9407b9c..b12a762 100644 --- a/shared/shared.py +++ b/shared/shared.py @@ -225,6 +225,8 @@ def retryRequest( else: print(f"URL: {response.url}") print(f"Status code: {response.status_code}") + print(f"Message: {response.reason}") + print(f"Response: {response.content}") print(f"Attempt {attempt + 1} failed") if attempt < retries: print(f"Retrying in {delay} seconds...") From 3846742b2bed1697ac82f083cdb9a7a08cbd18e8 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 14:10:47 -0400 Subject: [PATCH 28/33] Updates to blackhole watcher --- blackhole_watcher.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/blackhole_watcher.py b/blackhole_watcher.py index 0a9ad35..f2fd83b 100644 --- a/blackhole_watcher.py +++ b/blackhole_watcher.py @@ -5,18 +5,13 @@ class BlackholeHandler(FileSystemEventHandler): def __init__(self, is_radarr): super().__init__() - self.is_processing = False self.is_radarr = is_radarr self.path_name = getPath(is_radarr, create=True) - def on_created(self, event): - if not self.is_processing and not event.is_directory and event.src_path.lower().endswith((".torrent", ".magnet")): - self.is_processing = True - try: - start(self.is_radarr) - finally: - self.is_processing = False - + def on_created(self, event=None): + print("Watch found") + if not event or (not event.is_directory and event.src_path.lower().endswith((".torrent", ".magnet"))): + start(self.is_radarr) if __name__ == "__main__": print("Watching blackhole") @@ -24,6 +19,9 @@ def on_created(self, event): radarr_handler = BlackholeHandler(is_radarr=True) sonarr_handler = BlackholeHandler(is_radarr=False) + radarr_handler.on_created() + sonarr_handler.on_created() + radarr_observer = Observer() radarr_observer.schedule(radarr_handler, radarr_handler.path_name) From fe966476859b569d78ef4b6536cb5512c1564618 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 14:11:08 -0400 Subject: [PATCH 29/33] Remove print --- blackhole_watcher.py | 1 - 1 file changed, 1 deletion(-) diff --git a/blackhole_watcher.py b/blackhole_watcher.py index f2fd83b..681831f 100644 --- a/blackhole_watcher.py +++ b/blackhole_watcher.py @@ -9,7 +9,6 @@ def __init__(self, is_radarr): self.path_name = getPath(is_radarr, create=True) def on_created(self, event=None): - print("Watch found") if not event or (not event.is_directory and event.src_path.lower().endswith((".torrent", ".magnet"))): start(self.is_radarr) From 5314ca5e5ba844d2d06db19ecb81c6e4310d21ff Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Wed, 29 May 2024 15:53:12 -0400 Subject: [PATCH 30/33] Fail if queued --- shared/debrid.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/shared/debrid.py b/shared/debrid.py index 5b7b0cf..c719b6e 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -453,6 +453,10 @@ def _addFile(self, data=None, files=None): response = request.json() self.print('response info:', response) + + if response.get('detail') == 'queued': + return None + self.id = response['data']['torrent_id'] return self.id From a4264856514cc69a8b9b6eff1ad0a1750f60da97 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Thu, 30 May 2024 16:35:49 -0400 Subject: [PATCH 31/33] print/discord notifications and blackhole watcher startup fixes --- .env.template | 24 ++++++ blackhole.py | 10 +-- blackhole_watcher.py | 54 +++++++------ docker-compose.yml | 183 ++++++++++++++++++++++--------------------- shared/shared.py | 35 ++++++--- 5 files changed, 180 insertions(+), 126 deletions(-) diff --git a/.env.template b/.env.template index 3b354f7..b69dc36 100644 --- a/.env.template +++ b/.env.template @@ -15,10 +15,34 @@ SONARR_HOST= SONARR_API_KEY= SONARR_ROOT_FOLDER= +SONARR_HOST_4K= +SONARR_API_KEY_4K= +SONARR_ROOT_FOLDER_4K= + +SONARR_HOST_ANIME= +SONARR_API_KEY_ANIME= +SONARR_ROOT_FOLDER_ANIME= + +SONARR_HOST_MUX= +SONARR_API_KEY_MUX= +SONARR_ROOT_FOLDER_MUX= + RADARR_HOST= RADARR_API_KEY= RADARR_ROOT_FOLDER= +RADARR_HOST_4K= +RADARR_API_KEY_4K= +RADARR_ROOT_FOLDER_4K= + +RADARR_HOST_ANIME= +RADARR_API_KEY_ANIME= +RADARR_ROOT_FOLDER_ANIME= + +RADARR_HOST_MUX= +RADARR_API_KEY_MUX= +RADARR_ROOT_FOLDER_MUX= + TAUTULLI_HOST= TAUTULLI_API_KEY= diff --git a/blackhole.py b/blackhole.py index 2c7d74d..860f7d4 100644 --- a/blackhole.py +++ b/blackhole.py @@ -327,8 +327,9 @@ def print(*values: object): items = [item for item in history if item['data'].get('torrentInfoHash', '').casefold() == torrentHash.casefold() or cleanFileName(item['sourceTitle'].casefold()) == torrent.file.fileInfo.filenameWithoutExt.casefold()] if not items: - raise Exception("No history items found to cancel") - + message = "No history items found to mark as failed. Arr will not attempt to grab an alternative." + print(message) + discordError(message, torrent.file.fileInfo.filenameWithoutExt) for item in items: # TODO: See if we can fail without blacklisting as cached items constantly changes arr.failHistoryItem(item['id']) @@ -372,8 +373,5 @@ async def on_created(isRadarr): discordError(f"Error processing", e) print("Exit 'on_created'") -def start(isRadarr): - asyncio.run(on_created(isRadarr)) - if __name__ == "__main__": - start(isRadarr=sys.argv[1] == 'radarr') + asyncio.run(on_created(isRadarr=sys.argv[1] == 'radarr')) diff --git a/blackhole_watcher.py b/blackhole_watcher.py index 681831f..adb6286 100644 --- a/blackhole_watcher.py +++ b/blackhole_watcher.py @@ -1,6 +1,7 @@ +import asyncio from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler -from blackhole import start, getPath +from blackhole import on_created, getPath class BlackholeHandler(FileSystemEventHandler): def __init__(self, is_radarr): @@ -8,31 +9,40 @@ def __init__(self, is_radarr): self.is_radarr = is_radarr self.path_name = getPath(is_radarr, create=True) - def on_created(self, event=None): - if not event or (not event.is_directory and event.src_path.lower().endswith((".torrent", ".magnet"))): - start(self.is_radarr) + def on_created(self, event): + if not event.is_directory and event.src_path.lower().endswith((".torrent", ".magnet")): + asyncio.run(on_created(self.is_radarr)) -if __name__ == "__main__": - print("Watching blackhole") + async def on_created(self): + await on_created(self.is_radarr) + +async def main(): + print("Watching blackhole") - radarr_handler = BlackholeHandler(is_radarr=True) - sonarr_handler = BlackholeHandler(is_radarr=False) + radarr_handler = BlackholeHandler(is_radarr=True) + sonarr_handler = BlackholeHandler(is_radarr=False) - radarr_handler.on_created() - sonarr_handler.on_created() + radarr_observer = Observer() + radarr_observer.schedule(radarr_handler, radarr_handler.path_name) - radarr_observer = Observer() - radarr_observer.schedule(radarr_handler, radarr_handler.path_name) + sonarr_observer = Observer() + sonarr_observer.schedule(sonarr_handler, sonarr_handler.path_name) - sonarr_observer = Observer() - sonarr_observer.schedule(sonarr_handler, sonarr_handler.path_name) + try: + radarr_observer.start() + sonarr_observer.start() + + await asyncio.gather( + radarr_handler.on_created(), + sonarr_handler.on_created() + ) + except KeyboardInterrupt: + radarr_observer.stop() + sonarr_observer.stop() - try: - radarr_observer.start() - sonarr_observer.start() - except KeyboardInterrupt: - radarr_observer.stop() - sonarr_observer.stop() + radarr_observer.join() + sonarr_observer.join() - radarr_observer.join() - sonarr_observer.join() + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 054c2af..6f788b0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ x-blackhole: &blackhole context: . dockerfile: Dockerfile.blackhole image: ghcr.io/westsurname/scripts/blackhole:latest + pull_policy: always user: "${PUID:-}${PGID:+:${PGID}}" env_file: - .env @@ -13,6 +14,7 @@ x-repair: &repair context: . dockerfile: Dockerfile.scripts image: ghcr.io/westsurname/scripts/scripts:latest + pull_policy: always command: python repair.py --no-confirm env_file: - .env @@ -25,114 +27,114 @@ services: environment: - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} volumes: - - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} + - ${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}}:${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}} - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH}:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH}:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} profiles: [blackhole, blackhole_all, all] - # blackhole_4k: - # <<: *blackhole - # container_name: blackhole_4k_service - # environment: - # - SONARR_HOST=${SONARR_HOST_4K} - # - SONARR_API_KEY=${SONARR_API_KEY_4K} - # - RADARR_HOST=${RADARR_HOST_4K} - # - RADARR_API_KEY=${RADARR_API_KEY_4K} - # - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} - # volumes: - # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} 4k:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} - # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} 4k:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} - # profiles: [blackhole_4k, blackhole_all, all] + blackhole_4k: + <<: *blackhole + container_name: blackhole_4k_service + environment: + - SONARR_HOST=${SONARR_HOST_4K} + - SONARR_API_KEY=${SONARR_API_KEY_4K} + - RADARR_HOST=${RADARR_HOST_4K} + - RADARR_API_KEY=${RADARR_API_KEY_4K} + - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} + volumes: + - ${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}}:${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}} + - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} + - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} 4k:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} + - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} 4k:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} + profiles: [blackhole_4k, blackhole_all, all] - # blackhole_anime: - # <<: *blackhole - # container_name: blackhole_anime_service - # environment: - # - SONARR_HOST=${SONARR_HOST_ANIME} - # - SONARR_API_KEY=${SONARR_API_KEY_ANIME} - # - RADARR_HOST=${RADARR_HOST_ANIME} - # - RADARR_API_KEY=${RADARR_API_KEY_ANIME} - # - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} - # volumes: - # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} anime:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} - # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} anime:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} - # profiles: [blackhole_anime, blackhole_all, all] + blackhole_anime: + <<: *blackhole + container_name: blackhole_anime_service + environment: + - SONARR_HOST=${SONARR_HOST_ANIME} + - SONARR_API_KEY=${SONARR_API_KEY_ANIME} + - RADARR_HOST=${RADARR_HOST_ANIME} + - RADARR_API_KEY=${RADARR_API_KEY_ANIME} + - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} + volumes: + - ${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}}:${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}} + - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} + - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} anime:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} + - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} anime:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} + profiles: [blackhole_anime, blackhole_all, all] - # blackhole_mux: - # <<: *blackhole - # container_name: blackhole_mux_service - # environment: - # - SONARR_HOST=${SONARR_HOST_MUX} - # - SONARR_API_KEY=${SONARR_API_KEY_MUX} - # - RADARR_HOST=${RADARR_HOST_MUX} - # - RADARR_API_KEY=${RADARR_API_KEY_MUX} - # - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} - # volumes: - # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} mux:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} - # - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} mux:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} - # profiles: [blackhole_mux, blackhole_all, all] + blackhole_mux: + <<: *blackhole + container_name: blackhole_mux_service + environment: + - SONARR_HOST=${SONARR_HOST_MUX} + - SONARR_API_KEY=${SONARR_API_KEY_MUX} + - RADARR_HOST=${RADARR_HOST_MUX} + - RADARR_API_KEY=${RADARR_API_KEY_MUX} + - BLACKHOLE_BASE_WATCH_PATH=/${BLACKHOLE_BASE_WATCH_PATH} + volumes: + - ${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}}:${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}} + - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} + - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} mux:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_SONARR_PATH} + - ${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} mux:/${BLACKHOLE_BASE_WATCH_PATH}/${BLACKHOLE_RADARR_PATH} + profiles: [blackhole_mux, blackhole_all, all] repair_service: <<: *repair container_name: repair_service volumes: - - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} + - ${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}}:${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}} - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} - ${SONARR_ROOT_FOLDER}:${SONARR_ROOT_FOLDER} - ${RADARR_ROOT_FOLDER}:${RADARR_ROOT_FOLDER} profiles: [repair, repair_all, all] - # repair_4k: - # <<: *repair - # container_name: repair_4k_service - # environment: - # - SONARR_HOST=${SONARR_HOST_4K} - # - SONARR_API_KEY=${SONARR_API_KEY_4K} - # - RADARR_HOST=${RADARR_HOST_4K} - # - RADARR_API_KEY=${RADARR_API_KEY_4K} - # volumes: - # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${SONARR_ROOT_FOLDER_4K}:${SONARR_ROOT_FOLDER} - # - ${RADARR_ROOT_FOLDER_4K}:${RADARR_ROOT_FOLDER} - # profiles: [repair_4k, repair_all, all] + repair_4k: + <<: *repair + container_name: repair_4k_service + environment: + - SONARR_HOST=${SONARR_HOST_4K} + - SONARR_API_KEY=${SONARR_API_KEY_4K} + - RADARR_HOST=${RADARR_HOST_4K} + - RADARR_API_KEY=${RADARR_API_KEY_4K} + volumes: + - ${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}}:${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}} + - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} + - ${SONARR_ROOT_FOLDER_4K}:${SONARR_ROOT_FOLDER} + - ${RADARR_ROOT_FOLDER_4K}:${RADARR_ROOT_FOLDER} + profiles: [repair_4k, repair_all, all] - # repair_anime: - # <<: *repair - # container_name: repair_anime_service - # environment: - # - SONARR_HOST=${SONARR_HOST_ANIME} - # - SONARR_API_KEY=${SONARR_API_KEY_ANIME} - # - RADARR_HOST=${RADARR_HOST_ANIME} - # - RADARR_API_KEY=${RADARR_API_KEY_ANIME} - # volumes: - # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${SONARR_ROOT_FOLDER_ANIME}:${SONARR_ROOT_FOLDER} - # - ${RADARR_ROOT_FOLDER_ANIME}:${RADARR_ROOT_FOLDER} - # profiles: [repair_anime, repair_all, all] + repair_anime: + <<: *repair + container_name: repair_anime_service + environment: + - SONARR_HOST=${SONARR_HOST_ANIME} + - SONARR_API_KEY=${SONARR_API_KEY_ANIME} + - RADARR_HOST=${RADARR_HOST_ANIME} + - RADARR_API_KEY=${RADARR_API_KEY_ANIME} + volumes: + - ${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}}:${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}} + - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} + - ${SONARR_ROOT_FOLDER_ANIME}:${SONARR_ROOT_FOLDER} + - ${RADARR_ROOT_FOLDER_ANIME}:${RADARR_ROOT_FOLDER} + profiles: [repair_anime, repair_all, all] - # repair_mux: - # <<: *repair - # container_name: repair_mux_service - # environment: - # - SONARR_HOST=${SONARR_HOST_MUX} - # - SONARR_API_KEY=${SONARR_API_KEY_MUX} - # - RADARR_HOST=${RADARR_HOST_MUX} - # - RADARR_API_KEY=${RADARR_API_KEY_MUX} - # volumes: - # - ${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null}:${REALDEBRID_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} - # - ${SONARR_ROOT_FOLDER_MUX}:${SONARR_ROOT_FOLDER} - # - ${RADARR_ROOT_FOLDER_MUX}:${RADARR_ROOT_FOLDER} - # profiles: [repair_mux, repair_all, all] + repair_mux: + <<: *repair + container_name: repair_mux_service + environment: + - SONARR_HOST=${SONARR_HOST_MUX} + - SONARR_API_KEY=${SONARR_API_KEY_MUX} + - RADARR_HOST=${RADARR_HOST_MUX} + - RADARR_API_KEY=${RADARR_API_KEY_MUX} + volumes: + - ${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}}:${REALDEBRID_MOUNT_TORRENTS_PATH:-${BLACKHOLE_RD_MOUNT_TORRENTS_PATH:-/dev/null}} + - ${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null}:${TORBOX_MOUNT_TORRENTS_PATH:-/dev/null} + - ${SONARR_ROOT_FOLDER_MUX}:${SONARR_ROOT_FOLDER} + - ${RADARR_ROOT_FOLDER_MUX}:${RADARR_ROOT_FOLDER} + profiles: [repair_mux, repair_all, all] watchlist: build: @@ -140,6 +142,7 @@ services: dockerfile: Dockerfile.watchlist container_name: watchlist_service image: ghcr.io/westsurname/scripts/watchlist:latest + pull_policy: always volumes: - ./shared/tokens.json:/app/shared/tokens.json env_file: @@ -153,6 +156,7 @@ services: dockerfile: Dockerfile.plex_authentication container_name: plex_authentication_service image: ghcr.io/westsurname/scripts/plex_authentication:latest + pull_policy: always volumes: - ./shared/tokens.json:/app/shared/tokens.json ports: @@ -170,6 +174,7 @@ services: dockerfile: Dockerfile.plex_request container_name: plex_request_service image: ghcr.io/westsurname/scripts/plex_request:latest + pull_policy: always volumes: - ./shared/tokens.json:/app/shared/tokens.json ports: diff --git a/shared/shared.py b/shared/shared.py index b12a762..27029e6 100644 --- a/shared/shared.py +++ b/shared/shared.py @@ -4,6 +4,7 @@ import requests from typing import Callable, Optional from environs import Env +from shared.discord import discordError, discordUpdate env = Env() env.read_env() @@ -223,18 +224,34 @@ def retryRequest( if 200 <= response.status_code < 300: return response else: - print(f"URL: {response.url}") - print(f"Status code: {response.status_code}") - print(f"Message: {response.reason}") - print(f"Response: {response.content}") - print(f"Attempt {attempt + 1} failed") - if attempt < retries: + message = [ + f"URL: {response.url}", + f"Status code: {response.status_code}", + f"Message: {response.reason}", + f"Response: {response.content}", + f"Attempt {attempt + 1} failed" + ] + for line in message: + print(line) + if attempt == retries: + discordError("Request Failed", "\n".join(message)) + else: + update_message = message + [f"Retrying in {delay} seconds..."] + discordUpdate("Retrying Request", "\n".join(update_message)) print(f"Retrying in {delay} seconds...") time.sleep(delay) except requests.RequestException as e: - print(f"URL: {response.url if 'response' in locals() else 'unknown'}") - print(f"Attempt {attempt + 1} encountered an error: {e}") - if attempt < retries: + message = [ + f"URL: {response.url if 'response' in locals() else 'unknown'}", + f"Attempt {attempt + 1} encountered an error: {e}" + ] + for line in message: + print(line) + if attempt == retries: + discordError("Request Exception", "\n".join(message)) + else: + update_message = message + [f"Retrying in {delay} seconds..."] + discordUpdate("Retrying Request", "\n".join(update_message)) print(f"Retrying in {delay} seconds...") time.sleep(delay) From ca4dfad99d42b843522180b78b06631c79a99a25 Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Thu, 30 May 2024 16:47:45 -0400 Subject: [PATCH 32/33] Fix import loop --- shared/debrid.py | 3 ++- shared/requests.py | 60 +++++++++++++++++++++++++++++++++++++++++++++ shared/shared.py | 61 +--------------------------------------------- 3 files changed, 63 insertions(+), 61 deletions(-) create mode 100644 shared/requests.py diff --git a/shared/debrid.py b/shared/debrid.py index c719b6e..27dc8ed 100644 --- a/shared/debrid.py +++ b/shared/debrid.py @@ -8,7 +8,8 @@ from urllib.parse import urljoin from datetime import datetime from shared.discord import discordUpdate -from shared.shared import realdebrid, torbox, mediaExtensions, checkRequiredEnvs, retryRequest +from shared.requests import retryRequest +from shared.shared import realdebrid, torbox, mediaExtensions, checkRequiredEnvs def validateDebridEnabled(): if not realdebrid['enabled'] and not torbox['enabled']: diff --git a/shared/requests.py b/shared/requests.py new file mode 100644 index 0000000..fe689a8 --- /dev/null +++ b/shared/requests.py @@ -0,0 +1,60 @@ +import time +import requests +from typing import Callable, Optional +from shared.discord import discordError, discordUpdate + + +def retryRequest( + requestFunc: Callable[[], requests.Response], + print: Callable[..., None] = print, + retries: int = 1, + delay: int = 1 +) -> Optional[requests.Response]: + """ + Retry a request if the response status code is not in the 200 range. + + :param requestFunc: A callable that returns an HTTP response. + :param print: Optional print function for logging. + :param retries: The number of times to retry the request after the initial attempt. + :param delay: The delay between retries in seconds. + :return: The response object or None if all attempts fail. + """ + attempts = retries + 1 # Total attempts including the initial one + for attempt in range(attempts): + try: + response = requestFunc() + if 200 <= response.status_code < 300: + return response + else: + message = [ + f"URL: {response.url}", + f"Status code: {response.status_code}", + f"Message: {response.reason}", + f"Response: {response.content}", + f"Attempt {attempt + 1} failed" + ] + for line in message: + print(line) + if attempt == retries: + discordError("Request Failed", "\n".join(message)) + else: + update_message = message + [f"Retrying in {delay} seconds..."] + discordUpdate("Retrying Request", "\n".join(update_message)) + print(f"Retrying in {delay} seconds...") + time.sleep(delay) + except requests.RequestException as e: + message = [ + f"URL: {response.url if 'response' in locals() else 'unknown'}", + f"Attempt {attempt + 1} encountered an error: {e}" + ] + for line in message: + print(line) + if attempt == retries: + discordError("Request Exception", "\n".join(message)) + else: + update_message = message + [f"Retrying in {delay} seconds..."] + discordUpdate("Retrying Request", "\n".join(update_message)) + print(f"Retrying in {delay} seconds...") + time.sleep(delay) + + return None \ No newline at end of file diff --git a/shared/shared.py b/shared/shared.py index 27029e6..d9a61d3 100644 --- a/shared/shared.py +++ b/shared/shared.py @@ -1,10 +1,6 @@ import os import re -import time -import requests -from typing import Callable, Optional from environs import Env -from shared.discord import discordError, discordUpdate env = Env() env.read_env() @@ -200,59 +196,4 @@ def checkRequiredEnvs(requiredEnvs): else: previousSuccess = True else: - previousSuccess = True - -def retryRequest( - requestFunc: Callable[[], requests.Response], - print: Callable[..., None] = print, - retries: int = 1, - delay: int = 1 -) -> Optional[requests.Response]: - """ - Retry a request if the response status code is not in the 200 range. - - :param requestFunc: A callable that returns an HTTP response. - :param print: Optional print function for logging. - :param retries: The number of times to retry the request after the initial attempt. - :param delay: The delay between retries in seconds. - :return: The response object or None if all attempts fail. - """ - attempts = retries + 1 # Total attempts including the initial one - for attempt in range(attempts): - try: - response = requestFunc() - if 200 <= response.status_code < 300: - return response - else: - message = [ - f"URL: {response.url}", - f"Status code: {response.status_code}", - f"Message: {response.reason}", - f"Response: {response.content}", - f"Attempt {attempt + 1} failed" - ] - for line in message: - print(line) - if attempt == retries: - discordError("Request Failed", "\n".join(message)) - else: - update_message = message + [f"Retrying in {delay} seconds..."] - discordUpdate("Retrying Request", "\n".join(update_message)) - print(f"Retrying in {delay} seconds...") - time.sleep(delay) - except requests.RequestException as e: - message = [ - f"URL: {response.url if 'response' in locals() else 'unknown'}", - f"Attempt {attempt + 1} encountered an error: {e}" - ] - for line in message: - print(line) - if attempt == retries: - discordError("Request Exception", "\n".join(message)) - else: - update_message = message + [f"Retrying in {delay} seconds..."] - discordUpdate("Retrying Request", "\n".join(update_message)) - print(f"Retrying in {delay} seconds...") - time.sleep(delay) - - return None \ No newline at end of file + previousSuccess = True \ No newline at end of file From 42b2b848545bd007b6f706a7fa149fe209f607cf Mon Sep 17 00:00:00 2001 From: westsurname <155189104+westsurname@users.noreply.github.com> Date: Thu, 30 May 2024 16:52:04 -0400 Subject: [PATCH 33/33] Fix --- blackhole_watcher.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/blackhole_watcher.py b/blackhole_watcher.py index adb6286..180b687 100644 --- a/blackhole_watcher.py +++ b/blackhole_watcher.py @@ -13,7 +13,7 @@ def on_created(self, event): if not event.is_directory and event.src_path.lower().endswith((".torrent", ".magnet")): asyncio.run(on_created(self.is_radarr)) - async def on_created(self): + async def on_run(self): await on_created(self.is_radarr) async def main(): @@ -33,8 +33,8 @@ async def main(): sonarr_observer.start() await asyncio.gather( - radarr_handler.on_created(), - sonarr_handler.on_created() + radarr_handler.on_run(), + sonarr_handler.on_run() ) except KeyboardInterrupt: radarr_observer.stop()