From a5fc7043edbc4b8221c63abb24c4279b11109348 Mon Sep 17 00:00:00 2001 From: Josep Llort Date: Sat, 19 Feb 2022 19:40:34 +0100 Subject: [PATCH] download torrent timeout fix #33 --- README.md | 6 +++--- .../llort/torrent/util/PctmixWebParserV4.java | 20 +++++++++++-------- .../jllort/torrent/ParserCheckIssue31.java | 14 +++++++------ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 547ee6f..237f88c 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ Utility to get torrent download links from the web without advertisements: ``` mkdir /utils cd /utils -wget https://github.com/darkman97i/torrent-web-parser/releases/download/v.1.4/torrent-web-parser-1.4.zip -unzip torrent-web-parser-1.4.zip -java -jar torrent-web-parser -g /utils/torrent-web-parser-1.4/geckodriver -d /utils/torrent-web-parser-1.4 -f universe,dry +wget https://github.com/darkman97i/torrent-web-parser/releases/download/v.1.5/torrent-web-parser-1.5.zip +unzip torrent-web-parser-1.5.zip +java -jar torrent-web-parser -g /utils/torrent-web-parser-1.5/geckodriver -d /utils/torrent-web-parser-1.5 -f universe,dry ``` ### Parameters diff --git a/src/main/java/info/llort/torrent/util/PctmixWebParserV4.java b/src/main/java/info/llort/torrent/util/PctmixWebParserV4.java index eba7e94..39fe005 100644 --- a/src/main/java/info/llort/torrent/util/PctmixWebParserV4.java +++ b/src/main/java/info/llort/torrent/util/PctmixWebParserV4.java @@ -32,11 +32,11 @@ public static void capture(String urlWebToParse, String geckoDriverPath, List downloadTorrentLinks = downloadTorrentLinks(torrentPageLinks, driver, proxy); - for (PageLinkInfo pli : downloadTorrentLinks) { - Console.println("Download link: " + pli.getUrl(), WHITE); - downloadTorrentFile(pli, driver, downloadTimeOut, proxy); - } + Set downloadTorrentLinks = downloadTorrentLinks(torrentPageLinks, driver, proxy, downloadTimeOut); +// for (PageLinkInfo pli : downloadTorrentLinks) { +// Console.println("Download link: " + pli.getUrl(), WHITE); +// downloadTorrentFile(pli, driver, downloadTimeOut, proxy); +// } } public static Set findMainPageLinks(String url, String geckoDriverPath, List filters, WebDriver driver) throws IOException { @@ -97,7 +97,7 @@ public static Set findPageTorrentLinks(Set pageLinks return links; } - public static Set downloadTorrentLinks(Set pageLinks, WebDriver driver, BrowserMobProxy proxy) throws IOException { + public static Set downloadTorrentLinks(Set pageLinks, WebDriver driver, BrowserMobProxy proxy, long downloadTimeOut) throws IOException, InterruptedException { Set links = new HashSet<>(); for (PageLinkInfo pli : pageLinks) { // Setting referer before jump to the page @@ -125,12 +125,14 @@ public static Set downloadTorrentLinks(Set pageLinks Console.println("executing javascript: " + js, YELLOW); Object result = ((JavascriptExecutor) driver).executeScript(js); Console.println("Javascript result: " + js, GREEN); - String torrentFileLinkValue = "https://atomixhq.art/t_download/" + result; + String torrentFileLinkValue = "https://atomixhq.art" + result; Console.println("torrentFileLinkValue: " + torrentFileLinkValue, GREEN); PageLinkInfo newPli = new PageLinkInfo(); newPli.setUrl(torrentFileLinkValue); newPli.setReferer(pli.getUrl()); links.add(newPli); + // Must donwload here because downtime + downloadTorrentFile(newPli, driver, downloadTimeOut, proxy); } } else { @@ -143,7 +145,9 @@ public static Set downloadTorrentLinks(Set pageLinks public static void downloadTorrentFile(PageLinkInfo pli, WebDriver driver, long timeOut, BrowserMobProxy proxy) throws InterruptedException { try { // Referer must be atomtt - proxy.addHeader(HttpHeaders.REFERER, pli.getReferer()); + proxy.addHeader(HttpHeaders.REFERER, "https://atomtt.com/"); // Referer is the page what contains the torrent link //atomtt etc... + proxy.addHeader("authority", "atomixhq.art"); + proxy.addHeader("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"); // Set timeout otherwise the driver lock driver.manage().timeouts().scriptTimeout(Duration.ofSeconds(timeOut)); driver.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(timeOut)); diff --git a/src/test/java/info/jllort/torrent/ParserCheckIssue31.java b/src/test/java/info/jllort/torrent/ParserCheckIssue31.java index 8de1201..4a37f05 100644 --- a/src/test/java/info/jllort/torrent/ParserCheckIssue31.java +++ b/src/test/java/info/jllort/torrent/ParserCheckIssue31.java @@ -26,8 +26,8 @@ public static void main(String[] args) throws UnknownHostException, InterruptedE String dstPath = Config.FILESYSTEM_DOWNLOAD_PATH; long timeOut = Config.FILE_DOWNLOAD_TIMEOUT; - String referer = "https://atomixhq.art/descargar/torrent/serie/el-pacificador/temporada-1/capitulo-08/"; - String link = "https://atomtt.com/t_download/168542/el-pacificador---temporada-1/"; + String referer = "https://atomixhq.art/descargar/torrent/peliculas-castellano/venganza-a-golpes-2022-/blurayrip-ac3-5-1/"; + String link = "https://atomtt.com/t_download/168581/venganza-a-golpes--2022-/"; // Creating proxy BrowserMobProxy proxy = new BrowserMobProxyServer(); @@ -82,9 +82,9 @@ public static void main(String[] args) throws UnknownHostException, InterruptedE System.out.println("intValue found: " + intValue); if (driver instanceof JavascriptExecutor) { // Referer must be atomtt - proxy.addHeader(HttpHeaders.REFERER, "https://atomtt.com/t_download/168542/el-pacificador---temporada-1/"); + proxy.addHeader(HttpHeaders.REFERER, link); // Javascript request to be executed by selenium - String js = "var values = {'t':'168542'};\n"; // the variable + String js = "var values = {'t':'168581'};\n"; // the variable js += "var xhr = new XMLHttpRequest();\n"; js += "xhr.open('POST', 'https://atomtt.com/to.php', false);\n"; js += "xhr.setRequestHeader('Content-type', 'application/x-www-form-urlencoded');\n"; @@ -95,13 +95,15 @@ public static void main(String[] args) throws UnknownHostException, InterruptedE System.out.println("Result value of cal to top: " + result); // result = ((JavascriptExecutor) driver).executeScript("alert('Welcome to Guru99');"); // download URl = https://atomixhq.art/t_download/temp/17022022/168542/El-Pacificador---Temporada-1--HDTV.torrent?md5=b611CNys8mldMTSN_atN6A&expires=1645101721 - String torrentFileLinkValue = "https://atomixhq.art/t_download/" + result; + String torrentFileLinkValue = "https://atomixhq.art" + result; System.out.println(torrentFileLinkValue); driver.manage().timeouts().scriptTimeout(Duration.ofSeconds(timeOut)); driver.manage().timeouts().pageLoadTimeout(Duration.ofSeconds(timeOut)); driver.manage().timeouts().implicitlyWait(Duration.ofSeconds(timeOut)); // Other referer - proxy.addHeader(HttpHeaders.REFERER, link); // Referer is the page what contains the torrent link //atomtt etc... + proxy.addHeader(HttpHeaders.REFERER, "https://atomtt.com/"); // Referer is the page what contains the torrent link //atomtt etc... + proxy.addHeader("authority", "atomixhq.art"); + proxy.addHeader("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36"); driver.navigate().to(torrentFileLinkValue); } }