Merge branch 'release/v0.3.1'

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "archive-pdf-urls"
-version = "0.3.0"
+version = "0.3.1"
 authors = ["Javier Arias <[email protected]>"]
 edition = "2021"
 license = "Apache-2.0"
@@ -18,4 +18,4 @@ log = "0.4.21"
 lopdf = "0.32.0"
 regex = "1.10.4"
 tokio = { version = "1.36.0", features = ["full"] }
-waybackmachine-client = { version = "0.3.0", path = "waybackmachine-client"}
+waybackmachine-client = { version = "=0.3.1", path = "waybackmachine-client"}

src/main.rs

@@ -1,5 +1,5 @@
 use clap::{crate_authors, crate_version, Arg, ArgAction, Command};
-use log::{error, info};
+use log::{error, info, warn};
 use lopdf::{Dictionary, Document, Object};
 use regex::Regex;
 use std::collections::HashSet;
@@ -48,6 +48,9 @@ async fn main() {
         .collect();
 
     let links_set = extract_links(doc);
+    if links_set.is_empty() {
+        warn!("No page annotations found in this PDF file");
+    }
     let client = WaybackMachineClient::new(ClientConfig::default());
 
     let mut exit_code = 0;

waybackmachine-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "waybackmachine-client"
-version = "0.3.0"
+version = "0.3.1"
 authors = ["Javier Arias <[email protected]>"]
 edition = "2021"
 license = "Apache-2.0"

waybackmachine-client/src/archivableurl.rs

@@ -30,6 +30,9 @@ impl ArchivableUrl {
             Host::Domain(domain) if domain.contains("archive.org") => {
                 return Err(Error::ExcludedUrl(self.url.to_string()));
             }
+            Host::Domain(domain) if domain.contains("jstor.org") => {
+                return Err(Error::ExcludedUrl(self.url.to_string()));
+            }
             Host::Ipv4(ipv4)
                 if ipv4.is_loopback()
                     || ipv4.is_private()
@@ -134,4 +137,20 @@ mod tests {
         assert!(result.is_err());
         assert_eq!(result.err(), Some(Error::InvalidUrl(url.to_string())));
     }
+
+    #[test]
+    fn wayback_url() {
+        let url = "https://archive.org/some-book";
+        let result = ArchivableUrl::parse(url);
+        assert!(result.is_err());
+        assert_eq!(result.err(), Some(Error::ExcludedUrl(url.to_string())));
+    }
+
+    #[test]
+    fn jstor_url() {
+        let url = "https://jstor.org/some-book";
+        let result = ArchivableUrl::parse(url);
+        assert!(result.is_err());
+        assert_eq!(result.err(), Some(Error::ExcludedUrl(url.to_string())));
+    }
 }

waybackmachine-client/src/lib.rs

@@ -10,7 +10,7 @@ use serde::Deserialize;
 use url::Url;
 
 /// Maximum number of allowed request retries attempts.
-const DEFAULT_MAX_REQUEST_RETRIES: u32 = 5;
+const DEFAULT_MAX_REQUEST_RETRIES: u32 = 10;
 
 /// Default threshold for considering an archive as recent, in days.
 /// URLs with archives older than this threshold will be re-archived.