Proxy images from indexers

Closes #970

core/src/main/java/org/nzbhydra/NzbHydra.java

@@ -36,6 +36,7 @@
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.ImportRuntimeHints;
+import org.springframework.context.annotation.Primary;
 import org.springframework.context.event.EventListener;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.transaction.annotation.EnableTransactionManagement;
@@ -347,7 +348,8 @@ public void destroy() {
 
 
     @Bean
-    public CacheManager getCacheManager() {
+    @Primary
+    public CacheManager genericCacheManager() {
         return new CaffeineCacheManager("infos", "titles", "updates", "dev");
     }
 

core/src/main/java/org/nzbhydra/cache/DiskCache.java

@@ -0,0 +1,168 @@
+/*
+ *  (C) Copyright 2024 TheOtherP ([email protected])
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.nzbhydra.cache;
+
+import lombok.SneakyThrows;
+import org.apache.commons.io.FileUtils;
+import org.jetbrains.annotations.NotNull;
+import org.nzbhydra.logging.LoggingMarkers;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.cache.support.AbstractValueAdaptingCache;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.time.Instant;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+
+public class DiskCache extends AbstractValueAdaptingCache {
+
+    private static final Logger logger = LoggerFactory.getLogger(DiskCache.class);
+
+    private static final int MAX_ENTRIES = 500;
+    private static final int MAX_ENTRIES_SIZE_MB = 50;
+
+    private static final Map<String, Instant> ACCESS_MAP = new HashMap<>();
+
+    private final File cacheDir;
+    private final String name;
+
+    public DiskCache(File cacheDir, String name) {
+        super(false);
+        this.cacheDir = cacheDir;
+        this.name = name;
+        boolean created = cacheDir.mkdirs();
+        if (!cacheDir.exists()) {
+            throw new RuntimeException("Error creating cache dir " + cacheDir.getAbsolutePath());
+        }
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public Object getNativeCache() {
+        return this;
+    }
+
+    @Override
+    @SneakyThrows
+    public <T> T get(Object key, Callable<T> valueLoader) {
+        if (key instanceof String keyString) {
+            ACCESS_MAP.put(keyString, Instant.now());
+            File keyFile = buildKeyFile(keyString);
+            if (keyFile.exists()) {
+                return (T) Files.readAllBytes(keyFile.toPath());
+            } else {
+                T value = valueLoader.call();
+                if ((value instanceof byte[] bytes)) {
+                    Files.write(keyFile.toPath(), bytes);
+                    return value;
+                } else {
+                    throw new RuntimeException("Illegal type for value " + value.getClass());
+                }
+            }
+        } else {
+            throw new RuntimeException("Illegal type for key " + key.getClass());
+        }
+    }
+
+    @Override
+    @SneakyThrows
+    public void put(Object key, Object value) {
+        if ((key instanceof String keyString) && (value instanceof byte[] valueBytes)) {
+            logger.debug(LoggingMarkers.DISK_CACHE, "Writing entry with key {} and size {}", keyString, valueBytes.length);
+            Files.write(buildKeyFile(keyString).toPath(), valueBytes);
+            ACCESS_MAP.put(keyString, Instant.now());
+        } else {
+            throw new RuntimeException("Illegal type for key " + key.getClass() + " and/or value " + value.getClass());
+        }
+        cleanAfterPut();
+    }
+
+    private void cleanAfterPut() {
+        while (ACCESS_MAP.size() > MAX_ENTRIES) {
+            logger.debug(LoggingMarkers.CONFIG_READ_WRITE, "{} entries in cache - exceeds limit of {}", ACCESS_MAP.size(), MAX_ENTRIES);
+            deleteOldestEntry();
+        }
+        while (Arrays.stream(cacheDir.listFiles()).mapToDouble(File::length).sum() / (1024 * 1024) > MAX_ENTRIES_SIZE_MB) {
+            logger.debug(LoggingMarkers.CONFIG_READ_WRITE, "Cache takes up too much space (more than {})", MAX_ENTRIES_SIZE_MB);
+            deleteOldestEntry();
+        }
+    }
+
+    private void deleteOldestEntry() {
+        Optional<Map.Entry<String, Instant>> oldestEntry = ACCESS_MAP.entrySet().stream().min(Map.Entry.comparingByValue());
+        if (oldestEntry.isPresent()) {
+            logger.debug(LoggingMarkers.DISK_CACHE, "Removing oldest entry {}", oldestEntry.get());
+            ACCESS_MAP.remove(oldestEntry.get().getKey());
+            FileUtils.deleteQuietly(buildKeyFile(oldestEntry.get().getKey()));
+        }
+    }
+
+    @NotNull
+    private File buildKeyFile(String keyString) {
+        return new File(cacheDir, keyString);
+    }
+
+    @Override
+    public void evict(Object key) {
+        if (key instanceof String keyString) {
+            File keyFile = buildKeyFile(keyString);
+            if (keyFile.exists()) {
+                logger.debug(LoggingMarkers.CONFIG_READ_WRITE, "Evicting entry {}", keyString);
+                ACCESS_MAP.remove(keyString);
+                FileUtils.deleteQuietly(keyFile);
+            } else {
+                logger.debug(LoggingMarkers.CONFIG_READ_WRITE, "Can't evict not existing entry {}", keyString);
+            }
+        }
+    }
+
+    @Override
+    @SneakyThrows
+    public void clear() {
+        if (!cacheDir.exists()) {
+            return;
+        }
+        logger.debug(LoggingMarkers.CONFIG_READ_WRITE, "Clearing cache");
+        FileUtils.cleanDirectory(cacheDir);
+        ACCESS_MAP.clear();
+    }
+
+    @Override
+    @SneakyThrows
+    protected Object lookup(Object key) {
+        if (key instanceof String keyString) {
+            File keyFile = buildKeyFile(keyString);
+            if (keyFile.exists()) {
+                logger.debug(LoggingMarkers.CONFIG_READ_WRITE, "Found entry for key {}", keyString);
+                return Files.readAllBytes(keyFile.toPath());
+            } else {
+                logger.debug(LoggingMarkers.CONFIG_READ_WRITE, "Did not find entry for key {}", keyString);
+            }
+        }
+        return null;
+    }
+
+}

core/src/main/java/org/nzbhydra/cache/ImageCacheConfig.java

@@ -0,0 +1,44 @@
+/*
+ *  (C) Copyright 2024 TheOtherP ([email protected])
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.nzbhydra.cache;
+
+import org.nzbhydra.NzbHydra;
+import org.springframework.cache.Cache;
+import org.springframework.cache.CacheManager;
+import org.springframework.cache.support.AbstractCacheManager;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.List;
+
+@Configuration
+public class ImageCacheConfig {
+
+
+    @Bean
+    public CacheManager imageCacheManager() {
+        return new AbstractCacheManager() {
+            @Override
+            protected Collection<? extends Cache> loadCaches() {
+                return List.of(new DiskCache(new File(NzbHydra.getDataFolder(), "cache"), "images"));
+            }
+        };
+    }
+
+}

core/src/main/java/org/nzbhydra/cache/ProxyImagesWeb.java

@@ -0,0 +1,50 @@
+/*
+ *  (C) Copyright 2024 TheOtherP ([email protected])
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.nzbhydra.cache;
+
+import org.nzbhydra.webaccess.HydraOkHttp3ClientHttpRequestFactory;
+import org.springframework.cache.annotation.Cacheable;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.client.ClientHttpResponse;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+@RestController
+public class ProxyImagesWeb {
+
+    private final HydraOkHttp3ClientHttpRequestFactory hydraOkHttp3ClientHttpRequestFactory;
+
+    public ProxyImagesWeb(HydraOkHttp3ClientHttpRequestFactory hydraOkHttp3ClientHttpRequestFactory) {
+        this.hydraOkHttp3ClientHttpRequestFactory = hydraOkHttp3ClientHttpRequestFactory;
+    }
+
+    @RequestMapping(value = "/cache/{originalUrl}", method = RequestMethod.GET, produces = MediaType.IMAGE_JPEG_VALUE)
+    @Cacheable(cacheNames = "images", cacheManager = "imageCacheManager")
+    public byte[] proxyImage(@PathVariable String originalUrl) throws Exception {
+        try (ClientHttpResponse response = hydraOkHttp3ClientHttpRequestFactory.createRequest(new URI(new String(Base64.getDecoder().decode(originalUrl), StandardCharsets.UTF_8)), HttpMethod.GET).execute()) {
+            return response.getBody().readAllBytes();
+        }
+    }
+
+}

core/src/main/java/org/nzbhydra/indexers/Indexer.java

@@ -479,6 +479,7 @@ public Optional<Instant> tryParseDate(String dateString) {
     }
 
 
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -502,27 +503,27 @@ public String toString() {
     }
 
     protected void warn(String msg) {
-        getLogger().warn(getName() + ": " + msg);
+        getLogger().warn("{}: {}", getName(), msg);
     }
 
     protected void error(String msg) {
-        getLogger().error(getName() + ": " + msg);
+        getLogger().error("{}: {}", getName(), msg);
     }
 
     protected void error(String msg, Throwable t) {
-        getLogger().error(getName() + ": " + msg, t);
+        getLogger().error("{}: {}", getName(), msg, t);
     }
 
     protected void info(String msg, Object... arguments) {
-        getLogger().info(getName() + ": " + msg, arguments);
+        getLogger().info("{}: {}", getName(), msg, arguments);
     }
 
     protected void debug(String msg, Object... arguments) {
-        getLogger().debug(getName() + ": " + msg, arguments);
+        getLogger().debug("{}: {}", getName(), msg, arguments);
     }
 
     protected void debug(Marker marker, String msg, Object... arguments) {
-        getLogger().debug(marker, getName() + ": " + msg, arguments);
+        getLogger().debug(marker, "{}: {}", getName(), msg, arguments);
     }
 
     protected abstract Logger getLogger();

core/src/main/java/org/nzbhydra/logging/LoggingMarkers.java

@@ -7,6 +7,7 @@ public class LoggingMarkers {
 
     public static final Marker CATEGORY_MAPPING = MarkerFactory.getMarker("CATEGORY_MAPPING");
     public static final Marker CONFIG_READ_WRITE = MarkerFactory.getMarker("CONFIG_READ_WRITE");
+    public static final Marker DISK_CACHE = MarkerFactory.getMarker("DISK_CACHE");
     public static final Marker CUSTOM_MAPPING = MarkerFactory.getMarker("CUSTOM_MAPPING");
     public static final Marker DOWNLOADER_STATUS_UPDATE = MarkerFactory.getMarker("DOWNLOADER_STATUS_UPDATE");
     public static final Marker DOWNLOAD_STATUS_UPDATE = MarkerFactory.getMarker("DOWNLOAD_STATUS_UPDATE");

core/src/main/java/org/nzbhydra/mediainfo/InfoProvider.java

@@ -73,7 +73,7 @@ public boolean canConvertAny(Set<MediaIdType> from, Set<MediaIdType> to) {
         return from.stream().anyMatch(x -> canConvertMap.containsKey(x) && canConvertMap.get(x).stream().anyMatch(to::contains));
     }
 
-    @Cacheable(cacheNames = "infos", sync = true)
+    @Cacheable(cacheNames = "infos", sync = true, cacheManager = "genericCacheManager")
     public MediaInfo convert(Map<MediaIdType, String> identifiers) throws InfoProviderException {
         for (MediaIdType idType : REAL_ID_TYPES) {
             if (identifiers.containsKey(idType) && identifiers.get(idType) != null) {
@@ -89,7 +89,7 @@ public MediaInfo convert(Map<MediaIdType, String> identifiers) throws InfoProvid
     }
 
 
-    @Cacheable(cacheNames = "infos", sync = true)
+    @Cacheable(cacheNames = "infos", sync = true, cacheManager = "genericCacheManager")
     //sync=true is currently apparently not supported by Caffeine. synchronizing by method is good enough because we'll likely rarely hit this method concurrently with different parameters
     public synchronized MediaInfo convert(String value, MediaIdType fromType) throws InfoProviderException {
         if (value == null) {
@@ -181,7 +181,7 @@ public MovieInfo findMovieInfoInDatabase(Map<MediaIdType, String> ids) {
         return matchingInfos.stream().max(MovieInfo::compareTo).orElse(null);
     }
 
-    @Cacheable(cacheNames = "titles", sync = true)
+    @Cacheable(cacheNames = "titles", sync = true, cacheManager = "genericCacheManager")
     public List<MediaInfo> search(String title, MediaIdType titleType) throws InfoProviderException {
         try {
             List<MediaInfo> infos;

core/src/main/java/org/nzbhydra/mediainfo/MediaInfoWeb.java

@@ -23,6 +23,8 @@
 import org.springframework.web.bind.annotation.RestController;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -67,7 +69,13 @@ public List<MediaInfoTO> load(CacheKey key) throws Exception {
     @RequestMapping(value = "/internalapi/autocomplete/{type}", produces = "application/json")
     public List<MediaInfoTO> autocomplete(@PathVariable("type") AutocompleteType type, @RequestParam("input") String input) throws ExecutionException {
         try {
-            return autocompleteCache.get(new CacheKey(type, input));
+            List<MediaInfoTO> tos = autocompleteCache.get(new CacheKey(type, input));
+            if (configProvider.getBaseConfig().getMain().isProxyImages()) {
+                tos.stream().filter(to -> to.getPosterUrl() != null).forEach(to -> {
+                    to.setPosterUrl("cache/" + Base64.getEncoder().encodeToString(to.getPosterUrl().getBytes(StandardCharsets.UTF_8)));
+                });
+            }
+            return tos;
         } catch (ExecutionException e) {
             logger.warn("Error while trying to find autocomplete data for input {}: {}", input, e.getMessage());
             return Collections.emptyList();