Properly log hard crashes

See #892

.gitignore

@@ -28,3 +28,5 @@ heapdump*
 misc/rsyncToServers.sh
 /nzbhydra.yml
 /results
+other/wrapper/pyInstaller/windows/build
+other/wrapper/pyInstaller/windows_console/build

buildCore.cmd

@@ -4,8 +4,8 @@ setlocal
 
 call "C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Auxiliary\Build\vcvarsall.bat" x64
 
-set path=c:\Programme\graalvm-ce-java17-22.2.0\bin\;%PATH%;c:\Programme\graalvm-ce-java17-22.2.0\bin\
-set java_home=c:\Programme\graalvm-ce-java17-22.2.0\
+set path=c:\Programme\graalvm\graalvm-community-openjdk-21+35.1\bin\;%PATH%;c:\Programme\graalvm\graalvm-community-openjdk-21+35.1\bin\
+set java_home=c:\Programme\graalvm\graalvm-community-openjdk-21+35.1
 set HYDRA_NATIVE_BUILD=true
 call mvn -pl org.nzbhydra:core -Pnative clean native:compile -DskipTests
 

core/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.nzbhydra</groupId>
         <artifactId>nzbhydra2</artifactId>
-        <version>5.1.11</version>
+        <version>5.1.12-SNAPSHOT</version>
     </parent>
 
     <artifactId>core</artifactId>
@@ -39,7 +39,7 @@
                 <plugin>
                     <groupId>org.graalvm.buildtools</groupId>
                     <artifactId>native-maven-plugin</artifactId>
-                    <version>0.9.19</version>
+                    <version>0.9.26</version>
                     <extensions>true</extensions>
                 </plugin>
             </plugins>
@@ -99,7 +99,7 @@
         <dependency>
             <groupId>org.nzbhydra</groupId>
             <artifactId>mapping</artifactId>
-            <version>5.1.11</version>
+            <version>5.1.12-SNAPSHOT</version>
         </dependency>
 
         <!-- spring (boot) -->
@@ -499,6 +499,8 @@
                         </plugin>
                         <plugin>
                             <groupId>org.graalvm.buildtools</groupId>
+                            <!-- Fix Illegal char error on windows -->
+                            <version>0.9.27</version>
                             <artifactId>native-maven-plugin</artifactId>
                             <configuration>
                                 <buildArgs>

core/src/main/java/org/nzbhydra/DevEndpoint.java

@@ -34,7 +34,9 @@
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RestController;
+import sun.misc.Unsafe;
 
+import java.lang.reflect.Field;
 import java.math.BigInteger;
 import java.util.List;
 
@@ -104,7 +106,15 @@ public void sendIndexerDisabledNotification() {
         applicationEventPublisher.publishEvent(new IndexerDisabledNotificationEvent(configProvider.getBaseConfig().getIndexers().get(0).getName(), IndexerConfig.State.DISABLED_SYSTEM_TEMPORARY, "Some message generated by hydra6"));
         applicationEventPublisher.publishEvent(new IndexerDisabledNotificationEvent(configProvider.getBaseConfig().getIndexers().get(0).getName(), IndexerConfig.State.DISABLED_SYSTEM_TEMPORARY, "Some message generated by hydra7"));
         applicationEventPublisher.publishEvent(new IndexerDisabledNotificationEvent(configProvider.getBaseConfig().getIndexers().get(0).getName(), IndexerConfig.State.DISABLED_SYSTEM_TEMPORARY, "Some message generated by hydra8"));
+    }
+
 
+    @RequestMapping(value = "/dev/crash", method = RequestMethod.GET)
+    public void crashHard() throws Exception {
+        Field f = Unsafe.class.getDeclaredField("theUnsafe");
+        f.setAccessible(true);
+        Unsafe unsafe = (Unsafe) f.get(null);
+        unsafe.putAddress(0, 0);
     }
 
 

core/src/main/java/org/nzbhydra/NativeHints.java

@@ -53,6 +53,7 @@ public void registerHints(RuntimeHints hints, ClassLoader classLoader) {
         logger.info("Registering native hints");
 
         hints.resources().registerResourceBundle("joptsimple.ExceptionMessages");
+        hints.resources().registerResourceBundle("org.apache.xerces.impl.msg.XMLMessages");
 
         final Set<Class<?>> classes = getClassesToRegister();
         classes.add(HashSet.class);

core/src/main/java/org/nzbhydra/indexers/Newznab.java

@@ -222,6 +222,9 @@ protected void addFurtherParametersToUri(SearchRequest searchRequest, UriCompone
     }
 
     protected void calculateAndAddCategories(SearchRequest searchRequest, UriComponentsBuilder componentsBuilder) {
+        if (config.getCategoryMapping() == null) {
+            error("Category mapping unknown - caps check incomplete?");
+        }
         List<Integer> categoryIds = new ArrayList<>();
         if (searchRequest.getInternalData().getNewznabCategories().isEmpty() || configProvider.getBaseConfig().getSearching().isTransformNewznabCategories()) {
             if (searchRequest.getCategory().getSubtype() == Subtype.ANIME && config.getCategoryMapping().getAnime().isPresent()) {
@@ -276,7 +279,7 @@ protected String addForbiddenWords(SearchRequest searchRequest, String query) {
         allForbiddenWords.addAll(searchRequest.getCategory().getForbiddenWords());
         List<String> allPossibleForbiddenWords = allForbiddenWords.stream().filter(x -> !(x.contains(" ") || x.contains("-") || x.contains("."))).collect(Collectors.toList());
         if (allForbiddenWords.size() > allPossibleForbiddenWords.size()) {
-            logger.debug("Not using some forbidden words in query because characters forbidden by newznab are contained");
+            debug("Not using some forbidden words in query because characters forbidden by newznab are contained");
         }
         if (!allPossibleForbiddenWords.isEmpty()) {
             if (config.getBackend().equals(BackendType.NZEDB) || config.getBackend().equals(BackendType.NNTMUX) || config.getHost().toLowerCase().contains("omgwtf") || config.getHost().toLowerCase().contains("nzbfinder")) {
@@ -294,7 +297,7 @@ protected String addRequiredWords(SearchRequest searchRequest, String query) {
         allRequiredWords.addAll(searchRequest.getCategory().getRequiredWords());
         List<String> allPossibleRequiredWords = allRequiredWords.stream().filter(x -> !(x.contains(" ") || x.contains("-") || x.contains("."))).collect(Collectors.toList());
         if (allRequiredWords.size() > allPossibleRequiredWords.size()) {
-            logger.debug("Not using some forbidden words in query because characters forbidden by newznab are contained");
+            debug("Not using some forbidden words in query because characters forbidden by newznab are contained");
         }
         if (!allPossibleRequiredWords.isEmpty()) {
             query += (query.isEmpty() ? "" : " ") + Joiner.on(" ").join(allPossibleRequiredWords);
@@ -508,10 +511,10 @@ protected void completeIndexerSearchResult(Xml response, IndexerSearchResult ind
             indexerStatus.setOldestDownload(apiLimits.getGrabOldestTime());
 
             indexerStatusRepository.save(indexerStatus);
-            logger.debug(LoggingMarkers.LIMITS, "Indexer {}. Saving IndexerStatus data: {}", indexer.getName(), indexerStatus);
+            debug(LoggingMarkers.LIMITS, "Indexer {}. Saving IndexerStatus data: {}", indexer.getName(), indexerStatus);
 
         } else {
-            logger.debug(LoggingMarkers.LIMITS, "Indexer {}. No limits provided in response.", indexer.getName());
+            debug(LoggingMarkers.LIMITS, "Indexer {}. No limits provided in response.", indexer.getName());
         }
 
     }
@@ -723,14 +726,14 @@ private void putGroupMatchIfFound(SearchResultItem searchResultItem, Matcher mat
 
     protected void computeCategory(SearchResultItem searchResultItem, List<Integer> newznabCategories) {
         if (!newznabCategories.isEmpty()) {
-            logger.debug(LoggingMarkers.CATEGORY_MAPPING, "Result {} has newznab categories {} and self-reported category {}", searchResultItem.getTitle(), newznabCategories, searchResultItem.getCategory());
+            debug(LoggingMarkers.CATEGORY_MAPPING, "Result {} has newznab categories {} and self-reported category {}", searchResultItem.getTitle(), newznabCategories, searchResultItem.getCategory());
             Integer mostSpecific = newznabCategories.stream().max(Integer::compareTo).get();
             IndexerCategoryConfig mapping = config.getCategoryMapping();
             Category category;
             if (mapping == null) { //May be the case in some corner cases
                 category = categoryProvider.fromSearchNewznabCategories(newznabCategories, categoryProvider.getNotAvailable());
                 searchResultItem.setOriginalCategory(categoryProvider.getNotAvailable().getName());
-                logger.debug(LoggingMarkers.CATEGORY_MAPPING, "No mapping available. Using original category N/A and new category {} for result {}", category, searchResultItem.getTitle());
+                debug(LoggingMarkers.CATEGORY_MAPPING, "No mapping available. Using original category N/A and new category {} for result {}", category, searchResultItem.getTitle());
             } else {
                 category = idToCategory.computeIfAbsent(mostSpecific, x -> {
                     Optional<Category> categoryOptional = Optional.empty();
@@ -751,14 +754,14 @@ protected void computeCategory(SearchResultItem searchResultItem, List<Integer>
                 searchResultItem.setOriginalCategory(mapping.getNameFromId(mostSpecific));
             }
             if (category == null) {
-                logger.debug(LoggingMarkers.CATEGORY_MAPPING, "No category found for {}. Using N/A", searchResultItem.getTitle());
+                debug(LoggingMarkers.CATEGORY_MAPPING, "No category found for {}. Using N/A", searchResultItem.getTitle());
                 searchResultItem.setCategory(categoryProvider.getNotAvailable());
             } else {
-                logger.debug(LoggingMarkers.CATEGORY_MAPPING, "Determined category {} for {}", category, searchResultItem.getTitle());
+                debug(LoggingMarkers.CATEGORY_MAPPING, "Determined category {} for {}", category, searchResultItem.getTitle());
                 searchResultItem.setCategory(category);
             }
         } else {
-            logger.debug(LoggingMarkers.CATEGORY_MAPPING, "No newznab categories exist for {}. Using N/A", searchResultItem.getTitle());
+            debug(LoggingMarkers.CATEGORY_MAPPING, "No newznab categories exist for {}. Using N/A", searchResultItem.getTitle());
             searchResultItem.setCategory(categoryProvider.getNotAvailable());
         }
     }

core/src/main/resources/META-INF/native-image/resource-config.json

@@ -2840,6 +2840,13 @@
           "locales": [
               "en"
           ]
+      },
+    {
+      "name": "org.apache.xerces.impl.msg.XMLMessages",
+      "locales": [
+        "en",
+        "en_US"
+      ]
       }
   ]
 }

core/src/main/resources/changelog.yaml

@@ -1,6 +1,16 @@
 #@formatter:off
+-   version: "v5.2.0"
+    date: "2023-09-22"
+    changes:
+        -   type: "feature"
+            text: "In case of hard crashes of the main process it will automatically be restarted unless the last automatic restart was less than 15 seconds ago (to prevent loops)"
+        -   type: "note"
+            text: "Windows releases now only contain three exe files (and documents) and no more DLL files"
+        -   type: "fix"
+            text: "Hard crashes of the main process will now be logged properly. See #892"
+    final: true
 -   version: "v5.1.11"
-    date: "2023-09-1"
+    date: "2023-09-01"
     changes:
         -   type: "fix"
             text: "Allow to configure Radarr v5 automatically. See #889"

misc/buildLinuxCore/amd64/dockerfile

@@ -12,14 +12,14 @@ ENV M2_HOME=/opt/maven
 ENV MAVEN_HOME=/opt/maven
 ENV PATH=${M2_HOME}/bin:${PATH}
 
-RUN wget -nv --no-check-certificate https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-22.3.0/graalvm-ce-java17-linux-amd64-22.3.0.tar.gz
-RUN tar xzf graalvm-ce-java17-linux-amd64-22.3.0.tar.gz -C /
+RUN wget -nv --no-check-certificate https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-22.3.3/graalvm-ce-java17-linux-amd64-22.3.3.tar.gz
+RUN tar xzf graalvm-ce-java17-linux-amd64-22.3.3.tar.gz -C /
 ENV PATH=/graalvm-ce-java17-22.3.0/bin/:$PATH
 ENV JAVA_HOME=/graalvm-ce-java17-22.3.0
 
-RUN wget -nv --no-check-certificate https://github.com/upx/upx/releases/download/v4.0.1/upx-4.0.1-amd64_linux.tar.xz
-RUN tar -xf upx-4.0.1-amd64_linux.tar.xz
-ENV PATH=/tmp/upx-4.0.1-amd64_linux/:$PATH
+RUN wget -nv --no-check-certificate https://github.com/upx/upx/releases/download/v4.1.0/upx-4.1.0-amd64_linux.tar.xz
+RUN tar -xf upx-4.1.0-amd64_linux.tar.xz
+ENV PATH=/tmp/upx-4.1.0-amd64_linux/:$PATH
 
 ENV HYDRA_NATIVE_BUILD=true
 

other/mockserver/pom.xml

@@ -60,7 +60,7 @@
         <dependency>
             <groupId>org.nzbhydra</groupId>
             <artifactId>mapping</artifactId>
-            <version>5.1.11</version>
+            <version>5.1.12-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <artifactId>jaxb-impl</artifactId>

other/wrapper/nzbhydra2wrapperPy3.py

@@ -2,6 +2,7 @@
 import random
 import string
 import sys
+import time
 import traceback
 import webbrowser
 
@@ -55,6 +56,7 @@ class OsType(str, Enum):
 file_logger = None
 logger.setLevel(LOGGER_DEFAULT_LEVEL)
 consoleLines = []
+lastRestart = 0
 
 
 def getBasePath():
@@ -354,7 +356,8 @@ def startup():
             jarFile = jarFiles[0]
         else:
             latestFile = max(jarFiles, key=os.path.getmtime)
-            logger.warning("Expected the number of JAR files in folder %s to be 1 but it's %d. Will remove all JARs except the one last changed: %s", libFolder, len(jarFiles), latestFile)
+            logger.warning("Expected the number of JAR files in folder %s to be 1 but it's %d. Will remove all JARs except the one last changed: %s", libFolder, len(jarFiles),
+                           latestFile)
             for file in jarFiles:
                 if file is not latestFile:
                     logger.info("Deleting file %s", file)
@@ -472,7 +475,7 @@ def startup():
             if nextlineString != "":
                 consoleLines.append(nextlineString)
 
-            if len(consoleLines) > 100:
+            if len(consoleLines) > 1000:
                 consoleLines = consoleLines[-100:]
             if not args.quiet and sys.stdout is not None:
                 sys.stdout.write(nextlineString)
@@ -506,7 +509,8 @@ def determineReleaseType():
     if os.path.exists(os.path.join(getBasePath(), "lib")):
         releaseType = ReleaseType.GENERIC
         if os.path.exists(os.path.join(getBasePath(), "core")) or os.path.exists(os.path.join(getBasePath(), "core.exe")):
-            logger.warning("lib folder and core(.exe) found. Either delete the executable to use the generic release type (using java and ignoring the executable) or delete the lib folder to use the executable and not require java")
+            logger.warning(
+                "lib folder and core(.exe) found. Either delete the executable to use the generic release type (using java and ignoring the executable) or delete the lib folder to use the executable and not require java")
     elif os.path.exists(os.path.join(getBasePath(), "core")) or os.path.exists(os.path.join(getBasePath(), "core.exe")):
         releaseType = ReleaseType.NATIVE
     else:
@@ -540,8 +544,8 @@ def handleUnexpectedExit():
             message = "You seem to be trying to run NZBHydra with a wrong Java version. Please make sure to use at least Java 9"
         elif "java.lang.OutOfMemoryError" in x:
             message = "The main process has exited because it didn't have enough memory. Please increase the XMX value in the main config"
-    logger.error(message)
-    sys.exit(-1)
+    logger.error(message + "\nThe last 1000 lines from output:")
+    logger.error("".join(consoleLines).replace("\n\n", ""))
 
 
 def getJavaVersion(javaExecutable):
@@ -681,7 +685,7 @@ def main(arguments):
                 logger.debug("Shutting down because child process was terminated by us after getting signal")
                 sys.exit(0)
 
-            if process.returncode == 1:
+            if process.returncode in (1, 99):
                 handleUnexpectedExit()
             args.restarted = True
 
@@ -713,6 +717,16 @@ def main(arguments):
             elif controlCode == 22:
                 logger.info("NZBHydra main process has terminated for restart")
                 doStart = True
+            elif controlCode in (1, 99):
+                global lastRestart
+                difference = time.time() - lastRestart
+                if difference < 15:
+                    logger.warning("Last automatic restart was less than 15 seconds ago - quitting to prevent loop")
+                    doStart = False
+                else:
+                    logger.info("Restarting NZBHydra after hard crash")
+                    lastRestart = time.time()
+                    doStart = True
             elif controlCode == 33:
                 logger.info("NZBHydra main process has terminated for restoration")
                 doStart = restore()

pom.xml

@@ -4,7 +4,7 @@
     <groupId>org.nzbhydra</groupId>
     <artifactId>nzbhydra2</artifactId>
     <packaging>pom</packaging>
-    <version>5.1.11</version>
+    <version>5.1.12-SNAPSHOT</version>
 
     <modules>
         <module>shared</module>
@@ -112,7 +112,7 @@
         <spring-data-commons.version>3.0.3</spring-data-commons.version>
         <jakarta.persistence-api.version>3.1.0</jakarta.persistence-api.version>
         <jackson.version>2.14.0</jackson.version>
-        <lombok.version>1.18.24</lombok.version>
+        <lombok.version>1.18.30</lombok.version>
         <logback.version>1.4.5</logback.version>
         <guava.version>23.0</guava.version>
         <junit.version>4.13.2</junit.version>

releases/generic-release/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.nzbhydra</groupId>
         <artifactId>releases</artifactId>
-        <version>5.1.11</version>
+        <version>5.1.12-SNAPSHOT</version>
     </parent>
 
     <artifactId>generic-release</artifactId>
@@ -15,7 +15,7 @@
         <dependency>
             <groupId>org.nzbhydra</groupId>
             <artifactId>core</artifactId>
-            <version>5.1.11</version>
+            <version>5.1.12-SNAPSHOT</version>
         </dependency>
     </dependencies>
 

releases/linux-amd64-release/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.nzbhydra</groupId>
         <artifactId>releases</artifactId>
-        <version>5.1.11</version>
+        <version>5.1.12-SNAPSHOT</version>
     </parent>
 
     <artifactId>linux-amd64-release</artifactId>
@@ -15,7 +15,7 @@
         <dependency>
             <groupId>org.nzbhydra</groupId>
             <artifactId>core</artifactId>
-            <version>5.1.11</version>
+            <version>5.1.12-SNAPSHOT</version>
         </dependency>
     </dependencies>
 

releases/linux-arm64-release/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.nzbhydra</groupId>
         <artifactId>releases</artifactId>
-        <version>5.1.11</version>
+        <version>5.1.12-SNAPSHOT</version>
     </parent>
 
     <artifactId>linux-arm64-release</artifactId>
@@ -15,7 +15,7 @@
         <dependency>
             <groupId>org.nzbhydra</groupId>
             <artifactId>core</artifactId>
-            <version>5.1.11</version>
+            <version>5.1.12-SNAPSHOT</version>
         </dependency>
     </dependencies>
 

releases/pom.xml

@@ -4,12 +4,12 @@
     <parent>
         <groupId>org.nzbhydra</groupId>
         <artifactId>nzbhydra2</artifactId>
-        <version>5.1.11</version>
+        <version>5.1.12-SNAPSHOT</version>
     </parent>
 
     <artifactId>releases</artifactId>
     <packaging>pom</packaging>
-    <version>5.1.11</version>
+    <version>5.1.12-SNAPSHOT</version>
 
     <modules>
         <module>generic-release</module>