Merge pull request arduino#9023 from mattiabertorello/add-file-cache

Add file downloader cache to make faster the library/boards manager

.classpath

@@ -4,7 +4,6 @@
 	<classpathentry kind="src" path="app/test"/>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
 	<classpathentry kind="lib" path="app/lib/apple.jar"/>
-	<classpathentry kind="lib" path="app/lib/ecj.jar"/>
 	<classpathentry kind="lib" path="app/test-lib/junit-4.11.jar"/>
 	<classpathentry kind="lib" path="app/test-lib/fest-assert-1.2.jar"/>
 	<classpathentry kind="lib" path="app/test-lib/fest-reflect-1.2.jar"/>
@@ -16,10 +15,10 @@
 	<classpathentry kind="lib" path="app/lib/commons-httpclient-3.1.jar"/>
 	<classpathentry kind="lib" path="app/lib/commons-logging-1.0.4.jar"/>
 	<classpathentry kind="lib" path="app/lib/commons-net-3.3.jar"/>
-	<classpathentry kind="lib" path="app/lib/jmdns-3.5.1.jar"/>
+	<classpathentry kind="lib" path="app/lib/jmdns-3.5.3.jar"/>
 	<classpathentry kind="lib" path="app/lib/slf4j-api-1.7.22.jar"/>
 	<classpathentry kind="lib" path="app/lib/jsch-0.1.50.jar"/>
-	<classpathentry kind="lib" path="app/lib/jssc-2.8.0.jar"/>
+	<classpathentry kind="lib" path="app/lib/jssc-2.8.0-arduino3.jar"/>
 	<classpathentry kind="lib" path="app/lib/bcpg-jdk15on-152.jar"/>
 	<classpathentry kind="lib" path="app/lib/bcprov-jdk15on-152.jar"/>
 	<classpathentry kind="lib" path="app/lib/jackson-core-2.9.5.jar"/>

app/.classpath

@@ -39,6 +39,8 @@
 	<classpathentry kind="lib" path="lib/jmdns-3.5.3.jar"/>
 	<classpathentry kind="lib" path="lib/slf4j-api-1.7.22.jar"/>
 	<classpathentry kind="lib" path="lib/slf4j-simple-1.7.22.jar"/>
+  <classpathentry kind="lib" path="lib/log4j-api-2.12.0.jar"/>
+  <classpathentry kind="lib" path="lib/log4j-core-2.12.0.jar"/>
 	<classpathentry kind="lib" path="lib/jsch-0.1.50.jar"/>
 	<classpathentry kind="lib" path="lib/jssc-2.8.0-arduino3.jar"/>
 	<classpathentry kind="lib" path="lib/rsyntaxtextarea-3.0.3-SNAPSHOT.jar"/>

app/build.xml

@@ -80,6 +80,10 @@
 	   includeAntRuntime="false"
 	   debug="true"
 	   classpathref="class.path" />
+    <!-- If you want to add files in the jars -->
+    <copy todir="bin" overwrite="true" verbose="true">
+      <fileset dir="src" includes="log4j2.xml" />
+    </copy>
   </target>
 
   <target name="test" depends="compile" description="Runs the test">

app/src/log4j2.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="warn" name="Arduino" packages="cc.arduino">
+  <Appenders>
+
+    <!-- Console Appender -->
+    <Console name="Console" target="SYSTEM_ERR">
+      <PatternLayout pattern="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}{UTC} %p %c{1.} [%t] %m%n" />
+    </Console>
+
+    <!-- Rolling File Appender -->
+    <RollingFile name="RollingFile" fileName="${sys:log4j.saveDirectory}/logs/application.log"
+                 filePattern="${sys:log4j.saveDirectory}/logs/application-%d{MM-dd-yyyy}-%i.log.gz"
+                 ignoreExceptions="false">
+      <PatternLayout>
+        <Pattern>%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX}{UTC} %p %c{1.} [%t] %m%n</Pattern>
+      </PatternLayout>
+      <Policies>
+        <SizeBasedTriggeringPolicy size="50 MB"/>
+      </Policies>
+      <DefaultRolloverStrategy max="20"/>
+    </RollingFile>
+  </Appenders>
+  <Loggers>
+    <Root level="debug">
+      <AppenderRef ref="Console" level="info" />
+      <AppenderRef ref="RollingFile"/>
+    </Root>
+  </Loggers>
+</Configuration>

app/src/processing/app/Base.java

@@ -26,30 +26,29 @@
 import cc.arduino.Constants;
 import cc.arduino.UpdatableBoardsLibsFakeURLsHandler;
 import cc.arduino.UploaderUtils;
-import cc.arduino.packages.Uploader;
 import cc.arduino.contributions.*;
-import cc.arduino.contributions.libraries.*;
+import cc.arduino.contributions.libraries.ContributedLibrary;
+import cc.arduino.contributions.libraries.LibrariesIndexer;
+import cc.arduino.contributions.libraries.LibraryInstaller;
+import cc.arduino.contributions.libraries.LibraryOfSameTypeComparator;
 import cc.arduino.contributions.libraries.ui.LibraryManagerUI;
 import cc.arduino.contributions.packages.ContributedPlatform;
 import cc.arduino.contributions.packages.ContributionInstaller;
 import cc.arduino.contributions.packages.ContributionsIndexer;
 import cc.arduino.contributions.packages.ui.ContributionManagerUI;
 import cc.arduino.files.DeleteFilesOnShutdown;
 import cc.arduino.packages.DiscoveryManager;
+import cc.arduino.packages.Uploader;
 import cc.arduino.view.Event;
 import cc.arduino.view.JMenuUtils;
 import cc.arduino.view.SplashScreenHelper;
-
+import com.github.zafarkhaja.semver.Version;
 import org.apache.commons.compress.utils.IOUtils;
 import org.apache.commons.lang3.StringUtils;
-
-import com.github.zafarkhaja.semver.Version;
-
 import processing.app.debug.TargetBoard;
 import processing.app.debug.TargetPackage;
 import processing.app.debug.TargetPlatform;
 import processing.app.helpers.*;
-import processing.app.helpers.OSUtils;
 import processing.app.helpers.filefilters.OnlyDirs;
 import processing.app.helpers.filefilters.OnlyFilesWithExtension;
 import processing.app.javax.swing.filechooser.FileNameExtensionFilter;
@@ -67,9 +66,9 @@
 import java.awt.*;
 import java.awt.event.*;
 import java.io.*;
-import java.util.*;
 import java.util.List;
 import java.util.Timer;
+import java.util.*;
 import java.util.logging.Handler;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -208,6 +207,8 @@ public Base(String[] args) throws Exception {
     BaseNoGui.getPlatform().init();
 
     BaseNoGui.initPortableFolder();
+    // This configure the logs root folder
+    System.setProperty("log4j.saveDirectory", BaseNoGui.getSettingsFolder().getAbsolutePath());
 
     // Look for a possible "--preferences-file" parameter and load preferences
     BaseNoGui.initParameters(args);
@@ -286,8 +287,9 @@ public Base(String[] args) throws Exception {
     pdeKeywords = new PdeKeywords();
     pdeKeywords.reload();
 
-    contributionInstaller = new ContributionInstaller(BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier());
-    libraryInstaller = new LibraryInstaller(BaseNoGui.getPlatform());
+    final GPGDetachedSignatureVerifier gpgDetachedSignatureVerifier = new GPGDetachedSignatureVerifier();
+    contributionInstaller = new ContributionInstaller(BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier);
+    libraryInstaller = new LibraryInstaller(BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier);
 
     parser.parseArgumentsPhase2();
 
@@ -301,7 +303,7 @@ public Base(String[] args) throws Exception {
     if (parser.isInstallBoard()) {
       ContributionsIndexer indexer = new ContributionsIndexer(
           BaseNoGui.getSettingsFolder(), BaseNoGui.getHardwareFolder(),
-          BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier());
+          BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier);
       ProgressListener progressListener = new ConsoleProgressListener();
 
       List<String> downloadedPackageIndexFiles = contributionInstaller.updateIndex(progressListener);

arduino-core/.classpath

@@ -8,6 +8,8 @@
 	<classpathentry kind="lib" path="lib/jmdns-3.5.3.jar"/>
 	<classpathentry kind="lib" path="lib/slf4j-api-1.7.22.jar"/>
 	<classpathentry kind="lib" path="lib/slf4j-simple-1.7.22.jar"/>
+	<classpathentry kind="lib" path="lib/log4j-api-2.12.0.jar"/>
+	<classpathentry kind="lib" path="lib/log4j-core-2.12.0.jar"/>
 	<classpathentry kind="lib" path="lib/jssc-2.8.0-arduino3.jar"/>
 	<classpathentry kind="lib" path="lib/jsch-0.1.50.jar"/>
 	<classpathentry kind="lib" path="lib/commons-exec-1.1.jar"/>

arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java

@@ -30,32 +30,37 @@
 package cc.arduino.contributions;
 
 import cc.arduino.utils.FileHash;
+import cc.arduino.utils.MultiStepProgress;
 import cc.arduino.utils.Progress;
 import cc.arduino.utils.network.FileDownloader;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import processing.app.BaseNoGui;
+import processing.app.PreferencesData;
 
 import java.io.File;
 import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.LinkOption;
-import java.nio.file.Path;
-import java.nio.file.Paths;
+import java.nio.file.*;
+import java.util.Collection;
 
 import static processing.app.I18n.format;
 import static processing.app.I18n.tr;
 
 public class DownloadableContributionsDownloader {
+  private static Logger log = LogManager.getLogger(DownloadableContributionsDownloader.class);
 
   private final File stagingFolder;
 
   public DownloadableContributionsDownloader(File _stagingFolder) {
     stagingFolder = _stagingFolder;
   }
 
-  public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener) throws Exception {
-    return download(contribution, progress, statusText, progressListener, false);
+  public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener, boolean allowCache) throws Exception {
+    return download(contribution, progress, statusText, progressListener, false, allowCache);
   }
 
-  public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener, boolean noResume) throws Exception {
+  public File download(DownloadableContribution contribution, Progress progress, final String statusText, ProgressListener progressListener, boolean noResume, boolean allowCache) throws Exception {
     URL url = new URL(contribution.getUrl());
     Path outputFile = Paths.get(stagingFolder.getAbsolutePath(), contribution.getArchiveFileName());
 
@@ -70,7 +75,7 @@ public File download(DownloadableContribution contribution, Progress progress, f
     while (true) {
       // Need to download or resume downloading?
       if (!Files.isRegularFile(outputFile, LinkOption.NOFOLLOW_LINKS) || (Files.size(outputFile) < contribution.getSize())) {
-        download(url, outputFile.toFile(), progress, statusText, progressListener, noResume);
+        download(url, outputFile.toFile(), progress, statusText, progressListener, noResume, allowCache);
         downloaded = true;
       }
 
@@ -116,12 +121,12 @@ private boolean hasChecksum(DownloadableContribution contribution) {
     return algo != null && !algo.isEmpty();
   }
 
-  public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener) throws Exception {
-    download(url, tmpFile, progress, statusText, progressListener, false);
+  public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean allowCache) throws Exception {
+    download(url, tmpFile, progress, statusText, progressListener, false, allowCache);
   }
 
-  public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean noResume) throws Exception {
-    FileDownloader downloader = new FileDownloader(url, tmpFile);
+  public void download(URL url, File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean noResume, boolean allowCache) throws Exception {
+    final FileDownloader downloader = new FileDownloader(url, tmpFile, allowCache);
     downloader.addObserver((o, arg) -> {
       FileDownloader me = (FileDownloader) o;
       String msg = "";
@@ -140,4 +145,94 @@ public void download(URL url, File tmpFile, Progress progress, String statusText
     }
   }
 
+  public void downloadIndexAndSignature(MultiStepProgress progress, URL packageIndexUrl, ProgressListener progressListener, SignatureVerifier signatureVerifier) throws Exception {
+
+    // Extract the file name from the url
+    final String indexFileName = FilenameUtils.getName(packageIndexUrl.getPath());
+    final File packageIndex = BaseNoGui.indexer.getIndexFile(indexFileName);
+
+    final String statusText = tr("Downloading platforms index...");
+
+    // Create temp files
+    final File packageIndexTemp = File.createTempFile(indexFileName, ".tmp");
+    try {
+      // Download package index
+      download(packageIndexUrl, packageIndexTemp, progress, statusText, progressListener, true, true);
+      final URL signatureUrl = new URL(packageIndexUrl.toString() + ".sig");
+
+      if (verifyDomain(packageIndexUrl)) {
+        if (checkSignature(progress, signatureUrl, progressListener, signatureVerifier, statusText, packageIndexTemp)) {
+          Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING);
+        } else {
+          log.info("The cached files have been removed. {} {}", packageIndexUrl, signatureUrl);
+          FileDownloader.invalidateFiles(packageIndexUrl, signatureUrl);
+        }
+      } else {
+        // Move the package index to the destination when the signature is not necessary
+        Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING);
+        log.info("The domain is not selected to verify the signature. will be copied into this path {}, packageIndex url: {}", packageIndex, packageIndexUrl);
+      }
+    } catch (Exception e) {
+      log.error("Cannot download the package index from {} the package will be discard", packageIndexUrl, e);
+      throw e;
+    } finally {
+      // Delete useless temp file
+      Files.deleteIfExists(packageIndexTemp.toPath());
+    }
+  }
+
+  public boolean verifyDomain(URL url) {
+    final Collection<String> domain = PreferencesData.
+      getCollection("http.signature_verify_domains");
+    if (domain.size() == 0) {
+      // Default domain
+      domain.add("downloads.arduino.cc");
+    }
+    if (domain.contains(url.getHost())) {
+      return true;
+    } else {
+      log.info("The domain is not selected to verify the signature. domain list: {}, url: {}", domain, url);
+      return false;
+    }
+  }
+
+  public boolean checkSignature(MultiStepProgress progress, URL signatureUrl, ProgressListener progressListener, SignatureVerifier signatureVerifier, String statusText, File fileToVerify) throws Exception {
+
+    final boolean allowInsecurePackages =
+      PreferencesData.getBoolean("allow_insecure_packages", false);
+    if (allowInsecurePackages) {
+      log.info("Allow insecure packages is true the signature will be skip and return always verified");
+      return true;
+    }
+
+    // Signature file name
+    final String signatureFileName = FilenameUtils.getName(signatureUrl.getPath());
+    final File packageIndexSignature = BaseNoGui.indexer.getIndexFile(signatureFileName);
+    final File packageIndexSignatureTemp = File.createTempFile(signatureFileName, ".tmp");
+
+
+    try {
+      // Download signature
+      download(signatureUrl, packageIndexSignatureTemp, progress, statusText, progressListener, true);
+
+      // Verify the signature before move the files
+      final boolean signatureVerified = signatureVerifier.isSigned(fileToVerify, packageIndexSignatureTemp);
+      if (signatureVerified) {
+        log.info("Signature verified. url={}, signature url={}, file to verify={}, signature file={}", signatureUrl, signatureUrl, fileToVerify, packageIndexSignatureTemp);
+        // Move if the signature is ok
+        Files.move(packageIndexSignatureTemp.toPath(), packageIndexSignature.toPath(), StandardCopyOption.REPLACE_EXISTING);
+      } else {
+        log.error("{} file signature verification failed. File ignored.", signatureUrl);
+        System.err.println(format(tr("{0} file signature verification failed. File ignored."), signatureUrl.toString()));
+      }
+      return signatureVerified;
+    } catch (Exception e) {
+      log.error("Cannot download the signature from {} the package will be discard", signatureUrl, e);
+      throw e;
+    } finally {
+      Files.deleteIfExists(packageIndexSignatureTemp.toPath());
+    }
+
+  }
+
 }

arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java

@@ -29,13 +29,16 @@
 
 package cc.arduino.contributions;
 
+import cc.arduino.Constants;
 import cc.arduino.utils.Progress;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.apache.commons.compress.compressors.gzip.GzipUtils;
 import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FilenameUtils;
 
 import java.io.*;
 import java.net.URL;
+import java.nio.file.Files;
 
 public class GZippedJsonDownloader {
 
@@ -49,18 +52,22 @@ public GZippedJsonDownloader(DownloadableContributionsDownloader downloader, URL
     this.gzippedUrl = gzippedUrl;
   }
 
-  public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener) throws Exception {
+  public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean allowCache) throws Exception {
+    File gzipTmpFile = null;
     try {
-      File gzipTmpFile = new File(tmpFile.getParentFile(), GzipUtils.getCompressedFilename(tmpFile.getName()));
+      String tmpFileName = FilenameUtils.getName(new URL(Constants.LIBRARY_INDEX_URL_GZ).getPath());
+      gzipTmpFile = File.createTempFile(tmpFileName, GzipUtils.getCompressedFilename(tmpFile.getName()));
       // remove eventual leftovers from previous downloads
-      if (gzipTmpFile.exists()) {
-        gzipTmpFile.delete();
-      }
-      new JsonDownloader(downloader, gzippedUrl).download(gzipTmpFile, progress, statusText, progressListener);
+      Files.deleteIfExists(gzipTmpFile.toPath());
+
+      new JsonDownloader(downloader, gzippedUrl).download(gzipTmpFile, progress, statusText, progressListener, allowCache);
       decompress(gzipTmpFile, tmpFile);
-      gzipTmpFile.delete();
     } catch (Exception e) {
-      new JsonDownloader(downloader, url).download(tmpFile, progress, statusText, progressListener);
+      new JsonDownloader(downloader, url).download(tmpFile, progress, statusText, progressListener, allowCache);
+    } finally {
+      if (gzipTmpFile != null) {
+        Files.deleteIfExists(gzipTmpFile.toPath());
+      }
     }
   }
 

arduino-core/src/cc/arduino/contributions/JsonDownloader.java

@@ -44,9 +44,9 @@ public JsonDownloader(DownloadableContributionsDownloader downloader, URL url) {
     this.url = url;
   }
 
-  public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener) throws Exception {
+  public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener, boolean allowCache) throws Exception {
     try {
-      downloader.download(url, tmpFile, progress, statusText, progressListener);
+      downloader.download(url, tmpFile, progress, statusText, progressListener, allowCache);
     } catch (InterruptedException e) {
       // Download interrupted... just exit
     }

arduino-core/src/cc/arduino/contributions/SignatureVerifier.java

@@ -50,6 +50,15 @@ public boolean isSigned(File indexFile) {
     }
   }
 
+  public boolean isSigned(File indexFile, File signature) {
+    try {
+      return verify(indexFile, signature, new File(BaseNoGui.getContentFile("lib"), "public.gpg.key"));
+    } catch (Exception e) {
+      BaseNoGui.showWarning(e.getMessage(), e.getMessage(), e);
+      return false;
+    }
+  }
+
   protected abstract boolean verify(File signedFile, File signature, File publicKey) throws IOException;
 
 }