From 6592c42dcf7a4a905f1bfd93e0b77a064be7ba41 Mon Sep 17 00:00:00 2001 From: Mattia Bertorello Date: Fri, 28 Jun 2019 09:02:21 +0200 Subject: [PATCH 01/33] Add the file downloader cache to make faster the library/boards manager --- .../arduino/utils/network/FileDownloader.java | 102 +++++++--------- .../utils/network/FileDownloaderCache.java | 100 +++++++++++++++ .../utils/network/HttpConnectionManager.java | 114 ++++++++++++++++++ 3 files changed, 258 insertions(+), 58 deletions(-) create mode 100644 arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java create mode 100644 arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java index 8e25cc9227c..f2cbcd07f32 100644 --- a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java +++ b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java @@ -29,26 +29,26 @@ package cc.arduino.utils.network; -import cc.arduino.net.CustomProxySelector; -import org.apache.commons.codec.binary.Base64; import org.apache.commons.compress.utils.IOUtils; import processing.app.BaseNoGui; -import processing.app.PreferencesData; +import processing.app.helpers.FileUtils; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; -import java.net.HttpURLConnection; -import java.net.Proxy; -import java.net.SocketTimeoutException; -import java.net.URL; +import java.net.*; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Observable; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; public class FileDownloader extends Observable { + private static Logger log = Logger + .getLogger(FileDownloader.class.getName()); public enum Status { CONNECTING, // @@ -68,15 +68,12 @@ public enum Status { private final File outputFile; private InputStream stream = null; private Exception error; - private String userAgent; public FileDownloader(URL url, File file) { downloadUrl = url; outputFile = file; downloaded = 0; initialSize = 0; - userAgent = "ArduinoIDE/" + BaseNoGui.VERSION_NAME + " Java/" - + System.getProperty("java.version"); } public long getInitialSize() { @@ -144,12 +141,37 @@ private void saveLocalFile() { } private void downloadFile(boolean noResume) throws InterruptedException { - RandomAccessFile file = null; + RandomAccessFile randomAccessOutputFile = null; try { + setStatus(Status.CONNECTING); + + final File settingsFolder = BaseNoGui.getPlatform().getSettingsFolder(); + final String cacheFolder = Paths.get(settingsFolder.getPath(), "cache").toString(); + final FileDownloaderCache fileDownloaderCache = + new FileDownloaderCache(cacheFolder, downloadUrl); + + final boolean isChanged = fileDownloaderCache.checkIfTheFileIsChanged(); + + if (!isChanged) { + try { + final Optional fileFromCache = + fileDownloaderCache.getFileFromCache(); + if (fileFromCache.isPresent()) { + FileUtils.copyFile(fileFromCache.get(), outputFile); + setStatus(Status.COMPLETE); + return; + } + } catch (Exception e) { + log.log(Level.WARNING, + "Cannot get the file from the cache, will be downloaded a new one ", e.getCause()); + + } + } + // Open file and seek to the end of it - file = new RandomAccessFile(outputFile, "rw"); - initialSize = file.length(); + randomAccessOutputFile = new RandomAccessFile(outputFile, "rw"); + initialSize = randomAccessOutputFile.length(); if (noResume && initialSize > 0) { // delete file and restart downloading @@ -157,49 +179,11 @@ private void downloadFile(boolean noResume) throws InterruptedException { initialSize = 0; } - file.seek(initialSize); - - setStatus(Status.CONNECTING); - - Proxy proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(downloadUrl.toURI()); - if ("true".equals(System.getProperty("DEBUG"))) { - System.err.println("Using proxy " + proxy); - } - - HttpURLConnection connection = (HttpURLConnection) downloadUrl.openConnection(proxy); - connection.setRequestProperty("User-agent", userAgent); - if (downloadUrl.getUserInfo() != null) { - String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes())); - connection.setRequestProperty("Authorization", auth); - } - - connection.setRequestProperty("Range", "bytes=" + initialSize + "-"); - connection.setConnectTimeout(5000); - setDownloaded(0); - - // Connect - connection.connect(); - int resp = connection.getResponseCode(); - - if (resp == HttpURLConnection.HTTP_MOVED_PERM || resp == HttpURLConnection.HTTP_MOVED_TEMP) { - URL newUrl = new URL(connection.getHeaderField("Location")); + randomAccessOutputFile.seek(initialSize); - proxy = new CustomProxySelector(PreferencesData.getMap()).getProxyFor(newUrl.toURI()); - - // open the new connnection again - connection = (HttpURLConnection) newUrl.openConnection(proxy); - connection.setRequestProperty("User-agent", userAgent); - if (downloadUrl.getUserInfo() != null) { - String auth = "Basic " + new String(new Base64().encode(downloadUrl.getUserInfo().getBytes())); - connection.setRequestProperty("Authorization", auth); - } - - connection.setRequestProperty("Range", "bytes=" + initialSize + "-"); - connection.setConnectTimeout(5000); - - connection.connect(); - resp = connection.getResponseCode(); - } + final HttpURLConnection connection = new HttpConnectionManager(downloadUrl) + .makeConnection((c) -> setDownloaded(0)); + final int resp = connection.getResponseCode(); if (resp < 200 || resp >= 300) { throw new IOException("Received invalid http status code from server: " + resp); @@ -221,11 +205,11 @@ private void downloadFile(boolean noResume) throws InterruptedException { if (read == -1) break; - file.write(buffer, 0, read); + randomAccessOutputFile.write(buffer, 0, read); setDownloaded(getDownloaded() + read); if (Thread.interrupted()) { - file.close(); + randomAccessOutputFile.close(); throw new InterruptedException(); } } @@ -234,6 +218,8 @@ private void downloadFile(boolean noResume) throws InterruptedException { if (getDownloaded() < getDownloadSize()) throw new Exception("Incomplete download"); } + // Set the cache whe it finish to download the file + fileDownloaderCache.fillCache(outputFile); setStatus(Status.COMPLETE); } catch (InterruptedException e) { setStatus(Status.CANCELLED); @@ -249,7 +235,7 @@ private void downloadFile(boolean noResume) throws InterruptedException { setError(e); } finally { - IOUtils.closeQuietly(file); + IOUtils.closeQuietly(randomAccessOutputFile); synchronized (this) { IOUtils.closeQuietly(stream); diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java new file mode 100644 index 00000000000..bac621a4f22 --- /dev/null +++ b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java @@ -0,0 +1,100 @@ +package cc.arduino.utils.network; + +import com.sun.istack.internal.NotNull; +import processing.app.PreferencesData; +import processing.app.helpers.FileUtils; + +import javax.script.ScriptException; +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.ProtocolException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; +import java.util.logging.Logger; + +public class FileDownloaderCache { + private static Logger log = Logger + .getLogger(FileDownloaderCache.class.getName()); + private final String cacheFolder; + private final URL remoteURL; + private final Path cacheFilePath; + // Will be initialized by call the checkIfTheFileIsChanged function + private String eTag; + + // BaseNoGui.getSettingsFolder() + public FileDownloaderCache(@NotNull String cacheFolder, @NotNull URL remoteURL) { + this.cacheFolder = cacheFolder; + this.remoteURL = remoteURL; + String[] splitPath = remoteURL.getPath().split("/"); + if (splitPath.length > 0) { + this.cacheFilePath = Paths.get(cacheFolder, splitPath[splitPath.length - 1]); + } else { + this.cacheFilePath = null; + } + } + + public boolean checkIfTheFileIsChanged() + throws NoSuchMethodException, ScriptException, IOException, + URISyntaxException { + + final HttpURLConnection headRequest = new HttpConnectionManager(remoteURL) + .makeConnection((connection) -> { + try { + connection.setRequestMethod("HEAD"); + } catch (ProtocolException e) { + throw new RuntimeException(e); + } + }); + final int responseCode = headRequest.getResponseCode(); + // Something bad is happening return a conservative true to try to download the file + if (responseCode < 200 || responseCode >= 300) { + log.warning("The head request return a bad response code " + responseCode); + return true; + } + + final String remoteETag = headRequest.getHeaderField("ETag"); + final String localETag = PreferencesData.get(getPreferencesDataKey()); + + // If the header doesn't exist or the local cache doesn't exist you need to download the file + if (remoteETag == null || localETag == null) { + return true; + } + eTag = remoteETag.trim().replace("\"", ""); + + // If are different means that the file is change + return !eTag.equals(localETag); + } + + public Optional getFileFromCache() { + if (Optional.ofNullable(cacheFilePath).isPresent()) { + if (Files.exists(cacheFilePath)) { + return Optional.of(new File(cacheFilePath.toUri())); + } + } + return Optional.empty(); + + } + + public void fillCache(File fileToCache) throws Exception { + if (Optional.ofNullable(eTag).isPresent() && + Optional.ofNullable(cacheFilePath).isPresent()) { + + PreferencesData.set(getPreferencesDataKey(), eTag); + // If the cache directory does not exist create it + final Path cachePath = Paths.get(cacheFolder); + if (!Files.exists(cachePath)) { + Files.createDirectory(cachePath); + } + FileUtils.copyFile(fileToCache, cacheFilePath.toFile()); + } + } + + private String getPreferencesDataKey() { + return "cache.file." + remoteURL.getPath(); + } +} diff --git a/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java b/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java new file mode 100644 index 00000000000..9f2562485ba --- /dev/null +++ b/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java @@ -0,0 +1,114 @@ +package cc.arduino.utils.network; + +import cc.arduino.net.CustomProxySelector; +import org.apache.commons.codec.binary.Base64; +import processing.app.BaseNoGui; +import processing.app.PreferencesData; + +import javax.script.ScriptException; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.Proxy; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.function.Consumer; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class HttpConnectionManager { + private static Logger log = Logger + .getLogger(HttpConnectionManager.class.getName()); + private final URL requestURL; + private final String userAgent; + private int connectTimeout; + + + public HttpConnectionManager(URL requestURL) { + this.requestURL = requestURL; + final String defaultUserAgent = String.format( + "ArduinoIDE/%s (%s; %s; %s; %s) Java/%s (%s)", + BaseNoGui.VERSION_NAME, + System.getProperty("os.name"), + System.getProperty("os.version"), + System.getProperty("os.arch"), + System.getProperty("user.language"), + System.getProperty("java.version"), + System.getProperty("java.vendor") + ); + this.userAgent = PreferencesData.get("http.user_agent", defaultUserAgent); + try { + this.connectTimeout = + Integer.parseInt( + PreferencesData.get("http.connection_timeout", "5000")); + } catch (NumberFormatException e) { + log.log(Level.WARNING, + "Cannot parse the http.connection_timeout configuration switch to default 5000 milliseconds", e.getCause()); + this.connectTimeout = 5000; + } + + } + + public HttpURLConnection makeConnection(Consumer beforeConnection) + throws URISyntaxException, NoSuchMethodException, IOException, + ScriptException { + return makeConnection(this.requestURL, 0, beforeConnection); + } + + private HttpURLConnection makeConnection(URL requestURL, int movedTimes, + Consumer beforeConnection) + throws NoSuchMethodException, ScriptException, IOException, + URISyntaxException { + log.info("Prepare http request to " + requestURL); + if (requestURL == null) { + log.warning("Invalid request url is null"); + throw new RuntimeException("Invalid request url is null"); + } + if (movedTimes > 3) { + log.warning("Too many redirect " + requestURL); + throw new RuntimeException("Too many redirect " + requestURL); + } + + Proxy proxy = new CustomProxySelector(PreferencesData.getMap()) + .getProxyFor(requestURL.toURI()); + if ("true".equals(System.getProperty("DEBUG"))) { + System.err.println("Using proxy " + proxy); + } + + HttpURLConnection connection = (HttpURLConnection) requestURL + .openConnection(proxy); + connection.setRequestProperty("User-agent", userAgent); + if (requestURL.getUserInfo() != null) { + String auth = "Basic " + new String( + new Base64().encode(requestURL.getUserInfo().getBytes())); + connection.setRequestProperty("Authorization", auth); + } + + int initialSize = 0; + connection.setRequestProperty("Range", "bytes=" + initialSize + "-"); + connection.setConnectTimeout(connectTimeout); + beforeConnection.accept(connection); + + // Connect + log.info("Connect to " + requestURL); + + connection.connect(); + int resp = connection.getResponseCode(); + + if (resp == HttpURLConnection.HTTP_MOVED_PERM + || resp == HttpURLConnection.HTTP_MOVED_TEMP) { + + URL newUrl = new URL(connection.getHeaderField("Location")); + log.info("The response code was a 301,302 so try again with the new URL " + newUrl); + + return this.makeConnection(newUrl, movedTimes + 1, beforeConnection); + } + log.info("The response code was" + resp); + + return connection; + } + + public void setConnectTimeout(int connectTimeout) { + this.connectTimeout = connectTimeout; + } +} + From 8ca093b94551a42398f437d3c5f3d66c09f10c05 Mon Sep 17 00:00:00 2001 From: Mattia Bertorello Date: Fri, 28 Jun 2019 13:02:50 +0200 Subject: [PATCH 02/33] Add slf4j, optimize some code and fix reported lint problem --- .../arduino/utils/network/FileDownloader.java | 9 ++-- .../utils/network/FileDownloaderCache.java | 26 +++++------ .../utils/network/HttpConnectionManager.java | 45 +++++++++---------- arduino-core/src/processing/app/Platform.java | 32 ++++++------- 4 files changed, 51 insertions(+), 61 deletions(-) diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java index f2cbcd07f32..27f677bfd13 100644 --- a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java +++ b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java @@ -31,6 +31,8 @@ import org.apache.commons.compress.utils.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import processing.app.BaseNoGui; import processing.app.helpers.FileUtils; @@ -44,11 +46,9 @@ import java.util.Observable; import java.util.Optional; import java.util.logging.Level; -import java.util.logging.Logger; public class FileDownloader extends Observable { - private static Logger log = Logger - .getLogger(FileDownloader.class.getName()); + private static Logger log = LoggerFactory.getLogger(FileDownloader.class); public enum Status { CONNECTING, // @@ -163,7 +163,7 @@ private void downloadFile(boolean noResume) throws InterruptedException { return; } } catch (Exception e) { - log.log(Level.WARNING, + log.warn( "Cannot get the file from the cache, will be downloaded a new one ", e.getCause()); } @@ -219,6 +219,7 @@ private void downloadFile(boolean noResume) throws InterruptedException { throw new Exception("Incomplete download"); } // Set the cache whe it finish to download the file + IOUtils.closeQuietly(randomAccessOutputFile); fileDownloaderCache.fillCache(outputFile); setStatus(Status.COMPLETE); } catch (InterruptedException e) { diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java index bac621a4f22..b89a862b45f 100644 --- a/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java +++ b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java @@ -1,6 +1,8 @@ package cc.arduino.utils.network; import com.sun.istack.internal.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import processing.app.PreferencesData; import processing.app.helpers.FileUtils; @@ -15,12 +17,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; -import java.util.logging.Logger; public class FileDownloaderCache { - private static Logger log = Logger - .getLogger(FileDownloaderCache.class.getName()); - private final String cacheFolder; + private static Logger log = LoggerFactory.getLogger(FileDownloaderCache.class); private final URL remoteURL; private final Path cacheFilePath; // Will be initialized by call the checkIfTheFileIsChanged function @@ -28,11 +27,10 @@ public class FileDownloaderCache { // BaseNoGui.getSettingsFolder() public FileDownloaderCache(@NotNull String cacheFolder, @NotNull URL remoteURL) { - this.cacheFolder = cacheFolder; this.remoteURL = remoteURL; String[] splitPath = remoteURL.getPath().split("/"); if (splitPath.length > 0) { - this.cacheFilePath = Paths.get(cacheFolder, splitPath[splitPath.length - 1]); + this.cacheFilePath = Paths.get(cacheFolder, splitPath); } else { this.cacheFilePath = null; } @@ -47,13 +45,14 @@ public boolean checkIfTheFileIsChanged() try { connection.setRequestMethod("HEAD"); } catch (ProtocolException e) { - throw new RuntimeException(e); + log.error("Invalid protocol", e); } }); final int responseCode = headRequest.getResponseCode(); + headRequest.disconnect(); // Something bad is happening return a conservative true to try to download the file if (responseCode < 200 || responseCode >= 300) { - log.warning("The head request return a bad response code " + responseCode); + log.warn("The head request return a bad response code " + responseCode); return true; } @@ -71,10 +70,8 @@ public boolean checkIfTheFileIsChanged() } public Optional getFileFromCache() { - if (Optional.ofNullable(cacheFilePath).isPresent()) { - if (Files.exists(cacheFilePath)) { - return Optional.of(new File(cacheFilePath.toUri())); - } + if (Optional.ofNullable(cacheFilePath).isPresent() && Files.exists(cacheFilePath)) { + return Optional.of(new File(cacheFilePath.toUri())); } return Optional.empty(); @@ -86,9 +83,8 @@ public void fillCache(File fileToCache) throws Exception { PreferencesData.set(getPreferencesDataKey(), eTag); // If the cache directory does not exist create it - final Path cachePath = Paths.get(cacheFolder); - if (!Files.exists(cachePath)) { - Files.createDirectory(cachePath); + if (!Files.exists(cacheFilePath.getParent())) { + Files.createDirectories(cacheFilePath.getParent()); } FileUtils.copyFile(fileToCache, cacheFilePath.toFile()); } diff --git a/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java b/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java index 9f2562485ba..39a72bb646e 100644 --- a/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java +++ b/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java @@ -1,7 +1,12 @@ package cc.arduino.utils.network; import cc.arduino.net.CustomProxySelector; +import com.sun.istack.internal.NotNull; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.httpclient.CircularRedirectException; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import processing.app.BaseNoGui; import processing.app.PreferencesData; @@ -12,12 +17,9 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.function.Consumer; -import java.util.logging.Level; -import java.util.logging.Logger; public class HttpConnectionManager { - private static Logger log = Logger - .getLogger(HttpConnectionManager.class.getName()); + private static Logger log = LoggerFactory.getLogger(HttpConnectionManager.class); private final URL requestURL; private final String userAgent; private int connectTimeout; @@ -41,38 +43,35 @@ public HttpConnectionManager(URL requestURL) { Integer.parseInt( PreferencesData.get("http.connection_timeout", "5000")); } catch (NumberFormatException e) { - log.log(Level.WARNING, + log.warn( "Cannot parse the http.connection_timeout configuration switch to default 5000 milliseconds", e.getCause()); this.connectTimeout = 5000; } } - public HttpURLConnection makeConnection(Consumer beforeConnection) - throws URISyntaxException, NoSuchMethodException, IOException, - ScriptException { + public HttpURLConnection makeConnection(@NotNull Consumer beforeConnection) + throws IOException, NoSuchMethodException, ScriptException, URISyntaxException { return makeConnection(this.requestURL, 0, beforeConnection); } - private HttpURLConnection makeConnection(URL requestURL, int movedTimes, - Consumer beforeConnection) - throws NoSuchMethodException, ScriptException, IOException, - URISyntaxException { + + public HttpURLConnection makeConnection() + throws IOException, NoSuchMethodException, ScriptException, URISyntaxException { + return makeConnection(this.requestURL, 0, (c) -> {}); + } + + private HttpURLConnection makeConnection(@NotNull URL requestURL, int movedTimes, + @NotNull Consumer beforeConnection) throws IOException, URISyntaxException, ScriptException, NoSuchMethodException { log.info("Prepare http request to " + requestURL); - if (requestURL == null) { - log.warning("Invalid request url is null"); - throw new RuntimeException("Invalid request url is null"); - } if (movedTimes > 3) { - log.warning("Too many redirect " + requestURL); - throw new RuntimeException("Too many redirect " + requestURL); + log.warn("Too many redirect " + requestURL); + throw new CircularRedirectException("Too many redirect " + requestURL); } Proxy proxy = new CustomProxySelector(PreferencesData.getMap()) .getProxyFor(requestURL.toURI()); - if ("true".equals(System.getProperty("DEBUG"))) { - System.err.println("Using proxy " + proxy); - } + log.debug("Using proxy {}", proxy); HttpURLConnection connection = (HttpURLConnection) requestURL .openConnection(proxy); @@ -89,7 +88,7 @@ private HttpURLConnection makeConnection(URL requestURL, int movedTimes, beforeConnection.accept(connection); // Connect - log.info("Connect to " + requestURL); + log.info("Connect to {} with method {}", requestURL, connection.getRequestMethod()); connection.connect(); int resp = connection.getResponseCode(); @@ -102,7 +101,7 @@ private HttpURLConnection makeConnection(URL requestURL, int movedTimes, return this.makeConnection(newUrl, movedTimes + 1, beforeConnection); } - log.info("The response code was" + resp); + log.info("The response code {}, headers {}", resp, StringUtils.join(connection.getHeaderFields())); return connection; } diff --git a/arduino-core/src/processing/app/Platform.java b/arduino-core/src/processing/app/Platform.java index 535c31ed7d3..a60c1cc59ae 100644 --- a/arduino-core/src/processing/app/Platform.java +++ b/arduino-core/src/processing/app/Platform.java @@ -23,6 +23,11 @@ package processing.app; import cc.arduino.packages.BoardPort; +import cc.arduino.utils.network.HttpConnectionManager; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import processing.app.debug.TargetBoard; import processing.app.debug.TargetPackage; import processing.app.debug.TargetPlatform; @@ -31,19 +36,10 @@ import javax.swing.*; import java.io.File; import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.ArrayList; -import java.util.Arrays; - -import java.net.URL; -import java.net.URLConnection; -import java.net.HttpURLConnection; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.DeserializationFeature; import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.*; import static processing.app.I18n.tr; @@ -64,7 +60,7 @@ * know if name is proper Java package syntax.) */ public class Platform { - + private static Logger log = LoggerFactory.getLogger(Platform.class); /** * Set the default L & F. While I enjoy the bounty of the sixteen possible @@ -207,12 +203,9 @@ public synchronized void getBoardWithMatchingVidPidFromCloud(String vid, String mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); try { URL jsonUrl = new URL("http", "api-builder.arduino.cc", 80, "/builder/v1/boards/0x"+vid+"/0x"+pid); - URLConnection connection = jsonUrl.openConnection(); - String userAgent = "ArduinoIDE/" + BaseNoGui.VERSION_NAME + " Java/" - + System.getProperty("java.version"); - connection.setRequestProperty("User-agent", userAgent); - connection.connect(); - HttpURLConnection httpConnection = (HttpURLConnection) connection; + + final HttpURLConnection httpConnection = new HttpConnectionManager(jsonUrl) + .makeConnection(); int code = httpConnection.getResponseCode(); if (code == 404) { return; @@ -228,6 +221,7 @@ public synchronized void getBoardWithMatchingVidPidFromCloud(String vid, String } catch (Exception e) { // No connection no problem, fail silently //e.printStackTrace(); + } } From d08932334263bdc6534fa42e32e0fec6b907232d Mon Sep 17 00:00:00 2001 From: Mattia Bertorello Date: Fri, 28 Jun 2019 17:16:08 +0200 Subject: [PATCH 03/33] Fix possible empty files during the download of the package index --- app/src/processing/app/Base.java | 20 +++--- .../DownloadableContributionsDownloader.java | 66 +++++++++++++++++-- .../contributions/GZippedJsonDownloader.java | 15 +++-- .../contributions/SignatureVerifier.java | 9 +++ .../libraries/LibraryInstaller.java | 14 ++-- .../packages/ContributionInstaller.java | 56 ++++------------ .../arduino/utils/network/FileDownloader.java | 7 +- arduino-core/src/processing/app/Platform.java | 3 - 8 files changed, 116 insertions(+), 74 deletions(-) diff --git a/app/src/processing/app/Base.java b/app/src/processing/app/Base.java index ef4d9b30a4e..d37dcf4fd85 100644 --- a/app/src/processing/app/Base.java +++ b/app/src/processing/app/Base.java @@ -26,9 +26,11 @@ import cc.arduino.Constants; import cc.arduino.UpdatableBoardsLibsFakeURLsHandler; import cc.arduino.UploaderUtils; -import cc.arduino.packages.Uploader; import cc.arduino.contributions.*; -import cc.arduino.contributions.libraries.*; +import cc.arduino.contributions.libraries.ContributedLibrary; +import cc.arduino.contributions.libraries.LibrariesIndexer; +import cc.arduino.contributions.libraries.LibraryInstaller; +import cc.arduino.contributions.libraries.LibraryOfSameTypeComparator; import cc.arduino.contributions.libraries.ui.LibraryManagerUI; import cc.arduino.contributions.packages.ContributedPlatform; import cc.arduino.contributions.packages.ContributionInstaller; @@ -36,20 +38,17 @@ import cc.arduino.contributions.packages.ui.ContributionManagerUI; import cc.arduino.files.DeleteFilesOnShutdown; import cc.arduino.packages.DiscoveryManager; +import cc.arduino.packages.Uploader; import cc.arduino.view.Event; import cc.arduino.view.JMenuUtils; import cc.arduino.view.SplashScreenHelper; - +import com.github.zafarkhaja.semver.Version; import org.apache.commons.compress.utils.IOUtils; import org.apache.commons.lang3.StringUtils; - -import com.github.zafarkhaja.semver.Version; - import processing.app.debug.TargetBoard; import processing.app.debug.TargetPackage; import processing.app.debug.TargetPlatform; import processing.app.helpers.*; -import processing.app.helpers.OSUtils; import processing.app.helpers.filefilters.OnlyDirs; import processing.app.helpers.filefilters.OnlyFilesWithExtension; import processing.app.javax.swing.filechooser.FileNameExtensionFilter; @@ -67,9 +66,9 @@ import java.awt.*; import java.awt.event.*; import java.io.*; -import java.util.*; import java.util.List; import java.util.Timer; +import java.util.*; import java.util.logging.Handler; import java.util.logging.Level; import java.util.logging.Logger; @@ -286,7 +285,8 @@ public Base(String[] args) throws Exception { pdeKeywords = new PdeKeywords(); pdeKeywords.reload(); - contributionInstaller = new ContributionInstaller(BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier()); + final GPGDetachedSignatureVerifier gpgDetachedSignatureVerifier = new GPGDetachedSignatureVerifier(); + contributionInstaller = new ContributionInstaller(BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier); libraryInstaller = new LibraryInstaller(BaseNoGui.getPlatform()); parser.parseArgumentsPhase2(); @@ -301,7 +301,7 @@ public Base(String[] args) throws Exception { if (parser.isInstallBoard()) { ContributionsIndexer indexer = new ContributionsIndexer( BaseNoGui.getSettingsFolder(), BaseNoGui.getHardwareFolder(), - BaseNoGui.getPlatform(), new GPGDetachedSignatureVerifier()); + BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier); ProgressListener progressListener = new ConsoleProgressListener(); List downloadedPackageIndexFiles = contributionInstaller.updateIndex(progressListener); diff --git a/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java b/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java index 3157514f876..07cbbc1a860 100644 --- a/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java +++ b/arduino-core/src/cc/arduino/contributions/DownloadableContributionsDownloader.java @@ -30,20 +30,23 @@ package cc.arduino.contributions; import cc.arduino.utils.FileHash; +import cc.arduino.utils.MultiStepProgress; import cc.arduino.utils.Progress; import cc.arduino.utils.network.FileDownloader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import processing.app.BaseNoGui; import java.io.File; import java.net.URL; -import java.nio.file.Files; -import java.nio.file.LinkOption; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; +import java.util.List; import static processing.app.I18n.format; import static processing.app.I18n.tr; public class DownloadableContributionsDownloader { + private static Logger log = LoggerFactory.getLogger(DownloadableContributionsDownloader.class); private final File stagingFolder; @@ -140,4 +143,59 @@ public void download(URL url, File tmpFile, Progress progress, String statusText } } + public void downloadIndexAndSignature(MultiStepProgress progress, List downloadedFilesAccumulator, String packageIndexUrlString, ProgressListener progressListener, SignatureVerifier signatureVerifier) throws Exception { + + // Extract the file name from the url + URL packageIndexUrl = new URL(packageIndexUrlString); + URL packageIndexSignatureUrl = new URL(packageIndexUrlString + ".sig"); + String[] urlPathParts = packageIndexUrl.getFile().split("/"); + File packageIndex = BaseNoGui.indexer.getIndexFile(urlPathParts[urlPathParts.length - 1]); + // Signature file name + File packageIndexSignature = BaseNoGui.indexer.getIndexFile(urlPathParts[urlPathParts.length - 1] + ".sig"); + + final String statusText = tr("Downloading platforms index..."); + downloadedFilesAccumulator.add(packageIndex.getName()); + + // Create temp files + File packageIndexTemp = File.createTempFile(packageIndexUrl.getPath(), ".tmp"); + File packageIndexSignatureTemp = File.createTempFile(packageIndexSignatureUrl.getPath(), ".tmp"); + try { + // Download package index + download(packageIndexUrl, packageIndexTemp, progress, statusText, progressListener, true); + try { + // Download signature + download(packageIndexSignatureUrl, packageIndexSignatureTemp, progress, statusText, progressListener, true); + + // Verify the signature before move the files + boolean signatureVerified = signatureVerifier.isSigned(packageIndexTemp, packageIndexSignatureTemp); + if (signatureVerified) { + // Move if the signature is ok + Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING); + Files.move(packageIndexSignatureTemp.toPath(), packageIndexSignature.toPath(), StandardCopyOption.REPLACE_EXISTING); + downloadedFilesAccumulator.add(packageIndexSignature.getName()); + } else { + downloadedFilesAccumulator.remove(packageIndex.getName()); + log.error("{} file signature verification failed. File ignored.", packageIndexSignatureUrl); + System.err.println(format(tr("{0} file signature verification failed. File ignored."), packageIndexUrlString)); + + } + } catch (Exception e) { + log.error("Cannot download the signature from {} the package will be install in any case", packageIndexSignatureUrl, e); + if (packageIndexTemp.length() > 0) { + Files.move(packageIndexTemp.toPath(), packageIndex.toPath(), StandardCopyOption.REPLACE_EXISTING); + } else { + log.error("The temporarily package index file is empty (path:{},url:{}), It cannot be move there {} ", + packageIndexTemp.toPath(), packageIndexUrlString, packageIndex.toPath()); + } + } + + } catch (Exception e) { + downloadedFilesAccumulator.remove(packageIndex.getName()); + throw e; + } finally { + // Delete useless temp file + Files.deleteIfExists(packageIndexTemp.toPath()); + Files.deleteIfExists(packageIndexSignatureTemp.toPath()); + } + } } diff --git a/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java b/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java index 6b6f3812327..0eb53fc259d 100644 --- a/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java +++ b/arduino-core/src/cc/arduino/contributions/GZippedJsonDownloader.java @@ -29,6 +29,7 @@ package cc.arduino.contributions; +import cc.arduino.Constants; import cc.arduino.utils.Progress; import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; import org.apache.commons.compress.compressors.gzip.GzipUtils; @@ -36,6 +37,7 @@ import java.io.*; import java.net.URL; +import java.nio.file.Files; public class GZippedJsonDownloader { @@ -50,17 +52,20 @@ public GZippedJsonDownloader(DownloadableContributionsDownloader downloader, URL } public void download(File tmpFile, Progress progress, String statusText, ProgressListener progressListener) throws Exception { + File gzipTmpFile = null; try { - File gzipTmpFile = new File(tmpFile.getParentFile(), GzipUtils.getCompressedFilename(tmpFile.getName())); + gzipTmpFile = File.createTempFile(new URL(Constants.LIBRARY_INDEX_URL_GZ).getPath(), GzipUtils.getCompressedFilename(tmpFile.getName())); // remove eventual leftovers from previous downloads - if (gzipTmpFile.exists()) { - gzipTmpFile.delete(); - } + Files.deleteIfExists(gzipTmpFile.toPath()); + new JsonDownloader(downloader, gzippedUrl).download(gzipTmpFile, progress, statusText, progressListener); decompress(gzipTmpFile, tmpFile); - gzipTmpFile.delete(); } catch (Exception e) { new JsonDownloader(downloader, url).download(tmpFile, progress, statusText, progressListener); + } finally { + if (gzipTmpFile != null) { + Files.deleteIfExists(gzipTmpFile.toPath()); + } } } diff --git a/arduino-core/src/cc/arduino/contributions/SignatureVerifier.java b/arduino-core/src/cc/arduino/contributions/SignatureVerifier.java index 6e2a80626b1..a4ea7a7ba53 100644 --- a/arduino-core/src/cc/arduino/contributions/SignatureVerifier.java +++ b/arduino-core/src/cc/arduino/contributions/SignatureVerifier.java @@ -50,6 +50,15 @@ public boolean isSigned(File indexFile) { } } + public boolean isSigned(File indexFile, File signature) { + try { + return verify(indexFile, signature, new File(BaseNoGui.getContentFile("lib"), "public.gpg.key")); + } catch (Exception e) { + BaseNoGui.showWarning(e.getMessage(), e.getMessage(), e); + return false; + } + } + protected abstract boolean verify(File signedFile, File signature, File publicKey) throws IOException; } diff --git a/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java b/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java index 4b4fb7f7dbb..a9e35b00561 100644 --- a/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java +++ b/arduino-core/src/cc/arduino/contributions/libraries/LibraryInstaller.java @@ -43,6 +43,8 @@ import java.io.File; import java.io.IOException; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.Optional; import static processing.app.I18n.tr; @@ -61,10 +63,11 @@ public synchronized void updateIndex(ProgressListener progressListener) throws E DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.librariesIndexer.getStagingFolder()); // Step 1: Download index File outputFile = BaseNoGui.librariesIndexer.getIndexFile(); - File tmpFile = new File(outputFile.getAbsolutePath() + ".tmp"); + // Create temp files + File libraryIndexTemp = File.createTempFile(new URL(Constants.LIBRARY_INDEX_URL).getPath(), ".tmp"); try { GZippedJsonDownloader gZippedJsonDownloader = new GZippedJsonDownloader(downloader, new URL(Constants.LIBRARY_INDEX_URL), new URL(Constants.LIBRARY_INDEX_URL_GZ)); - gZippedJsonDownloader.download(tmpFile, progress, tr("Downloading libraries index..."), progressListener); + gZippedJsonDownloader.download(libraryIndexTemp, progress, tr("Downloading libraries index..."), progressListener); } catch (InterruptedException e) { // Download interrupted... just exit return; @@ -74,10 +77,9 @@ public synchronized void updateIndex(ProgressListener progressListener) throws E // TODO: Check downloaded index // Replace old index with the updated one - if (outputFile.exists()) - outputFile.delete(); - if (!tmpFile.renameTo(outputFile)) - throw new Exception(tr("An error occurred while updating libraries index!")); + if (libraryIndexTemp.length() > 0) { + Files.move(libraryIndexTemp.toPath(), outputFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } // Step 2: Parse index BaseNoGui.librariesIndexer.parseIndex(); diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java index c75d1352a8c..4bd8f766b29 100644 --- a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java +++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java @@ -41,6 +41,8 @@ import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.Executor; import org.apache.commons.exec.PumpStreamHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import processing.app.BaseNoGui; import processing.app.I18n; import processing.app.Platform; @@ -51,7 +53,6 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.net.URL; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -62,6 +63,7 @@ import static processing.app.I18n.tr; public class ContributionInstaller { + private static Logger log = LoggerFactory.getLogger(ContributionInstaller.class); private final Platform platform; private final SignatureVerifier signatureVerifier; @@ -122,10 +124,10 @@ public synchronized List install(ContributedPlatform contributedPlatform // all the temporary folders and abort installation. List> resolvedToolReferences = contributedPlatform - .getResolvedToolReferences().entrySet().stream() - .filter((entry) -> !entry.getValue().isInstalled() - || entry.getValue().isBuiltIn()) - .collect(Collectors.toList()); + .getResolvedToolReferences().entrySet().stream() + .filter((entry) -> !entry.getValue().isInstalled() + || entry.getValue().isBuiltIn()) + .collect(Collectors.toList()); int i = 1; for (Map.Entry entry : resolvedToolReferences) { @@ -265,9 +267,10 @@ public synchronized List remove(ContributedPlatform contributedPlatform) // now try to remove the containing TOOL_NAME folder // (and silently fail if another version of the tool is installed) try { - Files.delete(destFolder.getParentFile().toPath()); + Files.deleteIfExists(destFolder.getParentFile().toPath()); } catch (Exception e) { // ignore + log.error(e.getMessage(), e); } } @@ -282,7 +285,8 @@ public synchronized List updateIndex(ProgressListener progressListener) MultiStepProgress progress = new MultiStepProgress(1); List downloadedPackageIndexFilesAccumulator = new LinkedList<>(); - downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, Constants.PACKAGE_INDEX_URL, progressListener); + final DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.indexer.getStagingFolder()); + downloader.downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, Constants.PACKAGE_INDEX_URL, progressListener, signatureVerifier); Set packageIndexURLs = new HashSet<>(); String additionalURLs = PreferencesData.get(Constants.PREF_BOARDS_MANAGER_ADDITIONAL_URLS, ""); @@ -292,8 +296,9 @@ public synchronized List updateIndex(ProgressListener progressListener) for (String packageIndexURL : packageIndexURLs) { try { - downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, packageIndexURL, progressListener); + downloader.downloadIndexAndSignature(progress, downloadedPackageIndexFilesAccumulator, packageIndexURL, progressListener, signatureVerifier); } catch (Exception e) { + log.error(e.getMessage(), e); System.err.println(e.getMessage()); } } @@ -303,41 +308,6 @@ public synchronized List updateIndex(ProgressListener progressListener) return downloadedPackageIndexFilesAccumulator; } - private void downloadIndexAndSignature(MultiStepProgress progress, List downloadedPackagedIndexFilesAccumulator, String packageIndexUrl, ProgressListener progressListener) throws Exception { - File packageIndex = download(progress, packageIndexUrl, progressListener); - downloadedPackagedIndexFilesAccumulator.add(packageIndex.getName()); - try { - File packageIndexSignature = download(progress, packageIndexUrl + ".sig", progressListener); - boolean signatureVerified = signatureVerifier.isSigned(packageIndex); - if (signatureVerified) { - downloadedPackagedIndexFilesAccumulator.add(packageIndexSignature.getName()); - } else { - downloadedPackagedIndexFilesAccumulator.remove(packageIndex.getName()); - Files.delete(packageIndex.toPath()); - Files.delete(packageIndexSignature.toPath()); - System.err.println(I18n.format(tr("{0} file signature verification failed. File ignored."), packageIndexUrl)); - } - } catch (Exception e) { - //ignore errors - } - } - - private File download(MultiStepProgress progress, String packageIndexUrl, ProgressListener progressListener) throws Exception { - String statusText = tr("Downloading platforms index..."); - URL url = new URL(packageIndexUrl); - String[] urlPathParts = url.getFile().split("/"); - File outputFile = BaseNoGui.indexer.getIndexFile(urlPathParts[urlPathParts.length - 1]); - File tmpFile = new File(outputFile.getAbsolutePath() + ".tmp"); - DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.indexer.getStagingFolder()); - boolean noResume = true; - downloader.download(url, tmpFile, progress, statusText, progressListener, noResume); - - Files.deleteIfExists(outputFile.toPath()); - Files.move(tmpFile.toPath(), outputFile.toPath()); - - return outputFile; - } - public synchronized void deleteUnknownFiles(List downloadedPackageIndexFiles) throws IOException { File preferencesFolder = BaseNoGui.indexer.getIndexFile(".").getParentFile(); File[] additionalPackageIndexFiles = preferencesFolder.listFiles(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME)); diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java index 27f677bfd13..aefb2788f08 100644 --- a/arduino-core/src/cc/arduino/utils/network/FileDownloader.java +++ b/arduino-core/src/cc/arduino/utils/network/FileDownloader.java @@ -30,7 +30,6 @@ package cc.arduino.utils.network; import org.apache.commons.compress.utils.IOUtils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; import processing.app.BaseNoGui; @@ -40,12 +39,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.RandomAccessFile; -import java.net.*; +import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; +import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Observable; import java.util.Optional; -import java.util.logging.Level; public class FileDownloader extends Observable { private static Logger log = LoggerFactory.getLogger(FileDownloader.class); @@ -186,6 +186,7 @@ private void downloadFile(boolean noResume) throws InterruptedException { final int resp = connection.getResponseCode(); if (resp < 200 || resp >= 300) { + Files.delete(outputFile.toPath()); throw new IOException("Received invalid http status code from server: " + resp); } diff --git a/arduino-core/src/processing/app/Platform.java b/arduino-core/src/processing/app/Platform.java index a60c1cc59ae..14e60edfa5e 100644 --- a/arduino-core/src/processing/app/Platform.java +++ b/arduino-core/src/processing/app/Platform.java @@ -26,8 +26,6 @@ import cc.arduino.utils.network.HttpConnectionManager; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import processing.app.debug.TargetBoard; import processing.app.debug.TargetPackage; import processing.app.debug.TargetPlatform; @@ -60,7 +58,6 @@ * know if name is proper Java package syntax.) */ public class Platform { - private static Logger log = LoggerFactory.getLogger(Platform.class); /** * Set the default L & F. While I enjoy the bounty of the sixteen possible From e6f09123bc59121f7ebc85535e16ffaeb3e13676 Mon Sep 17 00:00:00 2001 From: Mattia Bertorello Date: Fri, 28 Jun 2019 17:49:05 +0200 Subject: [PATCH 04/33] Remove notNull annotation that failed the build --- .../src/cc/arduino/utils/network/FileDownloaderCache.java | 3 +-- .../cc/arduino/utils/network/HttpConnectionManager.java | 7 +++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java index b89a862b45f..3b51033aa16 100644 --- a/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java +++ b/arduino-core/src/cc/arduino/utils/network/FileDownloaderCache.java @@ -1,6 +1,5 @@ package cc.arduino.utils.network; -import com.sun.istack.internal.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import processing.app.PreferencesData; @@ -26,7 +25,7 @@ public class FileDownloaderCache { private String eTag; // BaseNoGui.getSettingsFolder() - public FileDownloaderCache(@NotNull String cacheFolder, @NotNull URL remoteURL) { + public FileDownloaderCache(String cacheFolder, URL remoteURL) { this.remoteURL = remoteURL; String[] splitPath = remoteURL.getPath().split("/"); if (splitPath.length > 0) { diff --git a/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java b/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java index 39a72bb646e..ad75c1bfa02 100644 --- a/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java +++ b/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java @@ -1,7 +1,6 @@ package cc.arduino.utils.network; import cc.arduino.net.CustomProxySelector; -import com.sun.istack.internal.NotNull; import org.apache.commons.codec.binary.Base64; import org.apache.commons.httpclient.CircularRedirectException; import org.apache.commons.lang3.StringUtils; @@ -50,7 +49,7 @@ public HttpConnectionManager(URL requestURL) { } - public HttpURLConnection makeConnection(@NotNull Consumer beforeConnection) + public HttpURLConnection makeConnection(Consumer beforeConnection) throws IOException, NoSuchMethodException, ScriptException, URISyntaxException { return makeConnection(this.requestURL, 0, beforeConnection); } @@ -61,8 +60,8 @@ public HttpURLConnection makeConnection() return makeConnection(this.requestURL, 0, (c) -> {}); } - private HttpURLConnection makeConnection(@NotNull URL requestURL, int movedTimes, - @NotNull Consumer beforeConnection) throws IOException, URISyntaxException, ScriptException, NoSuchMethodException { + private HttpURLConnection makeConnection(URL requestURL, int movedTimes, + Consumer beforeConnection) throws IOException, URISyntaxException, ScriptException, NoSuchMethodException { log.info("Prepare http request to " + requestURL); if (movedTimes > 3) { log.warn("Too many redirect " + requestURL); From d3e712232bdc74bfb3d3e063af2b999f5a8175da Mon Sep 17 00:00:00 2001 From: Mattia Bertorello Date: Fri, 28 Jun 2019 17:52:31 +0200 Subject: [PATCH 05/33] Convert CircularRedirectException in IOException --- .../src/cc/arduino/utils/network/HttpConnectionManager.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java b/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java index ad75c1bfa02..dd51fa93e09 100644 --- a/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java +++ b/arduino-core/src/cc/arduino/utils/network/HttpConnectionManager.java @@ -2,7 +2,6 @@ import cc.arduino.net.CustomProxySelector; import org.apache.commons.codec.binary.Base64; -import org.apache.commons.httpclient.CircularRedirectException; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,7 +64,7 @@ private HttpURLConnection makeConnection(URL requestURL, int movedTimes, log.info("Prepare http request to " + requestURL); if (movedTimes > 3) { log.warn("Too many redirect " + requestURL); - throw new CircularRedirectException("Too many redirect " + requestURL); + throw new IOException("Too many redirect " + requestURL); } Proxy proxy = new CustomProxySelector(PreferencesData.getMap()) From 934df81b2357f85c35c0512f8bec744a65d00808 Mon Sep 17 00:00:00 2001 From: Mattia Bertorello Date: Fri, 28 Jun 2019 18:22:27 +0200 Subject: [PATCH 06/33] Add slf4j option to show the datetime in the logs --- build/build.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/build.xml b/build/build.xml index 1e034b11274..5c825168c3e 100644 --- a/build/build.xml +++ b/build/build.xml @@ -438,6 +438,8 @@