From d9fede49c3c4ab2d9915a9bd4b2f444fe0b80c8b Mon Sep 17 00:00:00 2001 From: charvam Date: Thu, 3 Sep 2020 15:06:47 +0200 Subject: [PATCH 01/29] ET-626 Added Bitfinex API connector. --- plugin-base/build.gradle | 1 + .../impl/everytrade/BitfinexConnector.java | 212 ++++++++++++++++++ .../impl/everytrade/EveryTradePlugin.java | 6 +- 3 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitfinexConnector.java diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index 4ccb28e8..41d72673 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -18,6 +18,7 @@ dependencies { implementation "com.github.GENERALBYTESCOM.XChange:xchange-bittrex:$xchangeVersion" implementation "com.github.GENERALBYTESCOM.XChange:xchange-bitstamp:$xchangeVersion" implementation "com.github.GENERALBYTESCOM.XChange:xchange-coinmate:$xchangeVersion" + implementation 'org.knowm.xchange:xchange-bitfinex:5.0.1' } configurations.runtimeClasspath { diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitfinexConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitfinexConnector.java new file mode 100644 index 00000000..f599010b --- /dev/null +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitfinexConnector.java @@ -0,0 +1,212 @@ +package io.everytrade.server.plugin.impl.everytrade; + +import io.everytrade.server.model.SupportedExchange; +import io.everytrade.server.plugin.api.IPlugin; +import io.everytrade.server.plugin.api.connector.ConnectorDescriptor; +import io.everytrade.server.plugin.api.connector.ConnectorParameterDescriptor; +import io.everytrade.server.plugin.api.connector.ConnectorParameterType; +import io.everytrade.server.plugin.api.connector.DownloadResult; +import io.everytrade.server.plugin.api.connector.IConnector; +import io.everytrade.server.plugin.api.parser.ParseResult; +import org.knowm.xchange.Exchange; +import org.knowm.xchange.ExchangeFactory; +import org.knowm.xchange.ExchangeSpecification; +import org.knowm.xchange.bitfinex.BitfinexExchange; +import org.knowm.xchange.bitfinex.service.BitfinexTradeService; +import org.knowm.xchange.dto.trade.UserTrade; +import org.knowm.xchange.service.trade.TradeService; +import org.knowm.xchange.service.trade.params.TradeHistoryParamsSorted; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class BitfinexConnector implements IConnector { + + private static final String ID = EveryTradePlugin.ID + IPlugin.PLUGIN_PATH_SEPARATOR + "bitfinexApiConnector"; + //https://docs.bitfinex.com/reference#rest-public-trades - 30 request / 1 minute, than 60 s no resp. + private static final int MAX_REQUEST_COUNT = 5; + private static final int TX_PER_REQUEST = 1000; + private static final String TX_SPLITER = "|"; + private static final Pattern SPLIT_PATTERN = Pattern.compile(String.format("(.*)\\%s(.*)", TX_SPLITER)); + + private static final ConnectorParameterDescriptor PARAMETER_API_SECRET = + new ConnectorParameterDescriptor( + "apiSecret", + ConnectorParameterType.SECRET, + "API Secret", + "" + ); + + private static final ConnectorParameterDescriptor PARAMETER_API_KEY = + new ConnectorParameterDescriptor( + "apiKey", + ConnectorParameterType.STRING, + "API Key", + "" + ); + + public static final ConnectorDescriptor DESCRIPTOR = new ConnectorDescriptor( + ID, + "Bitfinex Connector", + SupportedExchange.BITFINEX.getInternalId(), + List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET) + ); + + private final String apiKey; + private final String apiSecret; + + public BitfinexConnector(Map parameters) { + Objects.requireNonNull(this.apiKey = parameters.get(PARAMETER_API_KEY.getId())); + Objects.requireNonNull(this.apiSecret = parameters.get(PARAMETER_API_SECRET.getId())); + } + + @Override + public String getId() { + return ID; + } + + @Override + public DownloadResult getTransactions(String lastTransactionId) { + final ExchangeSpecification exSpec = new BitfinexExchange().getDefaultExchangeSpecification(); + exSpec.setApiKey(apiKey); + exSpec.setSecretKey(apiSecret); + final Exchange exchange = ExchangeFactory.INSTANCE.createExchange(exSpec); + final TradeService tradeService = exchange.getTradeService(); + + final List userTrades = download(tradeService, lastTransactionId); + + final String actualLastTransactionId; + if (!userTrades.isEmpty()) { + final UserTrade userTrade = userTrades.get(userTrades.size() - 1); + final String timeStamp = userTrade.getTimestamp().toInstant().toString(); + actualLastTransactionId = String.format("%s%s%s", timeStamp, TX_SPLITER, userTrade.getId()); + } else { + actualLastTransactionId = lastTransactionId; + } + + final ParseResult parseResult = XChangeConnectorParser.getParseResult(userTrades, SupportedExchange.BITFINEX); + + return new DownloadResult(parseResult, actualLastTransactionId); + } + + private List download( + TradeService tradeService, + String lastTransactionUid + ) { + final BitfinexTradeService.BitfinexTradeHistoryParams tradeHistoryParams + = (BitfinexTradeService.BitfinexTradeHistoryParams) tradeService.createTradeHistoryParams(); + tradeHistoryParams.setLimit(TX_PER_REQUEST); + tradeHistoryParams.setOrder(TradeHistoryParamsSorted.Order.asc); + final List userTrades = new ArrayList<>(); + TransactionIdentifier lastBlockDownloadedTx = parseFrom(lastTransactionUid); + + int counter = 0; + while (counter++ < MAX_REQUEST_COUNT) { + tradeHistoryParams.setStartTime(lastBlockDownloadedTx.date); + final List userTradesBlock; + try { + userTradesBlock = tradeService.getTradeHistory(tradeHistoryParams).getUserTrades(); + } catch (IOException e) { + throw new IllegalStateException("User trade history download failed. ", e); + } + final List userTradesToAdd; + final int duplicityTxIndex = findDuplicity(lastBlockDownloadedTx, userTradesBlock); + if (duplicityTxIndex > -1) { + if (duplicityTxIndex < userTradesBlock.size() - 1) { + userTradesToAdd = userTradesBlock.subList(duplicityTxIndex + 1, userTradesBlock.size()); + } else { + userTradesToAdd = List.of(); + } + } else { + userTradesToAdd = userTradesBlock; + } + + if (userTradesToAdd.isEmpty()) { + break; + } + + final UserTrade userTradeLast = userTradesToAdd.get(userTradesToAdd.size() - 1); + lastBlockDownloadedTx = new TransactionIdentifier(userTradeLast.getTimestamp(), userTradeLast.getId()); + + userTrades.addAll(userTradesToAdd); + } + + return userTrades; + } + + private TransactionIdentifier parseFrom(String lastTransactionUid) { + if (lastTransactionUid == null) { + return new TransactionIdentifier(Date.from(Instant.EPOCH), null); + } + Matcher matcher = SPLIT_PATTERN.matcher(lastTransactionUid); + if (occurrenceCount(lastTransactionUid, TX_SPLITER) != 1 || !matcher.find()) { + throw new IllegalArgumentException( + String.format("Illegal value of lastTransactionUid '%s'.", lastTransactionUid) + ); + } + final String date = matcher.group(1); + try { + return new TransactionIdentifier( + Date.from(Instant.parse(date)), + matcher.group(2) + ); + } catch (Exception e) { + throw new IllegalArgumentException( + String.format( + "Illegal value of date part '%s' of lastTransactionUid '%s'.", + date, + lastTransactionUid + ), + e + ); + } + } + + private int findDuplicity(TransactionIdentifier transactionIdentifier, List userTradesBlock) { + for (int i = 0; i < userTradesBlock.size(); i++) { + final UserTrade userTrade = userTradesBlock.get(i); + if (userTrade.getTimestamp().equals(transactionIdentifier.date) + && userTrade.getId().equals(transactionIdentifier.id) + ) { + return i; + } + } + return -1; + } + + @Override + public void close() { + //AutoCloseable + } + + private static class TransactionIdentifier { + private final Date date; + private final String id; + + public TransactionIdentifier(Date date, String id) { + this.date = date; + this.id = id; + } + } + + public int occurrenceCount(String input, String search) { + int startIndex = 0; + int index = 0; + int counter = 0; + while (index > -1 && startIndex < input.length()) { + index = input.indexOf(search, startIndex); + if (index > -1) { + counter++; + } + startIndex = index + 1; + } + return counter; + } +} \ No newline at end of file diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java index 0d2f9730..7d416b39 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java @@ -18,7 +18,8 @@ public class EveryTradePlugin implements IPlugin { EveryTradeConnector.DESCRIPTOR, KrakenConnector.DESCRIPTOR, BitstampConnector.DESCRIPTOR, - CoinmateConnector.DESCRIPTOR + CoinmateConnector.DESCRIPTOR, + BitfinexConnector.DESCRIPTOR ).stream().collect(Collectors.toMap(ConnectorDescriptor::getId, it -> it)); @Override @@ -50,6 +51,9 @@ public IConnector createConnectorInstance(String connectorId, Map Date: Thu, 3 Sep 2020 16:22:00 +0200 Subject: [PATCH 02/29] ET-626 Fixed gradle dependencies. --- plugin-base/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index 41d72673..6ee1069e 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -12,13 +12,13 @@ dependencies { compileOnly 'org.slf4j:slf4j-api:1.7.30' implementation 'com.univocity:univocity-parsers:2.8.3' - implementation 'org.knowm.xchange:xchange-core:5.0.1' + implementation "com.github.GENERALBYTESCOM.XChange:xchange-core:$xchangeVersion" implementation "com.github.GENERALBYTESCOM.XChange:xchange-kraken:$xchangeVersion" implementation "com.github.GENERALBYTESCOM.XChange:xchange-binance:$xchangeVersion" implementation "com.github.GENERALBYTESCOM.XChange:xchange-bittrex:$xchangeVersion" implementation "com.github.GENERALBYTESCOM.XChange:xchange-bitstamp:$xchangeVersion" implementation "com.github.GENERALBYTESCOM.XChange:xchange-coinmate:$xchangeVersion" - implementation 'org.knowm.xchange:xchange-bitfinex:5.0.1' + implementation "com.github.GENERALBYTESCOM.XChange:xchange-bitfinex:$xchangeVersion" } configurations.runtimeClasspath { From 2c74e42718d4d774f6a4da8a9c54742abac8f5d7 Mon Sep 17 00:00:00 2001 From: charvam Date: Fri, 4 Sep 2020 10:45:11 +0200 Subject: [PATCH 03/29] ET-626 Added Binance API connector. --- .../impl/everytrade/BinanceConnector.java | 171 ++++++++++++++++++ .../impl/everytrade/BinanceDownloadState.java | 48 +++++ .../impl/everytrade/EveryTradePlugin.java | 6 +- 3 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java create mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloadState.java diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java new file mode 100644 index 00000000..ab4c422e --- /dev/null +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java @@ -0,0 +1,171 @@ +package io.everytrade.server.plugin.impl.everytrade; + +import io.everytrade.server.model.SupportedExchange; +import io.everytrade.server.plugin.api.IPlugin; +import io.everytrade.server.plugin.api.connector.ConnectorDescriptor; +import io.everytrade.server.plugin.api.connector.ConnectorParameterDescriptor; +import io.everytrade.server.plugin.api.connector.ConnectorParameterType; +import io.everytrade.server.plugin.api.connector.DownloadResult; +import io.everytrade.server.plugin.api.connector.IConnector; +import io.everytrade.server.plugin.api.parser.ParseResult; +import org.knowm.xchange.Exchange; +import org.knowm.xchange.ExchangeFactory; +import org.knowm.xchange.ExchangeSpecification; +import org.knowm.xchange.binance.BinanceExchange; +import org.knowm.xchange.binance.service.BinanceTradeHistoryParams; +import org.knowm.xchange.currency.Currency; +import org.knowm.xchange.currency.CurrencyPair; +import org.knowm.xchange.dto.trade.UserTrade; +import org.knowm.xchange.service.trade.TradeService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class BinanceConnector implements IConnector { + + private static final String ID = EveryTradePlugin.ID + IPlugin.PLUGIN_PATH_SEPARATOR + "binanceApiConnector"; + //org.knowm.xchange.binance.BinanceResilience - 1200 request / 1 minute --> 10 user in 1 minute will be enough + private static final int MAX_REQUEST_COUNT = 120; + private static final int TX_PER_REQUEST = 1000; + + private static final ConnectorParameterDescriptor PARAMETER_API_SECRET = + new ConnectorParameterDescriptor( + "apiSecret", + ConnectorParameterType.SECRET, + "API Secret", + "" + ); + + private static final ConnectorParameterDescriptor PARAMETER_API_KEY = + new ConnectorParameterDescriptor( + "apiKey", + ConnectorParameterType.STRING, + "API Key", + "" + ); + + private static final ConnectorParameterDescriptor PARAMETER_API_SYMBOLS = + new ConnectorParameterDescriptor( + "apiSymbols", + ConnectorParameterType.STRING, + "Trade pairs (e.g.: BTC/USDT,LTC/ETH)", + "" + ); + + public static final ConnectorDescriptor DESCRIPTOR = new ConnectorDescriptor( + ID, + "Binance Connector", + SupportedExchange.BINANCE.getInternalId(), + List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET, PARAMETER_API_SYMBOLS) + ); + + private final String apiKey; + private final String apiSecret; + private final String apiSymbols; + + public BinanceConnector(Map parameters) { + Objects.requireNonNull(this.apiKey = parameters.get(PARAMETER_API_KEY.getId())); + Objects.requireNonNull(this.apiSecret = parameters.get(PARAMETER_API_SECRET.getId())); + Objects.requireNonNull(this.apiSymbols = parameters.get(PARAMETER_API_SYMBOLS.getId())); + } + + @Override + public String getId() { + return ID; + } + + @Override + public DownloadResult getTransactions(String lastTransactionId) { + final ExchangeSpecification exSpec = new BinanceExchange().getDefaultExchangeSpecification(); + exSpec.setApiKey(apiKey); + exSpec.setSecretKey(apiSecret); + final Exchange exchange = ExchangeFactory.INSTANCE.createExchange(exSpec); + final TradeService tradeService = exchange.getTradeService(); + final BinanceDownloadState downloadState = BinanceDownloadState.parseFrom(lastTransactionId); + + List userTrades = download(tradeService, downloadState); + final ParseResult parseResult = XChangeConnectorParser.getParseResult(userTrades, SupportedExchange.BINANCE); + + return new DownloadResult(parseResult, downloadState.toLastTransactionId()); + } + + private List download( + TradeService tradeService, + BinanceDownloadState downloadState + ) { + + final BinanceTradeHistoryParams tradeHistoryParams + = (BinanceTradeHistoryParams) tradeService.createTradeHistoryParams(); + tradeHistoryParams.setLimit(TX_PER_REQUEST); + + final List currencyPairs = symbolsToPairs(apiSymbols); + final List userTrades = new ArrayList<>(); + int counter = 0; + + for (CurrencyPair currencyPair : currencyPairs) { + tradeHistoryParams.setCurrencyPair(currencyPair); + String lastDownloadedTx = downloadState.getLastTransactionId(currencyPair.toString()); + tradeHistoryParams.setStartId(lastDownloadedTx); + + while (counter++ < MAX_REQUEST_COUNT) { + final List userTradesBlock; + try { + userTradesBlock = tradeService.getTradeHistory(tradeHistoryParams).getUserTrades(); + } catch (Exception e) { + throw new IllegalStateException("User trade history download failed. ", e); + } + if ( + lastDownloadedTx != null + && !userTradesBlock.isEmpty() + && userTradesBlock.get(0).getId().equals(lastDownloadedTx) + ) { + userTradesBlock.remove(0); + } + if (userTradesBlock.isEmpty()) { + break; + } + userTrades.addAll(userTradesBlock); + lastDownloadedTx = userTradesBlock.get(userTradesBlock.size() - 1).getId(); + tradeHistoryParams.setStartId(lastDownloadedTx); + } + downloadState.updateLastTransactionId(currencyPair.toString(), lastDownloadedTx); + } + + return userTrades; + + } + + @Override + public void close() { + //AutoCloseable + } + + private List symbolsToPairs(String symbols) { + return Arrays.stream(symbols.split(",")) + .map(String::strip) + .map(this::createPair) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private CurrencyPair createPair(String symbol) { + if (symbol == null) { + return null; + } + final String[] split = symbol.split("/"); + if (split.length != 2) { + return null; + } + final Currency base = Currency.getInstanceNoCreate(split[0]); + final Currency quote = Currency.getInstanceNoCreate(split[1]); + if (base == null || quote == null) { + return null; + } + return new CurrencyPair(base, quote); + } + +} diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloadState.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloadState.java new file mode 100644 index 00000000..6cc3fc26 --- /dev/null +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloadState.java @@ -0,0 +1,48 @@ +package io.everytrade.server.plugin.impl.everytrade; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class BinanceDownloadState { + private final Map pairLastIds; + + private BinanceDownloadState(Map pairLastIds) { + this.pairLastIds = pairLastIds; + } + + public static BinanceDownloadState parseFrom(String lastTransactionId) { + return new BinanceDownloadState(convertToMap(lastTransactionId)); + } + + public String getLastTransactionId(String pair) { + Objects.requireNonNull(pair); + return pairLastIds.get(pair); + } + + public void updateLastTransactionId(String pair, String lastId) { + Objects.requireNonNull(pair); + pairLastIds.put(pair, lastId); + } + + public String toLastTransactionId() { + return convertFromMap(pairLastIds); + } + + private String convertFromMap(Map map) { + return map.keySet().stream() + .map(key -> key + "=" + map.get(key)) + .collect(Collectors.joining(":")); + } + + private static Map convertToMap(String mapAsString) { + if (mapAsString == null) { + return new HashMap<>(); + } + return Arrays.stream(mapAsString.split(":")) + .map(entry -> entry.split("=")) + .collect(Collectors.toMap(entry -> entry[0], entry -> entry[1])); + } +} \ No newline at end of file diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java index 7d416b39..a75ebca3 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java @@ -19,7 +19,8 @@ public class EveryTradePlugin implements IPlugin { KrakenConnector.DESCRIPTOR, BitstampConnector.DESCRIPTOR, CoinmateConnector.DESCRIPTOR, - BitfinexConnector.DESCRIPTOR + BitfinexConnector.DESCRIPTOR, + BinanceConnector.DESCRIPTOR ).stream().collect(Collectors.toMap(ConnectorDescriptor::getId, it -> it)); @Override @@ -54,6 +55,9 @@ public IConnector createConnectorInstance(String connectorId, Map Date: Fri, 4 Sep 2020 12:16:16 +0200 Subject: [PATCH 04/29] ET-626 Added Bittrex API connector. --- .../impl/everytrade/BittrexConnector.java | 152 ++++++++++++++++++ .../impl/everytrade/EveryTradePlugin.java | 6 +- 2 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BittrexConnector.java diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BittrexConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BittrexConnector.java new file mode 100644 index 00000000..3d6bb972 --- /dev/null +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BittrexConnector.java @@ -0,0 +1,152 @@ +package io.everytrade.server.plugin.impl.everytrade; + +import io.everytrade.server.model.SupportedExchange; +import io.everytrade.server.parser.exchange.XChangeApiTransactionBean; +import io.everytrade.server.plugin.api.IPlugin; +import io.everytrade.server.plugin.api.connector.ConnectorDescriptor; +import io.everytrade.server.plugin.api.connector.ConnectorParameterDescriptor; +import io.everytrade.server.plugin.api.connector.ConnectorParameterType; +import io.everytrade.server.plugin.api.connector.DownloadResult; +import io.everytrade.server.plugin.api.connector.IConnector; +import io.everytrade.server.plugin.api.parser.ConversionStatistic; +import io.everytrade.server.plugin.api.parser.ImportedTransactionBean; +import io.everytrade.server.plugin.api.parser.ParseResult; +import io.everytrade.server.plugin.api.parser.RowError; +import io.everytrade.server.plugin.api.parser.RowErrorType; +import org.knowm.xchange.Exchange; +import org.knowm.xchange.ExchangeFactory; +import org.knowm.xchange.ExchangeSpecification; +import org.knowm.xchange.bittrex.BittrexExchange; +import org.knowm.xchange.dto.trade.UserTrade; +import org.knowm.xchange.service.trade.TradeService; +import org.knowm.xchange.service.trade.params.TradeHistoryParams; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class BittrexConnector implements IConnector { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + private static final String ID = EveryTradePlugin.ID + IPlugin.PLUGIN_PATH_SEPARATOR + "bittrexApiConnector"; + + private static final ConnectorParameterDescriptor PARAMETER_API_SECRET = + new ConnectorParameterDescriptor( + "apiSecret", + ConnectorParameterType.SECRET, + "API Secret", + "" + ); + + private static final ConnectorParameterDescriptor PARAMETER_API_KEY = + new ConnectorParameterDescriptor( + "apiKey", + ConnectorParameterType.STRING, + "API Key", + "" + ); + + public static final ConnectorDescriptor DESCRIPTOR = new ConnectorDescriptor( + ID, + "Bittrex Connector", + SupportedExchange.BITTREX.getInternalId(), + List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET) + ); + + private final String apiKey; + private final String apiSecret; + + public BittrexConnector(Map parameters) { + Objects.requireNonNull(this.apiKey = parameters.get(PARAMETER_API_KEY.getId())); + Objects.requireNonNull(this.apiSecret = parameters.get(PARAMETER_API_SECRET.getId())); + } + + @Override + public String getId() { + return ID; + } + + @Override + public DownloadResult getTransactions(String lastTransactionId) { + final ExchangeSpecification exSpec = new BittrexExchange().getDefaultExchangeSpecification(); + exSpec.setApiKey(apiKey); + exSpec.setSecretKey(apiSecret); + final Exchange exchange = ExchangeFactory.INSTANCE.createExchange(exSpec); + final TradeService tradeService = exchange.getTradeService(); + + List userTrades = download(lastTransactionId, tradeService); + + final String actualLastTransactionId = userTrades.isEmpty() + ? lastTransactionId + : userTrades.get(userTrades.size() - 1).getId(); + + final ParseResult parseResult = getParseResult(userTrades); + + return new DownloadResult(parseResult, actualLastTransactionId); + } + + private List download(String lastTransactionId, TradeService tradeService) { + final TradeHistoryParams tradeHistoryParams = tradeService.createTradeHistoryParams(); + final List userTradesBlock; + try { + userTradesBlock = tradeService.getTradeHistory(tradeHistoryParams).getUserTrades(); + } catch (Exception e) { + throw new IllegalStateException("User trade history download failed.", e); + } + + if (lastTransactionId == null) { + return userTradesBlock; + } + + final List userTradesToAdd; + final int duplicityTxIndex = findDuplicity(lastTransactionId, userTradesBlock); + if (duplicityTxIndex > -1) { + if (duplicityTxIndex < userTradesBlock.size() - 1) { + userTradesToAdd = userTradesBlock.subList(duplicityTxIndex + 1, userTradesBlock.size()); + } else { + userTradesToAdd = List.of(); + } + } else { + userTradesToAdd = userTradesBlock; + } + + return userTradesToAdd; + } + + private ParseResult getParseResult(List userTrades) { + final List importedTransactionBeans = new ArrayList<>(); + final List errorRows = new ArrayList<>(); + for (UserTrade userTrade : userTrades) { + try { + XChangeApiTransactionBean xChangeApiTransactionBean + = new XChangeApiTransactionBean(userTrade, SupportedExchange.BITTREX); + importedTransactionBeans.add(xChangeApiTransactionBean.toImportedTransactionBean()); + } catch (Exception e) { + log.error("Error converting to ImportedTransactionBean.", e); + errorRows.add(new RowError(userTrade.toString(), e.getMessage(), RowErrorType.FAILED)); + } + } + + return new ParseResult( + importedTransactionBeans, + new ConversionStatistic(errorRows, 0) + ); + } + + private int findDuplicity(String transactionId, List userTradesBlock) { + for (int i = 0; i < userTradesBlock.size(); i++) { + final UserTrade userTrade = userTradesBlock.get(i); + if (userTrade.getId().equals(transactionId)) { + return i; + } + } + return -1; + } + + @Override + public void close() { + //AutoCloseable + } +} \ No newline at end of file diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java index a75ebca3..9fe5aad0 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java @@ -20,7 +20,8 @@ public class EveryTradePlugin implements IPlugin { BitstampConnector.DESCRIPTOR, CoinmateConnector.DESCRIPTOR, BitfinexConnector.DESCRIPTOR, - BinanceConnector.DESCRIPTOR + BinanceConnector.DESCRIPTOR, + BittrexConnector.DESCRIPTOR ).stream().collect(Collectors.toMap(ConnectorDescriptor::getId, it -> it)); @Override @@ -58,6 +59,9 @@ public IConnector createConnectorInstance(String connectorId, Map Date: Mon, 7 Sep 2020 09:48:51 +0200 Subject: [PATCH 05/29] ET-626 Change working and plugin directory structure. --- .gitignore | 3 ++- plugin-tester/build.gradle | 1 + .../src/main/java/io/everytrade/server/plugin/Tester.java | 5 ++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index ffffac36..e2e0529b 100644 --- a/.gitignore +++ b/.gitignore @@ -157,4 +157,5 @@ gradle-app.setting # gradle/wrapper/gradle-wrapper.properties # SDKMAN RC -.sdkmanrc \ No newline at end of file +.sdkmanrc +/plugin-tester/private/ diff --git a/plugin-tester/build.gradle b/plugin-tester/build.gradle index 3623be68..a923bd9f 100644 --- a/plugin-tester/build.gradle +++ b/plugin-tester/build.gradle @@ -22,5 +22,6 @@ task gatherPlugins(type: Sync) { task run(type: JavaExec) { dependsOn gatherPlugins classpath = sourceSets.main.runtimeClasspath + workingDir = project.rootDir mainClass = 'io.everytrade.server.plugin.Tester' } \ No newline at end of file diff --git a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java index 90baa6e7..d6976a22 100644 --- a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java +++ b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java @@ -31,7 +31,7 @@ public Tester(Path workDir) { this.workDir = workDir; final Properties properties = loadProperties("tester.properties").orElse(new Properties()); pluginDir = Paths.get( - properties.getProperty("tester.pluginDir", "build/testedPlugins") + properties.getProperty("tester.pluginDir", "plugin-tester/build/testedPlugins") ); } @@ -103,9 +103,8 @@ private void printResult(DownloadResult downloadResult) { } private Optional> loadParams(String id) { - final Optional properties = loadProperties(id + ".properties"); + final Optional properties = loadProperties("plugin-tester/private/" + id +".properties"); return properties.map(this::toMap); - } private Map toMap(Properties properties) { From 02b8095dd8321eb9cbeb873abfed580c34509696 Mon Sep 17 00:00:00 2001 From: charvam Date: Mon, 7 Sep 2020 09:50:47 +0200 Subject: [PATCH 06/29] ET-626 Added Bitfinex xchange dependency. --- plugin-base/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index edd4541b..d762df04 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -31,6 +31,7 @@ dependencies { pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-bittrex:$xchangeVersion" pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-bitstamp:$xchangeVersion" pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-coinmate:$xchangeVersion" + pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-bitfinex:$xchangeVersion" } compileJava { From 7efc65fb9e8c62b561803f3023f1e9a74b27dc36 Mon Sep 17 00:00:00 2001 From: charvam Date: Mon, 7 Sep 2020 10:13:47 +0200 Subject: [PATCH 07/29] ET-626 Changed transactions log to csv format. --- .../io/everytrade/server/plugin/Tester.java | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java index d6976a22..53cf7a29 100644 --- a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java +++ b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java @@ -31,7 +31,7 @@ public Tester(Path workDir) { this.workDir = workDir; final Properties properties = loadProperties("tester.properties").orElse(new Properties()); pluginDir = Paths.get( - properties.getProperty("tester.pluginDir", "plugin-tester/build/testedPlugins") + properties.getProperty("tester.pluginDir", "plugin-tester/build/testedPlugins") ); } @@ -49,15 +49,15 @@ public Optional loadProperties(String fileName) { public static void main(String[] args) { new Tester( - Paths.get( - args.length == 1 ? args[0] : "" - ) + Paths.get( + args.length == 1 ? args[0] : "" + ) ).test(); } private void test() { final EverytradePluginManager pluginManager = - new EverytradePluginManager(pluginDir); + new EverytradePluginManager(pluginDir); pluginManager.loadPlugins(); pluginManager.startPlugins(); @@ -91,9 +91,32 @@ private void testPlugin(IPlugin plugin) { private void printResult(DownloadResult downloadResult) { final ParseResult parseResult = downloadResult.getParseResult(); - for (ImportedTransactionBean importedTransactionBean : parseResult.getImportedTransactionBeans()) { - log.info("importedTransactionBean = " + importedTransactionBean); - } + StringBuilder stringBuilder = new StringBuilder(); + final String columnSeparator = ","; + final String lineSeparator = "\n"; + stringBuilder + .append("uid").append(columnSeparator) + .append("executed").append(columnSeparator) + .append("base").append(columnSeparator) + .append("quote").append(columnSeparator) + .append("action").append(columnSeparator) + .append("baseQuantity").append(columnSeparator) + .append("unitPrice").append(columnSeparator) + .append("transactionPrice").append(columnSeparator) + .append("feeQuote").append(lineSeparator); + + parseResult.getImportedTransactionBeans().forEach(b -> stringBuilder + .append(b.getUid()).append(columnSeparator) + .append(b.getExecuted()).append(columnSeparator) + .append(b.getBase()).append(columnSeparator) + .append(b.getQuote()).append(columnSeparator) + .append(b.getAction()).append(columnSeparator) + .append(b.getBaseQuantity()).append(columnSeparator) + .append(b.getUnitPrice()).append(columnSeparator) + .append(b.getTransactionPrice()).append(columnSeparator) + .append(b.getFeeQuote()).append(lineSeparator) + ); + log.info("importedTransactionBeans = \n" + stringBuilder.toString()); final ConversionStatistic conversionStatistic = parseResult.getConversionStatistic(); log.info("conversionStatistic = " + conversionStatistic); @@ -103,7 +126,7 @@ private void printResult(DownloadResult downloadResult) { } private Optional> loadParams(String id) { - final Optional properties = loadProperties("plugin-tester/private/" + id +".properties"); + final Optional properties = loadProperties("plugin-tester/private/" + id + ".properties"); return properties.map(this::toMap); } From 4cc7f5047961f6d2ce601be502508eef02c147fd Mon Sep 17 00:00:00 2001 From: charvam Date: Mon, 7 Sep 2020 10:20:06 +0200 Subject: [PATCH 08/29] ET-626 Changed transactions log to csv format. --- .../src/main/java/io/everytrade/server/plugin/Tester.java | 1 - 1 file changed, 1 deletion(-) diff --git a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java index 53cf7a29..0f8214fa 100644 --- a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java +++ b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java @@ -5,7 +5,6 @@ import io.everytrade.server.plugin.api.connector.DownloadResult; import io.everytrade.server.plugin.api.connector.IConnector; import io.everytrade.server.plugin.api.parser.ConversionStatistic; -import io.everytrade.server.plugin.api.parser.ImportedTransactionBean; import io.everytrade.server.plugin.api.parser.ParseResult; import io.everytrade.server.plugin.support.EverytradePluginManager; import org.slf4j.Logger; From 21afcfb0a3c0e7e026c4fba8ff5e6108d18c0b3c Mon Sep 17 00:00:00 2001 From: charvam Date: Wed, 9 Sep 2020 10:59:21 +0200 Subject: [PATCH 09/29] ET-626 Added CoinbasePro API connector. --- plugin-base/build.gradle | 1 + .../impl/everytrade/CoinbaseProConnector.java | 180 ++++++++++++++++++ .../everytrade/CoinbaseProDownloadState.java | 48 +++++ .../impl/everytrade/EveryTradePlugin.java | 6 +- 4 files changed, 234 insertions(+), 1 deletion(-) create mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java create mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloadState.java diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index d762df04..204ef175 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -32,6 +32,7 @@ dependencies { pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-bitstamp:$xchangeVersion" pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-coinmate:$xchangeVersion" pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-bitfinex:$xchangeVersion" + pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-coinbasepro:$xchangeVersion" } compileJava { diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java new file mode 100644 index 00000000..650b4df2 --- /dev/null +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java @@ -0,0 +1,180 @@ +package io.everytrade.server.plugin.impl.everytrade; + +import io.everytrade.server.model.SupportedExchange; +import io.everytrade.server.plugin.api.IPlugin; +import io.everytrade.server.plugin.api.connector.ConnectorDescriptor; +import io.everytrade.server.plugin.api.connector.ConnectorParameterDescriptor; +import io.everytrade.server.plugin.api.connector.ConnectorParameterType; +import io.everytrade.server.plugin.api.connector.DownloadResult; +import io.everytrade.server.plugin.api.connector.IConnector; +import io.everytrade.server.plugin.api.parser.ParseResult; +import org.knowm.xchange.Exchange; +import org.knowm.xchange.ExchangeFactory; +import org.knowm.xchange.ExchangeSpecification; +import org.knowm.xchange.coinbasepro.CoinbaseProExchange; +import org.knowm.xchange.coinbasepro.dto.trade.CoinbaseProTradeHistoryParams; +import org.knowm.xchange.currency.Currency; +import org.knowm.xchange.currency.CurrencyPair; +import org.knowm.xchange.dto.trade.UserTrade; +import org.knowm.xchange.service.trade.TradeService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class CoinbaseProConnector implements IConnector { + private static final String ID = EveryTradePlugin.ID + IPlugin.PLUGIN_PATH_SEPARATOR + "coinbaseProApiConnector"; + //https://docs.pro.coinbase.com/#rate-limits - max 5 request per user per second + private static final int TX_PER_REQUEST = 100; + private static final int MAX_REQUEST_COUNT = 3000; + private static final int MAX_REQUESTS_PER_SECOND = 5; + private static final int REQUESTS_WINDOW_MS = 1000; + private static final SupportedExchange SUPPORTED_EXCHANGE = SupportedExchange.COINBASE; + + private static final ConnectorParameterDescriptor PARAMETER_API_SECRET = + new ConnectorParameterDescriptor( + "apiSecret", + ConnectorParameterType.SECRET, + "API Secret", + "" + ); + + private static final ConnectorParameterDescriptor PARAMETER_API_KEY = + new ConnectorParameterDescriptor( + "apiKey", + ConnectorParameterType.STRING, + "API Key", + "" + ); + + private static final ConnectorParameterDescriptor PARAMETER_API_PASS_PHRASE = + new ConnectorParameterDescriptor( + "apiPassPhrase", + ConnectorParameterType.SECRET, + "API PassPhrase", + "" + ); + + private static final ConnectorParameterDescriptor PARAMETER_API_SYMBOLS = + new ConnectorParameterDescriptor( + "apiSymbols", + ConnectorParameterType.STRING, + "Trade pairs (e.g.: LTC/EUR,LTC/BTC)", + "" + ); + + public static final ConnectorDescriptor DESCRIPTOR = new ConnectorDescriptor( + ID, + "Coinbase Pro Connector", + SUPPORTED_EXCHANGE.getInternalId(), + List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET, PARAMETER_API_SYMBOLS, PARAMETER_API_PASS_PHRASE) + ); + + private final String apiKey; + private final String apiSecret; + private final String apiSymbols; + private final String apiPassPhrase; + + public CoinbaseProConnector(Map parameters) { + Objects.requireNonNull(this.apiKey = parameters.get(PARAMETER_API_KEY.getId())); + Objects.requireNonNull(this.apiSecret = parameters.get(PARAMETER_API_SECRET.getId())); + Objects.requireNonNull(this.apiSymbols = parameters.get(PARAMETER_API_SYMBOLS.getId())); + Objects.requireNonNull(this.apiPassPhrase = parameters.get(PARAMETER_API_PASS_PHRASE.getId())); + } + + + @Override + public String getId() { + return ID; + } + + @Override + public DownloadResult getTransactions(String lastTransactionId) { + final ExchangeSpecification exSpec = new CoinbaseProExchange().getDefaultExchangeSpecification(); + exSpec.setApiKey(apiKey); + exSpec.setSecretKey(apiSecret); + exSpec.setExchangeSpecificParametersItem("passphrase", apiPassPhrase); + final Exchange exchange = ExchangeFactory.INSTANCE.createExchange(exSpec); + final TradeService tradeService = exchange.getTradeService(); + final CoinbaseProDownloadState downloadState = CoinbaseProDownloadState.parseFrom(lastTransactionId); + + final List userTrades = download(tradeService, downloadState); + final ParseResult parseResult = XChangeConnectorParser.getParseResult(userTrades, SUPPORTED_EXCHANGE); + + return new DownloadResult(parseResult, downloadState.toLastTransactionId()); + } + + private List download(TradeService tradeService, CoinbaseProDownloadState downloadState) { + final CoinbaseProTradeHistoryParams tradeHistoryParams + = (CoinbaseProTradeHistoryParams) tradeService.createTradeHistoryParams(); + tradeHistoryParams.setLimit(TX_PER_REQUEST); + + final List currencyPairs = symbolsToPairs(apiSymbols); + final List userTrades = new ArrayList<>(); + int counter = 0; + + for (CurrencyPair currencyPair : currencyPairs) { + tradeHistoryParams.setCurrencyPair(currencyPair); + Integer lastDownloadedTx = downloadState.getLastTransactionId(currencyPair.toString()); + tradeHistoryParams.setBeforeTradeId(lastDownloadedTx == null ? 1 : lastDownloadedTx ); + + while (counter++ < MAX_REQUEST_COUNT) { + final List userTradesBlock; + downloadSleep(counter); + try { + userTradesBlock = tradeService.getTradeHistory(tradeHistoryParams).getUserTrades(); + } catch (Exception e) { + throw new IllegalStateException("User trade history download failed. ", e); + } + if (userTradesBlock.isEmpty()) { + break; + } + userTrades.addAll(userTradesBlock); + lastDownloadedTx = Integer.valueOf(userTradesBlock.get(userTradesBlock.size() - 1).getId()); + tradeHistoryParams.setBeforeTradeId(lastDownloadedTx); + } + if (lastDownloadedTx != null) { + downloadState.updateLastTransactionId(currencyPair.toString(), lastDownloadedTx); + } + } + + return userTrades; + } + + private void downloadSleep(int counter) { + if (counter % MAX_REQUESTS_PER_SECOND == 0) { + try { + Thread.sleep(REQUESTS_WINDOW_MS); + } catch (InterruptedException e) { + throw new IllegalStateException("User trade history download sleep interrupted.", e); + } + } + } + + private List symbolsToPairs(String symbols) { + return Arrays.stream(symbols.split(",")) + .map(String::strip) + .map(this::createPair) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private CurrencyPair createPair(String symbol) { + if (symbol == null) { + return null; + } + final String[] split = symbol.split("/"); + if (split.length != 2) { + return null; + } + final Currency base = Currency.getInstanceNoCreate(split[0]); + final Currency quote = Currency.getInstanceNoCreate(split[1]); + if (base == null || quote == null) { + return null; + } + return new CurrencyPair(base, quote); + } +} diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloadState.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloadState.java new file mode 100644 index 00000000..81000475 --- /dev/null +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloadState.java @@ -0,0 +1,48 @@ +package io.everytrade.server.plugin.impl.everytrade; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class CoinbaseProDownloadState { + private final Map currencyPairLastIds; + + private CoinbaseProDownloadState(Map currencyPairLastIds) { + Objects.requireNonNull(this.currencyPairLastIds = currencyPairLastIds); + } + + public static CoinbaseProDownloadState parseFrom(String lastTransactionId) { + return new CoinbaseProDownloadState(convertToMap(lastTransactionId)); + } + + public Integer getLastTransactionId(String pair) { + Objects.requireNonNull(pair); + return currencyPairLastIds.get(pair); + } + + public void updateLastTransactionId(String pair, Integer lastId) { + Objects.requireNonNull(pair); + currencyPairLastIds.put(pair, lastId); + } + + public String toLastTransactionId() { + return convertFromMap(currencyPairLastIds); + } + + private String convertFromMap(Map map) { + return map.keySet().stream() + .map(key -> key + "=" + map.get(key)) + .collect(Collectors.joining(":")); + } + + private static Map convertToMap(String mapAsString) { + if (mapAsString == null) { + return new HashMap<>(); + } + return Arrays.stream(mapAsString.split(":")) + .map(entry -> entry.split("=")) + .collect(Collectors.toMap(entry -> entry[0], entry -> Integer.parseInt(entry[1]))); + } +} \ No newline at end of file diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java index 9fe5aad0..0533bb83 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradePlugin.java @@ -21,7 +21,8 @@ public class EveryTradePlugin implements IPlugin { CoinmateConnector.DESCRIPTOR, BitfinexConnector.DESCRIPTOR, BinanceConnector.DESCRIPTOR, - BittrexConnector.DESCRIPTOR + BittrexConnector.DESCRIPTOR, + CoinbaseProConnector.DESCRIPTOR ).stream().collect(Collectors.toMap(ConnectorDescriptor::getId, it -> it)); @Override @@ -62,6 +63,9 @@ public IConnector createConnectorInstance(String connectorId, Map Date: Wed, 9 Sep 2020 15:09:01 +0200 Subject: [PATCH 10/29] ET-626 Used newer XChange Bittrex API V3. --- plugin-base/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index 204ef175..33d9423c 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -28,7 +28,7 @@ dependencies { pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-core:$xchangeVersion" pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-kraken:$xchangeVersion" pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-binance:$xchangeVersion" - pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-bittrex:$xchangeVersion" + pluginCompile 'org.knowm.xchange:xchange-bittrexV3:5.0.2' pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-bitstamp:$xchangeVersion" pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-coinmate:$xchangeVersion" pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-bitfinex:$xchangeVersion" From 8eaf2cd546a61bc4b6385d58898f91ef537ed4dd Mon Sep 17 00:00:00 2001 From: charvam Date: Fri, 11 Sep 2020 15:34:42 +0200 Subject: [PATCH 11/29] ET-626 Added checkstyle support and editorConfig file. --- .editorconfig | 268 ++++++++++++++++++++++++++ config/checkstyle/checkstyle.xml | 319 +++++++++++++++++++++++++++++++ gradle.properties | 3 +- plugin-api/build.gradle | 5 + plugin-base/build.gradle | 5 + plugin-support/build.gradle | 5 + plugin-template/build.gradle | 5 + plugin-tester/build.gradle | 5 + 8 files changed, 614 insertions(+), 1 deletion(-) create mode 100644 .editorconfig create mode 100644 config/checkstyle/checkstyle.xml diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..dcf01fb3 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,268 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +indent_size = 4 +tab_width = 4 +trim_trailing_whitespace = true +insert_final_newline = false +max_line_length = 120 +ij_continuation_indent_size = 4 +ij_formatter_off_tag = @formatter:off +ij_formatter_on_tag = @formatter:on +ij_formatter_tags_enabled = false +ij_smart_tabs = false +ij_wrap_on_typing = true + +[Dockerfile] +ij_wrap_on_typing = false + +[*.java] +ij_java_align_consecutive_assignments = false +ij_java_align_consecutive_variable_declarations = false +ij_java_align_group_field_declarations = false +ij_java_align_multiline_annotation_parameters = false +ij_java_align_multiline_array_initializer_expression = false +ij_java_align_multiline_assignment = false +ij_java_align_multiline_binary_operation = false +ij_java_align_multiline_chained_methods = false +ij_java_align_multiline_extends_list = false +ij_java_align_multiline_for = true +ij_java_align_multiline_method_parentheses = false +ij_java_align_multiline_parameters = true +ij_java_align_multiline_parameters_in_calls = false +ij_java_align_multiline_parenthesized_expression = false +ij_java_align_multiline_records = true +ij_java_align_multiline_resources = true +ij_java_align_multiline_ternary_operation = false +ij_java_align_multiline_text_blocks = false +ij_java_align_multiline_throws_list = false +ij_java_align_subsequent_simple_methods = false +ij_java_align_throws_keyword = false +ij_java_annotation_parameter_wrap = off +ij_java_array_initializer_new_line_after_left_brace = false +ij_java_array_initializer_right_brace_on_new_line = false +ij_java_array_initializer_wrap = off +ij_java_assert_statement_colon_on_next_line = false +ij_java_assert_statement_wrap = off +ij_java_assignment_wrap = off +ij_java_binary_operation_sign_on_next_line = false +ij_java_binary_operation_wrap = off +ij_java_blank_lines_after_anonymous_class_header = 0 +ij_java_blank_lines_after_class_header = 0 +ij_java_blank_lines_after_imports = 1 +ij_java_blank_lines_after_package = 1 +ij_java_blank_lines_around_class = 1 +ij_java_blank_lines_around_field = 0 +ij_java_blank_lines_around_field_in_interface = 0 +ij_java_blank_lines_around_initializer = 1 +ij_java_blank_lines_around_method = 1 +ij_java_blank_lines_around_method_in_interface = 1 +ij_java_blank_lines_before_class_end = 0 +ij_java_blank_lines_before_imports = 1 +ij_java_blank_lines_before_method_body = 0 +ij_java_blank_lines_before_package = 0 +ij_java_block_brace_style = end_of_line +ij_java_block_comment_at_first_column = true +ij_java_call_parameters_new_line_after_left_paren = false +ij_java_call_parameters_right_paren_on_new_line = false +ij_java_call_parameters_wrap = off +ij_java_case_statement_on_separate_line = true +ij_java_catch_on_new_line = false +ij_java_class_annotation_wrap = split_into_lines +ij_java_class_brace_style = end_of_line +ij_java_class_count_to_use_import_on_demand = 999 +ij_java_class_names_in_javadoc = 1 +ij_java_do_not_indent_top_level_class_members = false +ij_java_do_not_wrap_after_single_annotation = false +ij_java_do_while_brace_force = always +ij_java_doc_add_blank_line_after_description = true +ij_java_doc_add_blank_line_after_param_comments = false +ij_java_doc_add_blank_line_after_return = false +ij_java_doc_add_p_tag_on_empty_lines = true +ij_java_doc_align_exception_comments = true +ij_java_doc_align_param_comments = true +ij_java_doc_do_not_wrap_if_one_line = false +ij_java_doc_enable_formatting = true +ij_java_doc_enable_leading_asterisks = true +ij_java_doc_indent_on_continuation = false +ij_java_doc_keep_empty_lines = true +ij_java_doc_keep_empty_parameter_tag = true +ij_java_doc_keep_empty_return_tag = true +ij_java_doc_keep_empty_throws_tag = true +ij_java_doc_keep_invalid_tags = true +ij_java_doc_param_description_on_new_line = false +ij_java_doc_preserve_line_breaks = false +ij_java_doc_use_throws_not_exception_tag = true +ij_java_else_on_new_line = false +ij_java_entity_dd_suffix = EJB +ij_java_entity_eb_suffix = Bean +ij_java_entity_hi_suffix = Home +ij_java_entity_lhi_prefix = Local +ij_java_entity_lhi_suffix = Home +ij_java_entity_li_prefix = Local +ij_java_entity_pk_class = java.lang.String +ij_java_entity_vo_suffix = VO +ij_java_enum_constants_wrap = off +ij_java_extends_keyword_wrap = off +ij_java_extends_list_wrap = off +ij_java_field_annotation_wrap = split_into_lines +ij_java_finally_on_new_line = false +ij_java_for_brace_force = always +ij_java_for_statement_new_line_after_left_paren = false +ij_java_for_statement_right_paren_on_new_line = false +ij_java_for_statement_wrap = off +ij_java_generate_final_locals = false +ij_java_generate_final_parameters = false +ij_java_if_brace_force = always +ij_java_imports_layout = *,|,javax.**,java.**,|,$* +ij_java_indent_case_from_switch = true +ij_java_insert_inner_class_imports = false +ij_java_insert_override_annotation = true +ij_java_keep_blank_lines_before_right_brace = 2 +ij_java_keep_blank_lines_between_package_declaration_and_header = 2 +ij_java_keep_blank_lines_in_code = 2 +ij_java_keep_blank_lines_in_declarations = 2 +ij_java_keep_control_statement_in_one_line = true +ij_java_keep_first_column_comment = true +ij_java_keep_indents_on_empty_lines = false +ij_java_keep_line_breaks = true +ij_java_keep_multiple_expressions_in_one_line = false +ij_java_keep_simple_blocks_in_one_line = false +ij_java_keep_simple_classes_in_one_line = false +ij_java_keep_simple_lambdas_in_one_line = false +ij_java_keep_simple_methods_in_one_line = false +ij_java_label_indent_absolute = false +ij_java_label_indent_size = 0 +ij_java_lambda_brace_style = end_of_line +ij_java_layout_static_imports_separately = true +ij_java_line_comment_add_space = false +ij_java_line_comment_at_first_column = true +ij_java_message_dd_suffix = EJB +ij_java_message_eb_suffix = Bean +ij_java_method_annotation_wrap = split_into_lines +ij_java_method_brace_style = end_of_line +ij_java_method_call_chain_wrap = off +ij_java_method_parameters_new_line_after_left_paren = false +ij_java_method_parameters_right_paren_on_new_line = false +ij_java_method_parameters_wrap = off +ij_java_modifier_list_wrap = false +ij_java_names_count_to_use_import_on_demand = 999 +ij_java_new_line_after_lparen_in_record_header = false +ij_java_packages_to_use_import_on_demand = java.awt.*,javax.swing.* +ij_java_parameter_annotation_wrap = off +ij_java_parentheses_expression_new_line_after_left_paren = false +ij_java_parentheses_expression_right_paren_on_new_line = false +ij_java_place_assignment_sign_on_next_line = false +ij_java_prefer_longer_names = true +ij_java_prefer_parameters_wrap = false +ij_java_record_components_wrap = normal +ij_java_repeat_synchronized = true +ij_java_replace_instanceof_and_cast = false +ij_java_replace_null_check = true +ij_java_replace_sum_lambda_with_method_ref = true +ij_java_resource_list_new_line_after_left_paren = false +ij_java_resource_list_right_paren_on_new_line = false +ij_java_resource_list_wrap = off +ij_java_rparen_on_new_line_in_record_header = false +ij_java_session_dd_suffix = EJB +ij_java_session_eb_suffix = Bean +ij_java_session_hi_suffix = Home +ij_java_session_lhi_prefix = Local +ij_java_session_lhi_suffix = Home +ij_java_session_li_prefix = Local +ij_java_session_si_suffix = Service +ij_java_space_after_closing_angle_bracket_in_type_argument = false +ij_java_space_after_colon = true +ij_java_space_after_comma = true +ij_java_space_after_comma_in_type_arguments = true +ij_java_space_after_for_semicolon = true +ij_java_space_after_quest = true +ij_java_space_after_type_cast = true +ij_java_space_before_annotation_array_initializer_left_brace = false +ij_java_space_before_annotation_parameter_list = false +ij_java_space_before_array_initializer_left_brace = false +ij_java_space_before_catch_keyword = true +ij_java_space_before_catch_left_brace = true +ij_java_space_before_catch_parentheses = true +ij_java_space_before_class_left_brace = true +ij_java_space_before_colon = true +ij_java_space_before_colon_in_foreach = true +ij_java_space_before_comma = false +ij_java_space_before_do_left_brace = true +ij_java_space_before_else_keyword = true +ij_java_space_before_else_left_brace = true +ij_java_space_before_finally_keyword = true +ij_java_space_before_finally_left_brace = true +ij_java_space_before_for_left_brace = true +ij_java_space_before_for_parentheses = true +ij_java_space_before_for_semicolon = false +ij_java_space_before_if_left_brace = true +ij_java_space_before_if_parentheses = true +ij_java_space_before_method_call_parentheses = false +ij_java_space_before_method_left_brace = true +ij_java_space_before_method_parentheses = false +ij_java_space_before_opening_angle_bracket_in_type_parameter = false +ij_java_space_before_quest = true +ij_java_space_before_switch_left_brace = true +ij_java_space_before_switch_parentheses = true +ij_java_space_before_synchronized_left_brace = true +ij_java_space_before_synchronized_parentheses = true +ij_java_space_before_try_left_brace = true +ij_java_space_before_try_parentheses = true +ij_java_space_before_type_parameter_list = false +ij_java_space_before_while_keyword = true +ij_java_space_before_while_left_brace = true +ij_java_space_before_while_parentheses = true +ij_java_space_inside_one_line_enum_braces = false +ij_java_space_within_empty_array_initializer_braces = false +ij_java_space_within_empty_method_call_parentheses = false +ij_java_space_within_empty_method_parentheses = false +ij_java_spaces_around_additive_operators = true +ij_java_spaces_around_assignment_operators = true +ij_java_spaces_around_bitwise_operators = true +ij_java_spaces_around_equality_operators = true +ij_java_spaces_around_lambda_arrow = true +ij_java_spaces_around_logical_operators = true +ij_java_spaces_around_method_ref_dbl_colon = false +ij_java_spaces_around_multiplicative_operators = true +ij_java_spaces_around_relational_operators = true +ij_java_spaces_around_shift_operators = true +ij_java_spaces_around_type_bounds_in_type_parameters = true +ij_java_spaces_around_unary_operator = false +ij_java_spaces_within_angle_brackets = false +ij_java_spaces_within_annotation_parentheses = false +ij_java_spaces_within_array_initializer_braces = false +ij_java_spaces_within_braces = false +ij_java_spaces_within_brackets = false +ij_java_spaces_within_cast_parentheses = false +ij_java_spaces_within_catch_parentheses = false +ij_java_spaces_within_for_parentheses = false +ij_java_spaces_within_if_parentheses = false +ij_java_spaces_within_method_call_parentheses = false +ij_java_spaces_within_method_parentheses = false +ij_java_spaces_within_parentheses = false +ij_java_spaces_within_switch_parentheses = false +ij_java_spaces_within_synchronized_parentheses = false +ij_java_spaces_within_try_parentheses = false +ij_java_spaces_within_while_parentheses = false +ij_java_special_else_if_treatment = true +ij_java_subclass_name_suffix = Impl +ij_java_ternary_operation_signs_on_next_line = false +ij_java_ternary_operation_wrap = off +ij_java_test_name_suffix = Test +ij_java_throws_keyword_wrap = off +ij_java_throws_list_wrap = off +ij_java_use_external_annotations = false +ij_java_use_fq_class_names = false +ij_java_use_relative_indents = false +ij_java_use_single_class_imports = true +ij_java_variable_annotation_wrap = off +ij_java_visibility = public +ij_java_while_brace_force = always +ij_java_while_on_new_line = false +ij_java_wrap_comments = false +ij_java_wrap_first_method_in_call_chain = false +ij_java_wrap_long_lines = false \ No newline at end of file diff --git a/config/checkstyle/checkstyle.xml b/config/checkstyle/checkstyle.xml new file mode 100644 index 00000000..1e1f6b28 --- /dev/null +++ b/config/checkstyle/checkstyle.xml @@ -0,0 +1,319 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index e3be0e77..83a9d628 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,4 @@ pf4jVersion=3.4.0 xchangeVersion=0f636cbfff10de85d8c111e19e8e3d142b728951 -projectVersion=1.0.3 \ No newline at end of file +projectVersion=1.0.3 +checkstyleVersion=8.34 \ No newline at end of file diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 827f6064..37afd336 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -2,6 +2,7 @@ import java.nio.charset.StandardCharsets plugins { id 'java-library' + id 'checkstyle' } group 'io.everytrade' @@ -16,4 +17,8 @@ tasks.withType(JavaCompile) { dependencies { api "org.pf4j:pf4j:$pf4jVersion" +} + +checkstyle { + toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index 33d9423c..4c9bbc4f 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java-library' + id 'checkstyle' } group 'io.everytrade' @@ -52,4 +53,8 @@ jar { configurations.fatJar.collect { it.isDirectory() ? it : zipTree(it) } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} + +checkstyle { + toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-support/build.gradle b/plugin-support/build.gradle index 827f6064..37afd336 100644 --- a/plugin-support/build.gradle +++ b/plugin-support/build.gradle @@ -2,6 +2,7 @@ import java.nio.charset.StandardCharsets plugins { id 'java-library' + id 'checkstyle' } group 'io.everytrade' @@ -16,4 +17,8 @@ tasks.withType(JavaCompile) { dependencies { api "org.pf4j:pf4j:$pf4jVersion" +} + +checkstyle { + toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-template/build.gradle b/plugin-template/build.gradle index 53180be0..e77d2537 100644 --- a/plugin-template/build.gradle +++ b/plugin-template/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java-library' + id 'checkstyle' } group 'com.example' @@ -45,4 +46,8 @@ jar { configurations.fatJar.collect { it.isDirectory() ? it : zipTree(it) } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} + +checkstyle { + toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-tester/build.gradle b/plugin-tester/build.gradle index a923bd9f..3b01cafc 100644 --- a/plugin-tester/build.gradle +++ b/plugin-tester/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java' + id 'checkstyle' } configurations { @@ -24,4 +25,8 @@ task run(type: JavaExec) { classpath = sourceSets.main.runtimeClasspath workingDir = project.rootDir mainClass = 'io.everytrade.server.plugin.Tester' +} + +checkstyle { + toolVersion checkstyleVersion } \ No newline at end of file From a9bb6bfbf7c0cee3954c64994dace9fb232b13ca Mon Sep 17 00:00:00 2001 From: charvam Date: Fri, 11 Sep 2020 15:58:33 +0200 Subject: [PATCH 12/29] ET-626 Changed private dir ignore. --- .gitignore | 4 +++- .../src/main/java/io/everytrade/server/plugin/Tester.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index e2e0529b..86ad08ac 100644 --- a/.gitignore +++ b/.gitignore @@ -158,4 +158,6 @@ gradle-app.setting # SDKMAN RC .sdkmanrc -/plugin-tester/private/ + +# Private user dir (for plugins credentials, etc) +private/ diff --git a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java index 0f8214fa..060e844a 100644 --- a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java +++ b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java @@ -125,7 +125,7 @@ private void printResult(DownloadResult downloadResult) { } private Optional> loadParams(String id) { - final Optional properties = loadProperties("plugin-tester/private/" + id + ".properties"); + final Optional properties = loadProperties("private/" + id + ".properties"); return properties.map(this::toMap); } From 9d072fdb4b4fa795c4cbc40795ff89e9e39b9252 Mon Sep 17 00:00:00 2001 From: charvam Date: Tue, 15 Sep 2020 08:31:31 +0200 Subject: [PATCH 13/29] ET-626 Unified XChange lib version. --- plugin-base/build.gradle | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index 4c9bbc4f..9268118a 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -26,14 +26,15 @@ dependencies { compileOnly 'org.slf4j:slf4j-api:1.7.30' pluginCompile 'com.univocity:univocity-parsers:2.8.3' - pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-core:$xchangeVersion" - pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-kraken:$xchangeVersion" - pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-binance:$xchangeVersion" - pluginCompile 'org.knowm.xchange:xchange-bittrexV3:5.0.2' - pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-bitstamp:$xchangeVersion" - pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-coinmate:$xchangeVersion" - pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-bitfinex:$xchangeVersion" - pluginCompile "com.github.GENERALBYTESCOM.XChange:xchange-coinbasepro:$xchangeVersion" + //TODO Change it to GENERALBYTESCOM once PR (charvam-->GENERALBYTESCOM) will by done. + pluginCompile "com.github.charvam.XChange:xchange-core:268ec0acaa" + pluginCompile "com.github.charvam.XChange:xchange-kraken:268ec0acaa" + pluginCompile "com.github.charvam.XChange:xchange-binance:268ec0acaa" + pluginCompile "com.github.charvam.XChange:xchange-bittrexV3:268ec0acaa" + pluginCompile "com.github.charvam.XChange:xchange-bitstamp:268ec0acaa" + pluginCompile "com.github.charvam.XChange:xchange-coinmate:268ec0acaa" + pluginCompile "com.github.charvam.XChange:xchange-bitfinex:268ec0acaa" + pluginCompile "com.github.charvam.XChange:xchange-coinbasepro:268ec0acaa" } compileJava { From b84a94b5270ee6911c9d2eef600258dd54a186dc Mon Sep 17 00:00:00 2001 From: charvam Date: Tue, 15 Sep 2020 09:49:13 +0200 Subject: [PATCH 14/29] ET-626 Comment --- plugin-tester/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin-tester/build.gradle b/plugin-tester/build.gradle index 3b01cafc..07c46296 100644 --- a/plugin-tester/build.gradle +++ b/plugin-tester/build.gradle @@ -23,6 +23,7 @@ task gatherPlugins(type: Sync) { task run(type: JavaExec) { dependsOn gatherPlugins classpath = sourceSets.main.runtimeClasspath + // unified with default IDE working directory workingDir = project.rootDir mainClass = 'io.everytrade.server.plugin.Tester' } From ef9b5dc0dfb965a8934d31b0a724232515b8710d Mon Sep 17 00:00:00 2001 From: charvam Date: Tue, 15 Sep 2020 09:54:56 +0200 Subject: [PATCH 15/29] ET-626 Refactoring - renamed params --- .../impl/everytrade/BinanceConnector.java | 14 +++++----- .../impl/everytrade/CoinbaseProConnector.java | 26 +++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java index ab4c422e..1e4df996 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java @@ -48,11 +48,11 @@ public class BinanceConnector implements IConnector { "" ); - private static final ConnectorParameterDescriptor PARAMETER_API_SYMBOLS = + private static final ConnectorParameterDescriptor PARAMETER_CURRENCY_PAIRS = new ConnectorParameterDescriptor( - "apiSymbols", + "currencyPairs", ConnectorParameterType.STRING, - "Trade pairs (e.g.: BTC/USDT,LTC/ETH)", + "Trade currency pairs (e.g.: BTC/USDT,LTC/ETH)", "" ); @@ -60,17 +60,17 @@ public class BinanceConnector implements IConnector { ID, "Binance Connector", SupportedExchange.BINANCE.getInternalId(), - List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET, PARAMETER_API_SYMBOLS) + List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET, PARAMETER_CURRENCY_PAIRS) ); private final String apiKey; private final String apiSecret; - private final String apiSymbols; + private final String currencyPairs; public BinanceConnector(Map parameters) { Objects.requireNonNull(this.apiKey = parameters.get(PARAMETER_API_KEY.getId())); Objects.requireNonNull(this.apiSecret = parameters.get(PARAMETER_API_SECRET.getId())); - Objects.requireNonNull(this.apiSymbols = parameters.get(PARAMETER_API_SYMBOLS.getId())); + Objects.requireNonNull(this.currencyPairs = parameters.get(PARAMETER_CURRENCY_PAIRS.getId())); } @Override @@ -102,7 +102,7 @@ private List download( = (BinanceTradeHistoryParams) tradeService.createTradeHistoryParams(); tradeHistoryParams.setLimit(TX_PER_REQUEST); - final List currencyPairs = symbolsToPairs(apiSymbols); + final List currencyPairs = symbolsToPairs(this.currencyPairs); final List userTrades = new ArrayList<>(); int counter = 0; diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java index 650b4df2..736b5e22 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java @@ -50,19 +50,19 @@ public class CoinbaseProConnector implements IConnector { "" ); - private static final ConnectorParameterDescriptor PARAMETER_API_PASS_PHRASE = + private static final ConnectorParameterDescriptor PARAMETER_PASS_PHRASE = new ConnectorParameterDescriptor( - "apiPassPhrase", + "passPhrase", ConnectorParameterType.SECRET, - "API PassPhrase", + "passphrase", "" ); - private static final ConnectorParameterDescriptor PARAMETER_API_SYMBOLS = + private static final ConnectorParameterDescriptor PARAMETER_CURRENCY_PAIRS = new ConnectorParameterDescriptor( - "apiSymbols", + "currencyPairs", ConnectorParameterType.STRING, - "Trade pairs (e.g.: LTC/EUR,LTC/BTC)", + "Trade currency pairs (e.g. LTC/EUR,LTC/BTC)", "" ); @@ -70,19 +70,19 @@ public class CoinbaseProConnector implements IConnector { ID, "Coinbase Pro Connector", SUPPORTED_EXCHANGE.getInternalId(), - List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET, PARAMETER_API_SYMBOLS, PARAMETER_API_PASS_PHRASE) + List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET, PARAMETER_CURRENCY_PAIRS, PARAMETER_PASS_PHRASE) ); private final String apiKey; private final String apiSecret; - private final String apiSymbols; - private final String apiPassPhrase; + private final String currencyPairs; + private final String passPhrase; public CoinbaseProConnector(Map parameters) { Objects.requireNonNull(this.apiKey = parameters.get(PARAMETER_API_KEY.getId())); Objects.requireNonNull(this.apiSecret = parameters.get(PARAMETER_API_SECRET.getId())); - Objects.requireNonNull(this.apiSymbols = parameters.get(PARAMETER_API_SYMBOLS.getId())); - Objects.requireNonNull(this.apiPassPhrase = parameters.get(PARAMETER_API_PASS_PHRASE.getId())); + Objects.requireNonNull(this.currencyPairs = parameters.get(PARAMETER_CURRENCY_PAIRS.getId())); + Objects.requireNonNull(this.passPhrase = parameters.get(PARAMETER_PASS_PHRASE.getId())); } @@ -96,7 +96,7 @@ public DownloadResult getTransactions(String lastTransactionId) { final ExchangeSpecification exSpec = new CoinbaseProExchange().getDefaultExchangeSpecification(); exSpec.setApiKey(apiKey); exSpec.setSecretKey(apiSecret); - exSpec.setExchangeSpecificParametersItem("passphrase", apiPassPhrase); + exSpec.setExchangeSpecificParametersItem("passphrase", passPhrase); final Exchange exchange = ExchangeFactory.INSTANCE.createExchange(exSpec); final TradeService tradeService = exchange.getTradeService(); final CoinbaseProDownloadState downloadState = CoinbaseProDownloadState.parseFrom(lastTransactionId); @@ -112,7 +112,7 @@ private List download(TradeService tradeService, CoinbaseProDownloadS = (CoinbaseProTradeHistoryParams) tradeService.createTradeHistoryParams(); tradeHistoryParams.setLimit(TX_PER_REQUEST); - final List currencyPairs = symbolsToPairs(apiSymbols); + final List currencyPairs = symbolsToPairs(this.currencyPairs); final List userTrades = new ArrayList<>(); int counter = 0; From 7dbd39c39d55cdf80dc0da301528e4aa350bee81 Mon Sep 17 00:00:00 2001 From: charvam Date: Tue, 15 Sep 2020 16:59:12 +0200 Subject: [PATCH 16/29] ET-626 Moved download logic to separate class. --- .../impl/everytrade/BinanceConnector.java | 81 +--------- .../impl/everytrade/BinanceDownloadState.java | 48 ------ .../impl/everytrade/BinanceDownloader.java | 85 ++++++++++ .../impl/everytrade/CoinbaseProConnector.java | 146 ++++-------------- .../everytrade/CoinbaseProDownloadState.java | 48 ------ .../everytrade/CoinbaseProDownloader.java | 85 ++++++++++ .../impl/everytrade/ConnectorUtils.java | 23 +++ 7 files changed, 228 insertions(+), 288 deletions(-) delete mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloadState.java create mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloader.java delete mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloadState.java create mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java create mode 100644 plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/ConnectorUtils.java diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java index 1e4df996..67bc2e89 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java @@ -28,9 +28,6 @@ public class BinanceConnector implements IConnector { private static final String ID = EveryTradePlugin.ID + IPlugin.PLUGIN_PATH_SEPARATOR + "binanceApiConnector"; - //org.knowm.xchange.binance.BinanceResilience - 1200 request / 1 minute --> 10 user in 1 minute will be enough - private static final int MAX_REQUEST_COUNT = 120; - private static final int TX_PER_REQUEST = 1000; private static final ConnectorParameterDescriptor PARAMETER_API_SECRET = new ConnectorParameterDescriptor( @@ -52,7 +49,7 @@ public class BinanceConnector implements IConnector { new ConnectorParameterDescriptor( "currencyPairs", ConnectorParameterType.STRING, - "Trade currency pairs (e.g.: BTC/USDT,LTC/ETH)", + "Trade currency pairs (e.g. BTC/USDT,LTC/ETH)", "" ); @@ -85,58 +82,12 @@ public DownloadResult getTransactions(String lastTransactionId) { exSpec.setSecretKey(apiSecret); final Exchange exchange = ExchangeFactory.INSTANCE.createExchange(exSpec); final TradeService tradeService = exchange.getTradeService(); - final BinanceDownloadState downloadState = BinanceDownloadState.parseFrom(lastTransactionId); + final BinanceDownloader binanceDownloader = new BinanceDownloader(tradeService, lastTransactionId); - List userTrades = download(tradeService, downloadState); + List userTrades = binanceDownloader.download(currencyPairs); final ParseResult parseResult = XChangeConnectorParser.getParseResult(userTrades, SupportedExchange.BINANCE); - return new DownloadResult(parseResult, downloadState.toLastTransactionId()); - } - - private List download( - TradeService tradeService, - BinanceDownloadState downloadState - ) { - - final BinanceTradeHistoryParams tradeHistoryParams - = (BinanceTradeHistoryParams) tradeService.createTradeHistoryParams(); - tradeHistoryParams.setLimit(TX_PER_REQUEST); - - final List currencyPairs = symbolsToPairs(this.currencyPairs); - final List userTrades = new ArrayList<>(); - int counter = 0; - - for (CurrencyPair currencyPair : currencyPairs) { - tradeHistoryParams.setCurrencyPair(currencyPair); - String lastDownloadedTx = downloadState.getLastTransactionId(currencyPair.toString()); - tradeHistoryParams.setStartId(lastDownloadedTx); - - while (counter++ < MAX_REQUEST_COUNT) { - final List userTradesBlock; - try { - userTradesBlock = tradeService.getTradeHistory(tradeHistoryParams).getUserTrades(); - } catch (Exception e) { - throw new IllegalStateException("User trade history download failed. ", e); - } - if ( - lastDownloadedTx != null - && !userTradesBlock.isEmpty() - && userTradesBlock.get(0).getId().equals(lastDownloadedTx) - ) { - userTradesBlock.remove(0); - } - if (userTradesBlock.isEmpty()) { - break; - } - userTrades.addAll(userTradesBlock); - lastDownloadedTx = userTradesBlock.get(userTradesBlock.size() - 1).getId(); - tradeHistoryParams.setStartId(lastDownloadedTx); - } - downloadState.updateLastTransactionId(currencyPair.toString(), lastDownloadedTx); - } - - return userTrades; - + return new DownloadResult(parseResult, binanceDownloader.getLastTransactionId()); } @Override @@ -144,28 +95,4 @@ public void close() { //AutoCloseable } - private List symbolsToPairs(String symbols) { - return Arrays.stream(symbols.split(",")) - .map(String::strip) - .map(this::createPair) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - - private CurrencyPair createPair(String symbol) { - if (symbol == null) { - return null; - } - final String[] split = symbol.split("/"); - if (split.length != 2) { - return null; - } - final Currency base = Currency.getInstanceNoCreate(split[0]); - final Currency quote = Currency.getInstanceNoCreate(split[1]); - if (base == null || quote == null) { - return null; - } - return new CurrencyPair(base, quote); - } - } diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloadState.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloadState.java deleted file mode 100644 index 6cc3fc26..00000000 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloadState.java +++ /dev/null @@ -1,48 +0,0 @@ -package io.everytrade.server.plugin.impl.everytrade; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -public class BinanceDownloadState { - private final Map pairLastIds; - - private BinanceDownloadState(Map pairLastIds) { - this.pairLastIds = pairLastIds; - } - - public static BinanceDownloadState parseFrom(String lastTransactionId) { - return new BinanceDownloadState(convertToMap(lastTransactionId)); - } - - public String getLastTransactionId(String pair) { - Objects.requireNonNull(pair); - return pairLastIds.get(pair); - } - - public void updateLastTransactionId(String pair, String lastId) { - Objects.requireNonNull(pair); - pairLastIds.put(pair, lastId); - } - - public String toLastTransactionId() { - return convertFromMap(pairLastIds); - } - - private String convertFromMap(Map map) { - return map.keySet().stream() - .map(key -> key + "=" + map.get(key)) - .collect(Collectors.joining(":")); - } - - private static Map convertToMap(String mapAsString) { - if (mapAsString == null) { - return new HashMap<>(); - } - return Arrays.stream(mapAsString.split(":")) - .map(entry -> entry.split("=")) - .collect(Collectors.toMap(entry -> entry[0], entry -> entry[1])); - } -} \ No newline at end of file diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloader.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloader.java new file mode 100644 index 00000000..6f916578 --- /dev/null +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloader.java @@ -0,0 +1,85 @@ +package io.everytrade.server.plugin.impl.everytrade; + +import org.knowm.xchange.binance.service.BinanceTradeHistoryParams; +import org.knowm.xchange.coinbasepro.dto.trade.CoinbaseProTradeHistoryParams; +import org.knowm.xchange.currency.Currency; +import org.knowm.xchange.currency.CurrencyPair; +import org.knowm.xchange.dto.trade.UserTrade; +import org.knowm.xchange.service.trade.TradeService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class BinanceDownloader { + //org.knowm.xchange.binance.BinanceResilience - 1200 request / 1 minute --> 10 user in 1 minute will be enough + private static final int MAX_REQUEST_COUNT = 120; + private static final int TX_PER_REQUEST = 1000; + private final Map currencyPairLastIds; + private final TradeService tradeService; + private final BinanceTradeHistoryParams tradeHistoryParams; + + public BinanceDownloader(TradeService tradeService, String lastTransactionId) { + Objects.requireNonNull(this.tradeService = tradeService); + tradeHistoryParams = (BinanceTradeHistoryParams) tradeService.createTradeHistoryParams(); + tradeHistoryParams.setLimit(TX_PER_REQUEST); + if (lastTransactionId == null) { + currencyPairLastIds = new HashMap<>(); + } else { + currencyPairLastIds = Arrays.stream(lastTransactionId.split(":")) + .map(entry -> entry.split("=")) + .collect(Collectors.toMap(entry -> entry[0], entry -> entry[1])); + } + } + + public List download(String currencyPairs) { + final List pairs = Arrays.stream(currencyPairs.split(",")) + .map(String::strip) + .map(ConnectorUtils::createPair) + .collect(Collectors.toList()); + + final List userTrades = new ArrayList<>(); + int sentRequests = 0; + + for (CurrencyPair pair : pairs) { + tradeHistoryParams.setCurrencyPair(pair); + String lastDownloadedTx = currencyPairLastIds.get(pair.toString()); + tradeHistoryParams.setStartId(lastDownloadedTx); + + while (sentRequests < MAX_REQUEST_COUNT) { + final List userTradesBlock; + try { + userTradesBlock = tradeService.getTradeHistory(tradeHistoryParams).getUserTrades(); + } catch (Exception e) { + throw new IllegalStateException("User trade history download failed. ", e); + } + if ( + lastDownloadedTx != null + && !userTradesBlock.isEmpty() + && userTradesBlock.get(0).getId().equals(lastDownloadedTx) + ) { + userTradesBlock.remove(0); + } + if (userTradesBlock.isEmpty()) { + break; + } + userTrades.addAll(userTradesBlock); + lastDownloadedTx = userTradesBlock.get(userTradesBlock.size() - 1).getId(); + tradeHistoryParams.setStartId(lastDownloadedTx); + ++sentRequests; + } + currencyPairLastIds.put(pair.toString(), lastDownloadedTx); + } + return userTrades; + } + + public String getLastTransactionId() { + return currencyPairLastIds.keySet().stream() + .map(key -> key + "=" + currencyPairLastIds.get(key)) + .collect(Collectors.joining(":")); + } +} diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java index 736b5e22..6ca5891d 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProConnector.java @@ -12,65 +12,54 @@ import org.knowm.xchange.ExchangeFactory; import org.knowm.xchange.ExchangeSpecification; import org.knowm.xchange.coinbasepro.CoinbaseProExchange; -import org.knowm.xchange.coinbasepro.dto.trade.CoinbaseProTradeHistoryParams; -import org.knowm.xchange.currency.Currency; -import org.knowm.xchange.currency.CurrencyPair; import org.knowm.xchange.dto.trade.UserTrade; import org.knowm.xchange.service.trade.TradeService; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; public class CoinbaseProConnector implements IConnector { private static final String ID = EveryTradePlugin.ID + IPlugin.PLUGIN_PATH_SEPARATOR + "coinbaseProApiConnector"; - //https://docs.pro.coinbase.com/#rate-limits - max 5 request per user per second - private static final int TX_PER_REQUEST = 100; - private static final int MAX_REQUEST_COUNT = 3000; - private static final int MAX_REQUESTS_PER_SECOND = 5; - private static final int REQUESTS_WINDOW_MS = 1000; private static final SupportedExchange SUPPORTED_EXCHANGE = SupportedExchange.COINBASE; private static final ConnectorParameterDescriptor PARAMETER_API_SECRET = - new ConnectorParameterDescriptor( - "apiSecret", - ConnectorParameterType.SECRET, - "API Secret", - "" - ); + new ConnectorParameterDescriptor( + "apiSecret", + ConnectorParameterType.SECRET, + "API Secret", + "" + ); private static final ConnectorParameterDescriptor PARAMETER_API_KEY = - new ConnectorParameterDescriptor( - "apiKey", - ConnectorParameterType.STRING, - "API Key", - "" - ); + new ConnectorParameterDescriptor( + "apiKey", + ConnectorParameterType.STRING, + "API Key", + "" + ); private static final ConnectorParameterDescriptor PARAMETER_PASS_PHRASE = - new ConnectorParameterDescriptor( - "passPhrase", - ConnectorParameterType.SECRET, - "passphrase", - "" - ); + new ConnectorParameterDescriptor( + "passPhrase", + ConnectorParameterType.SECRET, + "passphrase", + "" + ); private static final ConnectorParameterDescriptor PARAMETER_CURRENCY_PAIRS = - new ConnectorParameterDescriptor( - "currencyPairs", - ConnectorParameterType.STRING, - "Trade currency pairs (e.g. LTC/EUR,LTC/BTC)", - "" - ); + new ConnectorParameterDescriptor( + "currencyPairs", + ConnectorParameterType.STRING, + "Trade currency pairs (e.g. LTC/EUR,LTC/BTC)", + "" + ); public static final ConnectorDescriptor DESCRIPTOR = new ConnectorDescriptor( - ID, - "Coinbase Pro Connector", - SUPPORTED_EXCHANGE.getInternalId(), - List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET, PARAMETER_CURRENCY_PAIRS, PARAMETER_PASS_PHRASE) + ID, + "Coinbase Pro Connector", + SUPPORTED_EXCHANGE.getInternalId(), + List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET, PARAMETER_CURRENCY_PAIRS, PARAMETER_PASS_PHRASE) ); private final String apiKey; @@ -85,7 +74,6 @@ public CoinbaseProConnector(Map parameters) { Objects.requireNonNull(this.passPhrase = parameters.get(PARAMETER_PASS_PHRASE.getId())); } - @Override public String getId() { return ID; @@ -99,82 +87,10 @@ public DownloadResult getTransactions(String lastTransactionId) { exSpec.setExchangeSpecificParametersItem("passphrase", passPhrase); final Exchange exchange = ExchangeFactory.INSTANCE.createExchange(exSpec); final TradeService tradeService = exchange.getTradeService(); - final CoinbaseProDownloadState downloadState = CoinbaseProDownloadState.parseFrom(lastTransactionId); - - final List userTrades = download(tradeService, downloadState); + final CoinbaseProDownloader coinbaseProDownloader = new CoinbaseProDownloader(tradeService, lastTransactionId); + final List userTrades = coinbaseProDownloader.download(currencyPairs); final ParseResult parseResult = XChangeConnectorParser.getParseResult(userTrades, SUPPORTED_EXCHANGE); - return new DownloadResult(parseResult, downloadState.toLastTransactionId()); - } - - private List download(TradeService tradeService, CoinbaseProDownloadState downloadState) { - final CoinbaseProTradeHistoryParams tradeHistoryParams - = (CoinbaseProTradeHistoryParams) tradeService.createTradeHistoryParams(); - tradeHistoryParams.setLimit(TX_PER_REQUEST); - - final List currencyPairs = symbolsToPairs(this.currencyPairs); - final List userTrades = new ArrayList<>(); - int counter = 0; - - for (CurrencyPair currencyPair : currencyPairs) { - tradeHistoryParams.setCurrencyPair(currencyPair); - Integer lastDownloadedTx = downloadState.getLastTransactionId(currencyPair.toString()); - tradeHistoryParams.setBeforeTradeId(lastDownloadedTx == null ? 1 : lastDownloadedTx ); - - while (counter++ < MAX_REQUEST_COUNT) { - final List userTradesBlock; - downloadSleep(counter); - try { - userTradesBlock = tradeService.getTradeHistory(tradeHistoryParams).getUserTrades(); - } catch (Exception e) { - throw new IllegalStateException("User trade history download failed. ", e); - } - if (userTradesBlock.isEmpty()) { - break; - } - userTrades.addAll(userTradesBlock); - lastDownloadedTx = Integer.valueOf(userTradesBlock.get(userTradesBlock.size() - 1).getId()); - tradeHistoryParams.setBeforeTradeId(lastDownloadedTx); - } - if (lastDownloadedTx != null) { - downloadState.updateLastTransactionId(currencyPair.toString(), lastDownloadedTx); - } - } - - return userTrades; - } - - private void downloadSleep(int counter) { - if (counter % MAX_REQUESTS_PER_SECOND == 0) { - try { - Thread.sleep(REQUESTS_WINDOW_MS); - } catch (InterruptedException e) { - throw new IllegalStateException("User trade history download sleep interrupted.", e); - } - } - } - - private List symbolsToPairs(String symbols) { - return Arrays.stream(symbols.split(",")) - .map(String::strip) - .map(this::createPair) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } - - private CurrencyPair createPair(String symbol) { - if (symbol == null) { - return null; - } - final String[] split = symbol.split("/"); - if (split.length != 2) { - return null; - } - final Currency base = Currency.getInstanceNoCreate(split[0]); - final Currency quote = Currency.getInstanceNoCreate(split[1]); - if (base == null || quote == null) { - return null; - } - return new CurrencyPair(base, quote); + return new DownloadResult(parseResult, coinbaseProDownloader.getLastTransactionId()); } } diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloadState.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloadState.java deleted file mode 100644 index 81000475..00000000 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloadState.java +++ /dev/null @@ -1,48 +0,0 @@ -package io.everytrade.server.plugin.impl.everytrade; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; - -public class CoinbaseProDownloadState { - private final Map currencyPairLastIds; - - private CoinbaseProDownloadState(Map currencyPairLastIds) { - Objects.requireNonNull(this.currencyPairLastIds = currencyPairLastIds); - } - - public static CoinbaseProDownloadState parseFrom(String lastTransactionId) { - return new CoinbaseProDownloadState(convertToMap(lastTransactionId)); - } - - public Integer getLastTransactionId(String pair) { - Objects.requireNonNull(pair); - return currencyPairLastIds.get(pair); - } - - public void updateLastTransactionId(String pair, Integer lastId) { - Objects.requireNonNull(pair); - currencyPairLastIds.put(pair, lastId); - } - - public String toLastTransactionId() { - return convertFromMap(currencyPairLastIds); - } - - private String convertFromMap(Map map) { - return map.keySet().stream() - .map(key -> key + "=" + map.get(key)) - .collect(Collectors.joining(":")); - } - - private static Map convertToMap(String mapAsString) { - if (mapAsString == null) { - return new HashMap<>(); - } - return Arrays.stream(mapAsString.split(":")) - .map(entry -> entry.split("=")) - .collect(Collectors.toMap(entry -> entry[0], entry -> Integer.parseInt(entry[1]))); - } -} \ No newline at end of file diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java new file mode 100644 index 00000000..82e1638b --- /dev/null +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java @@ -0,0 +1,85 @@ +package io.everytrade.server.plugin.impl.everytrade; + +import org.knowm.xchange.coinbasepro.dto.trade.CoinbaseProTradeHistoryParams; +import org.knowm.xchange.currency.Currency; +import org.knowm.xchange.currency.CurrencyPair; +import org.knowm.xchange.dto.trade.UserTrade; +import org.knowm.xchange.service.trade.TradeService; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +public class CoinbaseProDownloader { + //https://docs.pro.coinbase.com/#rate-limits - max 5 request per user per second --> 200 ms between requests + private static final int TX_PER_REQUEST = 100; + private static final int MAX_REQUEST_COUNT = 3000; + private static final int SLEEP_BETWEEN_REQUESTS_MS = 200; + private final Map currencyPairLastIds; + private final TradeService tradeService; + private final CoinbaseProTradeHistoryParams tradeHistoryParams; + + public CoinbaseProDownloader(TradeService tradeService, String lastTransactionId) { + Objects.requireNonNull(this.tradeService = tradeService); + tradeHistoryParams = (CoinbaseProTradeHistoryParams) tradeService.createTradeHistoryParams(); + tradeHistoryParams.setLimit(TX_PER_REQUEST); + if (lastTransactionId == null) { + currencyPairLastIds = new HashMap<>(); + } else { + currencyPairLastIds = Arrays.stream(lastTransactionId.split(":")) + .map(entry -> entry.split("=")) + .collect(Collectors.toMap(entry -> entry[0], entry -> Integer.parseInt(entry[1]))); + } + } + + public List download(String currencyPairs) { + final List pairs = Arrays.stream(currencyPairs.split(",")) + .map(String::strip) + .map(ConnectorUtils::createPair) + .collect(Collectors.toList()); + final List userTrades = new ArrayList<>(); + int sentRequests = 0; + + for (CurrencyPair pair : pairs) { + tradeHistoryParams.setCurrencyPair(pair); + final Integer lastDownloadedTxFound = currencyPairLastIds.get(pair.toString()); + int lastDownloadedTx = lastDownloadedTxFound == null ? 1 : lastDownloadedTxFound; + + while (sentRequests < MAX_REQUEST_COUNT) { + tradeHistoryParams.setBeforeTradeId(lastDownloadedTx); + final List userTradesBlock; + try { + Thread.sleep(SLEEP_BETWEEN_REQUESTS_MS); + } catch (InterruptedException e) { + throw new IllegalStateException("User trade history download sleep interrupted.", e); + } + try { + userTradesBlock = tradeService.getTradeHistory(tradeHistoryParams).getUserTrades(); + } catch (Exception e) { + throw new IllegalStateException("User trade history download failed. ", e); + } + if (userTradesBlock.isEmpty()) { + break; + } + userTrades.addAll(userTradesBlock); + lastDownloadedTx = Integer.parseInt(userTradesBlock.get(userTradesBlock.size() - 1).getId()); + + ++sentRequests; + } + currencyPairLastIds.put(pair.toString(), lastDownloadedTx); + } + + return userTrades; + + } + + public String getLastTransactionId() { + return currencyPairLastIds.keySet().stream() + .map(key -> key + "=" + currencyPairLastIds.get(key)) + .collect(Collectors.joining(":")); + } +} \ No newline at end of file diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/ConnectorUtils.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/ConnectorUtils.java new file mode 100644 index 00000000..42437780 --- /dev/null +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/ConnectorUtils.java @@ -0,0 +1,23 @@ +package io.everytrade.server.plugin.impl.everytrade; + +import org.knowm.xchange.currency.Currency; +import org.knowm.xchange.currency.CurrencyPair; + +public class ConnectorUtils { + + public static CurrencyPair createPair(String pair) { + final String[] split = pair.split("/"); + if (split.length != 2) { + throw new IllegalArgumentException(String.format("Illegal pair value '%s'.", pair)); + } + final Currency base = Currency.getInstanceNoCreate(split[0]); + final Currency quote = Currency.getInstanceNoCreate(split[1]); + if (base == null) { + throw new IllegalArgumentException(String.format("Illegal base value '%s'.", split[0])); + } + if (quote == null) { + throw new IllegalArgumentException(String.format("Illegal quote value '%s'.", split[1])); + } + return new CurrencyPair(base, quote); + } +} From 7d0194cd0289d135eebb0344ae46d108dbc780e8 Mon Sep 17 00:00:00 2001 From: charvam Date: Tue, 15 Sep 2020 17:49:36 +0200 Subject: [PATCH 17/29] ET-626 Added audit plugins, fix style. --- build.gradle | 29 ++++++++++++++++++- gradle.properties | 7 ++++- .../everytrade/server/model/CurrencyPair.java | 4 ++- .../plugin/api/connector/IConnector.java | 3 +- plugin-base/build.gradle | 5 ---- plugin-support/build.gradle | 5 ---- plugin-template/build.gradle | 5 ---- plugin-tester/build.gradle | 5 ---- 8 files changed, 39 insertions(+), 24 deletions(-) diff --git a/build.gradle b/build.gradle index e8e59edd..61d08166 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,16 @@ import java.nio.charset.StandardCharsets +plugins { + id 'checkstyle' + id "org.owasp.dependencycheck" version "$owaspDependencyCheckGradlePluginVersion" + id "org.sonatype.gradle.plugins.scan" version "$sonatypeScanGradlePluginVersion" + id "com.github.spotbugs" version "$spotbugsGradlePluginVersion" +} + +dependencies { + spotbugsPlugins "com.h3xstream.findsecbugs:findsecbugs-plugin:$findsecbugsVersion" +} + allprojects { repositories { jcenter() @@ -16,4 +27,20 @@ allprojects { wrapper { distributionType = Wrapper.DistributionType.ALL gradleVersion = "6.6.1" -} \ No newline at end of file +} + +checkstyle { + toolVersion checkstyleVersion +} + +final File privatePropertyFile = new File(project.rootDir, 'private/gradle.properties') +if (privatePropertyFile.exists()) { + final Properties privateProperties = new Properties() + privatePropertyFile.withReader { BufferedReader reader -> + privateProperties.load(reader) + privateProperties.stringPropertyNames().each { + project.ext[it] = privateProperties[it] + } + } +} + diff --git a/gradle.properties b/gradle.properties index 83a9d628..466acb46 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,9 @@ pf4jVersion=3.4.0 xchangeVersion=0f636cbfff10de85d8c111e19e8e3d142b728951 projectVersion=1.0.3 -checkstyleVersion=8.34 \ No newline at end of file +checkstyleVersion=8.34 +spotbugsVersion=4.1.2 +owaspDependencyCheckGradlePluginVersion=6.0.0 +sonatypeScanGradlePluginVersion=1.2.3 +spotbugsGradlePluginVersion=4.5.0 +findsecbugsVersion=1.10.1 \ No newline at end of file diff --git a/plugin-api/src/main/java/io/everytrade/server/model/CurrencyPair.java b/plugin-api/src/main/java/io/everytrade/server/model/CurrencyPair.java index 826f830f..9b93eeb8 100644 --- a/plugin-api/src/main/java/io/everytrade/server/model/CurrencyPair.java +++ b/plugin-api/src/main/java/io/everytrade/server/model/CurrencyPair.java @@ -42,7 +42,9 @@ public CurrencyPair reverse() { } public Instant getIntroduction() { - return base.getIntroduction().compareTo(quote.getIntroduction()) >= 0 ? base.getIntroduction() : quote.getIntroduction(); + return base.getIntroduction().compareTo(quote.getIntroduction()) >= 0 + ? base.getIntroduction() + : quote.getIntroduction(); } public static List getTradeablePairs() { diff --git a/plugin-api/src/main/java/io/everytrade/server/plugin/api/connector/IConnector.java b/plugin-api/src/main/java/io/everytrade/server/plugin/api/connector/IConnector.java index 1ea9da3b..7cbc61a3 100644 --- a/plugin-api/src/main/java/io/everytrade/server/plugin/api/connector/IConnector.java +++ b/plugin-api/src/main/java/io/everytrade/server/plugin/api/connector/IConnector.java @@ -18,5 +18,6 @@ public interface IConnector extends AutoCloseable { /** * {@inheritDoc} */ - default void close() {}; + default void close() { + } } diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index 9268118a..eb56ae5c 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java-library' - id 'checkstyle' } group 'io.everytrade' @@ -55,7 +54,3 @@ jar { } duplicatesStrategy = DuplicatesStrategy.EXCLUDE } - -checkstyle { - toolVersion checkstyleVersion -} \ No newline at end of file diff --git a/plugin-support/build.gradle b/plugin-support/build.gradle index 37afd336..827f6064 100644 --- a/plugin-support/build.gradle +++ b/plugin-support/build.gradle @@ -2,7 +2,6 @@ import java.nio.charset.StandardCharsets plugins { id 'java-library' - id 'checkstyle' } group 'io.everytrade' @@ -17,8 +16,4 @@ tasks.withType(JavaCompile) { dependencies { api "org.pf4j:pf4j:$pf4jVersion" -} - -checkstyle { - toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-template/build.gradle b/plugin-template/build.gradle index e77d2537..53180be0 100644 --- a/plugin-template/build.gradle +++ b/plugin-template/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java-library' - id 'checkstyle' } group 'com.example' @@ -46,8 +45,4 @@ jar { configurations.fatJar.collect { it.isDirectory() ? it : zipTree(it) } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE -} - -checkstyle { - toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-tester/build.gradle b/plugin-tester/build.gradle index 07c46296..506f502e 100644 --- a/plugin-tester/build.gradle +++ b/plugin-tester/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java' - id 'checkstyle' } configurations { @@ -27,7 +26,3 @@ task run(type: JavaExec) { workingDir = project.rootDir mainClass = 'io.everytrade.server.plugin.Tester' } - -checkstyle { - toolVersion checkstyleVersion -} \ No newline at end of file From a51987c6f8d64b2c757dd4ead3aa3e70c8c9aa14 Mon Sep 17 00:00:00 2001 From: charvam Date: Tue, 15 Sep 2020 18:12:23 +0200 Subject: [PATCH 18/29] ET-626 Removed duplicate plugin. --- plugin-api/build.gradle | 5 ----- 1 file changed, 5 deletions(-) diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 37afd336..827f6064 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -2,7 +2,6 @@ import java.nio.charset.StandardCharsets plugins { id 'java-library' - id 'checkstyle' } group 'io.everytrade' @@ -17,8 +16,4 @@ tasks.withType(JavaCompile) { dependencies { api "org.pf4j:pf4j:$pf4jVersion" -} - -checkstyle { - toolVersion checkstyleVersion } \ No newline at end of file From 0d30b3d1ef3421c044607e401ca58ed9a044219f Mon Sep 17 00:00:00 2001 From: charvam Date: Wed, 16 Sep 2020 11:24:08 +0200 Subject: [PATCH 19/29] ET-626 Fixed checkstyle plugin. --- build.gradle | 5 ----- plugin-api/build.gradle | 5 +++++ plugin-base/build.gradle | 5 +++++ .../server/parser/exchange/XChangeApiTransactionBean.java | 4 +++- plugin-support/build.gradle | 5 +++++ plugin-template/build.gradle | 5 +++++ plugin-tester/build.gradle | 5 +++++ 7 files changed, 28 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index 61d08166..65d7c7c3 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,6 @@ import java.nio.charset.StandardCharsets plugins { - id 'checkstyle' id "org.owasp.dependencycheck" version "$owaspDependencyCheckGradlePluginVersion" id "org.sonatype.gradle.plugins.scan" version "$sonatypeScanGradlePluginVersion" id "com.github.spotbugs" version "$spotbugsGradlePluginVersion" @@ -29,10 +28,6 @@ wrapper { gradleVersion = "6.6.1" } -checkstyle { - toolVersion checkstyleVersion -} - final File privatePropertyFile = new File(project.rootDir, 'private/gradle.properties') if (privatePropertyFile.exists()) { final Properties privateProperties = new Properties() diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 827f6064..37afd336 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -2,6 +2,7 @@ import java.nio.charset.StandardCharsets plugins { id 'java-library' + id 'checkstyle' } group 'io.everytrade' @@ -16,4 +17,8 @@ tasks.withType(JavaCompile) { dependencies { api "org.pf4j:pf4j:$pf4jVersion" +} + +checkstyle { + toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index eb56ae5c..4d1ae54b 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java-library' + id 'checkstyle' } group 'io.everytrade' @@ -54,3 +55,7 @@ jar { } duplicatesStrategy = DuplicatesStrategy.EXCLUDE } + +checkstyle { + toolVersion checkstyleVersion +} diff --git a/plugin-base/src/main/java/io/everytrade/server/parser/exchange/XChangeApiTransactionBean.java b/plugin-base/src/main/java/io/everytrade/server/parser/exchange/XChangeApiTransactionBean.java index ac7dfe1b..6802d0f0 100644 --- a/plugin-base/src/main/java/io/everytrade/server/parser/exchange/XChangeApiTransactionBean.java +++ b/plugin-base/src/main/java/io/everytrade/server/parser/exchange/XChangeApiTransactionBean.java @@ -99,7 +99,9 @@ private TransactionType getTransactionType(Order.OrderType orderType) { case BID: return TransactionType.BUY; default: - throw new DataValidationException("ExchangeBean.UNSUPPORTED_TRANSACTION_TYPE ".concat(orderType.name())); + throw new DataValidationException( + "ExchangeBean.UNSUPPORTED_TRANSACTION_TYPE ".concat(orderType.name()) + ); } } } diff --git a/plugin-support/build.gradle b/plugin-support/build.gradle index 827f6064..37afd336 100644 --- a/plugin-support/build.gradle +++ b/plugin-support/build.gradle @@ -2,6 +2,7 @@ import java.nio.charset.StandardCharsets plugins { id 'java-library' + id 'checkstyle' } group 'io.everytrade' @@ -16,4 +17,8 @@ tasks.withType(JavaCompile) { dependencies { api "org.pf4j:pf4j:$pf4jVersion" +} + +checkstyle { + toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-template/build.gradle b/plugin-template/build.gradle index 53180be0..e77d2537 100644 --- a/plugin-template/build.gradle +++ b/plugin-template/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java-library' + id 'checkstyle' } group 'com.example' @@ -45,4 +46,8 @@ jar { configurations.fatJar.collect { it.isDirectory() ? it : zipTree(it) } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} + +checkstyle { + toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-tester/build.gradle b/plugin-tester/build.gradle index 506f502e..122646d2 100644 --- a/plugin-tester/build.gradle +++ b/plugin-tester/build.gradle @@ -1,5 +1,6 @@ plugins { id 'java' + id 'checkstyle' } configurations { @@ -26,3 +27,7 @@ task run(type: JavaExec) { workingDir = project.rootDir mainClass = 'io.everytrade.server.plugin.Tester' } + +checkstyle { + toolVersion checkstyleVersion +} From 18f2d86c26c41f193bb64af9a59c1994a3fbb72b Mon Sep 17 00:00:00 2001 From: charvam Date: Wed, 16 Sep 2020 12:10:50 +0200 Subject: [PATCH 20/29] ET-626 Added dependecyCheck plugin to all subprojects. --- plugin-api/build.gradle | 1 + plugin-base/build.gradle | 1 + plugin-support/build.gradle | 1 + plugin-template/build.gradle | 1 + plugin-tester/build.gradle | 1 + 5 files changed, 5 insertions(+) diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 37afd336..65bdb33f 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -3,6 +3,7 @@ import java.nio.charset.StandardCharsets plugins { id 'java-library' id 'checkstyle' + id "org.owasp.dependencycheck" } group 'io.everytrade' diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index 4d1ae54b..e532deed 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java-library' id 'checkstyle' + id "org.owasp.dependencycheck" } group 'io.everytrade' diff --git a/plugin-support/build.gradle b/plugin-support/build.gradle index 37afd336..65bdb33f 100644 --- a/plugin-support/build.gradle +++ b/plugin-support/build.gradle @@ -3,6 +3,7 @@ import java.nio.charset.StandardCharsets plugins { id 'java-library' id 'checkstyle' + id "org.owasp.dependencycheck" } group 'io.everytrade' diff --git a/plugin-template/build.gradle b/plugin-template/build.gradle index e77d2537..eaa965c0 100644 --- a/plugin-template/build.gradle +++ b/plugin-template/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java-library' id 'checkstyle' + id "org.owasp.dependencycheck" } group 'com.example' diff --git a/plugin-tester/build.gradle b/plugin-tester/build.gradle index 122646d2..c6d7772a 100644 --- a/plugin-tester/build.gradle +++ b/plugin-tester/build.gradle @@ -1,6 +1,7 @@ plugins { id 'java' id 'checkstyle' + id "org.owasp.dependencycheck" } configurations { From 229a26bdfb124eab42f82a9d2aa72119bd4e61eb Mon Sep 17 00:00:00 2001 From: charvam Date: Wed, 16 Sep 2020 16:39:50 +0200 Subject: [PATCH 21/29] ET-626 Audit plugins fixes. --- build.gradle | 49 ++++++++++++++++--- plugin-api/build.gradle | 6 --- plugin-base/build.gradle | 11 ++--- .../impl/everytrade/EveryTradeApiDto.java | 4 +- plugin-support/build.gradle | 6 --- plugin-template/build.gradle | 6 --- plugin-tester/build.gradle | 5 -- 7 files changed, 47 insertions(+), 40 deletions(-) diff --git a/build.gradle b/build.gradle index 65d7c7c3..f4e909eb 100644 --- a/build.gradle +++ b/build.gradle @@ -1,16 +1,14 @@ import java.nio.charset.StandardCharsets plugins { - id "org.owasp.dependencycheck" version "$owaspDependencyCheckGradlePluginVersion" - id "org.sonatype.gradle.plugins.scan" version "$sonatypeScanGradlePluginVersion" - id "com.github.spotbugs" version "$spotbugsGradlePluginVersion" + id 'org.owasp.dependencycheck' version "$owaspDependencyCheckGradlePluginVersion" apply false + id 'org.sonatype.gradle.plugins.scan' version "$sonatypeScanGradlePluginVersion" apply false + id 'com.github.spotbugs' version "$spotbugsGradlePluginVersion" apply false } -dependencies { - spotbugsPlugins "com.h3xstream.findsecbugs:findsecbugs-plugin:$findsecbugsVersion" -} +boolean spotbugsEnabled = false -allprojects { +subprojects { repositories { jcenter() maven { url 'https://jitpack.io' } @@ -21,6 +19,42 @@ allprojects { options.encoding = StandardCharsets.UTF_8 options.incremental = true } + + apply plugin: 'checkstyle' + apply plugin: 'org.owasp.dependencycheck' + apply plugin: 'org.sonatype.gradle.plugins.scan' + + if (spotbugsEnabled) { + apply plugin: 'com.github.spotbugs' + + dependencies { + spotbugsPlugins "com.h3xstream.findsecbugs:findsecbugs-plugin:$findsecbugsVersion" + } + spotbugs { + toolVersion = spotbugsVersion + } + + afterEvaluate { + [spotbugsMain, spotbugsTest].each { + it.reports { + html { + enabled = true + stylesheet = 'fancy-hist.xsl' + } + } + } + } + } + + checkstyle { + toolVersion checkstyleVersion + } + + afterEvaluate { + tasks.named('check') { + dependsOn 'ossIndexAudit' + } + } } wrapper { @@ -38,4 +72,3 @@ if (privatePropertyFile.exists()) { } } } - diff --git a/plugin-api/build.gradle b/plugin-api/build.gradle index 65bdb33f..827f6064 100644 --- a/plugin-api/build.gradle +++ b/plugin-api/build.gradle @@ -2,8 +2,6 @@ import java.nio.charset.StandardCharsets plugins { id 'java-library' - id 'checkstyle' - id "org.owasp.dependencycheck" } group 'io.everytrade' @@ -18,8 +16,4 @@ tasks.withType(JavaCompile) { dependencies { api "org.pf4j:pf4j:$pf4jVersion" -} - -checkstyle { - toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index e532deed..3490dd8b 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -1,7 +1,5 @@ plugins { id 'java-library' - id 'checkstyle' - id "org.owasp.dependencycheck" } group 'io.everytrade' @@ -36,6 +34,9 @@ dependencies { pluginCompile "com.github.charvam.XChange:xchange-coinmate:268ec0acaa" pluginCompile "com.github.charvam.XChange:xchange-bitfinex:268ec0acaa" pluginCompile "com.github.charvam.XChange:xchange-coinbasepro:268ec0acaa" + //TODO Remove it once ResCU releases a newer version with updated dependency - due to security bugs in the older + // version that will attract ResCU + pluginCompile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.1' } compileJava { @@ -55,8 +56,4 @@ jar { configurations.fatJar.collect { it.isDirectory() ? it : zipTree(it) } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE -} - -checkstyle { - toolVersion checkstyleVersion -} +} \ No newline at end of file diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradeApiDto.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradeApiDto.java index 3e9ebc5b..a68670ac 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradeApiDto.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/EveryTradeApiDto.java @@ -12,11 +12,11 @@ public EveryTradeApiDto() { } public String[] getHeader() { - return header; + return header == null ? null : header.clone(); } public void setHeader(String[] header) { - this.header = header; + this.header = header == null ? null : header.clone(); } public List getTransactions() { diff --git a/plugin-support/build.gradle b/plugin-support/build.gradle index 65bdb33f..827f6064 100644 --- a/plugin-support/build.gradle +++ b/plugin-support/build.gradle @@ -2,8 +2,6 @@ import java.nio.charset.StandardCharsets plugins { id 'java-library' - id 'checkstyle' - id "org.owasp.dependencycheck" } group 'io.everytrade' @@ -18,8 +16,4 @@ tasks.withType(JavaCompile) { dependencies { api "org.pf4j:pf4j:$pf4jVersion" -} - -checkstyle { - toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-template/build.gradle b/plugin-template/build.gradle index eaa965c0..53180be0 100644 --- a/plugin-template/build.gradle +++ b/plugin-template/build.gradle @@ -1,7 +1,5 @@ plugins { id 'java-library' - id 'checkstyle' - id "org.owasp.dependencycheck" } group 'com.example' @@ -47,8 +45,4 @@ jar { configurations.fatJar.collect { it.isDirectory() ? it : zipTree(it) } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE -} - -checkstyle { - toolVersion checkstyleVersion } \ No newline at end of file diff --git a/plugin-tester/build.gradle b/plugin-tester/build.gradle index c6d7772a..9489e448 100644 --- a/plugin-tester/build.gradle +++ b/plugin-tester/build.gradle @@ -1,7 +1,5 @@ plugins { id 'java' - id 'checkstyle' - id "org.owasp.dependencycheck" } configurations { @@ -29,6 +27,3 @@ task run(type: JavaExec) { mainClass = 'io.everytrade.server.plugin.Tester' } -checkstyle { - toolVersion checkstyleVersion -} From bd3a3e6b5e6ce2ba99e6871450a5716fa0710c9e Mon Sep 17 00:00:00 2001 From: charvam Date: Wed, 16 Sep 2020 16:59:03 +0200 Subject: [PATCH 22/29] ET-626 Changed project version + small fixes. --- build.gradle | 4 ++-- gradle.properties | 2 +- plugin-base/build.gradle | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index f4e909eb..82426ac1 100644 --- a/build.gradle +++ b/build.gradle @@ -34,8 +34,8 @@ subprojects { toolVersion = spotbugsVersion } - afterEvaluate { - [spotbugsMain, spotbugsTest].each { + ['spotbugsMain', 'spotbugsTest'].each { String spotbugsTaksName -> + tasks.named(spotbugsTaksName) { it.reports { html { enabled = true diff --git a/gradle.properties b/gradle.properties index 466acb46..4083e2dc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,6 +1,6 @@ pf4jVersion=3.4.0 xchangeVersion=0f636cbfff10de85d8c111e19e8e3d142b728951 -projectVersion=1.0.3 +projectVersion=1.1.0 checkstyleVersion=8.34 spotbugsVersion=4.1.2 owaspDependencyCheckGradlePluginVersion=6.0.0 diff --git a/plugin-base/build.gradle b/plugin-base/build.gradle index 3490dd8b..83086c52 100644 --- a/plugin-base/build.gradle +++ b/plugin-base/build.gradle @@ -25,7 +25,7 @@ dependencies { compileOnly 'org.slf4j:slf4j-api:1.7.30' pluginCompile 'com.univocity:univocity-parsers:2.8.3' - //TODO Change it to GENERALBYTESCOM once PR (charvam-->GENERALBYTESCOM) will by done. + //TODO Change it to GENERALBYTESCOM once PR (charvam-->GENERALBYTESCOM) will be done. pluginCompile "com.github.charvam.XChange:xchange-core:268ec0acaa" pluginCompile "com.github.charvam.XChange:xchange-kraken:268ec0acaa" pluginCompile "com.github.charvam.XChange:xchange-binance:268ec0acaa" @@ -35,7 +35,7 @@ dependencies { pluginCompile "com.github.charvam.XChange:xchange-bitfinex:268ec0acaa" pluginCompile "com.github.charvam.XChange:xchange-coinbasepro:268ec0acaa" //TODO Remove it once ResCU releases a newer version with updated dependency - due to security bugs in the older - // version that will attract ResCU + // version used by ResCU pluginCompile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.1' } From 44e464ce0c8fc077a63c402c8583e03bb3a46cdd Mon Sep 17 00:00:00 2001 From: charvam Date: Thu, 17 Sep 2020 08:53:46 +0200 Subject: [PATCH 23/29] ET-626 Reformatting. --- .../io/everytrade/server/plugin/Tester.java | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java index 060e844a..9c68a3a4 100644 --- a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java +++ b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java @@ -30,7 +30,7 @@ public Tester(Path workDir) { this.workDir = workDir; final Properties properties = loadProperties("tester.properties").orElse(new Properties()); pluginDir = Paths.get( - properties.getProperty("tester.pluginDir", "plugin-tester/build/testedPlugins") + properties.getProperty("tester.pluginDir", "plugin-tester/build/testedPlugins") ); } @@ -48,15 +48,15 @@ public Optional loadProperties(String fileName) { public static void main(String[] args) { new Tester( - Paths.get( - args.length == 1 ? args[0] : "" - ) + Paths.get( + args.length == 1 ? args[0] : "" + ) ).test(); } private void test() { final EverytradePluginManager pluginManager = - new EverytradePluginManager(pluginDir); + new EverytradePluginManager(pluginDir); pluginManager.loadPlugins(); pluginManager.startPlugins(); @@ -94,26 +94,26 @@ private void printResult(DownloadResult downloadResult) { final String columnSeparator = ","; final String lineSeparator = "\n"; stringBuilder - .append("uid").append(columnSeparator) - .append("executed").append(columnSeparator) - .append("base").append(columnSeparator) - .append("quote").append(columnSeparator) - .append("action").append(columnSeparator) - .append("baseQuantity").append(columnSeparator) - .append("unitPrice").append(columnSeparator) - .append("transactionPrice").append(columnSeparator) - .append("feeQuote").append(lineSeparator); + .append("uid").append(columnSeparator) + .append("executed").append(columnSeparator) + .append("base").append(columnSeparator) + .append("quote").append(columnSeparator) + .append("action").append(columnSeparator) + .append("baseQuantity").append(columnSeparator) + .append("unitPrice").append(columnSeparator) + .append("transactionPrice").append(columnSeparator) + .append("feeQuote").append(lineSeparator); parseResult.getImportedTransactionBeans().forEach(b -> stringBuilder - .append(b.getUid()).append(columnSeparator) - .append(b.getExecuted()).append(columnSeparator) - .append(b.getBase()).append(columnSeparator) - .append(b.getQuote()).append(columnSeparator) - .append(b.getAction()).append(columnSeparator) - .append(b.getBaseQuantity()).append(columnSeparator) - .append(b.getUnitPrice()).append(columnSeparator) - .append(b.getTransactionPrice()).append(columnSeparator) - .append(b.getFeeQuote()).append(lineSeparator) + .append(b.getUid()).append(columnSeparator) + .append(b.getExecuted()).append(columnSeparator) + .append(b.getBase()).append(columnSeparator) + .append(b.getQuote()).append(columnSeparator) + .append(b.getAction()).append(columnSeparator) + .append(b.getBaseQuantity()).append(columnSeparator) + .append(b.getUnitPrice()).append(columnSeparator) + .append(b.getTransactionPrice()).append(columnSeparator) + .append(b.getFeeQuote()).append(lineSeparator) ); log.info("importedTransactionBeans = \n" + stringBuilder.toString()); From abeef8ee2fed6b59da17ffb04379e40a721ac586 Mon Sep 17 00:00:00 2001 From: charvam Date: Thu, 17 Sep 2020 09:31:11 +0200 Subject: [PATCH 24/29] ET-626 Refactoring - duplicate methods extracted, fields renamed, import optimized and code reformatted. --- .../impl/everytrade/BinanceConnector.java | 6 - .../impl/everytrade/BinanceDownloader.java | 2 - .../impl/everytrade/BitfinexConnector.java | 108 +++++++----------- .../impl/everytrade/BitstampConnector.java | 57 ++++----- .../impl/everytrade/BittrexConnector.java | 20 +--- .../impl/everytrade/CoinmateConnector.java | 5 +- .../impl/everytrade/ConnectorUtils.java | 30 +++++ .../impl/everytrade/KrakenConnector.java | 6 +- .../impl/everytrade/KrakenDownloadState.java | 18 +-- plugin-tester/build.gradle | 3 +- 10 files changed, 117 insertions(+), 138 deletions(-) diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java index 67bc2e89..eb8363a2 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceConnector.java @@ -12,18 +12,12 @@ import org.knowm.xchange.ExchangeFactory; import org.knowm.xchange.ExchangeSpecification; import org.knowm.xchange.binance.BinanceExchange; -import org.knowm.xchange.binance.service.BinanceTradeHistoryParams; -import org.knowm.xchange.currency.Currency; -import org.knowm.xchange.currency.CurrencyPair; import org.knowm.xchange.dto.trade.UserTrade; import org.knowm.xchange.service.trade.TradeService; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; public class BinanceConnector implements IConnector { diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloader.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloader.java index 6f916578..259a1a13 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloader.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BinanceDownloader.java @@ -1,8 +1,6 @@ package io.everytrade.server.plugin.impl.everytrade; import org.knowm.xchange.binance.service.BinanceTradeHistoryParams; -import org.knowm.xchange.coinbasepro.dto.trade.CoinbaseProTradeHistoryParams; -import org.knowm.xchange.currency.Currency; import org.knowm.xchange.currency.CurrencyPair; import org.knowm.xchange.dto.trade.UserTrade; import org.knowm.xchange.service.trade.TradeService; diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitfinexConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitfinexConnector.java index f599010b..fba552af 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitfinexConnector.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitfinexConnector.java @@ -27,6 +27,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static io.everytrade.server.plugin.impl.everytrade.ConnectorUtils.findDuplicate; +import static io.everytrade.server.plugin.impl.everytrade.ConnectorUtils.occurrenceCount; + public class BitfinexConnector implements IConnector { private static final String ID = EveryTradePlugin.ID + IPlugin.PLUGIN_PATH_SEPARATOR + "bitfinexApiConnector"; @@ -37,26 +40,26 @@ public class BitfinexConnector implements IConnector { private static final Pattern SPLIT_PATTERN = Pattern.compile(String.format("(.*)\\%s(.*)", TX_SPLITER)); private static final ConnectorParameterDescriptor PARAMETER_API_SECRET = - new ConnectorParameterDescriptor( - "apiSecret", - ConnectorParameterType.SECRET, - "API Secret", - "" - ); + new ConnectorParameterDescriptor( + "apiSecret", + ConnectorParameterType.SECRET, + "API Secret", + "" + ); private static final ConnectorParameterDescriptor PARAMETER_API_KEY = - new ConnectorParameterDescriptor( - "apiKey", - ConnectorParameterType.STRING, - "API Key", - "" - ); + new ConnectorParameterDescriptor( + "apiKey", + ConnectorParameterType.STRING, + "API Key", + "" + ); public static final ConnectorDescriptor DESCRIPTOR = new ConnectorDescriptor( - ID, - "Bitfinex Connector", - SupportedExchange.BITFINEX.getInternalId(), - List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET) + ID, + "Bitfinex Connector", + SupportedExchange.BITFINEX.getInternalId(), + List.of(PARAMETER_API_KEY, PARAMETER_API_SECRET) ); private final String apiKey; @@ -96,19 +99,24 @@ public DownloadResult getTransactions(String lastTransactionId) { return new DownloadResult(parseResult, actualLastTransactionId); } + @Override + public void close() { + //AutoCloseable + } + private List download( - TradeService tradeService, - String lastTransactionUid + TradeService tradeService, + String lastTransactionUid ) { final BitfinexTradeService.BitfinexTradeHistoryParams tradeHistoryParams - = (BitfinexTradeService.BitfinexTradeHistoryParams) tradeService.createTradeHistoryParams(); + = (BitfinexTradeService.BitfinexTradeHistoryParams) tradeService.createTradeHistoryParams(); tradeHistoryParams.setLimit(TX_PER_REQUEST); tradeHistoryParams.setOrder(TradeHistoryParamsSorted.Order.asc); final List userTrades = new ArrayList<>(); TransactionIdentifier lastBlockDownloadedTx = parseFrom(lastTransactionUid); - int counter = 0; - while (counter++ < MAX_REQUEST_COUNT) { + int sentRequests = 0; + while (sentRequests < MAX_REQUEST_COUNT) { tradeHistoryParams.setStartTime(lastBlockDownloadedTx.date); final List userTradesBlock; try { @@ -117,10 +125,10 @@ private List download( throw new IllegalStateException("User trade history download failed. ", e); } final List userTradesToAdd; - final int duplicityTxIndex = findDuplicity(lastBlockDownloadedTx, userTradesBlock); - if (duplicityTxIndex > -1) { - if (duplicityTxIndex < userTradesBlock.size() - 1) { - userTradesToAdd = userTradesBlock.subList(duplicityTxIndex + 1, userTradesBlock.size()); + final int duplicateTxIndex = findDuplicate(lastBlockDownloadedTx.id, userTradesBlock); + if (duplicateTxIndex > -1) { + if (duplicateTxIndex < userTradesBlock.size() - 1) { + userTradesToAdd = userTradesBlock.subList(duplicateTxIndex + 1, userTradesBlock.size()); } else { userTradesToAdd = List.of(); } @@ -136,6 +144,7 @@ private List download( lastBlockDownloadedTx = new TransactionIdentifier(userTradeLast.getTimestamp(), userTradeLast.getId()); userTrades.addAll(userTradesToAdd); + ++sentRequests; } return userTrades; @@ -148,44 +157,27 @@ private TransactionIdentifier parseFrom(String lastTransactionUid) { Matcher matcher = SPLIT_PATTERN.matcher(lastTransactionUid); if (occurrenceCount(lastTransactionUid, TX_SPLITER) != 1 || !matcher.find()) { throw new IllegalArgumentException( - String.format("Illegal value of lastTransactionUid '%s'.", lastTransactionUid) + String.format("Illegal value of lastTransactionUid '%s'.", lastTransactionUid) ); } final String date = matcher.group(1); try { return new TransactionIdentifier( - Date.from(Instant.parse(date)), - matcher.group(2) + Date.from(Instant.parse(date)), + matcher.group(2) ); } catch (Exception e) { throw new IllegalArgumentException( - String.format( - "Illegal value of date part '%s' of lastTransactionUid '%s'.", - date, - lastTransactionUid - ), - e + String.format( + "Illegal value of date part '%s' of lastTransactionUid '%s'.", + date, + lastTransactionUid + ), + e ); } } - private int findDuplicity(TransactionIdentifier transactionIdentifier, List userTradesBlock) { - for (int i = 0; i < userTradesBlock.size(); i++) { - final UserTrade userTrade = userTradesBlock.get(i); - if (userTrade.getTimestamp().equals(transactionIdentifier.date) - && userTrade.getId().equals(transactionIdentifier.id) - ) { - return i; - } - } - return -1; - } - - @Override - public void close() { - //AutoCloseable - } - private static class TransactionIdentifier { private final Date date; private final String id; @@ -195,18 +187,4 @@ public TransactionIdentifier(Date date, String id) { this.id = id; } } - - public int occurrenceCount(String input, String search) { - int startIndex = 0; - int index = 0; - int counter = 0; - while (index > -1 && startIndex < input.length()) { - index = input.indexOf(search, startIndex); - if (index > -1) { - counter++; - } - startIndex = index + 1; - } - return counter; - } } \ No newline at end of file diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitstampConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitstampConnector.java index 5cd87cf9..07fd7e4b 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitstampConnector.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BitstampConnector.java @@ -96,37 +96,38 @@ public DownloadResult getTransactions(String lastTransactionId) { } private List download(String lastTransactionId, TradeService tradeService) { - final BitstampTradeHistoryParams bitstampTradeHistoryParams = - (BitstampTradeHistoryParams) tradeService.createTradeHistoryParams(); - bitstampTradeHistoryParams.setStartId(lastTransactionId); - bitstampTradeHistoryParams.setPageLength(TXS_PER_REQUEST); - String lastDownloadedTx = lastTransactionId; - final List userTrades = new ArrayList<>(); - int counter = 0; - while (counter++ < MAX_REQUEST_COUNT) { - - final List userTradesBlock; - try { - userTradesBlock = tradeService.getTradeHistory(bitstampTradeHistoryParams).getUserTrades(); - } catch (IOException e) { - throw new IllegalStateException("Download user trade history failed.", e); - } - - if ( - !userTradesBlock.isEmpty() + final BitstampTradeHistoryParams bitstampTradeHistoryParams = + (BitstampTradeHistoryParams) tradeService.createTradeHistoryParams(); + bitstampTradeHistoryParams.setStartId(lastTransactionId); + bitstampTradeHistoryParams.setPageLength(TXS_PER_REQUEST); + String lastDownloadedTx = lastTransactionId; + final List userTrades = new ArrayList<>(); + int sentRequests = 0; + + while (sentRequests < MAX_REQUEST_COUNT) { + final List userTradesBlock; + try { + userTradesBlock = tradeService.getTradeHistory(bitstampTradeHistoryParams).getUserTrades(); + } catch (IOException e) { + throw new IllegalStateException("Download user trade history failed.", e); + } + + if ( + !userTradesBlock.isEmpty() && lastDownloadedTx != null && lastDownloadedTx.equals(userTradesBlock.get(0).getId()) - ) { - userTradesBlock.remove(0); - } - if (userTradesBlock.isEmpty()) { - break; - } - userTrades.addAll(userTradesBlock); - lastDownloadedTx = userTradesBlock.get(userTradesBlock.size() - 1).getId(); - bitstampTradeHistoryParams.setStartId(lastDownloadedTx); + ) { + userTradesBlock.remove(0); + } + if (userTradesBlock.isEmpty()) { + break; } - return userTrades; + userTrades.addAll(userTradesBlock); + lastDownloadedTx = userTradesBlock.get(userTradesBlock.size() - 1).getId(); + bitstampTradeHistoryParams.setStartId(lastDownloadedTx); + ++sentRequests; + } + return userTrades; } @Override diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BittrexConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BittrexConnector.java index 3d6bb972..bb3f7669 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BittrexConnector.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/BittrexConnector.java @@ -28,6 +28,8 @@ import java.util.Map; import java.util.Objects; +import static io.everytrade.server.plugin.impl.everytrade.ConnectorUtils.findDuplicate; + public class BittrexConnector implements IConnector { private final Logger log = LoggerFactory.getLogger(this.getClass()); private static final String ID = EveryTradePlugin.ID + IPlugin.PLUGIN_PATH_SEPARATOR + "bittrexApiConnector"; @@ -101,10 +103,10 @@ private List download(String lastTransactionId, TradeService tradeSer } final List userTradesToAdd; - final int duplicityTxIndex = findDuplicity(lastTransactionId, userTradesBlock); - if (duplicityTxIndex > -1) { - if (duplicityTxIndex < userTradesBlock.size() - 1) { - userTradesToAdd = userTradesBlock.subList(duplicityTxIndex + 1, userTradesBlock.size()); + final int duplicateTxIndex = findDuplicate(lastTransactionId, userTradesBlock); + if (duplicateTxIndex > -1) { + if (duplicateTxIndex < userTradesBlock.size() - 1) { + userTradesToAdd = userTradesBlock.subList(duplicateTxIndex + 1, userTradesBlock.size()); } else { userTradesToAdd = List.of(); } @@ -135,16 +137,6 @@ private ParseResult getParseResult(List userTrades) { ); } - private int findDuplicity(String transactionId, List userTradesBlock) { - for (int i = 0; i < userTradesBlock.size(); i++) { - final UserTrade userTrade = userTradesBlock.get(i); - if (userTrade.getId().equals(transactionId)) { - return i; - } - } - return -1; - } - @Override public void close() { //AutoCloseable diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinmateConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinmateConnector.java index 5efa6b2a..697532ad 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinmateConnector.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinmateConnector.java @@ -103,8 +103,8 @@ private List download(String lastTransactionId, TradeService tradeSer tradeHistoryParams.setLimit(TX_PER_REQUEST); final List userTrades = new ArrayList<>(); - int counter = 0; - while (counter++ < MAX_REQUEST_COUNT) { + int sentRequests = 0; + while (sentRequests < MAX_REQUEST_COUNT) { final List userTradesBlock; try { userTradesBlock = tradeService.getTradeHistory(tradeHistoryParams).getUserTrades(); @@ -117,6 +117,7 @@ private List download(String lastTransactionId, TradeService tradeSer userTrades.addAll(userTradesBlock); final String lastDownloadedTx = userTradesBlock.get(userTradesBlock.size() - 1).getId(); tradeHistoryParams.setStartId(lastDownloadedTx); + ++sentRequests; } return userTrades; } diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/ConnectorUtils.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/ConnectorUtils.java index 42437780..388d9b17 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/ConnectorUtils.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/ConnectorUtils.java @@ -2,9 +2,15 @@ import org.knowm.xchange.currency.Currency; import org.knowm.xchange.currency.CurrencyPair; +import org.knowm.xchange.dto.trade.UserTrade; + +import java.util.List; public class ConnectorUtils { + private ConnectorUtils() { + } + public static CurrencyPair createPair(String pair) { final String[] split = pair.split("/"); if (split.length != 2) { @@ -20,4 +26,28 @@ public static CurrencyPair createPair(String pair) { } return new CurrencyPair(base, quote); } + + public static int occurrenceCount(String input, String search) { + int startIndex = 0; + int index = 0; + int counter = 0; + while (index > -1 && startIndex < input.length()) { + index = input.indexOf(search, startIndex); + if (index > -1) { + counter++; + } + startIndex = index + 1; + } + return counter; + } + + public static int findDuplicate(String transactionId, List userTradesBlock) { + for (int i = 0; i < userTradesBlock.size(); i++) { + final UserTrade userTrade = userTradesBlock.get(i); + if (userTrade.getId().equals(transactionId)) { + return i; + } + } + return -1; + } } diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenConnector.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenConnector.java index 0c3636a2..2c1efc72 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenConnector.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenConnector.java @@ -85,14 +85,14 @@ public DownloadResult getTransactions(String lastTransactionUid) { final boolean firstDownload = downloadState.getLastContinuousTxUid() == null; final List userTrades = new ArrayList<>(); - int counter = 0; - while (counter < MAX_REQUESTS_COUNT) { + int sentRequests = 0; + while (sentRequests < MAX_REQUESTS_COUNT) { final List downloadResult = download(tradeService, firstDownload, downloadState); if (downloadResult.isEmpty()) { break; } userTrades.addAll(downloadResult); - counter++; + ++sentRequests; } return new DownloadResult(XChangeConnectorParser.getParseResult(userTrades, SupportedExchange.KRAKEN), diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenDownloadState.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenDownloadState.java index 289fdee0..36c97a46 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenDownloadState.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenDownloadState.java @@ -3,13 +3,14 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static io.everytrade.server.plugin.impl.everytrade.ConnectorUtils.occurrenceCount; + public class KrakenDownloadState { public static final int MAX_LENGTH_DOWNLOADED_TXUID = 255; private String lastContinuousTxUid; private String firstTxUidAfterGap; private String lastTxUidAfterGap; - public static final int TX_UID_PART_COUNT = 3; private static final String TX_ID_SEPARATOR = ":"; private static final Pattern SPLIT_PATTERN = Pattern.compile( String.format("(.*)%s(.*)%s(.*)", @@ -88,23 +89,8 @@ public static KrakenDownloadState parseFrom(String lastTransactionUid) { ); } - private static int occurrenceCount(String input, String search) { - int startIndex = 0; - int index = 0; - int counter = 0; - while (index > -1 && startIndex < input.length()) { - index = input.indexOf(search, startIndex); - if (index > -1) { - counter++; - } - startIndex = index + 1; - } - return counter; - } - private static String getGroupValueOrNull(Matcher matcher, int group) { final String part = matcher.group(group); return part.isEmpty() ? null : part; } - } diff --git a/plugin-tester/build.gradle b/plugin-tester/build.gradle index 9489e448..98bc252e 100644 --- a/plugin-tester/build.gradle +++ b/plugin-tester/build.gradle @@ -25,5 +25,4 @@ task run(type: JavaExec) { // unified with default IDE working directory workingDir = project.rootDir mainClass = 'io.everytrade.server.plugin.Tester' -} - +} \ No newline at end of file From 08821532844a48bfa2708cc6831b968ef22dd1aa Mon Sep 17 00:00:00 2001 From: charvam Date: Thu, 17 Sep 2020 12:22:59 +0200 Subject: [PATCH 25/29] ET-626 Optimized regex for finding match. --- .../server/plugin/impl/everytrade/KrakenDownloadState.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenDownloadState.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenDownloadState.java index 36c97a46..0ae8de60 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenDownloadState.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/KrakenDownloadState.java @@ -13,7 +13,10 @@ public class KrakenDownloadState { private static final String TX_ID_SEPARATOR = ":"; private static final Pattern SPLIT_PATTERN = Pattern.compile( - String.format("(.*)%s(.*)%s(.*)", + String.format("^([^%s]*)%s([^%s]*)%s([^%s]*)$", + TX_ID_SEPARATOR, + TX_ID_SEPARATOR, + TX_ID_SEPARATOR, TX_ID_SEPARATOR, TX_ID_SEPARATOR ) @@ -76,7 +79,7 @@ public static KrakenDownloadState parseFrom(String lastTransactionUid) { } Matcher matcher = SPLIT_PATTERN.matcher(lastTransactionUid); - if (occurrenceCount(lastTransactionUid, TX_ID_SEPARATOR) != 2 || !matcher.find()) { + if (!matcher.find()) { throw new IllegalArgumentException( String.format("Illegal value of lastTransactionUid '%s'.", lastTransactionUid) ); From 98db551b679819df01199afa590aae86a8492e8e Mon Sep 17 00:00:00 2001 From: charvam Date: Thu, 17 Sep 2020 12:47:06 +0200 Subject: [PATCH 26/29] ET-626 Implicit lambda to explicit lambda. --- .../src/main/java/io/everytrade/server/plugin/Tester.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java index 9c68a3a4..64f9240c 100644 --- a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java +++ b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java @@ -5,6 +5,7 @@ import io.everytrade.server.plugin.api.connector.DownloadResult; import io.everytrade.server.plugin.api.connector.IConnector; import io.everytrade.server.plugin.api.parser.ConversionStatistic; +import io.everytrade.server.plugin.api.parser.ImportedTransactionBean; import io.everytrade.server.plugin.api.parser.ParseResult; import io.everytrade.server.plugin.support.EverytradePluginManager; import org.slf4j.Logger; @@ -104,7 +105,7 @@ private void printResult(DownloadResult downloadResult) { .append("transactionPrice").append(columnSeparator) .append("feeQuote").append(lineSeparator); - parseResult.getImportedTransactionBeans().forEach(b -> stringBuilder + parseResult.getImportedTransactionBeans().forEach((ImportedTransactionBean b) -> stringBuilder .append(b.getUid()).append(columnSeparator) .append(b.getExecuted()).append(columnSeparator) .append(b.getBase()).append(columnSeparator) From 4a1541d1904c4e49b6e620276a836529f47eba2d Mon Sep 17 00:00:00 2001 From: charvam Date: Thu, 17 Sep 2020 13:03:52 +0200 Subject: [PATCH 27/29] ET-626 Refactoring - constant created. --- .../server/plugin/impl/everytrade/CoinbaseProDownloader.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java index 82e1638b..fcc45e16 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java @@ -19,6 +19,7 @@ public class CoinbaseProDownloader { private static final int TX_PER_REQUEST = 100; private static final int MAX_REQUEST_COUNT = 3000; private static final int SLEEP_BETWEEN_REQUESTS_MS = 200; + public static final int FIRST_XCHANGE_TX_ID = 1; private final Map currencyPairLastIds; private final TradeService tradeService; private final CoinbaseProTradeHistoryParams tradeHistoryParams; @@ -47,7 +48,7 @@ public List download(String currencyPairs) { for (CurrencyPair pair : pairs) { tradeHistoryParams.setCurrencyPair(pair); final Integer lastDownloadedTxFound = currencyPairLastIds.get(pair.toString()); - int lastDownloadedTx = lastDownloadedTxFound == null ? 1 : lastDownloadedTxFound; + int lastDownloadedTx = lastDownloadedTxFound == null ? FIRST_XCHANGE_TX_ID : lastDownloadedTxFound; while (sentRequests < MAX_REQUEST_COUNT) { tradeHistoryParams.setBeforeTradeId(lastDownloadedTx); From f1c532d66cb7435b36cbcca83eb8ff922d60321c Mon Sep 17 00:00:00 2001 From: charvam Date: Thu, 17 Sep 2020 13:04:52 +0200 Subject: [PATCH 28/29] ET-626 Refactoring - constant renamed. --- .../server/plugin/impl/everytrade/CoinbaseProDownloader.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java index fcc45e16..3bc102dd 100644 --- a/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java +++ b/plugin-base/src/main/java/io/everytrade/server/plugin/impl/everytrade/CoinbaseProDownloader.java @@ -1,7 +1,6 @@ package io.everytrade.server.plugin.impl.everytrade; import org.knowm.xchange.coinbasepro.dto.trade.CoinbaseProTradeHistoryParams; -import org.knowm.xchange.currency.Currency; import org.knowm.xchange.currency.CurrencyPair; import org.knowm.xchange.dto.trade.UserTrade; import org.knowm.xchange.service.trade.TradeService; @@ -19,7 +18,7 @@ public class CoinbaseProDownloader { private static final int TX_PER_REQUEST = 100; private static final int MAX_REQUEST_COUNT = 3000; private static final int SLEEP_BETWEEN_REQUESTS_MS = 200; - public static final int FIRST_XCHANGE_TX_ID = 1; + public static final int FIRST_COINBASE_TX_ID = 1; private final Map currencyPairLastIds; private final TradeService tradeService; private final CoinbaseProTradeHistoryParams tradeHistoryParams; @@ -48,7 +47,7 @@ public List download(String currencyPairs) { for (CurrencyPair pair : pairs) { tradeHistoryParams.setCurrencyPair(pair); final Integer lastDownloadedTxFound = currencyPairLastIds.get(pair.toString()); - int lastDownloadedTx = lastDownloadedTxFound == null ? FIRST_XCHANGE_TX_ID : lastDownloadedTxFound; + int lastDownloadedTx = lastDownloadedTxFound == null ? FIRST_COINBASE_TX_ID : lastDownloadedTxFound; while (sentRequests < MAX_REQUEST_COUNT) { tradeHistoryParams.setBeforeTradeId(lastDownloadedTx); From c683a17d4db48f6e1a23c3a115559235b9e578ed Mon Sep 17 00:00:00 2001 From: charvam Date: Thu, 17 Sep 2020 13:28:36 +0200 Subject: [PATCH 29/29] ET-626 Refactoring - loop instead lambda. --- .../io/everytrade/server/plugin/Tester.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java index 64f9240c..eaed42ff 100644 --- a/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java +++ b/plugin-tester/src/main/java/io/everytrade/server/plugin/Tester.java @@ -105,17 +105,18 @@ private void printResult(DownloadResult downloadResult) { .append("transactionPrice").append(columnSeparator) .append("feeQuote").append(lineSeparator); - parseResult.getImportedTransactionBeans().forEach((ImportedTransactionBean b) -> stringBuilder - .append(b.getUid()).append(columnSeparator) - .append(b.getExecuted()).append(columnSeparator) - .append(b.getBase()).append(columnSeparator) - .append(b.getQuote()).append(columnSeparator) - .append(b.getAction()).append(columnSeparator) - .append(b.getBaseQuantity()).append(columnSeparator) - .append(b.getUnitPrice()).append(columnSeparator) - .append(b.getTransactionPrice()).append(columnSeparator) - .append(b.getFeeQuote()).append(lineSeparator) - ); + for (ImportedTransactionBean b : parseResult.getImportedTransactionBeans()) { + stringBuilder + .append(b.getUid()).append(columnSeparator) + .append(b.getExecuted()).append(columnSeparator) + .append(b.getBase()).append(columnSeparator) + .append(b.getQuote()).append(columnSeparator) + .append(b.getAction()).append(columnSeparator) + .append(b.getBaseQuantity()).append(columnSeparator) + .append(b.getUnitPrice()).append(columnSeparator) + .append(b.getTransactionPrice()).append(columnSeparator) + .append(b.getFeeQuote()).append(lineSeparator); + } log.info("importedTransactionBeans = \n" + stringBuilder.toString()); final ConversionStatistic conversionStatistic = parseResult.getConversionStatistic();