Merge pull request #112 from Azulinho/next_release

Next release

Dockerfile

@@ -31,7 +31,10 @@ RUN apt-get update &&  \
   autogen \
   libtool \
   shtool \
-  nasm
+  nasm && \
+  apt-get clean autoclean && \
+  apt-get autoremove --yes && \
+  rm -rf /var/lib/apt/lists/*
 
 
 RUN useradd -d /cryptobot -u 1001 -ms /bin/bash cryptobot
@@ -63,7 +66,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq \
   gzip \
   pigz \
   bzip2 \
-  pbzip2
+  pbzip2 && \
+  apt-get clean autoclean && \
+  apt-get autoremove --yes && \
+  rm -rf /var/lib/apt/lists/*
 
 RUN useradd -d /cryptobot -u 1001 -ms /bin/bash cryptobot
 USER cryptobot

app.py

@@ -1,7 +1,6 @@
 """ CryptoBot for Binance """
 
 import argparse
-import gzip
 import importlib
 import json
 import logging
@@ -25,6 +24,7 @@
 from binance.client import Client
 from binance.exceptions import BinanceAPIException
 from filelock import SoftFileLock
+from isal import igzip
 from lz4.frame import open as lz4open
 from tenacity import retry, wait_exponential
 
@@ -1628,9 +1628,14 @@ def logmode(self) -> None:
     def split_logline(self, line: str) -> Tuple:
         """splits a log line into symbol, date, price"""
 
-        parts = line.split(" ", maxsplit=4)
-        symbol = parts[2]
-        # skip processing the line if we don't care about this coin
+        try:
+            symbol, price = line[27:].split(" ", maxsplit=1)
+            # ocasionally binance returns rubbish
+            # we just skip it
+            market_price = float(price)
+        except ValueError:
+            return (False, False, False)
+
         if symbol not in self.tickers:
             return (False, False, False)
 
@@ -1640,21 +1645,10 @@ def split_logline(self, line: str) -> Tuple:
         if len(self.wallet) >= self.max_coins:
             if symbol not in self.wallet:
                 return (False, False, False)
-        day = " ".join(parts[0:2])
-        try:
-            # datetime is very slow, discard the .microseconds and fetch a
-            # cached pre-calculated unix epoch timestamp
-            day = day.split(".", maxsplit=1)[0]
-            date = c_date_from(day)
-        except ValueError:
-            date = c_date_from(day)
 
-        try:
-            # ocasionally binance returns rubbish
-            # we just skip it
-            market_price = float(parts[3])
-        except ValueError:
-            return (False, False, False)
+        # datetime is very slow, discard the .microseconds and fetch a
+        # cached pre-calculated unix epoch timestamp
+        date = c_date_from(line[0:19])
 
         return (symbol, date, market_price)
 
@@ -1740,12 +1734,12 @@ def backtest_logfile(self, price_log: str) -> None:
         logging.info(f"exposure: {self.calculates_exposure()}")
         try:
             # we support .lz4 and .gz for our price.log files.
-            # gzip -3 files provide the fastest decompression times we were able
+            # gzip -3 files provide the fastest decompression times I was able
             # to measure.
             if price_log.endswith(".lz4"):
                 f = lz4open(price_log, mode="rt")
             else:
-                f = gzip.open(price_log, "rt")
+                f = igzip.open(price_log, "rt")
             while True:
                 # reading a chunk of lines like this speeds up backtesting
                 # by a large amount.

utils/automated-backtesting.py

@@ -1,7 +1,6 @@
 """automated-backtesting.py"""
 import argparse
 import glob
-import gzip
 import os
 import re
 import shutil
@@ -10,6 +9,7 @@
 from datetime import datetime
 from multiprocessing import Pool, get_context
 from string import Template
+from isal import igzip
 
 import yaml
 
@@ -22,7 +22,7 @@ def backup_backtesting_log():
 def compress_file(filename):
     """compresses coin price.log file"""
     with open(filename) as uncompressed:
-        with gzip.open(f"{filename}.gz", mode="wt") as compressed:
+        with igzip.open(f"{filename}.gz", mode="wt") as compressed:
             shutil.copyfileobj(uncompressed, compressed)
     os.remove(filename)
 
@@ -32,7 +32,7 @@ def split_logs_into_coins(filename, cfg):
     coinfiles = set()
     coinfh = {}
     pairing = cfg["DEFAULTS"]["PAIRING"]
-    with gzip.open(f"{filename}", "rt") as logfile:
+    with igzip.open(f"{filename}", "rt") as logfile:
         for line in logfile:
 
             # don't process all the lines, but only the ones related to our PAIR

utils/prove-backtesting.py

@@ -8,8 +8,8 @@
 
 import pandas
 import yaml
-import gzip
 
+from isal import igzip
 
 def cli():
     parser = argparse.ArgumentParser()
@@ -97,22 +97,20 @@ def run_automated_backtesting(config, min, sortby):
 def create_zipped_logfile(dates, pairing, symbols=[]):
     log_msg(f"creating gzip lastfewdays.log.gz")
 
-    cli = "zcat "
-    for day in dates:
-        log = f"log/{day}.log.gz"
-        if not os.path.exists(log):
-            log_msg(f"WARNING: {log} does not exist")
-            continue
-        cli += f" log/{day}.log.gz"
-    cli += " |grep -E '"
-    cli += pairing
-    for symbol in symbols:
-        cli += f"|{symbol}"
-    cli += "' "
-    cli += " | pigz > log/lastfewdays.log.gz"
-
-    subprocess.run(cli, shell=True)
-
+    with igzip.open("log/lastfewdays.log.gz", "wt") as w:
+        for day in dates:
+            log = f"log/{day}.log.gz"
+            if not os.path.exists(log):
+                log_msg(f"WARNING: {log} does not exist")
+                continue
+            with igzip.open(log, "rt") as r:
+                for line in r:
+                    if pairing not in line:
+                        continue
+                    if symbols:
+                        if not any(symbol in line for symbol in symbols):
+                            continue
+                    w.write(line)
 
 def main():
     """main"""