Merge pull request #87 from Azulinho/next_release

multiple fixes

README.md

@@ -30,10 +30,13 @@ A python based trading bot for Binance, which relies heavily on backtesting.
    * [TRADING_FEE](#trading_fee)
    * [PRICE_LOGS](#price_logs)
    * [ENABLE_PUMP_AND_DUMP_CHECKS](#enable_pump_and_dump_checks)
+   * [ENABLE_NEW_LISTING_CHECKS](#enable_new_listing_checks)
+   * [STOP_BOT_ON_LOSS](#stop_bot_on_loss)
 5. [Bot command center](#bot-command-center)
 6. [Automated Backtesting](#automated-backtesting)
 7. [Development/New features](#development/new-features)
 
+
 ## Overview
 
 The bot while running saves the current market price for all coins available in
@@ -585,15 +588,38 @@ The list of price logs to be used for backtesting.
 
 
 ### ENABLE_PUMP_AND_DUMP_CHECKS
+
 ```
 ENABLE_PUMP_AND_DUMP_CHECKS: True
+```
 
 defaults to True
 
 Checks the price of a coin over the last 2 hours and prevents the bot from
 buying if the price 2 hours ago was lower than 1 hour ago (pump) and the current
 price is higher than 2 hours ago (dump pending).
 
+### ENABLE_NEW_LISTING_CHECKS
+
+```
+ENABLE_NEW_LISTING_CHECKS: True
+```
+
+defaults to True
+
+Checks that we have at least 30 days of price data on a coin, if we don't we
+skip buying this coin.
+
+### STOP_BOT_ON_LOSS
+
+```
+STOP_BOT_ON_LOSS: True
+```
+
+defaults to False
+
+Stops the bot immediately after a STOP_LOSS
+
 ## Bot command center
 
 The bot is running a *pdb* endpoint on container port 5555.
@@ -663,7 +689,9 @@ make automated-backtesting LOGFILE=lastfewdays.USDT.log.gz CONFIG=automated-back
 This will generate a config.yaml with the coins sorted by which strategy
 returned the highest number of wins for each coin.
 
+```
 make automated-backtesting LOGFILE=lastfewdays.USDT.log.gz CONFIG=automated-backtesting.yaml MIN=10 FILTER='' SORTBY='wins'
+```
 
 
 ## Development/New features

app.py

@@ -376,6 +376,14 @@ def __init__(self, conn, config_file, config) -> None:
         self.enable_pump_and_dump_checks : bool = config.get(
             "ENABLE_PUMP_AND_DUMP_CHECKS", True
         )
+        self.enable_new_listing_checks : bool = config.get(
+            "ENABLE_NEW_LISTING_CHECKS", True
+        )
+        self.stop_bot_on_loss : bool = config.get(
+            "STOP_BOT_ON_LOSS", False
+        )
+        self.stop_flag: bool = False
+        self.quit : bool = False
 
     def run_strategy(self, coin) -> None:
         """runs a specific strategy against a coin"""
@@ -391,8 +399,9 @@ def run_strategy(self, coin) -> None:
         if len(self.wallet) == self.max_coins:
             return
 
-        if coin.new_listing(self.mode):
-            return
+        if self.enable_new_listing_checks:
+            if coin.new_listing(self.mode):
+                return
 
         if self.enable_pump_and_dump_checks:
             if coin.check_for_pump_and_dump():
@@ -750,6 +759,9 @@ def stop_loss(self, coin: Coin) -> bool:
             coin.naughty_date = coin.date  # pylint: disable=attribute-defined-outside-init
             self.clear_coin_stats(coin)
             coin.naughty = True  # pylint: disable=attribute-defined-outside-init
+            if self.stop_bot_on_loss:
+                # STOP_BOT_ON_LOSS is set, set a STOP flag to stop the bot
+                self.quit = True
             return True
         return False
 
@@ -1093,7 +1105,7 @@ def run(self) -> None:
             self.process_coins()
             self.save_coins()
             self.wait()
-            if exists(".stop"):
+            if exists(".stop") or self.quit:
                 logging.warning(".stop flag found. Stopping bot.")
                 return
 
@@ -1106,6 +1118,9 @@ def logmode(self) -> None:
     def process_line(self, line: str) -> None:
         """processes a backlog line"""
 
+        if self.quit:
+            return
+
         # skip processing the line if it doesn't not match our PAIRING settings
         if self.pairing not in line:
             return
@@ -1174,6 +1189,9 @@ def process_line(self, line: str) -> None:
 
     def backtest_logfile(self, price_log: str) -> None:
         """processes one price.log file for backtesting"""
+        if self.quit:
+            return
+
         logging.info(f"backtesting: {price_log}")
         logging.info(f"wallet: {self.wallet}")
         try:
@@ -1203,6 +1221,9 @@ def backtesting(self) -> None:
 
         for price_log in self.price_logs:
             self.backtest_logfile(price_log)
+            if exists(".stop") or self.quit:
+                logging.warning(".stop flag found. Stopping bot.")
+                break
 
         with open("log/backtesting.log", "a", encoding="utf-8") as f:
             current_exposure = float(0)
@@ -1257,18 +1278,27 @@ def load_klines_for_coin(self, coin) -> None:
             md5_query = md5(query.encode()).hexdigest()
             f_path = f"cache/{symbol}.{md5_query}"
 
-            if exists(f_path):
+            # wrap results in a try call, in case our cached files are corrupt
+            # and attempt to pull the required fields from our data.
+            try:
                 with open(f_path, "r") as f:
                     results = json.load(f)
-            else:
+                _, _, high, low, _, _, closetime, _, _, _, _,_ = results[0]
+            except Exception:  # pylint: disable=broad-except
                 results = requests_with_backoff(query).json()
                 # this can be fairly API intensive for a large number of tickers
+                # so we cache these calls on disk, each coin, period, start day
+                # is md5sum'd and stored on a dedicated file on /cache
                 if self.mode == "backtesting":
                     with open(f_path, "w") as f:
                         f.write(json.dumps(results))
 
             if self.debug:
-                logging.debug(f"{symbol} : last_{unit}:{results[-1:]}")
+                # ocasionally we obtain an invalid results obj here
+                if results:
+                    logging.debug(f"{symbol} : last_{unit}:{results[-1:]}")
+                else:
+                    logging.debug(f"{symbol} : last_{unit}:{results}")
 
             if timeslice != 0:
                 lowest = []

lib/helpers.py

@@ -4,13 +4,16 @@
 import sys
 from datetime import datetime
 from functools import lru_cache
+from os import getpid
 from os.path import exists, getctime
 import colorlog
 import requests
 import udatetime
 from binance.client import Client
 from tenacity import retry, wait_exponential
 
+
+PID = getpid()
 c_handler = colorlog.StreamHandler(sys.stdout)
 c_handler.setFormatter(
     colorlog.ColoredFormatter(
@@ -29,7 +32,7 @@
 
 logging.basicConfig(
     level=logging.DEBUG,
-    format="[%(levelname)s] %(lineno)d %(funcName)s %(message)s",
+    format=f"[%(levelname)s] {PID} %(lineno)d %(funcName)s %(message)s",
     handlers=[f_handler, c_handler],
 )
 

requirements.txt

@@ -1,36 +1,46 @@
 aiohttp==3.8.0
 aiosignal==1.2.0
 async-timeout==4.0.0
+asyncore-wsgi==0.0.10
 attrs==21.2.0
 backports.zoneinfo==0.2.1
+bottle==0.12.15
 certifi==2021.10.8
 charset-normalizer==2.0.7
 colorama==0.4.4
+colorlog==6.6.0
 dateparser==1.1.0
 frozenlist==1.2.0
 idna==3.3
+intervaltree==3.1.0
+isal==0.11.1
+lz4==3.1.3
 multidict==5.2.0
 mypy==0.910
 mypy-extensions==0.4.3
+networkx==2.8
+numpy==1.22.3
+pandas==1.4.2
 python-binance==1.0.15
 python-dateutil==2.8.2
 pytz==2021.3
 pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
 regex==2021.11.2
 requests==2.26.0
 six==1.16.0
+sortedcontainers==2.4.0
+ta==0.10.1
+tag==0.5
 tenacity==8.0.1
 toml==0.10.2
 typing-extensions==3.10.0.2
 tzdata==2021.5
 tzlocal==4.1
-ujson>=5.1.0
+udatetime==0.0.16
+ujson==5.2.0
 urllib3==1.26.7
+web-pdb==1.5.6
 websockets==9.1
-yarl==1.7.2
-pyyaml==6.0
 xopen==1.2.1
-colorlog==v6.6.0
-web-pdb==1.5.6
-lz4==3.1.3
-udatetime==0.0.16
+yarl==1.7.2

utils/automated-backtesting.py

@@ -153,6 +153,7 @@ def generate_coin_template_config_file(coin, strategy, cfg):
     "DEBUG": $DEBUG,
     "TRADING_FEE": $TRADING_FEE,
     "SELL_AS_SOON_IT_DROPS": $SELL_AS_SOON_IT_DROPS,
+    "STOP_BOT_ON_LOSS": $STOP_BOT_ON_LOSS,
     "TICKERS": {
       "$COIN": {
           "BUY_AT_PERCENTAGE": $BUY_AT_PERCENTAGE,
@@ -207,6 +208,7 @@ def generate_coin_template_config_file(coin, strategy, cfg):
                         "KLINES_SLICE_PERCENTAGE_CHANGE"
                     ],
                     "STRATEGY": strategy,
+                    "STOP_BOT_ON_LOSS": cfg["STOP_BOT_ON_LOSS"]
                 }
             )
         )
@@ -230,12 +232,10 @@ def generate_config_for_tuned_strategy_run(strategy, cfg, results, logfile):
     }"""
     )
 
-    print(f"\ncreating {coin} config for {strategy}\n")
     with open(f"configs/{strategy}.yaml", "wt") as f:
         f.write(
             tmpl.substitute(
                 {
-                    "COIN": coin,
                     "PAUSE_FOR": cfg["PAUSE_FOR"],
                     "INITIAL_INVESTMENT": cfg["INITIAL_INVESTMENT"],
                     "MAX_COINS": cfg["MAX_COINS"],
@@ -257,7 +257,8 @@ def generate_config_for_tuned_strategy_run(strategy, cfg, results, logfile):
         )
 
 
-if __name__ == "__main__":
+
+def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("-l", "--log", help="logfile")
     parser.add_argument("-c", "--cfgs", help="backtesting cfg")
@@ -275,7 +276,7 @@ def generate_config_for_tuned_strategy_run(strategy, cfg, results, logfile):
     if os.path.exists("cache/binance.client"):
         os.remove("cache/binance.client")
 
-    with mp.Pool(processes=os.cpu_count() * 2) as pool:
+    with mp.Pool(processes=os.cpu_count()) as pool:
         # process one strategy at a time
         for strategy in cfgs["STRATEGIES"]:
             # cleanup backtesting.log
@@ -290,6 +291,11 @@ def generate_config_for_tuned_strategy_run(strategy, cfg, results, logfile):
                         **cfgs["DEFAULTS"],
                         **cfgs["STRATEGIES"][strategy][run],
                     }
+                    # on 'wins' we don't want to keep on processing our logfiles
+                    # when we hit a STOP_LOSS
+                    if args.sortby == "wins":
+                        config["STOP_BOT_ON_LOSS"] = True
+
                     # and we generate a specific coin config file for that strategy
                     generate_coin_template_config_file(
                         symbol, strategy, config
@@ -338,3 +344,7 @@ def generate_config_for_tuned_strategy_run(strategy, cfg, results, logfile):
         os.remove(item)
     for item in glob.glob('log/coin.*.log.gz'):
         os.remove(item)
+
+
+if __name__ == "__main__":
+    main()