diff --git a/Makefile b/Makefile index 838b530..73936e8 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,7 @@ pip_packages: .venv .ONESHELL: pre-commit-checks: pip_packages + set -e source .venv/bin/activate black --check app.py black --check klines_caching_service.py diff --git a/app.py b/app.py index e7eb984..d62dde0 100644 --- a/app.py +++ b/app.py @@ -374,7 +374,7 @@ def new_listing(self, days): class Bot: """Bot Class""" - def __init__(self, conn, config_file, config) -> None: + def __init__(self, conn, config_file, config, logs_dir="log") -> None: """Bot object""" # Binance API handler @@ -481,6 +481,7 @@ def __init__(self, conn, config_file, config) -> None: self.sell_all_on_pull_config_change: bool = config.get( "SELL_ALL_ON_PULL_CONFIG_CHANGE", False ) + self.logs_dir = logs_dir def extract_order_data( # pylint: disable=no-self-use self, order_details, coin @@ -505,7 +506,7 @@ def extract_order_data( # pylint: disable=no-self-use avg = total / qty - volume = qty + volume = float(self.calculate_volume_size(coin)) logging.debug(f"{coin.symbol} -> volume:{volume} avgPrice:{avg}") return { @@ -971,9 +972,11 @@ def write_log(self, symbol: str, price: str) -> None: self.oldprice[symbol] = float(price) if self.mode == "testnet": - price_log = "log/testnet.log" + price_log = f"{self.logs_dir}/testnet.log" else: - price_log = f"log/{datetime.now().strftime('%Y%m%d')}.log" + price_log = ( + f"{self.logs_dir}/{datetime.now().strftime('%Y%m%d')}.log" + ) with open(price_log, "a", encoding="utf-8") as f: f.write(f"{datetime.now()} {symbol} {price}\n") @@ -1278,8 +1281,9 @@ def past_soft_limit(self, coin: Coin) -> bool: ) # make sure we never set the SELL_AT_PERCENTAGE below what we've - # had to pay in fees. It's quite likely however that if we didn't - # sell our coin by now, we are likely to hit HARD_LIMIT_HOLDING_TIME + # had to pay in fees. + # However, It's quite likely that if we didn't sell our coin by + # now, we are likely to hit HARD_LIMIT_HOLDING_TIME if coin.sell_at_percentage < add_100(2 * self.trading_fee): coin.sell_at_percentage = add_100(2 * self.trading_fee) @@ -1610,8 +1614,11 @@ def check_for_sale_conditions(self, coin: Coin) -> Tuple[bool, str]: return (False, "HOLD") - def buy_strategy(self, coin: Coin) -> bool: + def buy_strategy( + self, coin: Coin # pylint: disable=unused-argument,R0201 + ) -> bool: """buy strategy""" + return False def wait(self) -> None: """implements a pause""" @@ -1800,7 +1807,9 @@ def backtesting(self) -> None: break # now that we are done, lets record our results - with open("log/backtesting.log", "a", encoding="utf-8") as f: + with open( + f"{self.logs_dir}/backtesting.log", "a", encoding="utf-8" + ) as f: current_exposure = float(0) for symbol in self.wallet: current_exposure = current_exposure + self.coins[symbol].profit @@ -1991,16 +2000,6 @@ def process_control_flags(self): parser.add_argument( "-m", "--mode", help='bot mode ["live", "backtesting", "testnet"]' ) - # TODO: the args below are not currently consumed - parser.add_argument( - "-cd", "--config-dir", help="configs directory", default="configs" - ) - parser.add_argument( - "-rd", - "--results-dir", - help="results directory", - default="results", - ) parser.add_argument( "-ld", "--logs-dir", help="logs directory", default="log" ) @@ -2030,7 +2029,7 @@ def process_control_flags(self): c_handler.setLevel(logging.INFO) if cfg["DEBUG"]: - f_handler = logging.FileHandler("log/debug.log") + f_handler = logging.FileHandler(f"{args.logs_dir}/debug.log") f_handler.setLevel(logging.DEBUG) logging.basicConfig( diff --git a/pyproject.toml b/pyproject.toml index 213c00c..0f924e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ disable = [ "too-many-instance-attributes", "too-many-arguments", "logging-fstring-interpolation", + "no-self-use", "too-many-lines", "too-many-statements", "too-many-branches", diff --git a/requirements-dev.txt b/requirements-dev.txt index 765e623..fa86c9e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,7 +17,7 @@ pre-commit==2.20.0 py==1.11.0 pylint==2.13.9 pyparsing==2.4.7 -pytest==6.2.5 +pytest==7.2.0 pytest-forked==1.3.0 pytest-socket==0.4.1 pytest-timeout diff --git a/utils/prove-backtesting.py b/utils/prove-backtesting.py index ce31c19..590b72d 100644 --- a/utils/prove-backtesting.py +++ b/utils/prove-backtesting.py @@ -42,6 +42,9 @@ def cli(): parser.add_argument( "-ld", "--logs-dir", help="logs directory", default="log" ) + parser.add_argument( + "-x", "--concurrency", help="SMP_MULTIPLIER value", default="1.0" + ) args = parser.parse_args() return [ @@ -55,6 +58,7 @@ def cli(): args.config_dir, args.results_dir, args.logs_dir, + args.concurrency, ] @@ -113,13 +117,16 @@ def run_prove_backtesting(config, results_dir): ) -def run_automated_backtesting(config, min_profit, sortby, logs_dir="log"): +def run_automated_backtesting( + config, min_profit, sortby, logs_dir="log", env={} +): """calls automated-backtesting""" subprocess.run( f"python -u utils/automated-backtesting.py -l {logs_dir}/lastfewdays.log.gz " + f"-c configs/{config} -m {min_profit} -f '' -s {sortby} --run-final-backtest=False", shell=True, check=False, + env=env, ) @@ -175,6 +182,7 @@ def main(): config_dir, results_dir, logs_dir, + concurrency, ) = cli() log_msg( @@ -212,8 +220,22 @@ def main(): log_msg( f"starting automated_backtesting using {config_file} for {min_profit}" ) + # check for new values for SMP_MULTIPLIER and adjust as needed + # use this setting in cron jobs to increase/decrease the number of + # parallel backtesting processes based on the time of the day. + if os.path.exists("control/SMP_MULTIPLIER"): + with open("control/SMP_MULTIPLIER") as f: + concurrency = f.read().strip() + log_msg(f"control/SMP_MULTIPLIER contains {concurrency}") + os.unlink("control/SMP_MULTIPLIER") + # runs automated_backtesting on all strategies - run_automated_backtesting(config_file, min_profit, sortby) + run_automated_backtesting( + config=config_file, + min_profit=min_profit, + sortby=sortby, + env={**os.environ, "SMP_MULTIPLIER": concurrency}, + ) dates = prove_backtesting_dates( end_date=start_date, days=int(forward_days)