diff --git a/run b/run index 70e94cd..ac91e4a 100755 --- a/run +++ b/run @@ -10,7 +10,7 @@ function usage() { echo "./run lastfewdays DAYS=3 PAIR=USDT" echo "./run automated-backtesting LOGFILE=lastfewdays.log.gz CONFIG_FILE=backtesting.yaml MIN=10 FILTER='' SORTBY='profit|wins'" echo "./run download-price-logs FROM=20210101 TO=20211231" - echo "./run prove-backtesting CONFIG_FILE=myconfig.yaml FROM=20220101 BACKTRACK=90 MIN=20 FORWARD=30 TO=20220901 SORTBY=profit|wins" + echo "./run prove-backtesting CONFIG_FILE=myconfig.yaml FROM=20220101 BACKTRACK=90 MIN=20 FORWARD=30 TO=20220901 SORTBY=profit|wins FILTER=''" echo "./run config-endpoint-service BIND=0.0.0.0 CONFIG_FILE=myconfig.yaml BACKTRACK=30 PAIRING=USDT MIN=10 TUNED_CONFIG=BuyDropSellRecoveryStrategy.yaml SORTBY=wins|profit" echo "./run klines-caching-service BIND=0.0.0.0" echo "./run download_price_logs FROM=20220101 TO=20220131" @@ -175,7 +175,7 @@ function automated_backtesting() { # runs the automated backtesting ${DOCKER_MOUNTS} \ ${DOCKER_NETWORK} \ -e LOGFILE=/cryptobot/log/${LOGFILE} \ - -e CONFIG=/cryptobot/configs/${CONFIG_FILE} \ + -e CONFIG_FILE=/cryptobot/configs/${CONFIG_FILE} \ -e MIN=${MIN} \ -e FILTER=${FILTER} \ -e SORTBY=${SORTBY} \ @@ -218,7 +218,7 @@ function prove_backtesting() { # runs the prove backtesting fi RESULTS_LOG="${RESULTS_DIR}/prove-backtesting" - RESULTS_LOG="${RESULTS_LOG}.${CONFIG}" + RESULTS_LOG="${RESULTS_LOG}.${CONFIG_FILE}" RESULTS_LOG="${RESULTS_LOG}.min${MIN}" RESULTS_LOG="${RESULTS_LOG}.${SORTBY}" RESULTS_LOG="${RESULTS_LOG}.${FROM}_${TO}" @@ -237,6 +237,7 @@ function prove_backtesting() { # runs the prove backtesting -e MIN=${MIN} \ -e FORWARD=${FORWARD} \ -e TO=${TO} \ + -e FILTER=${FILTER} \ -e SORTBY=${SORTBY} \ -e SMP_MULTIPLIER=${SMP_MULTIPLIER} \ ${IMAGE}:${TAG} \ diff --git a/utils/automated-backtesting.py b/utils/automated-backtesting.py index 5cc3185..7a656d2 100644 --- a/utils/automated-backtesting.py +++ b/utils/automated-backtesting.py @@ -30,7 +30,7 @@ def compress_file(filename): os.remove(filename) -def split_logs_into_coins(filename, cfg, logs_dir="log"): +def split_logs_into_coins(filename, cfg, filterby, logs_dir="log"): """splits one price.log into individual coin price.log files""" coinfiles = set() coinfh = {} @@ -40,10 +40,14 @@ def split_logs_into_coins(filename, cfg, logs_dir="log"): # don't process all the lines, but only the ones related to our PAIR # 2021-01-01 00:00:01.0023 BTCUSDT 36000.00 - if not f"{pairing} " in line: + if not f"{pairing} " in str(line): continue - parts = line.split(" ") + # and constrain to our filterby + if not f"{filterby} " in str(line): + continue + + parts = str(line).split(" ") symbol = parts[2] # don't process any BEAR/BULL/UP/DOWN lines @@ -565,7 +569,7 @@ def main(): run_final_backtest, ) = cli() - coinfiles = split_logs_into_coins(logfile, cfgs, logs_dir="log") + coinfiles = split_logs_into_coins(logfile, cfgs, filterby, logs_dir="log") if os.path.exists("cache/binance.client"): os.remove("cache/binance.client") diff --git a/utils/automated-backtesting.sh b/utils/automated-backtesting.sh index a93d95d..f9b4e23 100755 --- a/utils/automated-backtesting.sh +++ b/utils/automated-backtesting.sh @@ -2,4 +2,4 @@ ulimit -n 65535 source /cryptobot/.venv/bin/activate python -u utils/automated-backtesting.py \ - -l ${LOGFILE} -c ${CONFIG} -m ${MIN} -f "${FILTER}" -s "${SORTBY}" + -l ${LOGFILE} -c ${CONFIG_FILE} -m ${MIN} -f "${FILTER}" -s "${SORTBY}" diff --git a/utils/prove-backtesting.py b/utils/prove-backtesting.py index bd4d52d..7f52e99 100644 --- a/utils/prove-backtesting.py +++ b/utils/prove-backtesting.py @@ -31,6 +31,9 @@ def cli(): parser.add_argument( "-s", "--sortby", help="sortby results by profit/wins", default="wins" ) + parser.add_argument( + "-x", "--filter", help="filter coins by word", default="" + ) # TODO: the args below are not currently consumed parser.add_argument( @@ -43,7 +46,7 @@ def cli(): "-ld", "--logs-dir", help="logs directory", default="log" ) parser.add_argument( - "-x", "--concurrency", help="SMP_MULTIPLIER value", default="1.0" + "-n", "--concurrency", help="SMP_MULTIPLIER value", default="1.0" ) args = parser.parse_args() @@ -59,6 +62,7 @@ def cli(): args.results_dir, args.logs_dir, args.concurrency, + args.filter, ] @@ -118,19 +122,21 @@ def run_prove_backtesting(config, results_dir): def run_automated_backtesting( - config, min_profit, sortby, logs_dir="log", env={} + config, min_profit, sortby, logs_dir="log", env={}, filterby="" ): """calls automated-backtesting""" subprocess.run( f"python -u utils/automated-backtesting.py -l {logs_dir}/lastfewdays.log.gz " - + f"-c configs/{config} -m {min_profit} -f '' -s {sortby} --run-final-backtest=False", + + f"-c configs/{config} -m {min_profit} -f '{filterby}' -s {sortby} --run-final-backtest=False", shell=True, check=False, env=env, ) -def create_zipped_logfile(dates, pairing, logs_dir="log", symbols=[]): +def create_zipped_logfile( + dates, pairing, logs_dir="log", symbols=[], filterby="" +): """ generates lastfewdays.log.gz from the provided list of price.log files excluding any symbol in the excluded list, and not matching pairing @@ -147,7 +153,11 @@ def create_zipped_logfile(dates, pairing, logs_dir="log", symbols=[]): continue with igzip.open(log, "rt") as r: for line in r: - if pairing not in line: + # skip line if not related to our PAIRING + if pairing not in str(line): + continue + # skip line if not in our filtered word + if filterby not in str(line): continue # don't process any BEAR/BULL/UP/DOWN lines excluded = [ @@ -156,7 +166,7 @@ def create_zipped_logfile(dates, pairing, logs_dir="log", symbols=[]): f"BEAR{pairing}", f"BULL{pairing}", ] - if any(symbol in line for symbol in excluded): + if any(symbol in str(line) for symbol in excluded): continue if symbols: @@ -183,6 +193,7 @@ def main(): results_dir, logs_dir, concurrency, + filterby, ) = cli() log_msg( @@ -216,7 +227,7 @@ def main(): ) dates = backtesting_dates(end_date=start_date, days=backtrack_days) log_msg(dates) - create_zipped_logfile(dates, pairing, logs_dir) + create_zipped_logfile(dates, pairing, logs_dir, filterby=filterby) log_msg( f"starting automated_backtesting using {config_file} for {min_profit}" ) @@ -235,6 +246,7 @@ def main(): min_profit=min_profit, sortby=sortby, env={**os.environ, "SMP_MULTIPLIER": concurrency}, + filterby=filterby, ) dates = prove_backtesting_dates( @@ -266,7 +278,9 @@ def main(): ) log_msg(dates) - create_zipped_logfile(dates, pairing, logs_dir, symbols) + create_zipped_logfile( + dates, pairing, logs_dir, symbols, filterby=filterby + ) for strategy in strategies: with open(f"{config_dir}/{strategy}.yaml") as f: diff --git a/utils/prove-backtesting.sh b/utils/prove-backtesting.sh index 147d3dd..04e390a 100755 --- a/utils/prove-backtesting.sh +++ b/utils/prove-backtesting.sh @@ -4,4 +4,4 @@ ulimit -n 65535 source /cryptobot/.venv/bin/activate python -u /cryptobot/utils/prove-backtesting.py \ -d ${FROM} -b ${BACKTRACK} -c ${CONFIG_FILE} -m ${MIN} \ - -f ${FORWARD} -e ${TO} -s ${SORTBY} + -f ${FORWARD} -e ${TO} -s ${SORTBY} -x "${FILTER}" -n ${SMP_MULTIPLIER}