FILTER was not being consumed

FILTER was not being consumed in prove-backtesting and
automated-backtesting runs.
this commit, makes sure we can filter on a word like 'BTCUSDT' and the
tooling will only create logfiles or run backtesting on lines that
contain that word.
This allows us to quickly backtesting a single coin

run

@@ -10,7 +10,7 @@ function usage() {
 	echo "./run lastfewdays DAYS=3 PAIR=USDT"
 	echo "./run automated-backtesting LOGFILE=lastfewdays.log.gz CONFIG_FILE=backtesting.yaml MIN=10 FILTER='' SORTBY='profit|wins'"
 	echo "./run download-price-logs FROM=20210101 TO=20211231"
-	echo "./run prove-backtesting CONFIG_FILE=myconfig.yaml FROM=20220101 BACKTRACK=90 MIN=20 FORWARD=30 TO=20220901 SORTBY=profit|wins"
+	echo "./run prove-backtesting CONFIG_FILE=myconfig.yaml FROM=20220101 BACKTRACK=90 MIN=20 FORWARD=30 TO=20220901 SORTBY=profit|wins FILTER=''"
 	echo "./run config-endpoint-service BIND=0.0.0.0 CONFIG_FILE=myconfig.yaml BACKTRACK=30 PAIRING=USDT MIN=10 TUNED_CONFIG=BuyDropSellRecoveryStrategy.yaml SORTBY=wins|profit"
 	echo "./run klines-caching-service BIND=0.0.0.0"
 	echo "./run download_price_logs FROM=20220101 TO=20220131"
@@ -175,7 +175,7 @@ function automated_backtesting() { # runs the automated backtesting
 		${DOCKER_MOUNTS} \
 		${DOCKER_NETWORK} \
 		-e LOGFILE=/cryptobot/log/${LOGFILE} \
-		-e CONFIG=/cryptobot/configs/${CONFIG_FILE} \
+		-e CONFIG_FILE=/cryptobot/configs/${CONFIG_FILE} \
 		-e MIN=${MIN} \
 		-e FILTER=${FILTER} \
 		-e SORTBY=${SORTBY} \
@@ -218,7 +218,7 @@ function prove_backtesting() { # runs the prove backtesting
 	fi
 
 	RESULTS_LOG="${RESULTS_DIR}/prove-backtesting"
-	RESULTS_LOG="${RESULTS_LOG}.${CONFIG}"
+	RESULTS_LOG="${RESULTS_LOG}.${CONFIG_FILE}"
 	RESULTS_LOG="${RESULTS_LOG}.min${MIN}"
 	RESULTS_LOG="${RESULTS_LOG}.${SORTBY}"
 	RESULTS_LOG="${RESULTS_LOG}.${FROM}_${TO}"
@@ -237,6 +237,7 @@ function prove_backtesting() { # runs the prove backtesting
 		-e MIN=${MIN} \
 		-e FORWARD=${FORWARD} \
 		-e TO=${TO} \
+		-e FILTER=${FILTER} \
 		-e SORTBY=${SORTBY} \
 		-e SMP_MULTIPLIER=${SMP_MULTIPLIER} \
 		${IMAGE}:${TAG} \

utils/automated-backtesting.py

@@ -30,7 +30,7 @@ def compress_file(filename):
     os.remove(filename)
 
 
-def split_logs_into_coins(filename, cfg, logs_dir="log"):
+def split_logs_into_coins(filename, cfg, filterby, logs_dir="log"):
     """splits one price.log into individual coin price.log files"""
     coinfiles = set()
     coinfh = {}
@@ -40,10 +40,14 @@ def split_logs_into_coins(filename, cfg, logs_dir="log"):
 
             # don't process all the lines, but only the ones related to our PAIR
             # 2021-01-01 00:00:01.0023 BTCUSDT 36000.00
-            if not f"{pairing} " in line:
+            if not f"{pairing} " in str(line):
                 continue
 
-            parts = line.split(" ")
+            # and constrain to our filterby
+            if not f"{filterby} " in str(line):
+                continue
+
+            parts = str(line).split(" ")
             symbol = parts[2]
 
             # don't process any BEAR/BULL/UP/DOWN lines
@@ -565,7 +569,7 @@ def main():
         run_final_backtest,
     ) = cli()
 
-    coinfiles = split_logs_into_coins(logfile, cfgs, logs_dir="log")
+    coinfiles = split_logs_into_coins(logfile, cfgs, filterby, logs_dir="log")
 
     if os.path.exists("cache/binance.client"):
         os.remove("cache/binance.client")

utils/automated-backtesting.sh

@@ -2,4 +2,4 @@
 ulimit -n 65535
 source /cryptobot/.venv/bin/activate
 python -u utils/automated-backtesting.py \
-	-l ${LOGFILE} -c ${CONFIG} -m ${MIN} -f "${FILTER}" -s "${SORTBY}"
+	-l ${LOGFILE} -c ${CONFIG_FILE} -m ${MIN} -f "${FILTER}" -s "${SORTBY}"

utils/prove-backtesting.py

@@ -31,6 +31,9 @@ def cli():
     parser.add_argument(
         "-s", "--sortby", help="sortby results by profit/wins", default="wins"
     )
+    parser.add_argument(
+        "-x", "--filter", help="filter coins by word", default=""
+    )
 
     # TODO: the args below are not currently consumed
     parser.add_argument(
@@ -43,7 +46,7 @@ def cli():
         "-ld", "--logs-dir", help="logs directory", default="log"
     )
     parser.add_argument(
-        "-x", "--concurrency", help="SMP_MULTIPLIER value", default="1.0"
+        "-n", "--concurrency", help="SMP_MULTIPLIER value", default="1.0"
     )
     args = parser.parse_args()
 
@@ -59,6 +62,7 @@ def cli():
         args.results_dir,
         args.logs_dir,
         args.concurrency,
+        args.filter,
     ]
 
 
@@ -118,19 +122,21 @@ def run_prove_backtesting(config, results_dir):
 
 
 def run_automated_backtesting(
-    config, min_profit, sortby, logs_dir="log", env={}
+    config, min_profit, sortby, logs_dir="log", env={}, filterby=""
 ):
     """calls automated-backtesting"""
     subprocess.run(
         f"python -u utils/automated-backtesting.py -l {logs_dir}/lastfewdays.log.gz "
-        + f"-c configs/{config} -m {min_profit} -f '' -s {sortby} --run-final-backtest=False",
+        + f"-c configs/{config} -m {min_profit} -f '{filterby}' -s {sortby} --run-final-backtest=False",
         shell=True,
         check=False,
         env=env,
     )
 
 
-def create_zipped_logfile(dates, pairing, logs_dir="log", symbols=[]):
+def create_zipped_logfile(
+    dates, pairing, logs_dir="log", symbols=[], filterby=""
+):
     """
     generates lastfewdays.log.gz from the provided list of price.log files
     excluding any symbol in the excluded list, and not matching pairing
@@ -147,7 +153,11 @@ def create_zipped_logfile(dates, pairing, logs_dir="log", symbols=[]):
                 continue
             with igzip.open(log, "rt") as r:
                 for line in r:
-                    if pairing not in line:
+                    # skip line if not related to our PAIRING
+                    if pairing not in str(line):
+                        continue
+                    # skip line if not in our filtered word
+                    if filterby not in str(line):
                         continue
                     # don't process any BEAR/BULL/UP/DOWN lines
                     excluded = [
@@ -156,7 +166,7 @@ def create_zipped_logfile(dates, pairing, logs_dir="log", symbols=[]):
                         f"BEAR{pairing}",
                         f"BULL{pairing}",
                     ]
-                    if any(symbol in line for symbol in excluded):
+                    if any(symbol in str(line) for symbol in excluded):
                         continue
 
                     if symbols:
@@ -183,6 +193,7 @@ def main():
         results_dir,
         logs_dir,
         concurrency,
+        filterby,
     ) = cli()
 
     log_msg(
@@ -216,7 +227,7 @@ def main():
         )
         dates = backtesting_dates(end_date=start_date, days=backtrack_days)
         log_msg(dates)
-        create_zipped_logfile(dates, pairing, logs_dir)
+        create_zipped_logfile(dates, pairing, logs_dir, filterby=filterby)
         log_msg(
             f"starting automated_backtesting using {config_file} for {min_profit}"
         )
@@ -235,6 +246,7 @@ def main():
             min_profit=min_profit,
             sortby=sortby,
             env={**os.environ, "SMP_MULTIPLIER": concurrency},
+            filterby=filterby,
         )
 
         dates = prove_backtesting_dates(
@@ -266,7 +278,9 @@ def main():
         )
         log_msg(dates)
 
-        create_zipped_logfile(dates, pairing, logs_dir, symbols)
+        create_zipped_logfile(
+            dates, pairing, logs_dir, symbols, filterby=filterby
+        )
 
         for strategy in strategies:
             with open(f"{config_dir}/{strategy}.yaml") as f:

utils/prove-backtesting.sh

@@ -4,4 +4,4 @@ ulimit -n 65535
 source /cryptobot/.venv/bin/activate
 python -u /cryptobot/utils/prove-backtesting.py \
 	-d ${FROM} -b ${BACKTRACK} -c ${CONFIG_FILE} -m ${MIN} \
-	-f ${FORWARD} -e ${TO} -s ${SORTBY}
+	-f ${FORWARD} -e ${TO} -s ${SORTBY} -x "${FILTER}" -n ${SMP_MULTIPLIER}