From 8998337ad0cf155d0277a58413bfc43a5a49c644 Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 16:30:22 +0200 Subject: [PATCH 01/12] consistency with ema bands min long --- njit_clock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/njit_clock.py b/njit_clock.py index ec3c05cce..71805a739 100644 --- a/njit_clock.py +++ b/njit_clock.py @@ -91,7 +91,7 @@ def calc_clock_entry_long( psize_long: float, pprice_long: float, highest_bid: float, - ema_band_lower: float, + emas: float, utc_now_ms: float, prev_clock_fill_ts_entry: float, inverse: bool, @@ -117,7 +117,7 @@ def calc_clock_entry_long( if wallet_exposure_long < wallet_exposure_limit * 0.99: # entry long bid_price_long = calc_clock_price_bid( - ema_band_lower, highest_bid, ema_dist_entry, price_step + emas.min(), highest_bid, ema_dist_entry, price_step ) qty_long = calc_clock_qty( balance, @@ -563,7 +563,7 @@ def backtest_clock( psize_long, pprice_long, closes[k - 1], - emas_long.min(), + emas_long, timestamps[k - 1], prev_clock_fill_ts_entry_long, inverse, From 6a51417823c47cca635eb892f342368abff4a413 Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 16:31:13 +0200 Subject: [PATCH 02/12] black formatting --- njit_clock.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/njit_clock.py b/njit_clock.py index 71805a739..b5b057b3f 100644 --- a/njit_clock.py +++ b/njit_clock.py @@ -116,9 +116,7 @@ def calc_clock_entry_long( wallet_exposure_long = qty_to_cost(psize_long, pprice_long, inverse, c_mult) / balance if wallet_exposure_long < wallet_exposure_limit * 0.99: # entry long - bid_price_long = calc_clock_price_bid( - emas.min(), highest_bid, ema_dist_entry, price_step - ) + bid_price_long = calc_clock_price_bid(emas.min(), highest_bid, ema_dist_entry, price_step) qty_long = calc_clock_qty( balance, wallet_exposure_long, From c8abe886eeecafe498c9cd24c60e7c777488cc46 Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 16:33:40 +0200 Subject: [PATCH 03/12] use exchange's server time consistency with ema bands --- passivbot.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/passivbot.py b/passivbot.py index df0068b6a..3427da61b 100644 --- a/passivbot.py +++ b/passivbot.py @@ -79,6 +79,7 @@ def __init__(self, config: dict): self.hedge_mode = self.config["hedge_mode"] = True self.set_config(self.config) + self.server_time = utc_ms() self.ts_locked = { k: 0.0 @@ -215,6 +216,7 @@ async def _init(self): self.init_exchange_config(), self.init_order_book(), self.init_emas(), + self.update_server_time(), ) print("done") if "price_step_custom" in self.config and self.config["price_step_custom"] is not None: @@ -634,8 +636,8 @@ def calc_orders(self): psize_long, pprice_long, self.ob[0], - min(self.emas_long), - utc_ms(), + self.emas_long, + self.server_time, 0 if psize_long == 0.0 else self.last_fills_timestamps["clock_entry_long"], @@ -695,8 +697,8 @@ def calc_orders(self): psize_long, pprice_long, self.ob[1], - max(self.emas_long), - utc_ms(), + self.emas_long, + self.server_time, self.last_fills_timestamps["clock_close_long"], self.xk["inverse"], self.xk["qty_step"], @@ -842,8 +844,8 @@ def calc_orders(self): psize_short, pprice_short, self.ob[1], - max(self.emas_short), - utc_ms(), + self.emas_short, + self.server_time, 0 if psize_short == 0.0 else self.last_fills_timestamps["clock_entry_short"], @@ -903,8 +905,8 @@ def calc_orders(self): psize_short, pprice_short, self.ob[0], - min(self.emas_short), - utc_ms(), + self.emas_short, + self.server_time, self.last_fills_timestamps["clock_close_short"], self.xk["inverse"], self.xk["qty_step"], @@ -1358,6 +1360,18 @@ async def update_last_fills_timestamps(self): finally: self.ts_released["update_last_fills_timestamps"] = time.time() + async def update_server_time(self): + server_time = None + try: + server_time = await self.get_server_time() + self.server_time = server_time + return True + except Exception as e: + self.server_time = utc_ms() + traceback.print_exc() + print_async_exception(server_time) + return False + async def start_ohlcv_mode(self): logging.info("starting bot...") while True: @@ -1391,9 +1405,14 @@ async def on_minute_mark(self): self.heartbeat_ts = time.time() self.prev_price = self.ob[0] prev_pos = self.position.copy() - to_update = [self.update_position(), self.update_open_orders(), self.init_order_book()] + to_update = [ + self.update_position(), + self.update_open_orders(), + self.init_order_book(), + ] if self.passivbot_mode == "clock": to_update.append(self.update_last_fills_timestamps()) + to_update.append(self.get_server_time()) res = await asyncio.gather(*to_update) self.update_emas(self.ob[0], self.prev_price) """ @@ -1405,7 +1424,7 @@ async def on_minute_mark(self): print(res) """ if not all(res): - reskeys = ["pos", "open orders", "order book", "last fills"] + reskeys = ["pos", "open orders", "order book", "last fills", "server_time"] line = "error with " for i in range(len(to_update)): if not to_update[i]: From 7793483a2d6a50b107554b35dafeafd646321d0f Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 16:35:18 +0200 Subject: [PATCH 04/12] use slimmer analysis for optimizing --- optimize.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/optimize.py b/optimize.py index 37e428f7a..41b5f7f58 100644 --- a/optimize.py +++ b/optimize.py @@ -13,7 +13,7 @@ from multiprocessing import Pool, shared_memory from njit_funcs import round_dynamic from pure_funcs import ( - analyze_fills, + analyze_fills_slim, denumpyize, get_template_live_config, ts_to_date, @@ -68,13 +68,8 @@ def backtest_wrap(config_: dict, ticks_caches: dict): try: assert "adg_n_subdivisions" in config fills_long, fills_short, stats = backtest(config, ticks) - longs, shorts, sdf, analysis = analyze_fills(fills_long, fills_short, stats, config) - logging.debug( - f"backtested {config['symbol']: <12} pa distance long {analysis['pa_distance_mean_long']:.6f} " - + f"pa distance short {analysis['pa_distance_mean_short']:.6f} adg long {analysis['adg_long']:.6f} " - + f"adg short {analysis['adg_short']:.6f} std long {analysis['pa_distance_std_long']:.5f} " - + f"std short {analysis['pa_distance_std_short']:.5f}" - ) + #longs, shorts, sdf, analysis = analyze_fills(fills_long, fills_short, stats, config) + analysis = analyze_fills_slim(fills_long, fills_short, stats, config) except Exception as e: analysis = get_empty_analysis() logging.error(f'error with {config["symbol"]} {e}') From d2f69508e3a401674316186e20a07027385ac55f Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 16:36:20 +0200 Subject: [PATCH 05/12] add get_server_time() --- exchanges/kucoin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/exchanges/kucoin.py b/exchanges/kucoin.py index cce543cbc..a7ce2c2e8 100644 --- a/exchanges/kucoin.py +++ b/exchanges/kucoin.py @@ -82,6 +82,7 @@ def init_market_type(self): "public_token_ws": "/api/v1/bullet-public", "private_token_ws": "/api/v1/bullet-private", "income": "/api/v1/recentFills", + "server_time": "/api/v1/timestamp", "recent_orders": "/api/v1/recentDoneOrders", } self.hedge_mode = self.config["hedge_mode"] = False @@ -253,7 +254,8 @@ async def transfer_from_derivatives_to_spot(self, coin: str, amount: float): raise "Not implemented" async def get_server_time(self): - raise "Not implemented" + server_time = await self.public_get(self.endpoints["server_time"]) + return server_time["data"] async def fetch_position(self) -> dict: positions, balance = None, None From d811cc3a057834796b30443fca7efd64739fb2de Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 16:37:08 +0200 Subject: [PATCH 06/12] update end date --- configs/backtest/default.hjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/backtest/default.hjson b/configs/backtest/default.hjson index f388784cb..725e35a88 100644 --- a/configs/backtest/default.hjson +++ b/configs/backtest/default.hjson @@ -13,7 +13,7 @@ # format YYYY-MM-DD start_date: 2020-01-01 - end_date: 2023-03-17 + end_date: 2023-05-05 # non default historical data path # defaults to local dir if unspecified From 07dd5189329217cdbe25888f676fb291ba0a2133 Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 16:41:58 +0200 Subject: [PATCH 07/12] up version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fe8f75495..3738752a1 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ :warning: **Use at own risk** :warning: -v5.9.6 +v5.9.7 ## Overview From 215598c85037f6e12198e0013de44fb1bb06c4b4 Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 16:52:15 +0200 Subject: [PATCH 08/12] use lower TWE default values --- configs/forager/example_config.hjson | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/forager/example_config.hjson b/configs/forager/example_config.hjson index 06326782e..e33d518ac 100644 --- a/configs/forager/example_config.hjson +++ b/configs/forager/example_config.hjson @@ -1,8 +1,8 @@ { // supported exchanges: [kucoin, okx, bybit, binance] user: binance_01 - twe_long: 3.2 - twe_short: 1.6 + twe_long: 1.6 + twe_short: 0.8 n_longs: 6 n_shorts: 3 max_min_cost: 5.0 From 52870937608e786f6ea01ab1809e065440ff4896 Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 17:40:41 +0200 Subject: [PATCH 09/12] bug fix, typing Tuple --- downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/downloader.py b/downloader.py index 551b9eda9..7327d790a 100644 --- a/downloader.py +++ b/downloader.py @@ -107,7 +107,7 @@ def __init__(self, config: dict): ) ) - def validate_dataframe(self, df: pd.DataFrame) -> tuple[bool, pd.DataFrame, pd.DataFrame]: + def validate_dataframe(self, df: pd.DataFrame) -> Tuple[bool, pd.DataFrame, pd.DataFrame]: """ Validates a dataframe and detects gaps in it. Also detects missing trades in the beginning and end. @param df: Dataframe to check for gaps. From 4faef24466eef77d04d5fac0546cf98804e9221a Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 20:51:27 +0200 Subject: [PATCH 10/12] add adg_per_exposure and shorten headers --- inspect_opt_results.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/inspect_opt_results.py b/inspect_opt_results.py index 12655581a..1b1139ae7 100755 --- a/inspect_opt_results.py +++ b/inspect_opt_results.py @@ -21,6 +21,19 @@ from njit_funcs import round_dynamic +def shorten(key): + key_ = key + for src, dst in [ + ("weighted", "w"), + ("exposure", "exp"), + ("distance", "dist"), + ("ratio", "rt"), + ("mean_of_10_worst", "10_worst_mean"), + ]: + key_ = key_.replace(src, dst) + return key_ + + def main(): parser = argparse.ArgumentParser(prog="view conf", description="inspect conf") @@ -96,6 +109,7 @@ def main(): scores_res["individual_scores"], scores_res["keys"], ) + keys = keys[:1] + [("adg_per_exposure", True)] + keys[1:] for side in sides: all_scores[-1][side] = { "config": cfg[side], @@ -121,7 +135,7 @@ def main(): if os.path.exists(table_filepath): os.remove(table_filepath) for side in sides: - row_headers = ["symbol"] + [k[0] for k in keys] + ["n_days", "score"] + row_headers = ["symbol"] + [shorten(k[0]) for k in keys] + ["n_days", "score"] table = PrettyTable(row_headers) for rh in row_headers: table.align[rh] = "l" @@ -139,7 +153,7 @@ def main(): [("-> " if sym in best_candidate[side]["symbols_to_include"] else "") + sym] + [round_dynamic(x, 4) if np.isfinite(x) else x for x in xs] + [round(best_candidate[side]["n_days"][sym], 2)] - + [best_candidate[side]["individual_scores"][sym]] + + [round_dynamic(best_candidate[side]["individual_scores"][sym], 12)] ) means = [ np.mean( @@ -160,7 +174,7 @@ def main(): ["mean"] + [round_dynamic(m, 4) if np.isfinite(m) else m for m in means] + [round(np.mean(list(best_candidate[side]["n_days"].values())), 2)] - + [ind_scores_mean] + + [round_dynamic(ind_scores_mean, 12)] ) with open(make_get_filepath(table_filepath), "a") as f: output = table.get_string(border=True, padding_width=1) From a06e0f7404c1eeb4518a616c55218560aa063374 Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 20:52:11 +0200 Subject: [PATCH 11/12] add three more metrics to slim analysis --- pure_funcs.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pure_funcs.py b/pure_funcs.py index 3c1a6a8cd..11db9c9f6 100644 --- a/pure_funcs.py +++ b/pure_funcs.py @@ -654,11 +654,15 @@ def analyze_fills_slim(fills_long: list, fills_short: list, stats: list, config: # eqbal_ratio_mean_of_10_worst, # eqbal_ratio_std, + # plus n_days, starting_balance and adg_per_exposure + if "adg_n_subdivisions" not in config: config["adg_n_subdivisions"] = 1 ms_per_day = 1000 * 60 * 60 * 24 + n_days = (stats[-1][0] - stats[0][0]) / ms_per_day + if stats[-1][10] <= 0.0: adg_long = adg_weighted_long = stats[-1][10] else: @@ -723,6 +727,10 @@ def analyze_fills_slim(fills_long: list, fills_short: list, stats: list, config: "adg_weighted_per_exposure_long": adg_weighted_long / config["long"]["wallet_exposure_limit"], "adg_weighted_per_exposure_short": adg_weighted_short / config["short"]["wallet_exposure_limit"], + "adg_per_exposure_long": adg_long / config["long"]["wallet_exposure_limit"], + "adg_per_exposure_short": adg_short / config["short"]["wallet_exposure_limit"], + "n_days": n_days, + "starting_balance": stats[0][10], "pa_distance_std_long": pa_dists_long.std(), "pa_distance_std_short": pa_dists_short.std(), "pa_distance_mean_long": pa_dists_long.mean(), From 93fe3f913612f3df14aa91b53b69c95540859419 Mon Sep 17 00:00:00 2001 From: enarjord Date: Fri, 5 May 2023 20:54:26 +0200 Subject: [PATCH 12/12] black formatting --- optimize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/optimize.py b/optimize.py index 41b5f7f58..2c95a22db 100644 --- a/optimize.py +++ b/optimize.py @@ -68,7 +68,6 @@ def backtest_wrap(config_: dict, ticks_caches: dict): try: assert "adg_n_subdivisions" in config fills_long, fills_short, stats = backtest(config, ticks) - #longs, shorts, sdf, analysis = analyze_fills(fills_long, fills_short, stats, config) analysis = analyze_fills_slim(fills_long, fills_short, stats, config) except Exception as e: analysis = get_empty_analysis() @@ -265,7 +264,7 @@ async def run_opt(args, config): if config["ohlcv"]: data = load_hlc_cache( symbol, - config['inverse'], + config["inverse"], config["start_date"], config["end_date"], base_dir=config["base_dir"], @@ -295,7 +294,8 @@ async def run_opt(args, config): for fname in os.listdir(args.starting_configs): try: """ - if config["symbols"][0] not in os.path.join(args.starting_configs, fname): + # uncomment to skip single starting configs with wrong symbol + if not any(x in os.path.join(args.starting_configs, fname) for x in [config["symbols"][0], "symbols"]): print("skipping", os.path.join(args.starting_configs, fname)) continue """