diff --git a/app.py b/app.py index 30baf0b..ab62591 100644 --- a/app.py +++ b/app.py @@ -36,6 +36,7 @@ floor_value, mean, percent, + requests_with_backoff, ) @@ -1840,19 +1841,21 @@ def backtesting(self) -> None: def load_klines_for_coin(self, coin) -> bool: """fetches from binance or a local cache klines for a coin""" + ok: bool = False + data: dict = {} # fetch all the available klines for this coin, for the last # 60min, 24h, and 1000 days - ok: bool = False - # TODO: only run this in backtesting mode, not live or testnet - # in live/testnet, go directly to binance. - response = requests.get( - self.klines_caching_service_url - + f"?symbol={coin.symbol}" - + f"&date={coin.date}" - + f"&mode={self.mode}" - + f"&debug={self.debug}" - ) - data = response.json() + if self.mode in ["testnet", "live"]: + data = self.fetch_klines_from_binance(coin.symbol, coin.date) + else: + response: requests.Response = requests.get( + self.klines_caching_service_url + + f"?symbol={coin.symbol}" + + f"&date={coin.date}" + + f"&mode={self.mode}" + + f"&debug={self.debug}" + ) + data = response.json() # TODO: rework this if data: ok = True @@ -1863,7 +1866,7 @@ def load_klines_for_coin(self, coin) -> bool: coin.highest = data["highest"] # trim values - unit_values = { + unit_values: dict = { "m": (60, 1), "h": (24, 60), # for 'Days' we retrieve 1000 days, binance API default @@ -1881,6 +1884,144 @@ def load_klines_for_coin(self, coin) -> bool: return ok + def fetch_klines_from_binance(self, symbol: str, date: int) -> dict: + """fetches from binance""" + + # when we initialise a coin, we pull a bunch of klines from binance. + # we pull klines for the last 60min, the last 24h, and the last 1000days + + # url to pull klines data from + api_url: str = ( + f"https://api.binance.com/api/v3/klines?symbol={symbol}&" + ) + + # build a dict to allows to calculate how far back in h,m,d we are going + # to pull klines data from + unit_values: dict = { + "m": (60, 1), + "h": (24, 60), + # for 'Days' we retrieve 1000 days, binance API default + "d": (1000, 60 * 24), + } + + # build all the query strings we need to fetch data from binance + binance_query_strings: dict = {} + for unit in ["m", "h", "d"]: + + # lets find out the from what date we need to pull klines from while in + # backtesting mode. + timeslice, minutes_before_now = unit_values[unit] + + backtest_end_time = date + end_unix_time: int = int( + (backtest_end_time - (60 * minutes_before_now)) * 1000 + ) + + query: str = f"{api_url}endTime={end_unix_time}&interval=1{unit}" + binance_query_strings[unit] = query + + # now we need to initialize a temp buckets{} with the + # lowest[], averages[], highest[] + buckets: dict = {} + for bucket in ["lowest", "averages", "highest"]: + buckets[bucket] = {} + for unit in ["m", "h", "d", "s"]: + buckets[bucket][unit] = [] + + # now we need to query binance and populate our buckets dict + for unit in ["m", "h", "d"]: + + # the call binance for list of klines for our loop var + # unit ('m', 'm', 'd') + ok, klines = self.call_binance_for_klines( + binance_query_strings[unit] + ) + if ok: + # and get a dict with the lowest, averages, highest lists from those + # binance raw klines + ok, low_avg_high = self.populate_values(klines, unit) + + if ok: + # we should now have a new dict containing list of our + # lowest, averages, highest values in low_avg_high + for bucket in ["lowest", "averages", "highest"]: + buckets[bucket][unit] = low_avg_high[bucket] + # we need to trim our lists, so that we don't keep more + # values that we should, + # like storing the last 1000 minutes + # + # keep 60 minutes on our minutes bucket + # 24 hours in our hours bucket + timeslice, _ = unit_values[unit] + while len(buckets[bucket][unit]) > timeslice: + buckets[bucket][unit].pop() + return buckets + + def call_binance_for_klines(self, query): + """calls upstream binance and retrieves the klines for a coin""" + logging.info(f"calling binance on {query}") + response = requests_with_backoff(query) + if response.status_code == 400: + # 400 typically means binance has no klines for this coin + logging.warning(f"got a 400 from binance for {query}") + return (True, []) + return (True, response.json()) + + def process_klines_line(self, kline): + """returns date, low, avg, high from a kline""" + (_, _, high, low, _, _, closetime, _, _, _, _, _) = kline + + date = float(c_from_timestamp(closetime / 1000).timestamp()) + low = float(low) + high = float(high) + avg = (low + high) / 2 + + return date, low, avg, high + + def populate_values(self, klines, unit) -> Tuple: + """builds averages[], lowest[], highest[] out of klines""" + _lowest: list = [] + _averages: list = [] + _highest: list = [] + + # retrieve and calculate the lowest, highest, averages + # from the klines data. + # we need to transform the dates into consumable timestamps + # that work for our bot. + for line in klines: + date, low, avg, high = self.process_klines_line(line) + _lowest.append((date, low)) + _averages.append((date, avg)) + _highest.append((date, high)) + + # finally, populate all the data coin buckets + buckets: dict = {} + for metric in ["lowest", "averages", "highest"]: + buckets[metric] = [] + + unit_buckets: Dict[str, int] = { + "m": 60, + "h": 24, + # for 'Days' we retrieve 1000 days, binance API default + "d": 1000, + } + + timeslice: int = unit_buckets[unit] + # we gather all the data we collected and only populate + # the required number of records we require. + # this could possibly be optimized, but at the same time + # this only runs the once when we initialise a coin + for d, v in _lowest[-timeslice:]: + buckets["lowest"].append((d, v)) + + for d, v in _averages[-timeslice:]: + buckets["averages"].append((d, v)) + + for d, v in _highest[-timeslice:]: + buckets["highest"].append((d, v)) + + return (True, buckets) + def print_final_balance_report(self): """calculates and outputs final balance"""