From 870aaeb777cfe10d34327197064dc12e3c15f321 Mon Sep 17 00:00:00 2001
From: Azulinho <mail@azulinho.com>
Date: Thu, 16 Mar 2023 18:28:10 +0000
Subject: [PATCH] retry get_index_json calls

---
 lib/helpers.py               | 23 ++++++++++++++++-------
 run                          | 35 ++++++++++++++++++++++++-----------
 tests/prove-backtesting.yaml |  8 ++++----
 utils/prove-backtesting.py   | 18 ++++++++++++++++--
 utils/prove-backtesting.sh   |  1 -
 5 files changed, 60 insertions(+), 25 deletions(-)

diff --git a/lib/helpers.py b/lib/helpers.py
index 55c6080..a09fb40 100644
--- a/lib/helpers.py
+++ b/lib/helpers.py
@@ -12,7 +12,10 @@
 from binance.client import Client
 from filelock import SoftFileLock
 from pyrate_limiter import Duration, Limiter, RequestRate
-from tenacity import retry, wait_exponential
+from tenacity import (
+    retry,
+    wait_exponential,
+)
 
 rate = RequestRate(600, Duration.MINUTE)  # 600 requests per minute
 limiter = Limiter(rate)
@@ -69,16 +72,22 @@ def requests_with_backoff(query: str):
     return response
 
 
-@retry(wait=wait_exponential(multiplier=1, max=3))
 def get_price_log(query: str):
     """retry wrapper for requests calls"""
-    response = requests.get(query, timeout=5)
-    status = response.status_code
-    if status != 200:
+
+    for w in [1, 2, 3]:
+        response: requests.Response = requests.get(query, timeout=5)
+        status: int = response.status_code
+        if status == 200:
+            return response
+
         with open("log/price_log_service.response.log", "at") as f:
             f.write(f"{query} {status} {response}\n")
-        response.raise_for_status()
-    return response
+        if status == 404:
+            response.raise_for_status()
+        sleep(6 * w)
+    response.raise_for_status()
+    return None
 
 
 def cached_binance_client(access_key: str, secret_key: str) -> Client:
diff --git a/run b/run
index 3c6858c..9d65f38 100755
--- a/run
+++ b/run
@@ -308,6 +308,20 @@ function github_actions_ci_pr_docker_tests() {
 	cp tests/fake.yaml secrets/fake.yaml
 	cp tests/price.log.gz log/tests.price.log.gz
 
+	mkdir -p log/ETHUSDT
+	mkdir -p log/BTCUSDT
+	for ta in 01 02 03 04 05 06 07 08 09
+	do
+		cat tests/price.log.gz | grep 2021-12-${ta}  |gzip -1 > log/202112${ta}.log.gz
+		cat tests/price.log.gz | grep ETHUSDT | grep 2021-12-${ta}  |gzip -1 > log/ETHUSDT/202112${ta}.log.gz
+		cat tests/price.log.gz | grep BTCUSDT | grep 2021-12-${ta}  |gzip -1 > log/BTCUSDT/202112${ta}.log.gz
+	done
+	cp tests/index.json.gz log/
+
+	export PRICE_LOG_PORT=$( cat ${STATE_DIR}/.price_log_service.port)
+	touch log/test.log
+	curl http://${DOCKER_IP}:${PRICE_LOG_PORT}/test.log
+
 	echo BuyMoonSellRecoveryStrategy.yaml
 	cp tests/BuyMoonSellRecoveryStrategy.yaml configs/
 	./run backtesting CONFIG_FILE=BuyMoonSellRecoveryStrategy.yaml TAG=pr
@@ -344,22 +358,21 @@ function github_actions_ci_pr_docker_tests() {
 	grep 'wins:184 losses:4 stales:596 holds:0' results/backtesting.BuyOnRecoveryAfterDropFromAverageStrategy.yaml.txt
 
 	echo prove-backtesting
-	for ta in 01 02 03 04 05 06 07 08 09
-	do
-		cat tests/price.log.gz | grep 2021-12-${ta}  |gzip -1 > log/202112${ta}.log.gz
-	done
-	cp tests/index.json.gz log/
-
 	cp tests/prove-backtesting.yaml configs/
 	./run prove-backtesting \
 		TAG=pr CONFIG_FILE=prove-backtesting.yaml
 
-	md5sum ${CONFIG_DIR}/BuyDropSellRecoveryStrategy.yaml \
-		|cut -f1 -d " " | grep '8e4a7b58c6194d9eafba0d1006da0e7a'
+	wc -l results/prove-backtesting.prove-backtesting.yaml.txt \
+		| grep '29'
+
+	for ta in 01 02 03 04 05 06 07 08 09
+	do
+		rm -f log/202112${ta}.log.gz
+		rm -f log/ETHUSDT/202112${ta}.log.gz
+		rm -f log/BTCUSDT/202112${ta}.log.gz
+	done
+	rm -f log/index.json.gz
 
-	export PRICE_LOG_PORT=$( cat ${STATE_DIR}/.price_log_service.port)
-	touch log/test.log
-	curl http://${DOCKER_IP}:${PRICE_LOG_PORT}/test.log
   ./run down
 }
 
diff --git a/tests/prove-backtesting.yaml b/tests/prove-backtesting.yaml
index 0016362..79fa303 100644
--- a/tests/prove-backtesting.yaml
+++ b/tests/prove-backtesting.yaml
@@ -20,10 +20,10 @@ CONCURRENCY: 1
 MIN: 1
 FILTER_BY: "ETH"
 SORT_BY: "greed"
-FROM_DATE: 20211201
-END_DATE: 20211209
-ROLL_BACKWARDS: 3
-ROLL_FORWARD: 3
+FROM_DATE: 20211202
+END_DATE: 20211206
+ROLL_BACKWARDS: 1
+ROLL_FORWARD: 1
 STOP_BOT_ON_LOSS: false
 STOP_BOT_ON_STALE: false
 
diff --git a/utils/prove-backtesting.py b/utils/prove-backtesting.py
index b3bd4a1..d7ef6a4 100644
--- a/utils/prove-backtesting.py
+++ b/utils/prove-backtesting.py
@@ -19,6 +19,20 @@
 import requests
 import yaml
 
+from tenacity import retry, wait_exponential
+
+
+@retry(wait=wait_exponential(multiplier=2, min=1, max=30))
+def get_index_json(query: str) -> requests.Response:
+    """retry wrapper for requests calls"""
+    response: requests.Response = requests.get(query, timeout=5)
+    status: int = response.status_code
+    if status != 200:
+        with open("log/price_log_service.response.log", "at") as l:
+            l.write(f"{query} {status} {response}\n")
+        response.raise_for_status()
+    return response
+
 
 def log_msg(msg) -> None:
     """logs out message prefixed with timestamp"""
@@ -324,8 +338,8 @@ def write_optimized_strategy_config(
     def write_all_coin_configs(self, dates, thisrun):
         """generate all coinfiles"""
 
-        r: requests.Response = requests.get(
-            f"{self.price_log_service_url}/index.json.gz", timeout=30
+        r: requests.Response = get_index_json(
+            f"{self.price_log_service_url}/index.json.gz"
         )
         coinfiles: Any = json.loads(r.content)
 
diff --git a/utils/prove-backtesting.sh b/utils/prove-backtesting.sh
index 246efb8..d9a81f2 100755
--- a/utils/prove-backtesting.sh
+++ b/utils/prove-backtesting.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-set -x
 ulimit -n 65535
 source /cryptobot/.venv/bin/activate
 python -u /cryptobot/utils/prove-backtesting.py -c configs/${CONFIG_FILE}