feat: add abandoned_time_limit to PublisherStalledCheck

pyproject.toml

@@ -4,7 +4,7 @@ ignore_missing_imports = true
 
 [tool.poetry]
 name = "pyth-observer"
-version = "0.2.9"
+version = "0.2.10"
 description = "Alerts and stuff"
 authors = []
 readme = "README.md"

pyth_observer/check/publisher.py

@@ -221,7 +221,7 @@ def error_message(self) -> dict:
             "symbol": self.__state.symbol,
             "publisher_price": f"{self.__state.price} ± {self.__state.confidence_interval}",
             "aggregate_price": f"{self.__state.price_aggregate} ± {self.__state.confidence_interval_aggregate}",
-            "deviation": deviation,
+            "deviation": f"{deviation:.2f}%",
         }
 
     # Returns the distance between the aggregate price and the closest side of the publisher's confidence interval
@@ -237,6 +237,7 @@ def __init__(self, state: PublisherState, config: PublisherCheckConfig):
         self.__stall_time_limit: int = int(
             config["stall_time_limit"]
         )  # Time in seconds
+        self.__abandoned_time_limit: int = int(config["abandoned_time_limit"])
         self.__max_slot_distance: int = int(config["max_slot_distance"])
 
     def state(self) -> PublisherState:
@@ -258,7 +259,7 @@ def run(self) -> bool:
             return True
 
         publisher_key = (self.__state.publisher_name, self.__state.symbol)
-        current_time = time.time()
+        current_time = int(time.time())
         previous_price, last_change_time = PUBLISHER_CACHE.get(
             publisher_key, (None, None)
         )
@@ -267,7 +268,10 @@ def run(self) -> bool:
             PUBLISHER_CACHE[publisher_key] = (self.__state.price, current_time)
             return True
 
-        if (current_time - last_change_time) > self.__stall_time_limit:
+        time_since_last_change = current_time - last_change_time
+        if time_since_last_change > self.__stall_time_limit:
+            if time_since_last_change > self.__abandoned_time_limit:
+                return True  # Abandon this check after the abandoned time limit
             return False
 
         return True
@@ -279,8 +283,7 @@ def error_message(self) -> dict:
             "publisher": self.__state.publisher_name,
             "symbol": self.__state.symbol,
             "price": self.__state.price,
-            "stall_duration": time.time()
-            - PUBLISHER_CACHE[(self.__state.publisher_name, self.__state.symbol)][1],
+            "stall_duration": f"{int(time.time()) - PUBLISHER_CACHE[(self.__state.publisher_name, self.__state.symbol)][1]} seconds",
         }
 
 

pyth_observer/event.py

@@ -132,9 +132,8 @@ async def send(self):
 
             formatted_message = ""
             for key, value in text.items():
-                value_str = f"{value:.2f}%" if key == "deviation" else f"{value}"
                 formatted_message += (
-                    f"*{key.capitalize().replace('_', ' ')}:* {value_str}\n"
+                    f"*{key.capitalize().replace('_', ' ')}:* {value}\n"
                 )
 
             message_data = {

sample.config.yaml

@@ -10,7 +10,7 @@ events:
   # NOTE: Uncomment to enable Datadog metrics, see README.md for datadog credential docs.
   # - DatadogEvent
   - LogEvent
-  - TelegramEvent
+  # - TelegramEvent
 checks:
   global:
     # Price feed checks
@@ -46,7 +46,8 @@ checks:
       max_aggregate_distance: 6
     PublisherStalledCheck:
       enable: true
-      stall_time_limit: 60
+      stall_time_limit: 30000000 # arbitrary large number to "disable" this check while allowing per feed config to override
+      abandoned_time_limit: 30000000 # arbitrary large number to "disable" this check while allowing per feed config to override
       max_slot_distance: 25
   # Per-symbol config
   Crypto.MNGO/USD:
@@ -57,4 +58,5 @@ checks:
       max_slot_distance: 10000
   Crypto.BTC/USD:
     PublisherStalledCheck:
-      stall_time_limit: 10 # This will override the global stall_time_limit for Crypto.BTC/USD
+      stall_time_limit: 30 # This will override the global stall_time_limit for Crypto.BTC/USD
+      abandoned_time_limit: 600 # 10 minutes

tests/test_checks_publisher.py

@@ -63,11 +63,12 @@ def simulate_time_pass(seconds):
         current_time += seconds
         return current_time
 
-    def setup_check(state, stall_time_limit, max_slot_distance):
+    def setup_check(state, stall_time_limit, abandoned_time_limit, max_slot_distance):
         check = PublisherStalledCheck(
             state,
             {
                 "stall_time_limit": stall_time_limit,
+                "abandoned_time_limit": abandoned_time_limit,
                 "max_slot_distance": max_slot_distance,
             },
         )
@@ -83,17 +84,17 @@ def run_check(check, seconds, expected):
 
     PUBLISHER_CACHE.clear()
     state_a = make_state(1, 100.0, 2.0, 1, 100.0, 1.0)
-    check_a = setup_check(state_a, 5, 25)
+    check_a = setup_check(state_a, 5, 25, 25)
     run_check(check_a, 5, True)  # Should pass as it hits the limit exactly
 
     PUBLISHER_CACHE.clear()
     state_b = make_state(1, 100.0, 2.0, 1, 100.0, 1.0)
-    check_b = setup_check(state_b, 5, 25)
+    check_b = setup_check(state_b, 5, 25, 25)
     run_check(check_b, 6, False)  # Should fail as it exceeds the limit
 
     PUBLISHER_CACHE.clear()
     state_c = make_state(1, 100.0, 2.0, 1, 100.0, 1.0)
-    check_c = setup_check(state_c, 5, 25)
+    check_c = setup_check(state_c, 5, 25, 25)
     run_check(check_c, 2, True)  # Initial check should pass
     state_c.price = 105.0  # Change the price
     run_check(check_c, 3, True)  # Should pass as price changes
@@ -108,5 +109,5 @@ def run_check(check, seconds, expected):
     state_d = make_state(1, 100.0, 2.0, 1, 100.0, 1.0)
     state_d.latest_block_slot = 25
     state_d.slot = 0
-    check_d = setup_check(state_d, 5, 25)
+    check_d = setup_check(state_d, 5, 25, 25)
     run_check(check_d, 10, True)  # Should pass as the publisher is offline