Skip to content

Commit

Permalink
fix how delayed events are sent/resolved
Browse files Browse the repository at this point in the history
  • Loading branch information
ayazabbas committed Jun 13, 2024
1 parent d57a393 commit cca6317
Showing 1 changed file with 14 additions and 11 deletions.
25 changes: 14 additions & 11 deletions pyth_observer/dispatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,11 +89,10 @@ async def run(self, states: List[State]):
"failures": 1,
"last_window_failures": None,
"sent": False,
"event_type": event_type,
}
else:
alert["failures"] += 1
self.delayed_events[alert_identifier] = event
self.delayed_events[f"{event_type}-{alert_identifier}"] = event
continue # Skip sending immediately for ZendutyEvent or TelegramEvent

sent_events.append(event.send())
Expand Down Expand Up @@ -188,10 +187,7 @@ async def process_zenduty_events(self, current_time):
):
logger.debug(f"Resolving Zenduty alert {identifier}")
resolved = True
if (
info["sent"]
and info.get("event_type", "ZendutyEvent") == "ZendutyEvent"
):
if info["sent"]:
response = await send_zenduty_alert(
identifier, identifier, resolved=True
)
Expand All @@ -212,16 +208,23 @@ async def process_zenduty_events(self, current_time):
logger.debug(f"Raising Zenduty alert {identifier}")
self.open_alerts[identifier]["sent"] = True
self.open_alerts[identifier]["last_alert"] = current_time.isoformat()
event = self.delayed_events.get(identifier)
if event:
to_alert.append(event.send())
for event_type in ["ZendutyEvent", "TelegramEvent"]:
key = f"{event_type}-{identifier}"
event = self.delayed_events.get(key)
if event:
to_alert.append(event.send())

# Send the alerts that were delayed due to thresholds
await asyncio.gather(*to_alert)

# Remove alerts that have been resolved
for identifier in to_remove:
if self.open_alerts.get(identifier):
del self.open_alerts[identifier]
if self.delayed_events.get(identifier):
del self.delayed_events[identifier]
for event_type in ["ZendutyEvent", "TelegramEvent"]:
key = f"{event_type}-{identifier}"
if self.delayed_events.get(key):
del self.delayed_events[key]

with open(self.open_alerts_file, "w") as file:
json.dump(self.open_alerts, file)

0 comments on commit cca6317

Please sign in to comment.