diff --git a/etc/slurm-mail/slurm-mail.conf b/etc/slurm-mail/slurm-mail.conf index e7a7e70..0d57fcf 100644 --- a/etc/slurm-mail/slurm-mail.conf +++ b/etc/slurm-mail/slurm-mail.conf @@ -28,3 +28,4 @@ smtpUserName = smtpPassword = tailExe = /usr/bin/tail includeOutputLines = 0 +retryOnFailure = 1 \ No newline at end of file diff --git a/src/slurmmail/cli.py b/src/slurmmail/cli.py index 87713a5..5267a2a 100644 --- a/src/slurmmail/cli.py +++ b/src/slurmmail/cli.py @@ -93,6 +93,7 @@ def __init__(self) -> None: self.tail_lines: int self.html_templates: Dict[str, pathlib.Path] self.text_templates: Dict[str, pathlib.Path] + self.retry_on_failure: bool = True def get_scontrol_values(input_str: str) -> Dict[str, str]: @@ -732,9 +733,22 @@ def __process_spool_file( options.smtp_port, ) - smtp_conn.sendmail( - options.email_from_address, user_email.split(","), msg.as_string() - ) + try: + smtp_conn.sendmail( + options.email_from_address, user_email.split(","), msg.as_string() + ) + except ( + smtplib.SMTPHeloError, + smtplib.SMTPRecipientsRefused, + smtplib.SMTPSenderRefused, + smtplib.SMTPNotSupportedError + ) as e: + logging.error("Failed to send e-mail: %s", e) + if options.retry_on_failure: + # Raise the original exception to prevent `delete_spool_file` from being + # executed at the end of the function. This way the spool file will be + # retried on the next run. + raise e delete_spool_file(json_file) @@ -840,6 +854,7 @@ def send_mail_main(): smtp_password = config.get(section, "smtpPassword") options.tail_exe = pathlib.Path(config.get(section, "tailExe")) options.tail_lines = config.getint(section, "includeOutputLines") + options.retry_on_failure = config.getboolean(section, "retryOnFailure") if config.has_option(section, "emailRegEx"): options.mail_regex = config.get(section, "emailRegEx")