reinitialize pwenc in vrrp_backup (#15208)

(cherry picked from commit dff2676)

Co-authored-by: Caleb <[email protected]>

src/middlewared/middlewared/plugins/failover_/event.py

@@ -20,6 +20,7 @@
 # from middlewared.plugins.failover_.zpool_cachefile import ZPOOL_CACHE_FILE
 from middlewared.plugins.failover_.event_exceptions import AllZpoolsFailedToImport, IgnoreFailoverEvent, FencedError
 from middlewared.plugins.failover_.scheduled_reboot_alert import WATCHDOG_ALERT_FILE
+from middlewared.plugins.pwenc import PWENC_FILE_SECRET
 
 logger = logging.getLogger('failover')
 FAILOVER_LOCK_NAME = 'vrrp_event'
@@ -901,6 +902,18 @@ def vrrp_backup(self, job, fobj, ifname, event):
         logger.warning('Stopping fenced')
         self.run_call('failover.fenced.stop')
 
+        # In the rare case where the pwenc_secret file doesn't match, we'll
+        # copy over the secret seed file from the active and reinitialize
+        try:
+            self.run_call(
+                'failover.call_remote',
+                'failover.send_small_file',
+                [PWENC_FILE_SECRET],
+                {'raise_connect_error': False}
+            )
+        except Exception:
+            self.logger.error('Failed to reinitialize pwenc', exc_info=True)
+
         # Now that fenced is stopped, attach NVMe/RoCE.
         logger.info('Start bring up of NVMe/RoCE')
         try: