From 5141b03fe7bffd493cd26c20412bbcf3041fa495 Mon Sep 17 00:00:00 2001
From: Klaus Wenninger <klaus.wenninger@aon.at>
Date: Mon, 8 Jul 2019 21:15:51 +0200
Subject: [PATCH 7/8] Fix: fence-history: resync fence-history after stonithd
 crash

Setting up a 30s fallback timer to trigger history-sync if the
sync via DC doesn't happen
---
 crmd/callbacks.c |  2 +-
 crmd/control.c   |  9 +++++-
 crmd/te_utils.c  | 78 +++++++++++++++++++++++++++++++++++++++++-------
 crmd/tengine.h   |  3 +-
 4 files changed, 79 insertions(+), 13 deletions(-)

diff --git a/crmd/callbacks.c b/crmd/callbacks.c
index c51b2152e..7560470b8 100644
--- a/crmd/callbacks.c
+++ b/crmd/callbacks.c
@@ -212,7 +212,7 @@ peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *d
                 } else {
                     crm_info("New peer %s we want to sync fence history with",
                              node->uname);
-                    te_trigger_stonith_history_sync();
+                    te_trigger_stonith_history_sync(FALSE);
                 }
             }
             break;
diff --git a/crmd/control.c b/crmd/control.c
index 488ea8801..04935c729 100644
--- a/crmd/control.c
+++ b/crmd/control.c
@@ -192,7 +192,12 @@ do_shutdown(long long action,
         clear_bit(fsa_input_register, R_ST_REQUIRED);
 
         crm_info("Disconnecting STONITH...");
-        stonith_api->cmds->disconnect(stonith_api);
+        if (stonith_api->state != stonith_disconnected) {
+            stonith_api->cmds->disconnect(stonith_api);
+        }
+        stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
+        stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_FENCE);
+        stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED);
     }
 }
 
@@ -369,6 +374,8 @@ crmd_exit(int rc)
     crm_timer_stop(wait_timer);
     crm_timer_stop(recheck_timer);
 
+    te_cleanup_stonith_history_sync(NULL, TRUE);
+
     free(transition_timer); transition_timer = NULL;
     free(integration_timer); integration_timer = NULL;
     free(finalization_timer); finalization_timer = NULL;
diff --git a/crmd/te_utils.c b/crmd/te_utils.c
index 2805ea9cb..29b411fcc 100644
--- a/crmd/te_utils.c
+++ b/crmd/te_utils.c
@@ -33,7 +33,33 @@
 
 crm_trigger_t *stonith_reconnect = NULL;
 static crm_trigger_t *stonith_history_sync_trigger = NULL;
-static mainloop_timer_t *stonith_history_sync_timer = NULL;
+static mainloop_timer_t *stonith_history_sync_timer_short = NULL;
+static mainloop_timer_t *stonith_history_sync_timer_long = NULL;
+
+void
+te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers)
+{
+    if (free_timers) {
+        mainloop_timer_del(stonith_history_sync_timer_short);
+        stonith_history_sync_timer_short = NULL;
+        mainloop_timer_del(stonith_history_sync_timer_long);
+        stonith_history_sync_timer_long = NULL;
+    } else {
+        mainloop_timer_stop(stonith_history_sync_timer_short);
+        mainloop_timer_stop(stonith_history_sync_timer_long);
+    }
+
+    if (st) {
+        st->cmds->remove_notification(st, T_STONITH_NOTIFY_HISTORY_SYNCED);
+    }
+}
+
+static void
+tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event)
+{
+    te_cleanup_stonith_history_sync(st, FALSE);
+    crm_debug("Fence-history synced - cancel all timers");
+}
 
 /*
  * stonith cleanup list
@@ -174,6 +200,8 @@ fail_incompletable_stonith(crm_graph_t * graph)
 static void
 tengine_stonith_connection_destroy(stonith_t * st, stonith_event_t * e)
 {
+    te_cleanup_stonith_history_sync(st, FALSE);
+
     if (is_set(fsa_input_register, R_ST_REQUIRED)) {
         crm_crit("Fencing daemon connection failed");
         mainloop_set_trigger(stonith_reconnect);
@@ -187,11 +215,12 @@ tengine_stonith_connection_destroy(stonith_t * st, stonith_event_t * e)
         /* the client API won't properly reconnect notifications
          * if they are still in the table - so remove them
          */
-        stonith_api->cmds->remove_notification(st, T_STONITH_NOTIFY_DISCONNECT);
-        stonith_api->cmds->remove_notification(st, T_STONITH_NOTIFY_FENCE);
         if (stonith_api->state != stonith_disconnected) {
             stonith_api->cmds->disconnect(st);
         }
+        stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_DISCONNECT);
+        stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_FENCE);
+        stonith_api->cmds->remove_notification(stonith_api, T_STONITH_NOTIFY_HISTORY_SYNCED);
     }
 
     if (AM_I_DC) {
@@ -211,6 +240,9 @@ char *te_client_id = NULL;
 #  include <sys/reboot.h>
 #endif
 
+static void
+tengine_stonith_history_synced(stonith_t *st, stonith_event_t *st_event);
+
 static void
 tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event)
 {
@@ -404,6 +436,7 @@ do_stonith_history_sync(gpointer user_data)
     if (stonith_api && (stonith_api->state != stonith_disconnected)) {
         stonith_history_t *history = NULL;
 
+        te_cleanup_stonith_history_sync(stonith_api, FALSE);
         stonith_api->cmds->history(stonith_api,
                                    st_opt_sync_call | st_opt_broadcast,
                                    NULL, &history, 5);
@@ -423,11 +456,18 @@ stonith_history_sync_set_trigger(gpointer user_data)
 }
 
 void
-te_trigger_stonith_history_sync(void)
+te_trigger_stonith_history_sync(bool long_timeout)
 {
     /* trigger a sync in 5s to give more nodes the
      * chance to show up so that we don't create
      * unnecessary stonith-history-sync traffic
+     *
+     * the long timeout of 30s is there as a fallback
+     * so that after a successful connection to fenced
+     * we will wait for 30s for the DC to trigger a
+     * history-sync
+     * if this doesn't happen we trigger a sync locally
+     * (e.g. fenced segfaults and is restarted by pacemakerd)
      */
 
     /* as we are finally checking the stonith-connection
@@ -441,13 +481,25 @@ te_trigger_stonith_history_sync(void)
                                  do_stonith_history_sync, NULL);
     }
 
-    if(stonith_history_sync_timer == NULL) {
-        stonith_history_sync_timer =
-            mainloop_timer_add("history_sync", 5000,
-                               FALSE, stonith_history_sync_set_trigger,
-                               NULL);
+    if (long_timeout) {
+        if(stonith_history_sync_timer_long == NULL) {
+            stonith_history_sync_timer_long =
+                mainloop_timer_add("history_sync_long", 30000,
+                                   FALSE, stonith_history_sync_set_trigger,
+                                   NULL);
+        }
+        crm_info("Fence history will be synchronized cluster-wide within 30 seconds");
+        mainloop_timer_start(stonith_history_sync_timer_long);
+    } else {
+        if(stonith_history_sync_timer_short == NULL) {
+            stonith_history_sync_timer_short =
+                mainloop_timer_add("history_sync_short", 5000,
+                                   FALSE, stonith_history_sync_set_trigger,
+                                   NULL);
+        }
+        crm_info("Fence history will be synchronized cluster-wide within 5 seconds");
+        mainloop_timer_start(stonith_history_sync_timer_short);
     }
-    mainloop_timer_start(stonith_history_sync_timer);
 }
 
 gboolean
@@ -496,6 +548,12 @@ te_connect_stonith(gpointer user_data)
     stonith_api->cmds->register_notification(stonith_api, T_STONITH_NOTIFY_FENCE,
                                              tengine_stonith_notify);
 
+    stonith_api->cmds->register_notification(stonith_api,
+                                             T_STONITH_NOTIFY_HISTORY_SYNCED,
+                                             tengine_stonith_history_synced);
+
+    te_trigger_stonith_history_sync(TRUE);
+
     crm_notice("Fencer successfully connected");
     return TRUE;
 }
diff --git a/crmd/tengine.h b/crmd/tengine.h
index 1a9b2d294..a20760ce2 100644
--- a/crmd/tengine.h
+++ b/crmd/tengine.h
@@ -72,7 +72,8 @@ extern void abort_transition_graph(int abort_priority, enum transition_action ab
 
 extern gboolean te_connect_stonith(gpointer user_data);
 
-extern void te_trigger_stonith_history_sync(void);
+extern void te_trigger_stonith_history_sync(bool long_timeout);
+extern void te_cleanup_stonith_history_sync(stonith_t *st, bool free_timers);
 
 extern crm_trigger_t *transition_trigger;
 extern crm_trigger_t *stonith_reconnect;
-- 
2.26.2

