commit e965dde0027517cf2c634dadb380daa0b5b2ffd5
Author: Gao,Yan <ygao@suse.com>
Date:   Wed Feb 11 15:48:44 2015 +0100

    Feature: fencing: cl#5134 - Support random fencing delay to avoid double fencing
    
    This prevents double fencing when using slow fencing devices such as sbd.
    A "pcmk_delay_max" parameter can be configured for a fencing resource to
    enable random delay for stonith actions and specify the maximum of
    random delay.

diff --git a/fencing/commands.c b/fencing/commands.c
index c193a9d..742b81e 100644
--- a/fencing/commands.c
+++ b/fencing/commands.c
@@ -81,8 +81,11 @@ typedef struct async_command_s {
     int pid;
     int fd_stdout;
     int options;
-    int default_timeout;
-    int timeout;
+    int default_timeout; /* seconds */
+    int timeout; /* seconds */
+
+    int start_delay; /* milliseconds */
+    int delay_id;
 
     char *op;
     char *origin;
@@ -128,6 +131,24 @@ is_action_required(const char *action, stonith_device_t *device)
 }
 
 static int
+get_action_delay_max(stonith_device_t * device, const char * action)
+{
+    const char *value = NULL;
+    int delay_max_ms = 0;
+
+    if (safe_str_neq(action, "off") && safe_str_neq(action, "reboot")) {
+        return 0;
+    }
+
+    value = g_hash_table_lookup(device->params, STONITH_ATTR_DELAY_MAX);
+    if (value) {
+       delay_max_ms = crm_get_msec(value);
+    }
+
+    return delay_max_ms;
+}
+
+static int
 get_action_timeout(stonith_device_t * device, const char *action, int default_timeout)
 {
     char buffer[512] = { 0, };
@@ -155,6 +176,11 @@ free_async_command(async_command_t * cmd)
     if (!cmd) {
         return;
     }
+
+    if (cmd->delay_id) {
+        g_source_remove(cmd->delay_id);
+    }
+
     cmd_list = g_list_remove(cmd_list, cmd);
 
     g_list_free_full(cmd->device_list, free);
@@ -222,8 +248,16 @@ stonith_device_execute(stonith_device_t * device)
     if (device->pending_ops) {
         GList *first = device->pending_ops;
 
-        device->pending_ops = g_list_remove_link(device->pending_ops, first);
         cmd = first->data;
+        if (cmd && cmd->delay_id) {
+            crm_trace
+                ("Operation %s%s%s on %s was asked to run too early, waiting for start_delay timeout of %dms",
+                 cmd->action, cmd->victim ? " for node " : "", cmd->victim ? cmd->victim : "",
+                 device->id, cmd->start_delay);
+            return TRUE;
+        }
+
+        device->pending_ops = g_list_remove_link(device->pending_ops, first);
         g_list_free_1(first);
     }
 
@@ -301,9 +335,27 @@ stonith_device_dispatch(gpointer user_data)
     return stonith_device_execute(user_data);
 }
 
+static gboolean
+start_delay_helper(gpointer data)
+{
+    async_command_t *cmd = data;
+    stonith_device_t *device = NULL;
+
+    cmd->delay_id = 0;
+    device = cmd->device ? g_hash_table_lookup(device_list, cmd->device) : NULL;
+
+    if (device) {
+        mainloop_set_trigger(device->work);
+    }
+
+    return FALSE;
+}
+
 static void
 schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
 {
+    int delay_max = 0;
+
     CRM_CHECK(cmd != NULL, return);
     CRM_CHECK(device != NULL, return);
 
@@ -330,6 +382,14 @@ schedule_stonith_command(async_command_t * cmd, stonith_device_t * device)
 
     device->pending_ops = g_list_append(device->pending_ops, cmd);
     mainloop_set_trigger(device->work);
+
+    delay_max = get_action_delay_max(device, cmd->action);
+    if (delay_max > 0) {
+        cmd->start_delay = rand() % delay_max;
+        crm_notice("Delaying %s on %s for %lldms (timeout=%ds)",
+                    cmd->action, device->id, cmd->start_delay, cmd->timeout);
+        cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
+    }
 }
 
 void
@@ -1333,6 +1393,7 @@ stonith_query_capable_device_cb(GList * devices, void *user_data)
     for (lpc = devices; lpc != NULL; lpc = lpc->next) {
         stonith_device_t *device = g_hash_table_lookup(device_list, lpc->data);
         int action_specific_timeout;
+        int delay_max;
 
         if (!device) {
             /* It is possible the device got unregistered while
@@ -1354,6 +1415,12 @@ stonith_query_capable_device_cb(GList * devices, void *user_data)
         if (action_specific_timeout) {
             crm_xml_add_int(dev, F_STONITH_ACTION_TIMEOUT, action_specific_timeout);
         }
+
+        delay_max = get_action_delay_max(device, query->action);
+        if (delay_max > 0) {
+            crm_xml_add_int(dev, F_STONITH_DELAY_MAX, delay_max / 1000);
+        }
+
         if (query->target == NULL) {
             xmlNode *attrs = create_xml_node(dev, XML_TAG_ATTRS);
 
diff --git a/fencing/main.c b/fencing/main.c
index 70b5bde..5249664 100644
--- a/fencing/main.c
+++ b/fencing/main.c
@@ -1294,6 +1294,15 @@ main(int argc, char **argv)
         printf("    <content type=\"string\" default=\"dynamic-list\"/>\n");
         printf("  </parameter>\n");
 
+        printf("  <parameter name=\"%s\" unique=\"0\">\n", STONITH_ATTR_DELAY_MAX);
+        printf
+            ("    <shortdesc lang=\"en\">Enable random delay for stonith actions and specify the maximum of random delay</shortdesc>\n");
+        printf
+            ("    <longdesc lang=\"en\">This prevents double fencing when using slow devices such as sbd.\n"
+             "Use this to enable random delay for stonith actions and specify the maximum of random delay.</longdesc>\n");
+        printf("    <content type=\"time\" default=\"0s\"/>\n");
+        printf("  </parameter>\n");
+
         for (lpc = 0; lpc < DIMOF(actions); lpc++) {
             printf("  <parameter name=\"pcmk_%s_action\" unique=\"0\">\n", actions[lpc]);
             printf
diff --git a/fencing/remote.c b/fencing/remote.c
index 63c0274..8331cec 100644
--- a/fencing/remote.c
+++ b/fencing/remote.c
@@ -53,6 +53,7 @@ typedef struct st_query_result_s {
     gboolean tried;
     GListPtr device_list;
     GHashTable *custom_action_timeouts;
+    GHashTable *delay_maxes;
     /* Subset of devices that peer has verified connectivity on */
     GHashTable *verified_devices;
 
@@ -84,6 +85,7 @@ free_remote_query(gpointer data)
         free(query->host);
         g_list_free_full(query->device_list, free);
         g_hash_table_destroy(query->custom_action_timeouts);
+        g_hash_table_destroy(query->delay_maxes);
         g_hash_table_destroy(query->verified_devices);
         free(query);
     }
@@ -879,14 +881,20 @@ static int
 get_device_timeout(st_query_result_t * peer, const char *device, int default_timeout)
 {
     gpointer res;
+    int delay_max = 0;
 
     if (!peer || !device) {
         return default_timeout;
     }
 
+    res = g_hash_table_lookup(peer->delay_maxes, device);
+    if (res && GPOINTER_TO_INT(res) > 0) {
+        delay_max = GPOINTER_TO_INT(res);
+    }
+
     res = g_hash_table_lookup(peer->custom_action_timeouts, device);
 
-    return res ? GPOINTER_TO_INT(res) : default_timeout;
+    return res ? GPOINTER_TO_INT(res) + delay_max : default_timeout + delay_max;
 }
 
 static int
@@ -1244,17 +1252,20 @@ process_remote_stonith_query(xmlNode * msg)
     result->host = strdup(host);
     result->devices = devices;
     result->custom_action_timeouts = g_hash_table_new_full(crm_str_hash, g_str_equal, free, NULL);
+    result->delay_maxes = g_hash_table_new_full(crm_str_hash, g_str_equal, free, NULL);
     result->verified_devices = g_hash_table_new_full(crm_str_hash, g_str_equal, free, NULL);
 
     for (child = __xml_first_child(dev); child != NULL; child = __xml_next(child)) {
         const char *device = ID(child);
         int action_timeout = 0;
+        int delay_max = 0;
         int verified = 0;
         int required = 0;
 
         if (device) {
             result->device_list = g_list_prepend(result->device_list, strdup(device));
             crm_element_value_int(child, F_STONITH_ACTION_TIMEOUT, &action_timeout);
+            crm_element_value_int(child, F_STONITH_DELAY_MAX, &delay_max);
             crm_element_value_int(child, F_STONITH_DEVICE_VERIFIED, &verified);
             crm_element_value_int(child, F_STONITH_DEVICE_REQUIRED, &required);
             if (action_timeout) {
@@ -1263,6 +1274,12 @@ process_remote_stonith_query(xmlNode * msg)
                 g_hash_table_insert(result->custom_action_timeouts,
                                     strdup(device), GINT_TO_POINTER(action_timeout));
             }
+            if (delay_max > 0) {
+                crm_trace("Peer %s with device %s returned maximum of random delay %d",
+                          result->host, device, delay_max);
+                g_hash_table_insert(result->delay_maxes,
+                                    strdup(device), GINT_TO_POINTER(delay_max));
+            }
             if (verified) {
                 crm_trace("Peer %s has confirmed a verified device %s", result->host, device);
                 g_hash_table_insert(result->verified_devices,
diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h
index 6d9eb12..7f00cfd 100644
--- a/include/crm/fencing/internal.h
+++ b/include/crm/fencing/internal.h
@@ -63,6 +63,8 @@ xmlNode *create_device_registration_xml(const char *id, const char *namespace, c
 #  define F_STONITH_TOLERANCE     "st_tolerance"
 /*! Action specific timeout period returned in query of fencing devices. */
 #  define F_STONITH_ACTION_TIMEOUT       "st_action_timeout"
+/*! Maximum of random fencing delay for a device */
+#  define F_STONITH_DELAY_MAX            "st_delay_max"
 /*! Has this device been verified using a monitor type
  *  operation (monitor, list, status) */
 #  define F_STONITH_DEVICE_VERIFIED   "st_monitor_verified"
@@ -103,6 +105,7 @@ xmlNode *create_device_registration_xml(const char *id, const char *namespace, c
 #  define STONITH_ATTR_HOSTMAP   "pcmk_host_map"
 #  define STONITH_ATTR_HOSTLIST  "pcmk_host_list"
 #  define STONITH_ATTR_HOSTCHECK "pcmk_host_check"
+#  define STONITH_ATTR_DELAY_MAX "pcmk_delay_max"
 
 #  define STONITH_ATTR_ACTION_OP   "action"
 
