commit 62ed004d828ad9b4e3592b684e7d0965b7046965
Author: Andrew Beekhof <andrew@beekhof.net>
Date:   Fri Mar 11 13:22:41 2016 +1100

    Fix: PE: Correctly handle the ordering of demote actions for failed and moving containers

Index: pacemaker/pengine/allocate.c
===================================================================
--- pacemaker.orig/pengine/allocate.c
+++ pacemaker/pengine/allocate.c
@@ -1727,16 +1727,18 @@ apply_remote_node_ordering(pe_working_se
                     NULL,
                     pe_order_preserve | pe_order_implies_first,
                     data_set);
-            } else {
+            }
 
-                custom_action_order(remote_rsc,
-                    generate_op_key(remote_rsc->id, RSC_START, 0),
-                    NULL,
-                    action->rsc,
-                    NULL,
-                    action,
-                    pe_order_preserve | pe_order_implies_then | pe_order_runnable_left,
-                    data_set);
+            if(container && is_set(container->flags, pe_rsc_failed)) {
+                /* Just like a stop, the demote is implied by the
+                 * container having failed/stopped
+                 *
+                 * If we really wanted to we would order the demote
+                 * after the stop, IFF the containers current role was
+                 * stopped (otherwise we re-introduce an ordering
+                 * loop)
+                 */
+                pe_set_action_bit(action, pe_action_pseudo);
             }
 
         } else if (safe_str_eq(action->task, "stop") &&
Index: pacemaker/pengine/test10/whitebox-ms-ordering.dot
===================================================================
--- pacemaker.orig/pengine/test10/whitebox-ms-ordering.dot
+++ pacemaker/pengine/test10/whitebox-ms-ordering.dot
@@ -42,7 +42,7 @@
 "lxc-ms_demote_0 lxc1" -> "lxc-ms-master_demoted_0" [ style = bold]
 "lxc-ms_demote_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold]
 "lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold]
-"lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="black"]
+"lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="orange"]
 "lxc-ms_monitor_0 18node1" -> "probe_complete 18node1" [ style = bold]
 "lxc-ms_monitor_0 18node1" [ style=bold color="green" fontcolor="black"]
 "lxc-ms_monitor_0 18node2" -> "probe_complete 18node2" [ style = bold]
@@ -67,7 +67,6 @@
 "lxc-ms_stop_0 lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold]
 "lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"]
 "lxc1_monitor_30000 18node1" [ style=bold color="green" fontcolor="black"]
-"lxc1_start_0 18node1" -> "lxc-ms_demote_0 lxc1" [ style = bold]
 "lxc1_start_0 18node1" -> "lxc-ms_promote_0 lxc1" [ style = bold]
 "lxc1_start_0 18node1" -> "lxc-ms_start_0 lxc1" [ style = bold]
 "lxc1_start_0 18node1" -> "lxc1_monitor_30000 18node1" [ style = bold]
Index: pacemaker/pengine/test10/whitebox-ms-ordering.summary
===================================================================
--- pacemaker.orig/pengine/test10/whitebox-ms-ordering.summary
+++ pacemaker/pengine/test10/whitebox-ms-ordering.summary
@@ -31,17 +31,17 @@ Executing cluster transition:
  * Pseudo action:   probe_complete
  * Resource action: container1      start on 18node1
  * Resource action: container2      start on 18node1
- * Resource action: lxc1            start on 18node1
- * Resource action: lxc2            start on 18node1
- * Resource action: lxc-ms          demote on lxc1
+ * Pseudo action:   lxc-ms_demote_0
  * Pseudo action:   lxc-ms-master_demoted_0
  * Pseudo action:   lxc-ms-master_stop_0
- * Resource action: lxc1            monitor=30000 on 18node1
- * Resource action: lxc2            monitor=30000 on 18node1
+ * Resource action: lxc1            start on 18node1
+ * Resource action: lxc2            start on 18node1
  * Pseudo action:   lxc-ms_stop_0
  * Pseudo action:   lxc-ms_stop_0
  * Pseudo action:   lxc-ms-master_stopped_0
  * Pseudo action:   lxc-ms-master_start_0
+ * Resource action: lxc1            monitor=30000 on 18node1
+ * Resource action: lxc2            monitor=30000 on 18node1
  * Pseudo action:   all_stopped
  * Resource action: lxc-ms          start on lxc1
  * Resource action: lxc-ms          start on lxc2
Index: pacemaker/pengine/test10/whitebox-ms-ordering.exp
===================================================================
--- pacemaker.orig/pengine/test10/whitebox-ms-ordering.exp
+++ pacemaker/pengine/test10/whitebox-ms-ordering.exp
@@ -88,7 +88,7 @@
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="21" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="18node1"/>
+        <pseudo_event id="21" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0"/>
       </trigger>
       <trigger>
         <rsc_op id="23" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:0_start_0" on_node="lxc1" on_node_uuid="lxc1" router_node="18node1"/>
@@ -134,7 +134,7 @@
         <pseudo_event id="3" operation="probe_complete" operation_key="probe_complete"/>
       </trigger>
       <trigger>
-        <rsc_op id="21" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="18node1"/>
+        <pseudo_event id="21" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0"/>
       </trigger>
       <trigger>
         <pseudo_event id="30" operation="stop" operation_key="lxc-ms-master_stop_0"/>
@@ -143,18 +143,14 @@
   </synapse>
   <synapse id="11">
     <action_set>
-      <rsc_op id="21" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="18node1">
-        <primitive id="lxc-ms" long-id="lxc-ms:0" class="ocf" provider="pacemaker" type="Stateful"/>
+      <pseudo_event id="21" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0">
         <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
-      </rsc_op>
+      </pseudo_event>
     </action_set>
     <inputs>
       <trigger>
         <pseudo_event id="34" operation="demote" operation_key="lxc-ms-master_demote_0"/>
       </trigger>
-      <trigger>
-        <rsc_op id="36" operation="start" operation_key="lxc1_start_0" on_node="18node1" on_node_uuid="1"/>
-      </trigger>
     </inputs>
   </synapse>
   <synapse id="12">
@@ -245,7 +241,7 @@
     </action_set>
     <inputs>
       <trigger>
-        <rsc_op id="21" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="18node1"/>
+        <pseudo_event id="21" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0"/>
       </trigger>
       <trigger>
         <pseudo_event id="34" operation="demote" operation_key="lxc-ms-master_demote_0"/>
