commit 0c15dfd61dfedbaf0d3d25c28f38b5cbf0563a21
Author: Ken Gaillot <kgaillot@redhat.com>
Date:   Fri Apr 7 16:24:15 2017 -0500

    Test: pengine: add test for per-op fail count

Index: pacemaker/pengine/regression.sh
===================================================================
--- pacemaker.orig/pengine/regression.sh
+++ pacemaker/pengine/regression.sh
@@ -526,6 +526,7 @@ do_test bug-5025-3 "Automatically clean
 do_test bug-5025-4 "Clear failcount when last failure is a start op and rsc attributes changed."
 do_test failcount "Ensure failcounts are correctly expired"
 do_test failcount-block "Ensure failcounts are not expired when on-fail=block is present"
+do_test per-op-failcount "Ensure per-operation failcount is handled and not passed to fence agent"
 do_test monitor-onfail-restart "bug-5058 - Monitor failure with on-fail set to restart"
 do_test monitor-onfail-stop    "bug-5058 - Monitor failure wiht on-fail set to stop"
 do_test bug-5059 "No need to restart p_stateful1:*"
Index: pacemaker/pengine/test10/per-op-failcount.dot
===================================================================
--- /dev/null
+++ pacemaker/pengine/test10/per-op-failcount.dot
@@ -0,0 +1,19 @@
+digraph "g" {
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
+"prmDummy_monitor_10000 rh73-02-snmp" [ style=bold color="green" fontcolor="black"]
+"prmDummy_start_0 rh73-02-snmp" -> "prmDummy_monitor_10000 rh73-02-snmp" [ style = bold]
+"prmDummy_start_0 rh73-02-snmp" [ style=bold color="green" fontcolor="black"]
+"prmDummy_stop_0 rh73-01-snmp" -> "all_stopped" [ style = bold]
+"prmDummy_stop_0 rh73-01-snmp" -> "prmDummy_start_0 rh73-02-snmp" [ style = bold]
+"prmDummy_stop_0 rh73-01-snmp" [ style=bold color="green" fontcolor="orange"]
+"prmStonith2-1_start_0 rh73-02-snmp" [ style=bold color="green" fontcolor="black"]
+"prmStonith2-1_stop_0 rh73-01-snmp" -> "all_stopped" [ style = bold]
+"prmStonith2-1_stop_0 rh73-01-snmp" -> "prmStonith2-1_start_0 rh73-02-snmp" [ style = bold]
+"prmStonith2-1_stop_0 rh73-01-snmp" [ style=bold color="green" fontcolor="orange"]
+"stonith 'reboot' rh73-01-snmp" -> "prmDummy_stop_0 rh73-01-snmp" [ style = bold]
+"stonith 'reboot' rh73-01-snmp" -> "stonith_complete" [ style = bold]
+"stonith 'reboot' rh73-01-snmp" [ style=bold color="green" fontcolor="black"]
+"stonith_complete" -> "all_stopped" [ style = bold]
+"stonith_complete" -> "prmDummy_start_0 rh73-02-snmp" [ style = bold]
+"stonith_complete" [ style=bold color="green" fontcolor="orange"]
+}
Index: pacemaker/pengine/test10/per-op-failcount.exp
===================================================================
--- /dev/null
+++ pacemaker/pengine/test10/per-op-failcount.exp
@@ -0,0 +1,105 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
+  <synapse id="0">
+    <action_set>
+      <rsc_op id="5" operation="monitor" operation_key="prmDummy_monitor_10000" on_node="rh73-02-snmp" on_node_uuid="3232238266">
+        <primitive id="prmDummy" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_fail="restart" CRM_meta_on_node="rh73-02-snmp" CRM_meta_on_node_uuid="3232238266" CRM_meta_timeout="60000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="4" operation="start" operation_key="prmDummy_start_0" on_node="rh73-02-snmp" on_node_uuid="3232238266"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="1">
+    <action_set>
+      <rsc_op id="4" operation="start" operation_key="prmDummy_start_0" on_node="rh73-02-snmp" on_node_uuid="3232238266">
+        <primitive id="prmDummy" class="ocf" provider="pacemaker" type="Dummy"/>
+        <attributes CRM_meta_name="start" CRM_meta_on_fail="restart" CRM_meta_on_node="rh73-02-snmp" CRM_meta_on_node_uuid="3232238266" CRM_meta_timeout="60000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="3" operation="stop" operation_key="prmDummy_stop_0"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="10" operation="stonith_complete" operation_key="stonith_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="2">
+    <action_set>
+      <pseudo_event id="3" operation="stop" operation_key="prmDummy_stop_0">
+        <attributes CRM_meta_name="stop" CRM_meta_on_fail="fence" CRM_meta_timeout="60000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <crm_event id="11" operation="stonith" operation_key="stonith-rh73-01-snmp-reboot" on_node="rh73-01-snmp" on_node_uuid="3232238265"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="3">
+    <action_set>
+      <rsc_op id="9" operation="start" operation_key="prmStonith2-1_start_0" on_node="rh73-02-snmp" on_node_uuid="3232238266">
+        <primitive id="prmStonith2-1" class="stonith" type="external/ssh"/>
+        <attributes CRM_meta_name="start" CRM_meta_on_fail="restart" CRM_meta_on_node="rh73-02-snmp" CRM_meta_on_node_uuid="3232238266" CRM_meta_timeout="60000"  hostlist="rh73-02-snmp" pcmk_reboot_retries="1" pcmk_reboot_timeout="40s"/>
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="8" operation="stop" operation_key="prmStonith2-1_stop_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="4">
+    <action_set>
+      <pseudo_event id="8" operation="stop" operation_key="prmStonith2-1_stop_0">
+        <attributes CRM_meta_name="stop" CRM_meta_on_fail="ignore" CRM_meta_timeout="60000"  hostlist="rh73-02-snmp" pcmk_reboot_retries="1" pcmk_reboot_timeout="40s"/>
+      </pseudo_event>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="5">
+    <action_set>
+      <crm_event id="11" operation="stonith" operation_key="stonith-rh73-01-snmp-reboot" on_node="rh73-01-snmp" on_node_uuid="3232238265">
+        <attributes CRM_meta_on_node="rh73-01-snmp" CRM_meta_on_node_uuid="3232238265" CRM_meta_op_no_wait="true" CRM_meta_shutdown="0" CRM_meta_stonith_action="reboot" />
+        <downed>
+          <node id="3232238265"/>
+        </downed>
+      </crm_event>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="6">
+    <action_set>
+      <pseudo_event id="10" operation="stonith_complete" operation_key="stonith_complete">
+        <attributes />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <crm_event id="11" operation="stonith" operation_key="stonith-rh73-01-snmp-reboot" on_node="rh73-01-snmp" on_node_uuid="3232238265"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="7">
+    <action_set>
+      <pseudo_event id="2" operation="all_stopped" operation_key="all_stopped">
+        <attributes />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="3" operation="stop" operation_key="prmDummy_stop_0"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="8" operation="stop" operation_key="prmStonith2-1_stop_0"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="10" operation="stonith_complete" operation_key="stonith_complete"/>
+      </trigger>
+    </inputs>
+  </synapse>
+</transition_graph>
Index: pacemaker/pengine/test10/per-op-failcount.scores
===================================================================
--- /dev/null
+++ pacemaker/pengine/test10/per-op-failcount.scores
@@ -0,0 +1,8 @@
+Allocation scores:
+Using the original execution date of: 2017-04-06 09:04:22Z
+native_color: prmDummy allocation score on rh73-01-snmp: -INFINITY
+native_color: prmDummy allocation score on rh73-02-snmp: 200
+native_color: prmStonith1-1 allocation score on rh73-01-snmp: 0
+native_color: prmStonith1-1 allocation score on rh73-02-snmp: INFINITY
+native_color: prmStonith2-1 allocation score on rh73-01-snmp: INFINITY
+native_color: prmStonith2-1 allocation score on rh73-02-snmp: 0
Index: pacemaker/pengine/test10/per-op-failcount.summary
===================================================================
--- /dev/null
+++ pacemaker/pengine/test10/per-op-failcount.summary
@@ -0,0 +1,33 @@
+Using the original execution date of: 2017-04-06 09:04:22Z
+
+Current cluster status:
+Node rh73-01-snmp (3232238265): UNCLEAN (online)
+Online: [ rh73-02-snmp ]
+
+ prmDummy	(ocf::pacemaker:Dummy):	FAILED rh73-01-snmp
+ prmStonith1-1	(stonith:external/ssh):	Started rh73-02-snmp
+ prmStonith2-1	(stonith:external/ssh):	Started rh73-01-snmp
+
+Transition Summary:
+ * Recover prmDummy	(Started rh73-01-snmp -> rh73-02-snmp)
+ * Move    prmStonith2-1	(Started rh73-01-snmp -> rh73-02-snmp)
+
+Executing cluster transition:
+ * Pseudo action:   prmStonith2-1_stop_0
+ * Fencing rh73-01-snmp (reboot)
+ * Pseudo action:   stonith_complete
+ * Pseudo action:   prmDummy_stop_0
+ * Resource action: prmStonith2-1   start on rh73-02-snmp
+ * Pseudo action:   all_stopped
+ * Resource action: prmDummy        start on rh73-02-snmp
+ * Resource action: prmDummy        monitor=10000 on rh73-02-snmp
+Using the original execution date of: 2017-04-06 09:04:22Z
+
+Revised cluster status:
+Online: [ rh73-02-snmp ]
+OFFLINE: [ rh73-01-snmp ]
+
+ prmDummy	(ocf::pacemaker:Dummy):	Started rh73-02-snmp
+ prmStonith1-1	(stonith:external/ssh):	Started rh73-02-snmp
+ prmStonith2-1	(stonith:external/ssh):	Started rh73-02-snmp
+
Index: pacemaker/pengine/test10/per-op-failcount.xml
===================================================================
--- /dev/null
+++ pacemaker/pengine/test10/per-op-failcount.xml
@@ -0,0 +1,118 @@
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.7" epoch="4" num_updates="16" admin_epoch="0" cib-last-written="Thu Apr  6 18:04:01 2017" update-origin="rh73-01-snmp" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="3232238265" execution-date="1491469462">
+  <configuration>
+    <crm_config>
+      <cluster_property_set id="cib-bootstrap-options">
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-71dbd12"/>
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
+        <nvpair name="no-quorum-policy" value="ignore" id="cib-bootstrap-options-no-quorum-policy"/>
+        <nvpair name="stonith-enabled" value="true" id="cib-bootstrap-options-stonith-enabled"/>
+        <nvpair name="startup-fencing" value="false" id="cib-bootstrap-options-startup-fencing"/>
+      </cluster_property_set>
+    </crm_config>
+    <nodes>
+      <node id="3232238265" uname="rh73-01-snmp"/>
+      <node id="3232238266" uname="rh73-02-snmp"/>
+    </nodes>
+    <resources>
+      <primitive id="prmDummy" class="ocf" provider="pacemaker" type="Dummy">
+        <operations>
+          <op name="start" interval="0s" timeout="60s" on-fail="restart" id="prmDummy-start-0s"/>
+          <op name="monitor" interval="10s" timeout="60s" on-fail="restart" id="prmDummy-monitor-10s"/>
+          <op name="stop" interval="0s" timeout="60s" on-fail="fence" id="prmDummy-stop-0s"/>
+        </operations>
+      </primitive>
+      <primitive id="prmStonith1-1" class="stonith" type="external/ssh">
+        <instance_attributes id="prmStonith1-1-instance_attributes">
+          <nvpair name="pcmk_reboot_retries" value="1" id="prmStonith1-1-instance_attributes-pcmk_reboot_retries"/>
+          <nvpair name="pcmk_reboot_timeout" value="40s" id="prmStonith1-1-instance_attributes-pcmk_reboot_timeout"/>
+          <nvpair name="hostlist" value="rh73-01-snmp" id="prmStonith1-1-instance_attributes-hostlist"/>
+        </instance_attributes>
+        <operations>
+          <op name="start" interval="0s" timeout="60s" on-fail="restart" id="prmStonith1-1-start-0s"/>
+          <op name="stop" interval="0s" timeout="60s" on-fail="ignore" id="prmStonith1-1-stop-0s"/>
+        </operations>
+      </primitive>
+      <primitive id="prmStonith2-1" class="stonith" type="external/ssh">
+        <instance_attributes id="prmStonith2-1-instance_attributes">
+          <nvpair name="pcmk_reboot_retries" value="1" id="prmStonith2-1-instance_attributes-pcmk_reboot_retries"/>
+          <nvpair name="pcmk_reboot_timeout" value="40s" id="prmStonith2-1-instance_attributes-pcmk_reboot_timeout"/>
+          <nvpair name="hostlist" value="rh73-02-snmp" id="prmStonith2-1-instance_attributes-hostlist"/>
+        </instance_attributes>
+        <operations>
+          <op name="start" interval="0s" timeout="60s" on-fail="restart" id="prmStonith2-1-start-0s"/>
+          <op name="stop" interval="0s" timeout="60s" on-fail="ignore" id="prmStonith2-1-stop-0s"/>
+        </operations>
+      </primitive>
+    </resources>
+    <constraints>
+      <rsc_location id="rsc_location-1" rsc="prmDummy">
+        <!--### Resource Location ###-->
+        <rule score="300" id="rsc_location-1-rule">
+          <expression attribute="#uname" operation="eq" value="rh73-01-snmp" id="rsc_location-1-rule-expression"/>
+        </rule>
+        <rule score="200" id="rsc_location-1-rule-0">
+          <expression attribute="#uname" operation="eq" value="rh73-02-snmp" id="rsc_location-1-rule-0-expression"/>
+        </rule>
+      </rsc_location>
+    </constraints>
+    <fencing-topology>
+      <fencing-level devices="prmStonith1-1" index="1" target="rh73-01-snmp" id="fencing"/>
+      <fencing-level devices="prmStonith2-1" index="1" target="rh73-02-snmp" id="fencing-0"/>
+    </fencing-topology>
+    <rsc_defaults>
+      <meta_attributes id="rsc-options">
+        <nvpair name="resource-stickiness" value="INFINITY" id="rsc-options-resource-stickiness"/>
+        <nvpair name="migration-threshold" value="INFINITY" id="rsc-options-migration-threshold"/>
+      </meta_attributes>
+    </rsc_defaults>
+  </configuration>
+  <status>
+    <node_state id="3232238266" uname="rh73-02-snmp" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="3232238266">
+        <lrm_resources>
+          <lrm_resource id="prmStonith1-1" type="external/ssh" class="stonith">
+            <lrm_rsc_op id="prmStonith1-1_last_0" operation_key="prmStonith1-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.13" transition-key="10:1:0:b7473df5-063c-4fcb-8a21-ad97057cde4e" transition-magic="0:0;10:1:0:b7473df5-063c-4fcb-8a21-ad97057cde4e" on_node="rh73-02-snmp" call-id="14" rc-code="0" op-status="0" interval="0" last-run="1491469441" last-rc-change="1491469441" exec-time="1095" queue-time="0" op-digest="f5a04fe6e81a361a76eb0c07358e3c8c"/>
+          </lrm_resource>
+          <lrm_resource id="prmStonith2-1" type="external/ssh" class="stonith">
+            <lrm_rsc_op id="prmStonith2-1_last_0" operation_key="prmStonith2-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.13" transition-key="7:1:7:b7473df5-063c-4fcb-8a21-ad97057cde4e" transition-magic="0:7;7:1:7:b7473df5-063c-4fcb-8a21-ad97057cde4e" on_node="rh73-02-snmp" call-id="13" rc-code="7" op-status="0" interval="0" last-run="1491469440" last-rc-change="1491469440" exec-time="0" queue-time="0" op-digest="509683ffd094721b85b9d2805ce42080"/>
+          </lrm_resource>
+          <lrm_resource id="prmDummy" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="prmDummy_last_0" operation_key="prmDummy_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.13" transition-key="5:1:7:b7473df5-063c-4fcb-8a21-ad97057cde4e" transition-magic="0:7;5:1:7:b7473df5-063c-4fcb-8a21-ad97057cde4e" on_node="rh73-02-snmp" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1491469440" last-rc-change="1491469440" exec-time="41" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state  passwd  op_sleep  envfile " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="3232238266">
+        <instance_attributes id="status-3232238266">
+          <nvpair id="status-3232238266-shutdown" name="shutdown" value="0"/>
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+    <node_state id="3232238265" uname="rh73-01-snmp" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <lrm id="3232238265">
+        <lrm_resources>
+          <lrm_resource id="prmStonith1-1" type="external/ssh" class="stonith">
+            <lrm_rsc_op id="prmStonith1-1_last_0" operation_key="prmStonith1-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.13" transition-key="3:1:7:b7473df5-063c-4fcb-8a21-ad97057cde4e" transition-magic="0:7;3:1:7:b7473df5-063c-4fcb-8a21-ad97057cde4e" on_node="rh73-01-snmp" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1491469442" last-rc-change="1491469442" exec-time="7" queue-time="0" op-digest="f5a04fe6e81a361a76eb0c07358e3c8c"/>
+          </lrm_resource>
+          <lrm_resource id="prmStonith2-1" type="external/ssh" class="stonith">
+            <lrm_rsc_op id="prmStonith2-1_last_0" operation_key="prmStonith2-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.13" transition-key="11:1:0:b7473df5-063c-4fcb-8a21-ad97057cde4e" transition-magic="0:0;11:1:0:b7473df5-063c-4fcb-8a21-ad97057cde4e" on_node="rh73-01-snmp" call-id="15" rc-code="0" op-status="0" interval="0" last-run="1491469442" last-rc-change="1491469442" exec-time="1144" queue-time="0" op-digest="509683ffd094721b85b9d2805ce42080"/>
+          </lrm_resource>
+          <lrm_resource id="prmDummy" type="Dummy" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="prmDummy_last_0" operation_key="prmDummy_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.13" transition-key="2:3:0:b7473df5-063c-4fcb-8a21-ad97057cde4e" transition-magic="7:5;2:3:0:b7473df5-063c-4fcb-8a21-ad97057cde4e" on_node="rh73-01-snmp" call-id="18" rc-code="5" op-status="7" interval="0" last-run="1491469462" last-rc-change="1491469462" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" state  passwd  op_sleep  envfile " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="prmDummy_monitor_10000" operation_key="prmDummy_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.13" transition-key="9:1:0:b7473df5-063c-4fcb-8a21-ad97057cde4e" transition-magic="0:0;9:1:0:b7473df5-063c-4fcb-8a21-ad97057cde4e" on_node="rh73-01-snmp" call-id="16" rc-code="0" op-status="0" interval="10000" last-rc-change="1491469442" exec-time="36" queue-time="0" op-digest="873ed4f07792aa8ff18f3254244675ea" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="prmDummy_last_failure_0" operation_key="prmDummy_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.13" transition-key="2:3:0:b7473df5-063c-4fcb-8a21-ad97057cde4e" transition-magic="7:5;2:3:0:b7473df5-063c-4fcb-8a21-ad97057cde4e" on_node="rh73-01-snmp" call-id="18" rc-code="5" op-status="7" interval="0" last-rc-change="1491469462" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" last-run="1491469462"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+      <transient_attributes id="3232238265">
+        <instance_attributes id="status-3232238265">
+          <nvpair id="status-3232238265-shutdown" name="shutdown" value="0"/>
+          <nvpair id="status-3232238265-fail-count-prmDummy.monitor_10000" name="fail-count-prmDummy#monitor_10000" value="1"/>
+          <nvpair id="status-3232238265-last-failure-prmDummy.monitor_10000" name="last-failure-prmDummy#monitor_10000" value="1491469462"/>
+          <nvpair id="status-3232238265-fail-count-prmDummy.stop_0" name="fail-count-prmDummy#stop_0" value="INFINITY"/>
+          <nvpair id="status-3232238265-last-failure-prmDummy.stop_0" name="last-failure-prmDummy#stop_0" value="1491469462"/>
+        </instance_attributes>
+      </transient_attributes>
+    </node_state>
+  </status>
+</cib>
