From 8306ac714ab58786f678dd78a6485b998d5828ea Mon Sep 17 00:00:00 2001
From: "Gao,Yan" <ygao@suse.com>
Date: Fri, 2 Dec 2022 18:37:09 +0100
Subject: [PATCH 1/6] Test: cts-scheduler: add test for preventing a leftover
 pending monitor from causing unexpected stop of other instances

- Monitor of stateful-1 failed on node-1 so that the pending monitor
  didn't get overridden.
- While stateful-1 was stopping on node-1, node-2 was rejoining and
  pending.
- stateful-1 was stopped on node-1 and scheduler would unexpectedly
  decide to stop the promoted instance on node-3.
---
 pengine/regression.sh                          |  1 +
 .../leftover-pending-monitor.dot          | 36 ++++++++
 .../leftover-pending-monitor.exp          | 81 ++++++++++++++++++
 .../leftover-pending-monitor.scores    | 28 +++++++
 .../leftover-pending-monitor.summary  | 35 ++++++++
 .../leftover-pending-monitor.xml          | 82 +++++++++++++++++++
 6 files changed, 263 insertions(+)
 create mode 100644 pengine/test10/leftover-pending-monitor.dot
 create mode 100644 pengine/test10/leftover-pending-monitor.exp
 create mode 100644 pengine/test10/leftover-pending-monitor.scores
 create mode 100644 pengine/test10/leftover-pending-monitor.summary
 create mode 100644 pengine/test10/leftover-pending-monitor.xml

Index: pacemaker-2.0.1+20190417.13d370ca9/pengine/regression.sh
===================================================================
--- pacemaker-2.0.1+20190417.13d370ca9.orig/pengine/regression.sh
+++ pacemaker-2.0.1+20190417.13d370ca9/pengine/regression.sh
@@ -833,6 +833,7 @@ do_test master-notify "Master promotion
 do_test master-score-startup "Use permanent master scores without LRM history"
 do_test failed-demote-recovery "Recover resource in slave role after demote fails"
 do_test failed-demote-recovery-master "Recover resource in master role after demote fails"
+do_test leftover-pending-monitor "Prevent a leftover pending monitor from causing unexpected stop of other instances"
 
 echo ""
 do_test history-1 "Correctly parse stateful-1 resource state"
Index: pacemaker-2.0.1+20190417.13d370ca9/pengine/test10/leftover-pending-monitor.dot
===================================================================
--- /dev/null
+++ pacemaker-2.0.1+20190417.13d370ca9/pengine/test10/leftover-pending-monitor.dot
@@ -0,0 +1,36 @@
+ digraph "g" {
+"promotable-1_demote_0" -> "promotable-1_demoted_0" [ style = bold]
+"promotable-1_demote_0" -> "stateful-1_demote_0 node-3" [ style = bold]
+"promotable-1_demote_0" [ style=bold color="green" fontcolor="orange"]
+"promotable-1_demoted_0" -> "promotable-1_start_0" [ style = dashed]
+"promotable-1_demoted_0" -> "promotable-1_stop_0" [ style = bold]
+"promotable-1_demoted_0" [ style=bold color="green" fontcolor="orange"]
+"promotable-1_running_0" [ style=dashed color="red" fontcolor="orange"]
+"promotable-1_start_0" -> "promotable-1_running_0" [ style = dashed]
+"promotable-1_start_0" -> "stateful-1:2_start_0 node-1" [ style = dashed]
+"promotable-1_start_0" -> "stateful-1_start_0 node-3" [ style = dashed]
+"promotable-1_start_0" [ style=dashed color="red" fontcolor="orange"]
+"promotable-1_stop_0" -> "promotable-1_stopped_0" [ style = bold]
+"promotable-1_stop_0" -> "stateful-1_stop_0 node-3" [ style = bold]
+"promotable-1_stop_0" [ style=bold color="green" fontcolor="orange"]
+"promotable-1_stopped_0" -> "promotable-1_start_0" [ style = dashed]
+"promotable-1_stopped_0" [ style=bold color="green" fontcolor="orange"]
+"st-sbd_monitor_0 node-2" [ style=dashed color="red" fontcolor="black"]
+"stateful-1:2_monitor_5000 node-1" [ style=dashed color="red" fontcolor="black"]
+"stateful-1:2_start_0 node-1" -> "promotable-1_running_0" [ style = dashed]
+"stateful-1:2_start_0 node-1" -> "stateful-1:2_monitor_5000 node-1" [ style = dashed]
+"stateful-1:2_start_0 node-1" [ style=dashed color="red" fontcolor="black"]
+"stateful-1_demote_0 node-3" -> "promotable-1_demoted_0" [ style = bold]
+"stateful-1_demote_0 node-3" -> "stateful-1_stop_0 node-3" [ style = bold]
+"stateful-1_demote_0 node-3" [ style=bold color="green" fontcolor="black"]
+"stateful-1_monitor_0 node-2" -> "promotable-1_start_0" [ style = dashed]
+"stateful-1_monitor_0 node-2" -> "promotable-1_stopped_0" [ style = dashed]
+"stateful-1_monitor_0 node-2" [ style=dashed color="red" fontcolor="black"]
+"stateful-1_monitor_10000 node-3" [ style=dashed color="red" fontcolor="black"]
+"stateful-1_start_0 node-3" -> "promotable-1_running_0" [ style = dashed]
+"stateful-1_start_0 node-3" -> "stateful-1_monitor_10000 node-3" [ style = dashed]
+"stateful-1_start_0 node-3" [ style=dashed color="red" fontcolor="black"]
+"stateful-1_stop_0 node-3" -> "promotable-1_stopped_0" [ style = bold]
+"stateful-1_stop_0 node-3" -> "stateful-1_start_0 node-3" [ style = dashed]
+"stateful-1_stop_0 node-3" [ style=bold color="green" fontcolor="black"]
+}
Index: pacemaker-2.0.1+20190417.13d370ca9/pengine/test10/leftover-pending-monitor.exp
===================================================================
--- /dev/null
+++ pacemaker-2.0.1+20190417.13d370ca9/pengine/test10/leftover-pending-monitor.exp
@@ -0,0 +1,81 @@
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
+  <synapse id="0">
+    <action_set>
+      <rsc_op id="8" operation="stop" operation_key="stateful-1_stop_0" internal_operation_key="stateful-1:1_stop_0" on_node="node-3" on_node_uuid="3">
+        <primitive id="stateful-1" long-id="stateful-1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="node-3" CRM_meta_on_node_uuid="3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="7" operation="demote" operation_key="stateful-1_demote_0" internal_operation_key="stateful-1:1_demote_0" on_node="node-3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="14" operation="stop" operation_key="promotable-1_stop_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="1">
+    <action_set>
+      <rsc_op id="7" operation="demote" operation_key="stateful-1_demote_0" internal_operation_key="stateful-1:1_demote_0" on_node="node-3" on_node_uuid="3">
+        <primitive id="stateful-1" long-id="stateful-1:1" class="ocf" provider="pacemaker" type="Stateful"/>
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_on_node="node-3" CRM_meta_on_node_uuid="3" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+      </rsc_op>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="18" operation="demote" operation_key="promotable-1_demote_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="2" priority="1000000">
+    <action_set>
+      <pseudo_event id="19" operation="demoted" operation_key="promotable-1_demoted_0">
+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="7" operation="demote" operation_key="stateful-1_demote_0" internal_operation_key="stateful-1:1_demote_0" on_node="node-3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="18" operation="demote" operation_key="promotable-1_demote_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="3">
+    <action_set>
+      <pseudo_event id="18" operation="demote" operation_key="promotable-1_demote_0">
+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs/>
+  </synapse>
+  <synapse id="4" priority="1000000">
+    <action_set>
+      <pseudo_event id="15" operation="stopped" operation_key="promotable-1_stopped_0">
+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <rsc_op id="8" operation="stop" operation_key="stateful-1_stop_0" internal_operation_key="stateful-1:1_stop_0" on_node="node-3" on_node_uuid="3"/>
+      </trigger>
+      <trigger>
+        <pseudo_event id="14" operation="stop" operation_key="promotable-1_stop_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+  <synapse id="5">
+    <action_set>
+      <pseudo_event id="14" operation="stop" operation_key="promotable-1_stop_0">
+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_promoted_max="1" CRM_meta_promoted_node_max="1" CRM_meta_timeout="20000" />
+      </pseudo_event>
+    </action_set>
+    <inputs>
+      <trigger>
+        <pseudo_event id="19" operation="demoted" operation_key="promotable-1_demoted_0"/>
+      </trigger>
+    </inputs>
+  </synapse>
+</transition_graph>
Index: pacemaker-2.0.1+20190417.13d370ca9/pengine/test10/leftover-pending-monitor.scores
===================================================================
--- /dev/null
+++ pacemaker-2.0.1+20190417.13d370ca9/pengine/test10/leftover-pending-monitor.scores
@@ -0,0 +1,28 @@
+
+clone_color: promotable-1 allocation score on node-1: 0
+clone_color: promotable-1 allocation score on node-2: 0
+clone_color: promotable-1 allocation score on node-3: 0
+clone_color: stateful-1:0 allocation score on node-1: 0
+clone_color: stateful-1:0 allocation score on node-2: 0
+clone_color: stateful-1:0 allocation score on node-3: 0
+clone_color: stateful-1:1 allocation score on node-1: 0
+clone_color: stateful-1:1 allocation score on node-2: 0
+clone_color: stateful-1:1 allocation score on node-3: 11
+clone_color: stateful-1:2 allocation score on node-1: 0
+clone_color: stateful-1:2 allocation score on node-2: 0
+clone_color: stateful-1:2 allocation score on node-3: 10
+native_color: st-sbd allocation score on node-1: 1
+native_color: st-sbd allocation score on node-2: 0
+native_color: st-sbd allocation score on node-3: 0
+native_color: stateful-1:0 allocation score on node-1: -INFINITY
+native_color: stateful-1:0 allocation score on node-2: -INFINITY
+native_color: stateful-1:0 allocation score on node-3: -INFINITY
+native_color: stateful-1:1 allocation score on node-1: 0
+native_color: stateful-1:1 allocation score on node-2: -INFINITY
+native_color: stateful-1:1 allocation score on node-3: 11
+native_color: stateful-1:2 allocation score on node-1: 0
+native_color: stateful-1:2 allocation score on node-2: -INFINITY
+native_color: stateful-1:2 allocation score on node-3: -INFINITY
+stateful-1:0 promotion score on none: 0
+stateful-1:1 promotion score on node-3: INFINITY
+stateful-1:2 promotion score on node-1: -1
Index: pacemaker-2.0.1+20190417.13d370ca9/pengine/test10/leftover-pending-monitor.summary
===================================================================
--- /dev/null
+++ pacemaker-2.0.1+20190417.13d370ca9/pengine/test10/leftover-pending-monitor.summary
@@ -0,0 +1,35 @@
+Using the original execution date of: 2022-12-02 17:04:52Z
+Current cluster status:
+  * Node List:
+    * Node node-2: pending
+    * Online: [ node-1 node-3 ]
+
+  * Full List of Resources:
+    * st-sbd	(stonith:external/sbd):	 Started node-1
+    * Clone Set: promotable-1 [stateful-1] (promotable):
+      * stateful-1	(ocf:pacemaker:Stateful):	 Stopped node-1 (Monitoring)
+      * Promoted: [ node-3 ]
+      * Stopped: [ node-1 node-2 ]
+
+Transition Summary:
+  * Stop       stateful-1:1     ( Promoted node-3 )  due to unrunnable stateful-1:0 monitor
+  * Start      stateful-1:2     (          node-1 )  due to unrunnable stateful-1:0 monitor (blocked)
+
+Executing Cluster Transition:
+  * Pseudo action:   promotable-1_demote_0
+  * Resource action: stateful-1      demote on node-3
+  * Pseudo action:   promotable-1_demoted_0
+  * Pseudo action:   promotable-1_stop_0
+  * Resource action: stateful-1      stop on node-3
+  * Pseudo action:   promotable-1_stopped_0
+Using the original execution date of: 2022-12-02 17:04:52Z
+
+Revised Cluster Status:
+  * Node List:
+    * Node node-2: pending
+    * Online: [ node-1 node-3 ]
+
+  * Full List of Resources:
+    * st-sbd	(stonith:external/sbd):	 Started node-1
+    * Clone Set: promotable-1 [stateful-1] (promotable):
+      * Stopped: [ node-1 node-2 node-3 ]
Index: pacemaker-2.0.1+20190417.13d370ca9/pengine/test10/leftover-pending-monitor.xml
===================================================================
--- /dev/null
+++ pacemaker-2.0.1+20190417.13d370ca9/pengine/test10/leftover-pending-monitor.xml
@@ -0,0 +1,82 @@
+<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.9" epoch="8" num_updates="53" admin_epoch="0" cib-last-written="Fri Dec  2 18:00:35 2022" update-origin="node-1" update-client="crm_resource" update-user="root" have-quorum="1" dc-uuid="1" execution-date="1670000692">
+  <configuration>
+    <crm_config>
+      <cluster_property_set id="cib-bootstrap-options">
+        <nvpair name="have-watchdog" value="true" id="cib-bootstrap-options-have-watchdog"/>
+        <nvpair name="dc-version" value="2.1.2+20211124.ada5c3b36-150400.29.16-2.1.2+20211124.ada5c3b36" id="cib-bootstrap-options-dc-version"/>
+        <nvpair name="cluster-infrastructure" value="corosync" id="cib-bootstrap-options-cluster-infrastructure"/>
+      </cluster_property_set>
+    </crm_config>
+    <nodes>
+      <node id="1" uname="node-1"/>
+      <node id="2" uname="node-2"/>
+      <node id="3" uname="node-3"/>
+    </nodes>
+    <resources>
+      <primitive id="st-sbd" class="stonith" type="external/sbd"/>
+      <master id="promotable-1">
+        <meta_attributes id="promotable-1-meta_attributes">
+          <nvpair name="promotable" value="true" id="promotable-1-meta_attributes-promotable"/>
+          <nvpair name="interleave" value="true" id="promotable-1-meta_attributes-interleave"/>
+        </meta_attributes>
+        <primitive id="stateful-1" class="ocf" provider="pacemaker" type="Stateful">
+          <operations>
+            <op name="monitor" interval="5" id="stateful-1-monitor-5"/>
+            <op name="monitor" interval="10" role="Master" id="stateful-1-monitor-10"/>
+          </operations>
+        </primitive>
+      </master>
+    </resources>
+    <constraints>
+      <rsc_location id="cli-prefer-promotable-1" rsc="promotable-1" role="Master" node="node-3" score="INFINITY"/>
+    </constraints>
+    <rsc_defaults>
+      <meta_attributes id="build-resource-defaults">
+        <nvpair id="build-resource-stickiness" name="resource-stickiness" value="1"/>
+      </meta_attributes>
+    </rsc_defaults>
+  </configuration>
+  <status>
+    <node_state id="1" uname="node-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
+      <transient_attributes id="1">
+        <instance_attributes id="status-1">
+          <nvpair id="status-1-.feature-set" name="#feature-set" value="3.11.0"/>
+          <nvpair id="status-1-fail-count-stateful-1.monitor_5000" name="fail-count-stateful-1#monitor_5000" value="2"/>
+          <nvpair id="status-1-last-failure-stateful-1.monitor_5000" name="last-failure-stateful-1#monitor_5000" value="1670000676"/>
+        </instance_attributes>
+      </transient_attributes>
+      <lrm id="1">
+        <lrm_resources>
+          <lrm_resource id="st-sbd" type="external/sbd" class="stonith">
+            <lrm_rsc_op id="st-sbd_last_0" operation_key="st-sbd_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="3:1:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:0;3:1:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-1" call-id="11" rc-code="0" op-status="0" interval="0" last-rc-change="1670000509" exec-time="1418" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="stateful-1" type="Stateful" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="stateful-1_last_0" operation_key="stateful-1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="2:7:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:0;2:7:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-1" call-id="19" rc-code="0" op-status="0" interval="0" last-rc-change="1670000676" exec-time="15102" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="stateful-1_monitor_5000" operation_key="stateful-1_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="6:6:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="-1:193;6:6:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-1" call-id="-1" rc-code="193" op-status="-1" interval="5000" last-rc-change="1670000661" exec-time="0" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+            <lrm_rsc_op id="stateful-1_last_failure_0" operation_key="stateful-1_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="6:6:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:7;6:6:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-1" call-id="17" rc-code="7" op-status="0" interval="5000" last-rc-change="1670000676" exec-time="15023" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+    <node_state id="3" in_ccm="true" crmd="online" crm-debug-origin="post_cache_update" uname="node-3" join="member" expected="member">
+      <transient_attributes id="3">
+        <instance_attributes id="status-3">
+          <nvpair id="status-3-.feature-set" name="#feature-set" value="3.11.0"/>
+          <nvpair id="status-3-master-stateful-1" name="master-stateful-1" value="10"/>
+        </instance_attributes>
+      </transient_attributes>
+      <lrm id="3">
+        <lrm_resources>
+          <lrm_resource id="st-sbd" type="external/sbd" class="stonith">
+            <lrm_rsc_op id="st-sbd_last_0" operation_key="st-sbd_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="1:1:7:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:7;1:1:7:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-3" call-id="5" rc-code="7" op-status="0" interval="0" last-rc-change="1670000497" exec-time="5" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+          </lrm_resource>
+          <lrm_resource id="stateful-1" type="Stateful" class="ocf" provider="pacemaker">
+            <lrm_rsc_op id="stateful-1_last_0" operation_key="stateful-1_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="8:2:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:0;8:2:0:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-3" call-id="12" rc-code="0" op-status="0" interval="0" last-rc-change="1670000510" exec-time="65" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart="  state  " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
+            <lrm_rsc_op id="stateful-1_monitor_10000" operation_key="stateful-1_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.11.0" transition-key="10:3:8:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" transition-magic="0:8;10:3:8:85fd6005-3b05-4b2a-a0d6-3635783e4e1d" exit-reason="" on_node="node-3" call-id="13" rc-code="8" op-status="0" interval="10000" last-rc-change="1670000510" exec-time="17" queue-time="0" op-digest="4811cef7f7f94e3a35a70be7916cb2fd"/>
+          </lrm_resource>
+        </lrm_resources>
+      </lrm>
+    </node_state>
+    <node_state id="2" uname="node-2" crmd="offline" crm-debug-origin="post_cache_update" in_ccm="true" join="down" expected="down"/>
+  </status>
+</cib>
