From 8e9c08122ae738eec189abe402ba7358b57182cd Mon Sep 17 00:00:00 2001
From: Joel Colledge <joel.colledge@linbit.com>
Date: Tue, 9 Nov 2021 14:38:43 +0100
Subject: [PATCH] drbd: avoid blocking application IO by continually locking
 resync extent

This fixes an issue where, under certain timing conditions, we block
application writes for long time periods. This can occur as follows:
* We are SyncTarget.
* SyncSource is Primary and under continuous write load.
* We lock a resync extent which overlaps with the activity log extent
  for an active write.
* We receive the peer write, but it must wait for the AL extent.
* SyncSource cancels our resync requests.
* We receive these cancellations, release the resync extent and
  reschedule the resync work.
* Due to some scheduling quirk, the resync work reliably runs before the
  submitter work has been woken und had the chance to lock the AL
  extent.
* We lock the resync extent again, and the process repeats.

Avoid this livelock issue by only scheduling the resync work immediately
when we have definitely made some progress.
---
 drbd/drbd_int.h      |  1 +
 drbd/drbd_receiver.c | 21 ++++++++++++++++-----
 drbd/drbd_sender.c   |  2 ++
 drbd/drbd_state.c    |  2 ++
 4 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/drbd/drbd_int.h b/drbd/drbd_int.h
index 8aef56d601e4..109e75877917 100644
--- a/drbd/drbd_int.h
+++ b/drbd/drbd_int.h
@@ -1161,6 +1161,7 @@ struct drbd_peer_device {
 	enum drbd_disk_state resync_finished_pdsk; /* Finished while starting resync */
 	int resync_again; /* decided to resync again while resync running */
 	unsigned long resync_next_bit; /* bitmap bit to search from for next resync request */
+	unsigned long last_resync_next_bit; /* value of resync_next_bit before last set of resync requests */
 	struct mutex resync_next_bit_mutex;
 
 	atomic_t ap_pending_cnt; /* AP data packets on the wire, ack expected */
diff --git a/drbd/drbd_receiver.c b/drbd/drbd_receiver.c
index 1502343cbacc..17255e284a38 100644
--- a/drbd/drbd_receiver.c
+++ b/drbd/drbd_receiver.c
@@ -228,11 +228,22 @@ static void rs_sectors_came_in(struct drbd_peer_device *peer_device, int size)
 {
 	int rs_sect_in = atomic_add_return(size >> 9, &peer_device->rs_sect_in);
 
-	/* In case resync runs faster than anticipated, run the resync_work early */
-	if (rs_sect_in >= peer_device->rs_in_flight)
-		drbd_queue_work_if_unqueued(
-			&peer_device->connection->sender_work,
-			&peer_device->resync_work);
+	/* When resync runs faster than anticipated, consider running the
+	 * resync_work early. */
+	if (rs_sect_in >= peer_device->rs_in_flight) {
+		mutex_lock(&peer_device->resync_next_bit_mutex);
+		/* Only run resync_work early if we are definitely making
+		 * progress. Otherwise we might continually lock a resync
+		 * extent even when all the requests are canceled. This can
+		 * cause application IO to be blocked for an indefinitely long
+		 * time. */
+		if (peer_device->repl_state[NOW] == L_VERIFY_S ||
+				peer_device->resync_next_bit > peer_device->last_resync_next_bit)
+			drbd_queue_work_if_unqueued(
+					&peer_device->connection->sender_work,
+					&peer_device->resync_work);
+		mutex_unlock(&peer_device->resync_next_bit_mutex);
+	}
 }
 
 /* kick lower level device, if we have more than (arbitrary number)
diff --git a/drbd/drbd_sender.c b/drbd/drbd_sender.c
index 58ef3b49cfe4..72387307797c 100644
--- a/drbd/drbd_sender.c
+++ b/drbd/drbd_sender.c
@@ -754,6 +754,8 @@ static int make_resync_request(struct drbd_peer_device *peer_device, int cancel)
 	 * just because the first reply came "fast", ... */
 	peer_device->rs_in_flight += number * BM_SECT_PER_BIT;
 
+	peer_device->last_resync_next_bit = peer_device->resync_next_bit;
+
 	for (i = 0; i < number; i++) {
 		bool send_buffer_ok = true;
 		/* Stop generating RS requests, when half of the send buffer is filled */
diff --git a/drbd/drbd_state.c b/drbd/drbd_state.c
index c5404dcac5c6..aeaf36a9ba05 100644
--- a/drbd/drbd_state.c
+++ b/drbd/drbd_state.c
@@ -2291,6 +2291,7 @@ static void initialize_resync(struct drbd_peer_device *peer_device)
 	unsigned long tw = drbd_bm_total_weight(peer_device);
 	unsigned long now = jiffies;
 
+	peer_device->last_resync_next_bit = 0;
 	peer_device->rs_failed = 0;
 	peer_device->rs_paused = 0;
 	peer_device->rs_same_csum = 0;
@@ -2498,6 +2499,7 @@ static void finish_state_change(struct drbd_resource *resource, struct completio
 				peer_device->resync_next_bit = 0;
 				if (repl_state[NEW] == L_SYNC_TARGET)
 					mod_timer(&peer_device->resync_timer, jiffies);
+				peer_device->last_resync_next_bit = 0;
 			}
 
 			if ((repl_state[OLD] == L_SYNC_TARGET  || repl_state[OLD] == L_SYNC_SOURCE) &&
-- 
2.26.2

