From db21a275c912a82d833ebc0d694c06218142a05c Mon Sep 17 00:00:00 2001
From: Philipp Reisner <philipp.reisner@linbit.com>
Date: Tue, 9 Oct 2018 17:21:45 +0200
Subject: [PATCH 1/2] drbd: Enforce new current UUID written before writes and
 check peers before generating a new one

Make sure no write makes it to the backing device before new current-UUID
In case we need to generate a new UUID before the next write we need to
make sure that this update to the meta-data block is persistent before
any write is submitted to the local backing disk.

So far the code ensured that for the first write, but a second one
following the first one was potentially submitted the the backend
immediately.

Improve that by keeping the NEW_CUR_UUID flag set until the meta-data
update is finished, and creating a new flag (WRITING_NEW_CUR_UUID)
to ensure only one context is actually doing it.

AND Check peers just befor creating a new current UUID

When a primary node gets isolated it might see one of its peers
still alive while one peers is already gone. Now check the liveliness
of the peers considerd as alive before creating the new
UUID. In case we loose quorum in that process, do not create
the new current UUID.
---
 drbd/drbd_int.h    |  3 +++
 drbd/drbd_nl.c     |  2 +-
 drbd/drbd_req.c    | 27 ++++++-----------------
 drbd/drbd_sender.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 64 insertions(+), 21 deletions(-)

diff --git a/drbd/drbd_int.h b/drbd/drbd_int.h
index 2e6c1a7a..e4dacf13 100644
--- a/drbd/drbd_int.h
+++ b/drbd/drbd_int.h
@@ -544,6 +544,7 @@ enum {
 	FORCE_DETACH,		/* Force-detach from local disk, aborting any pending local IO */
 	NEW_CUR_UUID,		/* Create new current UUID when thawing IO or issuing local IO */
 	__NEW_CUR_UUID,		/* Set NEW_CUR_UUID as soon as state change visible */
+	WRITING_NEW_CUR_UUID,	/* Set while the new current ID gets generated. */
 	AL_SUSPENDED,		/* Activity logging is currently suspended. */
 	UNREGISTERED,
 	FLUSH_PENDING,		/* if set, device->flush_jif is when we submitted that flush
@@ -556,6 +557,7 @@ enum {
         GO_DISKLESS,            /* tell worker to schedule cleanup before detach */
         DESTROY_DISK,           /* tell worker to close backing devices and destroy related structures. */
 	MD_SYNC,		/* tell worker to call drbd_md_sync() */
+	MAKE_NEW_CUR_UUID,	/* tell worker to ping peers and eventually write new current uuid */
 
 	HAVE_LDEV,
 	STABLE_RESYNC,		/* One peer_device finished the resync stable! */
@@ -760,6 +762,7 @@ enum {
 	CONN_DISCARD_MY_DATA,
 	SEND_STATE_AFTER_AHEAD_C,
 	NOTIFY_PEERS_LOST_PRIMARY,
+	CHECKING_PEER,		/* used by make_new_urrent_uuid() to check liveliness */
 };
 
 /* flag bits per resource */
diff --git a/drbd/drbd_nl.c b/drbd/drbd_nl.c
index 35b305b8..3548414c 100644
--- a/drbd/drbd_nl.c
+++ b/drbd/drbd_nl.c
@@ -1161,8 +1161,8 @@ retry:
 
 		idr_for_each_entry(&resource->devices, device, vnr) {
 			if (forced) {
-				clear_bit(NEW_CUR_UUID, &device->flags);
 				drbd_uuid_new_current(device, true);
+				clear_bit(NEW_CUR_UUID, &device->flags);
 			}
 		}
 	}
diff --git a/drbd/drbd_req.c b/drbd/drbd_req.c
index 7f4880d2..0ae20a8a 100644
--- a/drbd/drbd_req.c
+++ b/drbd/drbd_req.c
@@ -1620,11 +1620,6 @@ drbd_request_prepare(struct drbd_device *device, struct bio *bio,
 	    (bio_op(bio) == REQ_OP_DISCARD))
 		goto queue_for_submitter_thread;
 
-	/* Unconditionally defer to worker,
-	 * if we still need to bumpt our data generation id */
-	if (test_bit(NEW_CUR_UUID, &device->flags))
-		goto queue_for_submitter_thread;
-
 	if (req->private_bio && !test_bit(AL_SUSPENDED, &device->flags)) {
 		if (!drbd_al_begin_io_fastpath(device, &req->i))
 			goto queue_for_submitter_thread;
@@ -1875,6 +1870,13 @@ static bool inc_ap_bio_cond(struct drbd_device *device, int rw)
 	bool rv = false;
 	unsigned int nr_requests;
 
+	if (test_bit(NEW_CUR_UUID, &device->flags)) {
+		if (!test_and_set_bit(WRITING_NEW_CUR_UUID, &device->flags))
+			drbd_device_post_work(device, MAKE_NEW_CUR_UUID);
+
+		return false;
+	}
+
 	spin_lock_irq(&device->resource->req_lock);
 	nr_requests = device->resource->res_opts.nr_requests;
 	rv = may_inc_ap_bio(device) && atomic_read(&device->ap_bio_cnt[rw]) < nr_requests;
@@ -2093,17 +2095,6 @@ static void send_and_submit_pending(struct drbd_device *device, struct waiting_f
 	blk_finish_plug(&plug);
 }
 
-
-static void ensure_current_uuid(struct drbd_device *device)
-{
-	if (test_and_clear_bit(NEW_CUR_UUID, &device->flags)) {
-		struct drbd_resource *resource = device->resource;
-		mutex_lock(&resource->conf_update);
-		drbd_uuid_new_current(device, false);
-		mutex_unlock(&resource->conf_update);
-	}
-}
-
 /* more: for non-blocking fill-up # of updates in the transaction */
 static bool grab_new_incoming_requests(struct drbd_device *device, struct waiting_for_act_log *wfa, bool more)
 {
@@ -2135,8 +2126,6 @@ void do_submit(struct work_struct *ws)
 	for (;;) {
 		DEFINE_WAIT(wait);
 
-		ensure_current_uuid(device);
-
 		/* move used-to-be-postponed back to front of incoming */
 		wfa_splice_init(&wfa, later, incoming);
 		submit_fast_path(device, &wfa);
@@ -2237,8 +2226,6 @@ void do_submit(struct work_struct *ws)
 
 		drbd_al_begin_io_commit(device);
 
-		ensure_current_uuid(device);
-
 		send_and_submit_pending(device, &wfa);
 	}
 	drbd_kick_lo(device);
diff --git a/drbd/drbd_sender.c b/drbd/drbd_sender.c
index 2bdb59c6..64a4e0bc 100644
--- a/drbd/drbd_sender.c
+++ b/drbd/drbd_sender.c
@@ -45,6 +45,7 @@ static int make_ov_request(struct drbd_peer_device *, int);
 static int make_resync_request(struct drbd_peer_device *, int);
 static bool should_send_barrier(struct drbd_connection *, unsigned int epoch);
 static void maybe_send_barrier(struct drbd_connection *, unsigned int);
+static unsigned long get_work_bits(const unsigned long mask, unsigned long *flags);
 
 /* endio handlers:
  *   drbd_md_endio (defined here)
@@ -2231,6 +2232,55 @@ void __update_timing_details(
 	++(*cb_nr);
 }
 
+static bool all_peers_responded(struct drbd_device *device)
+{
+	struct drbd_resource *resource = device->resource;
+	struct drbd_connection *connection;
+	bool all_responded = true;
+
+	rcu_read_lock();
+	for_each_connection_rcu(connection, resource) {
+		if (!test_bit(CHECKING_PEER, &connection->flags))
+			continue;
+		if (connection->cstate[NOW] < C_CONNECTED) {
+			clear_bit(CHECKING_PEER, &connection->flags);
+			continue;
+		}
+		if (!test_bit(GOT_PING_ACK, &connection->flags)) {
+			all_responded = false;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	return all_responded;
+}
+
+static void make_new_current_uuid(struct drbd_device *device)
+{
+	struct drbd_resource *resource = device->resource;
+	struct drbd_connection *connection;
+	u64 im;
+
+	for_each_connection_ref(connection, im, resource) {
+		if (connection->cstate[NOW] < C_CONNECTED)
+			continue;
+		clear_bit(GOT_PING_ACK, &connection->flags);
+		set_bit(CHECKING_PEER, &connection->flags);
+		request_ping(connection);
+	}
+	wait_event(resource->state_wait, all_peers_responded(device));
+
+	if (device->have_quorum[NOW]) { /* or quorum not enabled is implicit */
+		mutex_lock(&resource->conf_update);
+		drbd_uuid_new_current(device, false);
+		mutex_unlock(&resource->conf_update);
+	}
+
+	get_work_bits(1UL << NEW_CUR_UUID | 1UL << WRITING_NEW_CUR_UUID, &device->flags);
+	wake_up(&device->misc_wait);
+}
+
 static void do_device_work(struct drbd_device *device, const unsigned long todo)
 {
 	if (test_bit(MD_SYNC, &todo))
@@ -2239,6 +2289,8 @@ static void do_device_work(struct drbd_device *device, const unsigned long todo)
 		go_diskless(device);
 	if (test_bit(DESTROY_DISK, &todo))
 		drbd_ldev_destroy(device);
+	if (test_bit(MAKE_NEW_CUR_UUID, &todo))
+		make_new_current_uuid(device);
 }
 
 static void do_peer_device_work(struct drbd_peer_device *peer_device, const unsigned long todo)
@@ -2259,6 +2311,7 @@ static void do_peer_device_work(struct drbd_peer_device *peer_device, const unsi
 	((1UL << GO_DISKLESS)	\
 	|(1UL << DESTROY_DISK)	\
 	|(1UL << MD_SYNC)	\
+	|(1UL << MAKE_NEW_CUR_UUID)\
 	)
 
 #define DRBD_PEER_DEVICE_WORK_MASK	\
-- 
2.34.1

