From 19bbc74d8d90178060202fe2bde53dc2a269f97c Mon Sep 17 00:00:00 2001
From: Noel Power <noel.power@suse.com>
Date: Tue, 30 Jan 2024 14:44:59 +0000
Subject: [PATCH] DEBUG: if we fail to find server_id of process that has
 exclusive lock

---
 lib/messaging/messages_dgm.c | 37 ++++++++++++++++++++++++++++++++--
 lib/messaging/messages_dgm.h |  1 +
 lib/util/samba_util.h        |  1 +
 lib/util/util.c              | 19 ++++++++++++++++--
 source3/include/serverid.h   |  1 +
 source3/lib/g_lock.c         | 10 ++++++++-
 source3/lib/serverid.c       | 39 +++++++++++++++++++++++++++++++-----
 7 files changed, 98 insertions(+), 10 deletions(-)

diff --git a/lib/messaging/messages_dgm.c b/lib/messaging/messages_dgm.c
index ef065def4d9..f9f8c121ffb 100644
--- a/lib/messaging/messages_dgm.c
+++ b/lib/messaging/messages_dgm.c
@@ -1507,13 +1507,17 @@ static int messaging_dgm_read_unique(int fd, uint64_t *punique)
 	return 0;
 }
 
-int messaging_dgm_get_unique(pid_t pid, uint64_t *unique)
+int messaging_dgm_get_unique_debug(pid_t pid, uint64_t *unique, bool debug)
 {
 	struct messaging_dgm_context *ctx = global_dgm_context;
 	struct sun_path_buf lockfile_name;
 	int ret, fd;
+	int saveerrno;
 
 	if (ctx == NULL) {
+		if (debug) {
+			DBG_ERR("ctx is NULL\n");
+		}
 		return EBADF;
 	}
 
@@ -1523,28 +1527,57 @@ int messaging_dgm_get_unique(pid_t pid, uint64_t *unique)
 		/*
 		 * Protect against losing our own lock
 		 */
-		return messaging_dgm_read_unique(ctx->lockfile_fd, unique);
+		ret = messaging_dgm_read_unique(ctx->lockfile_fd, unique);
+		if (debug) {
+			DBG_ERR("messaging_dgm_read_unique returns %d\n",
+					ret);
+		}
+		return ret;
 	}
 
 	ret = snprintf(lockfile_name.buf, sizeof(lockfile_name.buf),
 		       "%s/%u", ctx->lockfile_dir.buf, (int)pid);
 	if (ret < 0) {
+		if (debug) {
+			DBG_ERR("snprintf failed with ret\n");
+		}
 		return errno;
 	}
 	if ((size_t)ret >= sizeof(lockfile_name.buf)) {
+		if (debug) {
+			DBG_ERR("Buf name %d exceedes buf size %d with dir %s\n",
+					ret, (int)sizeof(lockfile_name.buf),
+					ctx->lockfile_dir.buf);
+		}
 		return ENAMETOOLONG;
 	}
 
 	fd = open(lockfile_name.buf, O_NONBLOCK|O_RDONLY, 0);
 	if (fd == -1) {
+		if (debug) {
+			saveerrno = errno;
+			DBG_ERR("failed to open %s errno %d %s\n",
+				lockfile_name.buf,
+				(int)saveerrno,
+				strerror(errno));
+
+		}
 		return errno;
 	}
 
 	ret = messaging_dgm_read_unique(fd, unique);
+	if (debug) {
+		DBG_ERR("messaging_dgm_read_unique returns %d\n", ret);
+	}
 	close(fd);
 	return ret;
 }
 
+int messaging_dgm_get_unique(pid_t pid, uint64_t *unique)
+{
+	return messaging_dgm_get_unique_debug(pid, unique, false);
+}
+
 int messaging_dgm_cleanup(pid_t pid)
 {
 	struct messaging_dgm_context *ctx = global_dgm_context;
diff --git a/lib/messaging/messages_dgm.h b/lib/messaging/messages_dgm.h
index 7221c72660b..eb711f07dbe 100644
--- a/lib/messaging/messages_dgm.h
+++ b/lib/messaging/messages_dgm.h
@@ -37,6 +37,7 @@ int messaging_dgm_init(struct tevent_context *ev,
 		       void *recv_cb_private_data);
 void messaging_dgm_destroy(void);
 int messaging_dgm_get_unique(pid_t pid, uint64_t *unique);
+int messaging_dgm_get_unique_debug(pid_t pid, uint64_t *unique, bool debug);
 int messaging_dgm_send(pid_t pid,
 		       const struct iovec *iov, int iovlen,
 		       const int *fds, size_t num_fds);
diff --git a/lib/util/samba_util.h b/lib/util/samba_util.h
index 4e279b72df1..bb985d4c354 100644
--- a/lib/util/samba_util.h
+++ b/lib/util/samba_util.h
@@ -497,6 +497,7 @@ _PUBLIC_ char* get_myname(TALLOC_CTX *mem_ctx);
 /**
  Check if a process exists. Does this work on all unixes?
 **/
+_PUBLIC_ bool process_exists_by_pid_debug(pid_t pid, bool debug);
 _PUBLIC_ bool process_exists_by_pid(pid_t pid);
 
 /**
diff --git a/lib/util/util.c b/lib/util/util.c
index c066406d320..71852765b4d 100644
--- a/lib/util/util.c
+++ b/lib/util/util.c
@@ -383,14 +383,29 @@ _PUBLIC_ char *get_myname(TALLOC_CTX *ctx)
  Check if a process exists. Does this work on all unixes?
 **/
 
-_PUBLIC_ bool process_exists_by_pid(pid_t pid)
+_PUBLIC_ bool process_exists_by_pid_debug(pid_t pid, bool debug)
 {
 	/* Doing kill with a non-positive pid causes messages to be
 	 * sent to places we don't want. */
+	int ret;
+	int saved_errno;
 	if (pid <= 0) {
+		if (debug) {
+			DBG_ERR("%d < 0, return false\n", (int)pid);
+		}
 		return false;
 	}
-	return(kill(pid,0) == 0 || errno != ESRCH);
+	ret = kill(pid, 0);
+	saved_errno = errno;
+	if (debug) {
+		DBG_ERR("kill(%d, 0) -> %d - %d\n", (int)pid, ret, saved_errno);
+	}
+	return(ret == 0 || saved_errno != ESRCH);
+}
+
+_PUBLIC_ bool process_exists_by_pid(pid_t pid)
+{
+	return process_exists_by_pid_debug(pid, false);
 }
 
 /**
diff --git a/source3/include/serverid.h b/source3/include/serverid.h
index 89487cfa62d..2bbe59e0947 100644
--- a/source3/include/serverid.h
+++ b/source3/include/serverid.h
@@ -27,5 +27,6 @@
  * Check existence of a server id
  */
 bool serverid_exists(const struct server_id *id);
+bool serverid_exists_debug(const struct server_id *id, bool debug);
 
 #endif
diff --git a/source3/lib/g_lock.c b/source3/lib/g_lock.c
index 7eae374fbcf..5a2cb32ed4a 100644
--- a/source3/lib/g_lock.c
+++ b/source3/lib/g_lock.c
@@ -24,6 +24,7 @@
 #include "lib/util/talloc_stack.h"
 #include "lib/util/samba_util.h"
 #include "lib/util_path.h"
+#include "lib/messaging/messages_dgm.h"
 #include "dbwrap/dbwrap.h"
 #include "dbwrap/dbwrap_open.h"
 #include "dbwrap/dbwrap_watch.h"
@@ -390,8 +391,14 @@ static NTSTATUS g_lock_trylock(
 		bool self_exclusive = server_id_equal(&self, &lck.exclusive);
 
 		if (!self_exclusive) {
-			bool exists = serverid_exists(&lck.exclusive);
+			bool exists = serverid_exists_debug(&lck.exclusive,
+							    true);
 			if (!exists) {
+				struct server_id_buf tmp1;
+				DBG_ERR("it appears lck.exclusive pid %s doesn't exist,"
+					"taking over the lock\n",
+					server_id_str_buf(lck.exclusive,
+							  &tmp1));
 				lck.exclusive = (struct server_id) { .pid=0 };
 				goto noexclusive;
 			}
@@ -827,6 +834,7 @@ NTSTATUS g_lock_lock(struct g_lock_ctx *ctx, TDB_DATA key,
 		 */
 	}
 
+	DBG_ERR("Falling back to g_lock_trylock...\n");
 	frame = talloc_stackframe();
 	status = NT_STATUS_NO_MEMORY;
 
diff --git a/source3/lib/serverid.c b/source3/lib/serverid.c
index 70739c79d2f..f9de09b321f 100644
--- a/source3/lib/serverid.c
+++ b/source3/lib/serverid.c
@@ -25,13 +25,24 @@
 #include "lib/messages_ctdb.h"
 #include "lib/messaging/messages_dgm.h"
 
-static bool serverid_exists_local(const struct server_id *id)
+static bool serverid_exists_local(const struct server_id *id, bool debug)
 {
-	bool exists = process_exists_by_pid(id->pid);
 	uint64_t unique;
+	struct server_id_buf tmp;
+	bool exists;
 	int ret;
+	if (debug) {
+		DBG_ERR("Checking for %s locally clustering enabled ? %s\n",
+			server_id_str_buf(*id, &tmp),
+			lp_clustering() ? "yes" : "no");
+	}
+
+	exists = process_exists_by_pid_debug(id->pid, debug);
 
 	if (!exists) {
+		if (debug) {
+			DBG_ERR("%s does not exists locally\n",
+				  server_id_str_buf(*id, &tmp)); }
 		return false;
 	}
 
@@ -39,18 +50,31 @@ static bool serverid_exists_local(const struct server_id *id)
 		return true;
 	}
 
-	ret = messaging_dgm_get_unique(id->pid, &unique);
+	ret = messaging_dgm_get_unique_debug(id->pid, &unique, debug);
+
 	if (ret != 0) {
+		if (debug) {
+			DBG_ERR("Failed to get unique: %d\n", ret);
+		}
 		return false;
 	}
+	if (debug) {
+		DBG_ERR("unique: %"PRIu64", id->unique_id: %"PRIu64"\n",
+			  unique, id->unique_id);
+	}
 
 	return (unique == id->unique_id);
 }
 
-bool serverid_exists(const struct server_id *id)
+bool serverid_exists_debug(const struct server_id *id, bool debug)
 {
 	if (procid_is_local(id)) {
-		return serverid_exists_local(id);
+		return serverid_exists_local(id, debug);
+	} else {
+		if (debug) {
+			DBG_ERR("procid_is_local() failed, id->vnn(%d) != "
+				"my_vnn (%d)\n", id->vnn, get_my_vnn());
+		}
 	}
 
 	if (lp_clustering()) {
@@ -60,3 +84,8 @@ bool serverid_exists(const struct server_id *id)
 
 	return false;
 }
+
+bool serverid_exists(const struct server_id *id)
+{
+	return serverid_exists_debug(id, false);
+}
-- 
2.35.3

