From 9ae84ad187e2ba440c40f44b8eb21c82c2fdbf12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <christian.koenig@amd.com>
Date: Wed, 15 May 2013 11:21:36 +0200
Subject: [PATCH] fix deadlock with xcb_take_socket/return_socket v3
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

To prevent different threads from stealing the socket from each other the
caller of "xcb_take_socket" must hold a lock that is also acquired in
"return_socket". Unfortunately xcb tries to prevent calling return_socket
from multiple threads and this can lead to a deadlock situation.

A simple example:
- X11 has taken the socket
- Thread A has locked the display.
- Thread B does xcb_no_operation() and thus ends up in libX11's return_socket(),
  waiting for the display lock.
- Thread A calls e.g. xcb_no_operation(), too, ends up in return_socket() and
  because socket_moving == 1, ends up waiting for thread B
=> Deadlock

This patch allows calling return_socket from different threads at the same time
an so resolves the deadlock situation.

Partially fixes: https://bugs.freedesktop.org/show_bug.cgi?id=20708

v2: fixes additional pthread_cond_wait dependencies,
    rework comments and patch description

v3: separate pthread_cond_wait dependencies and unrelated whitespace
    change into their own patch, use unsigned for socket_seq

Signed-off-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Uli Schlachter <psychon@znc.in>
---
 src/xcb_out.c | 38 ++++++++++++++++++++------------------
 src/xcbext.h  |  1 +
 src/xcbint.h  |  7 ++++---
 3 files changed, 25 insertions(+), 21 deletions(-)

Index: libxcb-1.1/src/xcb_out.c
===================================================================
--- libxcb-1.1.orig/src/xcb_out.c
+++ libxcb-1.1/src/xcb_out.c
@@ -82,21 +82,24 @@ static void send_sync(xcb_connection_t *
 
 static void get_socket_back(xcb_connection_t *c)
 {
-    while(c->out.return_socket && c->out.socket_moving)
-        pthread_cond_wait(&c->out.socket_cond, &c->iolock);
-    if(!c->out.return_socket)
-        return;
-
-    c->out.socket_moving = 1;
-    pthread_mutex_unlock(&c->iolock);
-    c->out.return_socket(c->out.socket_closure);
-    pthread_mutex_lock(&c->iolock);
-    c->out.socket_moving = 0;
-
-    pthread_cond_broadcast(&c->out.socket_cond);
-    c->out.return_socket = 0;
-    c->out.socket_closure = 0;
-    _xcb_in_replies_done(c);
+    while (c->out.return_socket) {
+        /* we are about to release the lock,
+           so make a copy of the current status */
+        xcb_return_socket_func_t return_socket = c->out.return_socket;
+        void *socket_closure = c->out.socket_closure;
+        int socket_seq = c->out.socket_seq;
+
+        pthread_mutex_unlock(&c->iolock);
+        return_socket(socket_closure);
+        pthread_mutex_lock(&c->iolock);
+
+        /* make sure nobody else has acquired the socket */
+        if (socket_seq == c->out.socket_seq) {
+            c->out.return_socket = 0;
+            c->out.socket_closure = 0;
+            _xcb_in_replies_done(c);
+        }
+    }
 }
 
 /* Public interface */
@@ -274,6 +277,7 @@ int xcb_take_socket(xcb_connection_t *c,
     {
         c->out.return_socket = return_socket;
         c->out.socket_closure = closure;
+        ++c->out.socket_seq;
         if(flags)
             _xcb_in_expect_reply(c, c->out.request, WORKAROUND_EXTERNAL_SOCKET_OWNER, flags);
         assert(c->out.request == c->out.request_written);
@@ -310,11 +314,9 @@ int xcb_flush(xcb_connection_t *c)
 
 int _xcb_out_init(_xcb_out *out)
 {
-    if(pthread_cond_init(&out->socket_cond, 0))
-        return 0;
     out->return_socket = 0;
     out->socket_closure = 0;
-    out->socket_moving = 0;
+    out->socket_seq = 0;
 
     if(pthread_cond_init(&out->cond, 0))
         return 0;
Index: libxcb-1.1/src/xcbint.h
===================================================================
--- libxcb-1.1.orig/src/xcbint.h
+++ libxcb-1.1/src/xcbint.h
@@ -76,14 +76,15 @@ void *_xcb_map_remove(_xcb_map *q, unsig
 
 /* xcb_out.c */
 
+typedef void (*xcb_return_socket_func_t)(void *closure);
+
 typedef struct _xcb_out {
     pthread_cond_t cond;
     int writing;
 
-    pthread_cond_t socket_cond;
-    void (*return_socket)(void *closure);
+    xcb_return_socket_func_t return_socket;
     void *socket_closure;
-    int socket_moving;
+    unsigned int socket_seq;
 
     char queue[4096];
     int queue_len;
