>From 63f0eb0889c8f8a82ba06a02a8a92d695902baad Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Thu, 28 Sep 2017 10:41:38 +0100
Subject: [PATCH] Wait up to 5 seconds for running threads to complete before
 exiting.

There is a race condition where nbdkit exits (calling plugin_cleanup()
which unloads the plugin), while the plugin is in its plugin.close()
function in another thread.

This keeps track of the number of running threads and waits up to 5
seconds for the running threads to exit before exiting the main
program.

For further analysis, see also:
https://www.redhat.com/archives/libguestfs/2017-September/msg00226.html
---
 src/internal.h    |  3 +++
 src/main.c        | 13 +++++++++++++
 src/threadlocal.c | 35 +++++++++++++++++++++++++++++++++++
 3 files changed, 51 insertions(+)

diff --git a/src/internal.h b/src/internal.h
index 1fc5d69..b98be3d 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -182,6 +182,9 @@ extern size_t threadlocal_get_instance_num (void);
 extern void threadlocal_set_error (int err);
 extern int threadlocal_get_error (void);
 /*extern void threadlocal_get_sockaddr ();*/
+extern size_t get_running_threads (void);
+extern void incr_running_threads (void);
+extern void decr_running_threads (void);
 
 /* Declare program_name. */
 #if HAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME == 1
diff --git a/src/main.c b/src/main.c
index c4e41e4..010a21a 100644
--- a/src/main.c
+++ b/src/main.c
@@ -184,6 +184,7 @@ main (int argc, char *argv[])
   int option_index;
   int help = 0, version = 0, dump_plugin = 0;
   int tls_set_on_cli = 0;
+  size_t count;
 
   threadlocal_init ();
 
@@ -486,6 +487,12 @@ main (int argc, char *argv[])
 
   start_serving ();
 
+  /* Wait, but not forever, for all threads to complete. */
+  debug ("waiting for running threads to complete");
+  for (count = 0; count < 5 && get_running_threads (); ++count)
+    sleep (1);
+  debug ("waited %zus for running threads to complete", count);
+
   plugin_cleanup ();
 
   free (unixsocket);
@@ -631,6 +638,12 @@ start_serving (void)
     threadlocal_new_server_thread ();
     if (handle_single_connection (0, 1) == -1)
       exit (EXIT_FAILURE);
+    /* When handling a single connection we reuse the main thread as
+     * the server thread.  We don't want to count it as a running
+     * thread, but it hasn't called pthread_exit(), so we need to
+     * explicitly decrement the running threads here.
+     */
+    decr_running_threads ();
     return;
   }
 
diff --git a/src/threadlocal.c b/src/threadlocal.c
index d6e3942..cc19d48 100644
--- a/src/threadlocal.c
+++ b/src/threadlocal.c
@@ -71,6 +71,8 @@ free_threadlocal (void *threadlocalv)
 
   free (threadlocal->addr);
   free (threadlocal);
+
+  decr_running_threads ();
 }
 
 void
@@ -91,6 +93,8 @@ threadlocal_new_server_thread (void)
 {
   struct threadlocal *threadlocal;
 
+  incr_running_threads ();
+
   threadlocal = calloc (1, sizeof *threadlocal);
   if (threadlocal == NULL) {
     perror ("malloc");
@@ -177,3 +181,34 @@ threadlocal_get_error (void)
   errno = err;
   return threadlocal ? threadlocal->err : 0;
 }
+
+/* These functions keep track of the number of running threads. */
+static pthread_mutex_t running_threads_lock = PTHREAD_MUTEX_INITIALIZER;
+static size_t running_threads = 0;
+
+size_t
+get_running_threads (void)
+{
+  size_t r;
+
+  pthread_mutex_lock (&running_threads_lock);
+  r = running_threads;
+  pthread_mutex_unlock (&running_threads_lock);
+  return r;
+}
+
+void
+incr_running_threads (void)
+{
+  pthread_mutex_lock (&running_threads_lock);
+  running_threads++;
+  pthread_mutex_unlock (&running_threads_lock);
+}
+
+void
+decr_running_threads (void)
+{
+  pthread_mutex_lock (&running_threads_lock);
+  running_threads--;
+  pthread_mutex_unlock (&running_threads_lock);
+}
-- 
2.13.2