mpm: Fix more gevent errors on SIGTERM

Sometimes when running usrp_hwd.py in a terminal and then canceling it
with Ctrl+C, it prints a really large stacktrace into the terminal
resulting from an uncaught gevent BlockingSwitchOutError. This comes
from trying to block on Process#join inside a gevent signal handler.

This commit resolves this issue by simply triggering an event in the
signal handler which prompts a different non-daemon thread to join the
subprocesses and end the parent process.

Signed-off-by: Samuel O'Brien <sam.obrien@ni.com>

1 files changed, 19 insertions, 6 deletions

diff --git a/mpm/python/usrp_hwd.py b/mpm/python/usrp_hwd.py
index 3523fa9b4..fb8c1b94e 100755
--- a/mpm/python/usrp_hwd.py
+++ b/mpm/python/usrp_hwd.py
@@ -12,15 +12,18 @@ import sys
 import time
 import argparse
 from gevent import signal
-try:
-    from gevent.hub import BlockingSwitchOutError
-except ImportError:
-    from gevent.exceptions import BlockingSwitchOutError
 import usrp_mpm as mpm
 from usrp_mpm.mpmtypes import SharedState
 from usrp_mpm.sys_utils import watchdog
 
+# pylint: disable=wrong-import-order
+# We have to import threading here because it must be imported after
+# gevent.monkey.patch_all is called in rpc_server.py.
+# (imported in the usrp_mpm __init__.py)
+from threading import Event, Thread
+
 _PROCESSES = []
+_KILL_EVENT = Event()
 
 def setup_arg_parser():
     """
@@ -92,17 +95,23 @@ def parse_args():
 
 def kill_time(sig, frame):
     """
-    kill all processes
+    kill all processes by setting _KILL_EVENT
     to be used in a signal handler
+    """
+    _KILL_EVENT.set()
 
+def kill_thread():
+    """
+    Kill all processes after _KILL_EVENT is triggered
     If all processes are properly terminated, this will exit
     """
+    _KILL_EVENT.wait()
     log = mpm.get_main_logger().getChild('kill')
     for proc in _PROCESSES:
         proc.terminate()
         log.info("Terminating pid: {0}".format(proc.pid))
     for proc in _PROCESSES:
-            proc.join()
+        proc.join()
     log.info("System exiting")
     sys.exit(0)
 
@@ -154,6 +163,10 @@ def spawn_processes(log, args):
     )
     log.debug("Discovery process has PID: %d", _PROCESSES[-1].pid)
     log.info("Processes launched. Registering signal handlers.")
+    # Launch the kill thread
+    # This is used because we cannot block in a signal handler,
+    # meaning we cannot join threads
+    Thread(target=kill_thread, daemon=False).start()
     signal.signal(signal.SIGTERM, kill_time)
     signal.signal(signal.SIGINT, kill_time)
     for proc in _PROCESSES: