Remove daemon.py exit logging, since it caused problems:

After including these logging statements, orphaned workers might stay alive
after the driver died.  My current theory is that the print to sys.stderr
failed due to Java closing the file and an exception was thrown that managed
to propagate to to the uncaught exception handler, causing daemon.py to exit
before it could send SIGHUP to its children.

Author: JoshRosen

python/pyspark/daemon.py

@@ -47,6 +47,7 @@ def worker(sock):
 
     signal.signal(SIGHUP, SIG_DFL)
     signal.signal(SIGCHLD, SIG_DFL)
+    signal.signal(SIGTERM, SIG_DFL)
 
     # Blocks until the socket is closed by draining the input stream
     # until it raises an exception or returns EOF.
@@ -93,10 +94,9 @@ def shutdown(code):
         os.kill(0, SIGHUP)
         exit(code)
 
-    def sig_term(signum, frame):
-        print >> sys.stderr, "daemon.py shutting down due to SIGTERM"
+    def handle_sigterm(*args):
         shutdown(1)
-    signal.signal(SIGTERM, sig_term)  # Gracefully exit on SIGTERM
+    signal.signal(SIGTERM, handle_sigterm)  # Gracefully exit on SIGTERM
     signal.signal(SIGHUP, SIG_IGN)  # Don't die on SIGHUP
 
     # Cleanup zombie children
@@ -124,7 +124,6 @@ def handle_sigchld(*args):
                     raise
             if 0 in ready_fds:
                 # Spark told us to exit by closing stdin
-                print >> sys.stderr, "daemon.py shutting down because Java closed stdin"
                 shutdown(0)
             if listen_sock in ready_fds:
                 sock, addr = listen_sock.accept()
@@ -141,7 +140,6 @@ def handle_sigchld(*args):
                 else:
                     sock.close()
     finally:
-        print >> sys.stderr, "daemon.py shutting down due to uncaught exception"
         shutdown(1)