Suppress automatic GIL acquire to avoid deadlock

CHANGELOG.md

@@ -13,6 +13,7 @@
 - PR #3004: Remove Single Process Multi GPU (SPMG) code
 
 ## Bug Fixes
+- PR #3037: Avoid logging deadlock in multi-threaded C code
 - PR #2983: Fix seeding of KISS99 RNG
 - PR #3011: Fix unused initialize_embeddings parameter in Barnes-Hut t-SNE
 - PR #3008: Check number of columns in check_array validator

python/cuml/common/logger.pyx

@@ -75,7 +75,7 @@ level_critical = CUML_LEVEL_CRITICAL
 """Disables all log messages"""
 level_off = CUML_LEVEL_OFF
 
-cdef void _log_callback(int lvl, const char * msg) with gil:
+cdef void _log_callback(int lvl, const char * msg) nogil:
     """
     Default spdlogs callback function to redirect logs correctly to sys.stdout
 
@@ -86,14 +86,35 @@ cdef void _log_callback(int lvl, const char * msg) with gil:
     msg : char *
         Message to be logged
     """
-    print(msg.decode('utf-8'), end='')
+    with gil:
+        print(msg.decode('utf-8'), end='')
 
-cdef void _log_flush() with gil:
+
+cdef void _nogil_log_callback(int lvl, const char * msg) nogil:
+    """
+    Wrapper for _log_callback to explicitly disable Cython's automatic GIL
+    acquire
+    """
+    with nogil:
+        _log_callback(lvl, msg)
+
+
+cdef void _log_flush() nogil:
     """
     Default spdlogs callback function to flush logs
     """
-    if sys.stdout is not None:
-        sys.stdout.flush()
+    with gil:
+        if sys.stdout is not None:
+            sys.stdout.flush()
+
+
+cdef void _nogil_log_flush() nogil:
+    """
+    Wrapper for _log_flush to explicitly disable Cython's automatic GIL
+    acquire
+    """
+    with nogil:
+        _log_flush()
 
 
 class LogLevelSetter:
@@ -345,5 +366,5 @@ def flush():
 
 
 # Set callback functions to handle redirected sys.stdout in Python
-Logger.get().setCallback(_log_callback)
-Logger.get().setFlush(_log_flush)
+Logger.get().setCallback(_nogil_log_callback)
+Logger.get().setFlush(_nogil_log_flush)