From 06fd6c8a41ba01e92afcc3ba3501cf8544faa2f6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 03:18:52 +0100 Subject: [PATCH 01/10] implement PSUTIL_DEBUG from C module --- psutil/_psutil_common.c | 3 +++ psutil/_psutil_common.h | 1 + 2 files changed, 4 insertions(+) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index cf9899f11..d023f12b0 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -11,6 +11,7 @@ // Global vars. +int PSUTIL_DEBUG = 0; int PSUTIL_TESTING = 0; @@ -80,6 +81,8 @@ psutil_set_testing(PyObject *self, PyObject *args) { */ void psutil_setup(void) { + if (getenv("PSUTIL_DEBUG") != NULL) + PSUTIL_DEBUG = 1; if (getenv("PSUTIL_TESTING") != NULL) PSUTIL_TESTING = 1; } diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index a609b886a..3c36a6fee 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -7,6 +7,7 @@ #include extern int PSUTIL_TESTING; +extern int PSUTIL_DEBUG; // a signaler for connections without an actual status static const int PSUTIL_CONN_NONE = 128; From 8f2b5a1a2d281056afdc06e5edf190703ca205ef Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 03:29:34 +0100 Subject: [PATCH 02/10] update doc --- HISTORY.rst | 5 +++++ docs/index.rst | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 48adbe523..6dadb0312 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,11 @@ *XXXX-XX-XX* +**Enhancements** + +- 1173_: introduced PSUTIL_DEBUG environment variable which can be set in order + to print useful debug messages on stderr (useful in case of nasty errors). + **Bug fixes** - 1152_: [Windows] disk_io_counters() may return an empty dict. diff --git a/docs/index.rst b/docs/index.rst index ce798e923..befd5dcad 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2300,6 +2300,28 @@ Constants ---- +Debug mode +========== + +In case you bump into nasty errors which look like being psutil's fault you may +want to run psutil in debug mode. psutil may (or may not) print some useful +message on stderr before crashing with an exception. +To enable debug mode on UNIX: + +:: + + PSUTIL_DEBUG=1 python script.py + +On Windows: + +:: + + set PSUTIL_DEBUG=1 && C:\python36\python.exe script.py + +.. versionadded:: 5.4.2 + +---- + Unicode ======= From 81deaa81713656c48320b0ca0e89384a3eca0616 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 03:42:18 +0100 Subject: [PATCH 03/10] add psutil_debug() utility function --- psutil/_psutil_common.c | 10 ++++++++++ psutil/_psutil_common.h | 1 + 2 files changed, 11 insertions(+) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index d023f12b0..74a0479d3 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -76,6 +76,16 @@ psutil_set_testing(PyObject *self, PyObject *args) { } +/* + * Print a debug message on stderr. No-op if PSUTIL_DEBUG env var is not set. + */ +void +psutil_debug(char *msg) { + if (PSUTIL_DEBUG) + fprintf(stderr, "psutil-debug> %s\n", msg); +} + + /* * Called on module import on all platforms. */ diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 3c36a6fee..c90cae7c0 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -21,4 +21,5 @@ PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); PyObject* psutil_set_testing(PyObject *self, PyObject *args); +void psutil_debug(char *msg); void psutil_setup(void); From 589e1c52e2ee2a5bd45fe27fe95cb212e743fd9e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 04:04:59 +0100 Subject: [PATCH 04/10] update doc --- docs/index.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index befd5dcad..620fea464 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2305,16 +2305,17 @@ Debug mode In case you bump into nasty errors which look like being psutil's fault you may want to run psutil in debug mode. psutil may (or may not) print some useful -message on stderr before crashing with an exception. +message on stderr before crashing with an exception +(see `original motivation `__). To enable debug mode on UNIX: -:: +.. code-block:: bash PSUTIL_DEBUG=1 python script.py On Windows: -:: +.. code-block:: bat set PSUTIL_DEBUG=1 && C:\python36\python.exe script.py From 8bb5b98f42355297ae7f00de6e1ebcc960430bbf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 04:54:13 +0100 Subject: [PATCH 05/10] enable PSUTIL_DEBUG for tests --- Makefile | 2 +- psutil/_psutil_windows.c | 6 ++++++ scripts/internal/winmake.py | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e8ff51ca3..7b2eca867 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ DEPS = \ # In not in a virtualenv, add --user options for install commands. INSTALL_OPTS = `$(PYTHON) -c "import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"` -TEST_PREFIX = PYTHONWARNINGS=all PSUTIL_TESTING=1 +TEST_PREFIX = PYTHONWARNINGS=all PSUTIL_TESTING=1 PSUTIL_DEBUG=1 all: test diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 574885b64..d9129322e 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -341,6 +341,8 @@ psutil_proc_kill(PyObject *self, PyObject *args) { if (hProcess == NULL) { if (GetLastError() == ERROR_INVALID_PARAMETER) { // see https://github.com/giampaolo/psutil/issues/24 + psutil_debug("OpenProcess -> ERROR_INVALID_PARAMETER turned " + "into NoSuchProcess"); NoSuchProcess(); } else { @@ -2409,10 +2411,14 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { // 1364/job/ascpdi271b06jle3 // Assume it means we're dealing with some exotic disk // and go on. + psutil_debug("DeviceIoControl -> ERROR_INVALID_FUNCTION; " + "ignore a disk"); goto next; } else if (GetLastError() == ERROR_NOT_SUPPORTED) { // Again, let's assume we're dealing with some exotic disk. + psutil_debug("DeviceIoControl -> ERROR_NOT_SUPPORTED; " + "ignore a disk"); goto next; } // XXX: it seems we should also catch ERROR_INVALID_PARAMETER: diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 35ce059cf..548f7a8ed 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -177,6 +177,7 @@ def recursive_rm(*patterns): def test_setup(): os.environ['PYTHONWARNINGS'] = 'all' os.environ['PSUTIL_TESTING'] = '1' + os.environ['PSUTIL_DEBUG'] = '1' # =================================================================== From 2bb283db8b3b60409e5ff4977bd870fc41f985cf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 04:59:26 +0100 Subject: [PATCH 06/10] update appveyor.yml --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index f17b5b878..092dc23a8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -83,7 +83,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "set PYTHONWARNINGS=all && set PSUTIL_TESTING=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" + - "set PYTHONWARNINGS=all && set PSUTIL_TESTING=1 && set PSUTIL_DEBUG=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" From a9d1ad4b64d00fcd82fd22e257227b3cbdb8c926 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 12:07:21 +0100 Subject: [PATCH 07/10] change psutil_debug() signature so that it can accept variable num of args --- psutil/_psutil_common.c | 8 +++++--- psutil/_psutil_common.h | 2 +- psutil/_psutil_windows.c | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 74a0479d3..3cf0ded67 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -80,9 +80,11 @@ psutil_set_testing(PyObject *self, PyObject *args) { * Print a debug message on stderr. No-op if PSUTIL_DEBUG env var is not set. */ void -psutil_debug(char *msg) { - if (PSUTIL_DEBUG) - fprintf(stderr, "psutil-debug> %s\n", msg); +psutil_debug(const char* format, ...) { + va_list argptr; + va_start(argptr, format); + vfprintf(stderr, format, argptr); + va_end(argptr); } diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index c90cae7c0..965966af7 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -21,5 +21,5 @@ PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); PyObject* psutil_set_testing(PyObject *self, PyObject *args); -void psutil_debug(char *msg); +void psutil_debug(const char* format, ...); void psutil_setup(void); diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index d9129322e..e95ab45c2 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2412,13 +2412,13 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { // Assume it means we're dealing with some exotic disk // and go on. psutil_debug("DeviceIoControl -> ERROR_INVALID_FUNCTION; " - "ignore a disk"); + "ignore PhysicalDrive%i", devNum); goto next; } else if (GetLastError() == ERROR_NOT_SUPPORTED) { // Again, let's assume we're dealing with some exotic disk. psutil_debug("DeviceIoControl -> ERROR_NOT_SUPPORTED; " - "ignore a disk"); + "ignore PhysicalDrive%i", devNum); goto next; } // XXX: it seems we should also catch ERROR_INVALID_PARAMETER: @@ -2433,7 +2433,7 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { goto error; } - sprintf_s(szDeviceDisplay, MAX_PATH, "PhysicalDrive%d", devNum); + sprintf_s(szDeviceDisplay, MAX_PATH, "PhysicalDrive%i", devNum); py_tuple = Py_BuildValue( "(IILLKK)", diskPerformance.ReadCount, From 77e77e6115799648c01f67ef033a4e778c542732 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 13:19:28 +0100 Subject: [PATCH 08/10] provide DEBUG info in psutil_raise_for_pid() --- psutil/_psutil_bsd.c | 2 +- psutil/_psutil_osx.c | 13 +++++++------ psutil/_psutil_posix.c | 15 +++++++++++---- psutil/arch/freebsd/proc_socks.c | 2 +- psutil/arch/freebsd/specific.c | 6 +++--- psutil/arch/netbsd/specific.c | 2 +- psutil/arch/openbsd/specific.c | 4 ++-- psutil/arch/osx/process_info.c | 2 +- 8 files changed, 27 insertions(+), 19 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index f1adb1d37..9a2ed04bc 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -475,7 +475,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); goto error; } diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 4ff301d4f..bb98c1cc1 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -266,7 +266,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (pid == 0) AccessDenied(); else - psutil_raise_for_pid(pid, "proc_pidpath() syscall failed"); + psutil_raise_for_pid(pid, "proc_pidpath()"); return NULL; } return PyUnicode_DecodeFSDefault(buf); @@ -336,7 +336,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { err = task_for_pid(mach_task_self(), (pid_t)pid, &task); if (err != KERN_SUCCESS) { - psutil_raise_for_pid(pid, "task_for_pid() failed"); + psutil_raise_for_pid(pid, "task_for_pid()"); goto error; } @@ -376,8 +376,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { errno = 0; proc_regionfilename((pid_t)pid, address, buf, sizeof(buf)); if ((errno != 0) || ((sizeof(buf)) <= 0)) { - psutil_raise_for_pid( - pid, "proc_regionfilename() syscall failed"); + psutil_raise_for_pid(pid, "proc_regionfilename()"); goto error; } @@ -1147,7 +1146,8 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { continue; } else { - psutil_raise_for_pid(pid, "proc_pidinfo() syscall failed"); + psutil_raise_for_pid( + pid, "proc_pidinfo(PROC_PIDFDVNODEPATHINFO)"); goto error; } } @@ -1259,7 +1259,8 @@ psutil_proc_connections(PyObject *self, PyObject *args) { continue; } else { - psutil_raise_for_pid(pid, "proc_pidinfo() syscall failed"); + psutil_raise_for_pid( + pid, "proc_pidinfo(PROC_PIDFDSOCKETINFO)"); goto error; } } diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index ea0fba534..8ccd8a91e 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -112,16 +112,23 @@ psutil_pid_exists(long pid) { * This will always set a Python exception and return NULL. */ int -psutil_raise_for_pid(long pid, char *msg) { +psutil_raise_for_pid(long pid, char *syscall_name) { // Set exception to AccessDenied if pid exists else NoSuchProcess. if (errno != 0) { + // Unlikely we get here. + psutil_debug("%s syscall failed and set errno to %i; raising " + "OSError(errno)", syscall_name, errno); PyErr_SetFromErrno(PyExc_OSError); return 0; } - if (psutil_pid_exists(pid) == 0) + else if (psutil_pid_exists(pid) == 0) { + psutil_debug("%s syscall failed and PID %i no longer exists; " + "assume NoSuchProcess", syscall_name, pid); NoSuchProcess(); - else - PyErr_SetString(PyExc_RuntimeError, msg); + } + else { + PyErr_Format(PyExc_RuntimeError, "%s syscall failed", syscall_name); + } return 0; } diff --git a/psutil/arch/freebsd/proc_socks.c b/psutil/arch/freebsd/proc_socks.c index c5b19a0de..a458a01e5 100644 --- a/psutil/arch/freebsd/proc_socks.c +++ b/psutil/arch/freebsd/proc_socks.c @@ -212,7 +212,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); goto error; } diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index 8d09ad89a..ff128e65f 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -548,7 +548,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); goto error; } @@ -597,7 +597,7 @@ psutil_proc_num_fds(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); return NULL; } free(freep); @@ -765,7 +765,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getvmmap(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getvmmap() failed"); + psutil_raise_for_pid(pid, "kinfo_getvmmap()"); goto error; } for (i = 0; i < cnt; i++) { diff --git a/psutil/arch/netbsd/specific.c b/psutil/arch/netbsd/specific.c index 1dc2080ee..0a32139d3 100644 --- a/psutil/arch/netbsd/specific.c +++ b/psutil/arch/netbsd/specific.c @@ -502,7 +502,7 @@ psutil_proc_num_fds(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); return NULL; } free(freep); diff --git a/psutil/arch/openbsd/specific.c b/psutil/arch/openbsd/specific.c index de30c4d7d..2a0d30cea 100644 --- a/psutil/arch/openbsd/specific.c +++ b/psutil/arch/openbsd/specific.c @@ -404,7 +404,7 @@ psutil_proc_num_fds(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); return NULL; } free(freep); @@ -509,7 +509,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); goto error; } diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/osx/process_info.c index 7c715be81..f0a011320 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -354,7 +354,7 @@ psutil_proc_pidinfo(long pid, int flavor, uint64_t arg, void *pti, int size) { errno = 0; int ret = proc_pidinfo((int)pid, flavor, arg, pti, size); if ((ret <= 0) || ((unsigned long)ret < sizeof(pti))) { - psutil_raise_for_pid(pid, "proc_pidinfo() syscall failed"); + psutil_raise_for_pid(pid, "proc_pidinfo()"); return 0; } return ret; From 648ebe12265d20025ec1557af3f302f6fe5c2802 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 13:35:15 +0100 Subject: [PATCH 09/10] properly print debug message --- psutil/_psutil_common.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 3cf0ded67..e9fce85e6 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -83,7 +83,9 @@ void psutil_debug(const char* format, ...) { va_list argptr; va_start(argptr, format); + fprintf(stderr, "psutil-dubug> "); vfprintf(stderr, format, argptr); + fprintf(stderr, "\n"); va_end(argptr); } From 3c5438978f621f18b8e9b0b70fde4182221d7cb3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 13:40:24 +0100 Subject: [PATCH 10/10] do not print too much --- psutil/_psutil_posix.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index 8ccd8a91e..76cf2db03 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -116,8 +116,6 @@ psutil_raise_for_pid(long pid, char *syscall_name) { // Set exception to AccessDenied if pid exists else NoSuchProcess. if (errno != 0) { // Unlikely we get here. - psutil_debug("%s syscall failed and set errno to %i; raising " - "OSError(errno)", syscall_name, errno); PyErr_SetFromErrno(PyExc_OSError); return 0; }