From 1ced30a4b5e4e1aadacdc7f0d168a261a2d1c782 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 15 Nov 2016 19:13:26 +0100 Subject: [PATCH 001/922] update README --- README.rst | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ecd642ac3..76db114a7 100644 --- a/README.rst +++ b/README.rst @@ -215,6 +215,16 @@ Process management >>> p.cmdline() ['/usr/bin/python', 'main.py'] >>> + >>> p.pid + 7055 + >>> p.ppid() + 7054 + >>> p.parent() + + >>> p.children() + [, + ] + >>> >>> p.status() 'running' >>> p.username() @@ -242,10 +252,10 @@ Process management >>> >>> p.memory_info() pmem(rss=10915840, vms=67608576, shared=3313664, text=2310144, lib=0, data=7262208, dirty=0) - >>> >>> p.memory_full_info() # "real" USS memory usage (Linux, OSX, Win only) pfullmem(rss=10199040, vms=52133888, shared=3887104, text=2867200, lib=0, data=5967872, dirty=0, uss=6545408, pss=6872064, swap=0) - >>> + >>> p.memory_percent() + 0.7823 >>> p.memory_maps() [pmmap_grouped(path='/lib/x8664-linux-gnu/libutil-2.15.so', rss=32768, size=2125824, pss=32768, shared_clean=0, shared_dirty=0, private_clean=20480, private_dirty=12288, referenced=32768, anonymous=12288, swap=0), pmmap_grouped(path='/lib/x8664-linux-gnu/libc-2.15.so', rss=3821568, size=3842048, pss=3821568, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=3821568, referenced=3575808, anonymous=3821568, swap=0), @@ -297,6 +307,10 @@ Process management 'XDG_CONFIG_DIRS': '/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg', 'COLORTERM': 'gnome-terminal', ...} >>> + >>> p.as_dict() + {'status': 'running', 'num_ctx_switches': pctxsw(voluntary=63, involuntary=1), 'pid': 5457, ...} + >>> p.is_running() + True >>> p.suspend() >>> p.resume() >>> From 1a17c0ddf5e70c9966e1ce35bfee9ef4c0bc42c6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 18 Nov 2016 18:08:48 +0100 Subject: [PATCH 002/922] update README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 76db114a7..fcdc8cd9e 100644 --- a/README.rst +++ b/README.rst @@ -10,7 +10,7 @@ :target: https://coveralls.io/github/giampaolo/psutil?branch=master :alt: Test coverage (coverall.io) -.. image:: https://img.shields.io/pypi/v/psutil.svg?label=version +.. image:: https://img.shields.io/pypi/v/psutil.svg?label=pypi :target: https://pypi.python.org/pypi/psutil/ :alt: Latest version @@ -70,7 +70,7 @@ Also see https://github.com/giampaolo/psutil/tree/master/scripts. Projects using psutil ===================== -At the time of writing there are currently almost +At the time of writing there are over `4000 projects `__ on github which depend from psutil. Here's some I find particularly interesting: From 6c3e3e188652781224c063e907658948600e7872 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 19 Nov 2016 04:02:30 +0100 Subject: [PATCH 003/922] #693 / win / users(): use WTS_CURRENT_SERVER_HANDLE instead of WTSOpenServer() --- psutil/_psutil_windows.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 00bd22342..4d939aff6 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2552,7 +2552,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { */ static PyObject * psutil_users(PyObject *self, PyObject *args) { - HANDLE hServer = NULL; + HANDLE hServer = WTS_CURRENT_SERVER_HANDLE; LPTSTR buffer_user = NULL; LPTSTR buffer_addr = NULL; PWTS_SESSION_INFO sessions = NULL; @@ -2581,12 +2581,6 @@ psutil_users(PyObject *self, PyObject *args) { WinStationQueryInformationW = (PWINSTATIONQUERYINFORMATIONW) \ GetProcAddress(hInstWinSta, "WinStationQueryInformationW"); - hServer = WTSOpenServer('\0'); - if (hServer == NULL) { - PyErr_SetFromWindowsErr(0); - goto error; - } - if (WTSEnumerateSessions(hServer, 0, 1, &sessions, &count) == 0) { PyErr_SetFromWindowsErr(0); goto error; @@ -2671,7 +2665,6 @@ psutil_users(PyObject *self, PyObject *args) { Py_XDECREF(py_tuple); } - WTSCloseServer(hServer); WTSFreeMemory(sessions); WTSFreeMemory(buffer_user); WTSFreeMemory(buffer_addr); @@ -2686,8 +2679,6 @@ psutil_users(PyObject *self, PyObject *args) { if (hInstWinSta != NULL) FreeLibrary(hInstWinSta); - if (hServer != NULL) - WTSCloseServer(hServer); if (sessions != NULL) WTSFreeMemory(sessions); if (buffer_user != NULL) From 856b3bff92d2fdf56d14ff9690be6f11f35dc4d6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 25 Nov 2016 00:40:07 +0100 Subject: [PATCH 004/922] #371: implement hardware temperatures on Linux --- psutil/__init__.py | 33 +++++++++++++++++++++++++++++ psutil/_common.py | 2 ++ psutil/_pslinux.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+) diff --git a/psutil/__init__.py b/psutil/__init__.py index 3287a2c7e..2ab8de417 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2100,6 +2100,39 @@ def users(): return _psplatform.users() +if hasattr(_psplatform, "temperatures"): + + def temperatures(fahrenheit=False): + """Return hardware temperatures as a list of named tuples. + Each entry represents a "sensor" monitoring a certain hardware + resource. + The hardware resource may be a CPU, an hard disk or something + else, depending on the OS and its configuration. + All temperatures are expressed in celsius unless 'fahrenheit' + parameter is specified. + This function may raise NotImplementedError in case the OS + is not configured in order to provide these metrics. + """ + def to_fahrenheit(n): + return (n * 9 / 5) + 32 + + ret = [] + for rawtuple in _psplatform.temperatures(): + name, label, current, high, critical = rawtuple + if fahrenheit: + current = to_fahrenheit(current) + if high is not None: + high = to_fahrenheit(high) + if critical is not None: + critical = to_fahrenheit(critical) + if high and not critical: + critical = high + elif critical and not high: + high = critical + ret.append(_common.shwtemp(name, label, current, high, critical)) + return ret + + # ===================================================================== # --- Windows services # ===================================================================== diff --git a/psutil/_common.py b/psutil/_common.py index 3879a1d73..3e5f07f53 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -156,6 +156,8 @@ class NicDuplex(enum.IntEnum): # psutil.cpu_stats() scpustats = namedtuple( 'scpustats', ['ctx_switches', 'interrupts', 'soft_interrupts', 'syscalls']) +shwtemp = namedtuple( + 'shwtemp', ['name', 'label', 'current', 'high', 'critical']) # --- for Process methods diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 91fdae4f8..143dd5f26 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -9,6 +9,7 @@ import base64 import errno import functools +import glob import os import re import socket @@ -63,6 +64,7 @@ HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) HAS_PRLIMIT = hasattr(cext, "linux_prlimit") +_DEFAULT = object() # RLIMIT_* constants, not guaranteed to be present on all kernels if HAS_PRLIMIT: @@ -1059,6 +1061,56 @@ def boot_time(): "line 'btime' not found in %s/stat" % get_procfs_path()) +def temperatures(): + """Return hardware (CPU and others) temperatures as a list + of named tuples including name, label, current, max and + critical temperatures. + + Implementation notes: + - /sys/class/hwmon looks like the most recent interface to + retrieve this info, and this implementation relies on it + only (old distros will probably use something else) + - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon + - /sys/class/thermal/thermal_zone* is another one but it's more + difficult to parse + """ + def cat(fname, replace=_DEFAULT): + try: + f = open(fname) + except IOError: + if replace != _DEFAULT: + return replace + else: + raise + else: + with f: + return f.read().strip() + + path = '/sys/class/hwmon' + if not os.path.exists(path): + raise NotImplementedError( + "%s hwmon fs does not exist on this platform" % path) + + ret = [] + basenames = sorted(set( + [x.split('_')[0] for x in + glob.glob('/sys/class/hwmon/hwmon*/temp*_*')])) + for base in basenames: + name = cat(os.path.join(os.path.dirname(base), 'name')) + label = cat(base + '_label', replace='') + current = int(cat(base + '_input')) / 1000.0 + high = cat(base + '_max', replace=None) + if high is not None: + high = int(high) / 1000.0 + critical = cat(base + '_crit', replace=None) + if critical is not None: + critical = int(critical) / 1000.0 + + ret.append((name, label, current, high, critical)) + + return ret + + # ===================================================================== # --- processes # ===================================================================== From 95b3d9680052366202a2b2bfac63431884826e70 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 25 Nov 2016 19:17:22 +0100 Subject: [PATCH 005/922] #357: implement process cpu_num() on Linux --- docs/index.rst | 12 ++++++++++++ psutil/__init__.py | 12 ++++++++++++ psutil/_pslinux.py | 5 +++++ psutil/tests/test_process.py | 9 +++++++++ 4 files changed, 38 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 7fd79a1e3..4f0416753 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1180,6 +1180,18 @@ Process class .. versionchanged:: 2.2.0 added support for FreeBSD + .. method:: cpu_num() + + Return what CPU this process is currently running on. + The returned number should be ``<=`` :func:`psutil.cpu_count()` and + ``<= len(psutil.cpu_percent(percpu=True))``. + It may be used in conjunction with ``psutil.cpu_percent(percpu=True)`` to + observe the system workload distributed across multiple CPUs. + + Availability: Linux + + .. versionadded:: 5.1.0 + .. method:: memory_info() Return a namedtuple with variable fields depending on the platform diff --git a/psutil/__init__.py b/psutil/__init__.py index 3287a2c7e..0ab337834 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -836,6 +836,18 @@ def cpu_affinity(self, cpus=None): else: self._proc.cpu_affinity_set(list(set(cpus))) + if hasattr(_psplatform.Process, "cpu_num"): + + def cpu_num(self): + """Return what CPU this process is currently running on. + The returned number should be <= psutil.cpu_count() + and <= len(psutil.cpu_percent(percpu=True)). + It may be used in conjunction with + psutil.cpu_percent(percpu=True) to observe the system + workload distributed across CPUs. + """ + return self._proc.cpu_num() + # Linux, OSX and Windows only if hasattr(_psplatform.Process, "environ"): diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 91fdae4f8..c2c17a9ac 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1239,6 +1239,11 @@ def cpu_times(self): children_stime = float(values[15]) / CLOCK_TICKS return _common.pcputimes(utime, stime, children_utime, children_stime) + @wrap_exceptions + def cpu_num(self): + """What CPU the process is on.""" + return int(self._parse_stat_file()[37]) + @wrap_exceptions def wait(self, timeout=None): try: diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index b01760839..49aa4d862 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -281,6 +281,12 @@ def test_cpu_times_2(self): if (max([kernel_time, ktime]) - min([kernel_time, ktime])) > 0.1: self.fail("expected: %s, found: %s" % (ktime, kernel_time)) + @unittest.skipUnless(hasattr(psutil.Process, "cpu_num"), + "platform not supported") + def test_cpu_num(self): + p = psutil.Process() + self.assertIn(p.cpu_num(), range(psutil.cpu_count())) + def test_create_time(self): sproc = get_test_subprocess() now = time.time() @@ -1728,6 +1734,9 @@ def cpu_times(self, ret, proc): self.assertTrue(ret.user >= 0) self.assertTrue(ret.system >= 0) + def cpu_num(self, ret, proc): + self.assertIn(ret, range(psutil.cpu_count())) + def memory_info(self, ret, proc): for name in ret._fields: self.assertGreaterEqual(getattr(ret, name), 0) From c0d605dfd198b76810e0f52eb425c9784a76f63e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 25 Nov 2016 20:06:22 +0100 Subject: [PATCH 006/922] #357: implement process cpu_num() on FreeBSD --- psutil/_psbsd.py | 7 ++++++- psutil/_psutil_bsd.c | 17 ++++++++++++++--- psutil/tests/test_process.py | 7 +++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index a33015048..7cb9f2cd8 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -121,7 +121,8 @@ memtext=20, memdata=21, memstack=22, - name=23, + cpunum=23, + name=24, ) @@ -589,6 +590,10 @@ def cpu_times(self): rawtuple[kinfo_proc_map['ch_user_time']], rawtuple[kinfo_proc_map['ch_sys_time']]) + @wrap_exceptions + def cpu_num(self): + return self.oneshot()[kinfo_proc_map['cpunum']] + @wrap_exceptions def memory_info(self): rawtuple = self.oneshot() diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 33448ccf1..483912794 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -203,6 +203,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { long memtext; long memdata; long memstack; + unsigned char oncpu; kinfo_proc kp; long pagesize = sysconf(_SC_PAGESIZE); char str[1000]; @@ -257,11 +258,18 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { memstack = (long)kp.p_vm_ssize * pagesize; #endif + // what CPU we're on; top was used as an example: + // https://svnweb.freebsd.org/base/head/usr.bin/top/machine.c? + // view=markup&pathrev=273835 + if (kp.ki_stat == SRUN && kp.ki_oncpu != NOCPU) + oncpu = kp.ki_oncpu; + else + oncpu = kp.ki_lastcpu; + // Return a single big tuple with all process info. py_retlist = Py_BuildValue( - "(lillllllidllllddddlllllO)", + "(lillllllidllllddddlllllbO)", #ifdef __FreeBSD__ - // (long)kp.ki_ppid, // (long) ppid (int)kp.ki_stat, // (int) status // UIDs @@ -292,8 +300,9 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { memtext, // (long) mem text memdata, // (long) mem data memstack, // (long) mem stack + // others + oncpu, // (unsigned char) the CPU we are on #elif defined(__OpenBSD__) || defined(__NetBSD__) - // (long)kp.p_ppid, // (long) ppid (int)kp.p_stat, // (int) status // UIDs @@ -326,6 +335,8 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { memtext, // (long) mem text memdata, // (long) mem data memstack, // (long) mem stack + // others + oncpu, // (unsigned char) the CPU we are on #endif py_name // (pystr) name ); diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 49aa4d862..afbdc6901 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -285,6 +285,10 @@ def test_cpu_times_2(self): "platform not supported") def test_cpu_num(self): p = psutil.Process() + num = p.cpu_num() + self.assertGreaterEqual(num, 0) + if psutil.cpu_count() == 1: + self.assertEqual(num, 0) self.assertIn(p.cpu_num(), range(psutil.cpu_count())) def test_create_time(self): @@ -1735,6 +1739,9 @@ def cpu_times(self, ret, proc): self.assertTrue(ret.system >= 0) def cpu_num(self, ret, proc): + self.assertGreaterEqual(ret, 0) + if psutil.cpu_count() == 1: + self.assertEqual(ret, 0) self.assertIn(ret, range(psutil.cpu_count())) def memory_info(self, ret, proc): From be81f1d48c46b0b149fbb7b1f977de8896e9be08 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 25 Nov 2016 20:10:23 +0100 Subject: [PATCH 007/922] #357: add test which compares cpu_num() against cpu_affinity() --- psutil/tests/test_process.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index afbdc6901..62fa36ff1 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -868,6 +868,10 @@ def test_cpu_affinity(self): if hasattr(os, "sched_getaffinity"): self.assertEqual(p.cpu_affinity(), list(os.sched_getaffinity(p.pid))) + # also test num_cpu() + if hasattr(p, "num_cpu"): + self.assertEqual(p.cpu_affinity()[0], p.num_cpu()) + # p.cpu_affinity(all_cpus) self.assertEqual(p.cpu_affinity(), all_cpus) @@ -1814,6 +1818,9 @@ def is_running(self, ret, proc): def cpu_affinity(self, ret, proc): assert ret != [], ret + cpus = range(psutil.cpu_count()) + for n in ret: + self.assertIn(n, cpus) def terminal(self, ret, proc): if ret is not None: From fb9bd7f445081b024c7a1c7dfab8949d2e6496fb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 25 Nov 2016 22:45:33 +0100 Subject: [PATCH 008/922] skip test which fails on travis --- psutil/tests/test_linux.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index fe2f08106..d063fb404 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -534,6 +534,7 @@ def test_ctx_switches(self): psutil_value = psutil.cpu_stats().ctx_switches self.assertAlmostEqual(vmstat_value, psutil_value, delta=500) + @unittest.skipIf(TRAVIS, "fails on Travis") def test_interrupts(self): vmstat_value = vmstat("interrupts") psutil_value = psutil.cpu_stats().interrupts From 907a19e123bfe2a8bfdf1b69c16d73c61f27e7a3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 26 Nov 2016 00:26:23 +0100 Subject: [PATCH 009/922] GIT pre-commit script: exit if line ends with space --- .git-pre-commit | 20 ++++++++++---------- docs/conf.py | 1 + scripts/nettop.py | 1 + 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.git-pre-commit b/.git-pre-commit index e15884d16..071957d14 100755 --- a/.git-pre-commit +++ b/.git-pre-commit @@ -54,23 +54,23 @@ def main(): py_files = [x for x in out.split(b'\n') if x.endswith(b'.py') and os.path.exists(x)] + lineno = 0 for path in py_files: with open(path) as f: - data = f.read() - - # pdb - if "pdb.set_trace" in data: - for lineno, line in enumerate(data.split('\n'), 1): + for line in f: + lineno += 1 + # space at end of line + if line.endswith(' '): + print("%s:%s %r" % (path, lineno, line)) + return exit( + "commit aborted: space at end of line") line = line.rstrip() + # pdb if "pdb.set_trace" in line: print("%s:%s %s" % (path, lineno, line)) return exit( "commit aborted: you forgot a pdb in your python code") - - # bare except clause - if "except:" in data: - for lineno, line in enumerate(data.split('\n'), 1): - line = line.rstrip() + # bare except clause if "except:" in line and not line.endswith("# NOQA"): print("%s:%s %s" % (path, lineno, line)) return exit("commit aborted: bare except clause") diff --git a/docs/conf.py b/docs/conf.py index 9fa163b65..2267b5d10 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,6 +35,7 @@ def get_version(): else: raise ValueError("couldn't find version string") + VERSION = get_version() # If your documentation needs a minimal Sphinx version, state it here. diff --git a/scripts/nettop.py b/scripts/nettop.py index 97f80aadc..e13903c11 100755 --- a/scripts/nettop.py +++ b/scripts/nettop.py @@ -49,6 +49,7 @@ def tear_down(): curses.echo() curses.endwin() + win = curses.initscr() atexit.register(tear_down) curses.endwin() From 4b52141bd4f7c477d4ad3c36142ebeddb92eab18 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 26 Nov 2016 02:27:01 +0100 Subject: [PATCH 010/922] #357: does not support cpu_num() on Net and Open BSD --- docs/index.rst | 2 +- psutil/_psbsd.py | 7 ++++--- psutil/_psutil_bsd.c | 8 ++++++++ scripts/internal/bench_oneshot.py | 7 +++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4f0416753..360ef1587 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -837,7 +837,7 @@ Process class +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ | :meth:`memory_maps` | | | | | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | *speedup: +2.5x* | *speedup: +1.8x / +6.5x* | *speedup: +1.9x* | *speedup: +2.0x* | *speedup: +1.3x* | + | *speedup: +2.6x* | *speedup: +1.8x / +6.5x* | *speedup: +1.9x* | *speedup: +2.0x* | *speedup: +1.3x* | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ .. versionadded:: 5.0.0 diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 7cb9f2cd8..0a14bf80c 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -590,9 +590,10 @@ def cpu_times(self): rawtuple[kinfo_proc_map['ch_user_time']], rawtuple[kinfo_proc_map['ch_sys_time']]) - @wrap_exceptions - def cpu_num(self): - return self.oneshot()[kinfo_proc_map['cpunum']] + if FREEBSD: + @wrap_exceptions + def cpu_num(self): + return self.oneshot()[kinfo_proc_map['cpunum']] @wrap_exceptions def memory_info(self): diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 483912794..b1bce487a 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -258,6 +258,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { memstack = (long)kp.p_vm_ssize * pagesize; #endif +#ifdef __FreeBSD__ // what CPU we're on; top was used as an example: // https://svnweb.freebsd.org/base/head/usr.bin/top/machine.c? // view=markup&pathrev=273835 @@ -265,6 +266,13 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { oncpu = kp.ki_oncpu; else oncpu = kp.ki_lastcpu; +#else + // On Net/OpenBSD we have kp.p_cpuid but it appears it's always + // set to KI_NOCPU. Even if it's not, ki_lastcpu does not exist + // so there's no way to determine where "sleeping" processes + // were. Not supported. + oncpu = -1; +#endif // Return a single big tuple with all process info. py_retlist = Py_BuildValue( diff --git a/scripts/internal/bench_oneshot.py b/scripts/internal/bench_oneshot.py index cf8497f89..639e9ad76 100755 --- a/scripts/internal/bench_oneshot.py +++ b/scripts/internal/bench_oneshot.py @@ -37,10 +37,11 @@ if psutil.LINUX: names += [ - 'cpu_times', - 'gids', # 'memory_full_info', # 'memory_maps', + 'cpu_num', + 'cpu_times', + 'gids', 'name', 'num_ctx_switches', 'num_threads', @@ -63,6 +64,8 @@ 'terminal', 'uids', ] + if psutil.FREEBSD: + names.append('cpu_num') elif psutil.SUNOS: names += [ 'cmdline', From 64f9e17d812e626e00a1687bd0f3fb33503247f5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 26 Nov 2016 12:40:11 +0100 Subject: [PATCH 011/922] update README --- README.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.rst b/README.rst index fcdc8cd9e..91d41cdb0 100644 --- a/README.rst +++ b/README.rst @@ -247,9 +247,6 @@ Process management [0, 1, 2, 3] >>> p.cpu_affinity([0]) # set >>> - >>> p.memory_percent() - 0.63423 - >>> >>> p.memory_info() pmem(rss=10915840, vms=67608576, shared=3313664, text=2310144, lib=0, data=7262208, dirty=0) >>> p.memory_full_info() # "real" USS memory usage (Linux, OSX, Win only) From ba065041f8f687b0d44b2b5e712e65e2a2ee6b88 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 26 Nov 2016 17:47:12 +0100 Subject: [PATCH 012/922] process cpu_times(): be unambiguous when sum()muing values --- psutil/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 3287a2c7e..384a1937d 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1003,7 +1003,8 @@ def timer(): return _timer() * num_cpus else: def timer(): - return sum(cpu_times()) + t = cpu_times() + return sum((t.user, t.system)) if blocking: st1 = timer() From 253607002a1a9a70048391b0f81cd14144f2ff14 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 28 Nov 2016 23:54:40 +0100 Subject: [PATCH 013/922] fix #940 / linux / cpu_percent(): correctly take guest, guest_nice and iowait times into account --- HISTORY.rst | 3 +++ docs/index.rst | 32 ++++++++++++++++++++------------ psutil/__init__.py | 45 ++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index defd94d93..f93e09745 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,9 @@ **Bug fixes** - 936_: [Windows] fix compilation error on VS 2013 (patch by Max Bélanger). +- 940_: [Linux] cpu_percent() and cpu_times_percent() was calculated + incorrectly as "iowait", "guest" and "guest_nice" times were not properly + taken into account. 5.0.0 diff --git a/docs/index.rst b/docs/index.rst index 7fd79a1e3..2eb7738ee 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -61,21 +61,29 @@ CPU Every attribute represents the seconds the CPU has spent in the given mode. The attributes availability varies depending on the platform: - - **user** - - **system** - - **idle** + - **user**: time spent by normal processes executing in user mode; on Linux + this also includes **guest** time + - **system**: time spent by processes executing in kernel mode + - **idle**: time spent doing nothing Platform-specific fields: - - **nice** *(UNIX)* - - **iowait** *(Linux)* - - **irq** *(Linux, BSD)* - - **softirq** *(Linux)* - - **steal** *(Linux 2.6.11+)* - - **guest** *(Linux 2.6.24+)* - - **guest_nice** *(Linux 3.2.0+)* - - **interrupt** *(Windows)* - - **dpc** *(Windows)* + - **nice** *(UNIX)*: time spent by niced processes executing in user mode; + on Linux this also includes **guest_nice** time + - **iowait** *(Linux)*: time spent waiting for I/O to complete + - **irq** *(Linux, BSD)*: time spent for servicing hardware interrupts + - **softirq** *(Linux)*: time spent for servicing software interrupts + - **steal** *(Linux 2.6.11+)*: time spent by other operating systems when + running in a virtualized environment + - **guest** *(Linux 2.6.24+)*: time spent running a virtual CPU for guest + operating systems under the control of the Linux kernel + - **guest_nice** *(Linux 3.2.0+)*: time spent running a niced guest + (virtual CPU for guest operating systems under the control of the Linux + kernel) + - **interrupt** *(Windows)*: time spent for servicing hardware interrupts ( + similar to "irq" on UNIX) + - **dpc** *(Windows)*: time spent servicing deferred procedure calls (DPCs); + DPCs are interrupts that run at a lower priority than standard interrupts. When *percpu* is ``True`` return a list of namedtuples for each logical CPU on the system. diff --git a/psutil/__init__.py b/psutil/__init__.py index 384a1937d..3269c8574 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1630,6 +1630,41 @@ def cpu_times(percpu=False): traceback.print_exc() +def _cpu_tot_time(times): + """Given a cpu_time() ntuple calculates the total CPU time + (including idle time). + """ + tot = sum(times) + if LINUX: + # On Linux guest times are already accounted in "user" or + # "nice" times, so we subtract them from total. + # Htop does the same. References: + # https://github.com/giampaolo/psutil/pull/940 + # http://unix.stackexchange.com/questions/178045 + # https://github.com/torvalds/linux/blob/ + # 447976ef4fd09b1be88b316d1a81553f1aa7cd07/kernel/sched/ + # cputime.c#L158 + tot -= getattr(times, "guest", 0) # Linux 2.6.24+ + tot -= getattr(times, "guest_nice", 0) # Linux 3.2.0+ + return tot + + +def _cpu_busy_time(times): + """Given a cpu_time() ntuple calculates the busy CPU time. + We do so by subtracting all idle CPU times. + """ + busy = _cpu_tot_time(times) + busy -= times.idle + # Linux: "iowait" is time during which the CPU does not do anything + # (waits for IO to complete). On Linux IO wait is *not* accounted + # in "idle" time so we subtract it. Htop does the same. + # References: + # https://github.com/torvalds/linux/blob/ + # 447976ef4fd09b1be88b316d1a81553f1aa7cd07/kernel/sched/cputime.c#L244 + busy -= getattr(times, "iowait", 0) + return busy + + def cpu_percent(interval=None, percpu=False): """Return a float representing the current system-wide CPU utilization as a percentage. @@ -1672,11 +1707,11 @@ def cpu_percent(interval=None, percpu=False): raise ValueError("interval is not positive (got %r)" % interval) def calculate(t1, t2): - t1_all = sum(t1) - t1_busy = t1_all - t1.idle + t1_all = _cpu_tot_time(t1) + t1_busy = _cpu_busy_time(t1) - t2_all = sum(t2) - t2_busy = t2_all - t2.idle + t2_all = _cpu_tot_time(t2) + t2_busy = _cpu_busy_time(t2) # this usually indicates a float precision issue if t2_busy <= t1_busy: @@ -1748,7 +1783,7 @@ def cpu_times_percent(interval=None, percpu=False): def calculate(t1, t2): nums = [] - all_delta = sum(t2) - sum(t1) + all_delta = _cpu_tot_time(t2) - _cpu_tot_time(t1) for field in t1._fields: field_delta = getattr(t2, field) - getattr(t1, field) try: From 34c261da4995f6a4baf2142d59a4e879de89ea68 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 30 Nov 2016 23:52:43 +0100 Subject: [PATCH 014/922] update doc --- docs/index.rst | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 2eb7738ee..d2b5ddf7f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -864,8 +864,8 @@ Process class .. method:: name() The process name. On Windows the return value is cached after first - call. Not on POSIX because the process - `name may change `__. + call. Not on POSIX because the process name + `may change `__. .. method:: exe() @@ -875,7 +875,8 @@ Process class .. method:: cmdline() - The command line this process has been called with. + The command line this process has been called with. The return value is not + cached because the cmdline of a process may change. .. method:: environ() @@ -1143,7 +1144,7 @@ Process class multiple threads running on different CPU cores. .. note:: - the returned value is explcitly **not** split evenly between all CPUs + the returned value is explicitly **not** split evenly between all CPUs cores (differently from :func:`psutil.cpu_percent()`). This means that a busy loop process running on a system with 2 CPU cores will be reported as having 100% CPU utilization instead of 50%. @@ -1165,9 +1166,12 @@ Process class Get or set process current `CPU affinity `__. CPU affinity consists in telling the OS to run a certain process on a - limited set of CPUs only. The number of eligible CPUs can be obtained with - ``list(range(psutil.cpu_count()))``. ``ValueError`` will be raise on set - in case an invalid CPU number is specified. + limited set of CPUs only. + On Linux this is done via the ``taskset`` command. + The number of eligible CPUs can be obtained with + ``list(range(psutil.cpu_count()))``. + ``ValueError`` will be raised on set in case an invalid CPU number is + specified. >>> import psutil >>> psutil.cpu_count() @@ -1270,7 +1274,7 @@ Process class pmem(rss=15491072, vms=84025344, shared=5206016, text=2555904, lib=0, data=9891840, dirty=0) .. versionchanged:: - 4.0.0 mutiple fields are returned, not only `rss` and `vms`. + 4.0.0 multiple fields are returned, not only `rss` and `vms`. .. method:: memory_info_ex() @@ -1290,7 +1294,7 @@ Process class It does so by passing through the whole process address. As such it usually requires higher user privileges than :meth:`memory_info` and is considerably slower. - On platforms where extra fields are not implented this simply returns the + On platforms where extra fields are not implemented this simply returns the same metrics as :meth:`memory_info`. - **uss** *(Linux, OSX, Windows)*: @@ -1860,7 +1864,7 @@ Constants Availability: Linux .. versionchanged:: - 3.0.0 on Python >= 3.4 thse constants are + 3.0.0 on Python >= 3.4 these constants are `enums `__ instead of a plain integer. From df588555d2c3efff7b9bec5380c70b0f8c712e61 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 1 Dec 2016 20:37:22 +0100 Subject: [PATCH 015/922] update doc --- docs/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index d2b5ddf7f..bd2773da6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -138,6 +138,8 @@ CPU :func:`psutil.cpu_times(percpu=True)`. *interval* and *percpu* arguments have the same meaning as in :func:`cpu_percent()`. + On Linux "guest" and "guest_nice" percentages are not accounted in "user" + and "user_nice" percentages. .. warning:: the first time this function is called with *interval* = ``0.0`` or From 6967c8d6cf8072a2bcd911e47ebbfa37d606394e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 1 Dec 2016 23:37:49 +0100 Subject: [PATCH 016/922] add tests for cpu_count(logical=True) on Linux --- psutil/tests/test_linux.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index d063fb404..871cb74eb 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -461,6 +461,21 @@ def test_cpu_times(self): else: self.assertNotIn('guest_nice', fields) + @unittest.skipUnless(os.path.exists("/sys/devices/system/cpu/online"), + "/sys/devices/system/cpu/online does not exist") + def test_cpu_count_logical_w_sysdev_cpu_online(self): + with open("/sys/devices/system/cpu/online") as f: + value = f.read().strip() + value = int(value.split('-')[1]) + 1 + self.assertEqual(psutil.cpu_count(), value) + + @unittest.skipUnless(os.path.exists("/sys/devices/system/cpu"), + "/sys/devices/system/cpu does not exist") + def test_cpu_count_logical_w_sysdev_cpu_num(self): + ls = os.listdir("/sys/devices/system/cpu") + count = len([x for x in ls if re.search("cpu\d+$", x) is not None]) + self.assertEqual(psutil.cpu_count(), count) + @unittest.skipUnless(which("nproc"), "nproc utility not available") def test_cpu_count_logical_w_nproc(self): num = int(sh("nproc --all")) From 8d83d5f3669e00af4f50a4434fd0d3e26d0c96a4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 2 Dec 2016 02:46:35 +0100 Subject: [PATCH 017/922] #941: implement cpu_freq() for Linux --- psutil/__init__.py | 13 +++++++++++++ psutil/_common.py | 2 ++ psutil/_pslinux.py | 35 +++++++++++++++++++++++++++++++++++ psutil/tests/test_system.py | 10 ++++++++++ 4 files changed, 60 insertions(+) diff --git a/psutil/__init__.py b/psutil/__init__.py index 3269c8574..d3d7bda2b 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1846,6 +1846,19 @@ def cpu_stats(): return _psplatform.cpu_stats() +if hasattr(_psplatform, "cpu_freq"): + + def cpu_freq(): + """Return CPU frequencies as a list of nameduples including + current, min and max CPU frequency. + The CPUs order is supposed to be consistent with other CPU + functions having a 'percpu' argument and returning results for + multiple CPUs (cpu_times(), cpu_percent(), cpu_times_percent()). + Values are expressed in Mhz. + """ + return _psplatform.cpu_freq() + + # ===================================================================== # --- system memory related functions # ===================================================================== diff --git a/psutil/_common.py b/psutil/_common.py index 3879a1d73..a8ae27ac1 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -156,6 +156,8 @@ class NicDuplex(enum.IntEnum): # psutil.cpu_stats() scpustats = namedtuple( 'scpustats', ['ctx_switches', 'interrupts', 'soft_interrupts', 'syscalls']) +# psutil.cpu_freq() +scpufreq = namedtuple('scpufreq', ['curr', 'min', 'max']) # --- for Process methods diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 91fdae4f8..c2247ae00 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -9,6 +9,7 @@ import base64 import errno import functools +import glob import os import re import socket @@ -133,6 +134,8 @@ class IOPriority(enum.IntEnum): "0B": _common.CONN_CLOSING } +_DEFAULT = object() + # ===================================================================== # -- exceptions @@ -276,6 +279,18 @@ def set_scputimes_ntuple(procfs_path): return scputimes +def cat(fname, fallback=_DEFAULT, binary=True): + """Return file content.""" + try: + with open_binary(fname) if binary else open_text(fname) as f: + return f.read() + except IOError: + if fallback != _DEFAULT: + return fallback + else: + raise + + try: scputimes = set_scputimes_ntuple("/proc") except Exception: @@ -607,6 +622,26 @@ def cpu_stats(): ctx_switches, interrupts, soft_interrupts, syscalls) +if os.path.exists("/sys/devices/system/cpu/cpufreq"): + + def cpu_freq(): + # scaling_* files seem preferable to cpuinfo_*, see: + # http://unix.stackexchange.com/a/87537/168884 + ret = [] + ls = glob.glob("/sys/devices/system/cpu/cpufreq/policy[0-9]*") + # Sort the list so that '10' comes after '2'. This should + # ensure the CPU order is consistent with other CPU functions + # having a 'percpu' argument and returning results for multiple + # CPUs (cpu_times(), cpu_percent(), cpu_times_percent()). + ls.sort(key=lambda x: int(os.path.basename(x)[6:])) + for path in ls: + curr = int(cat(os.path.join(path, "scaling_cur_freq"))) / 1000 + max_ = int(cat(os.path.join(path, "scaling_max_freq"))) / 1000 + min_ = int(cat(os.path.join(path, "scaling_min_freq"))) / 1000 + ret.append(_common.scpufreq(curr, min_, max_)) + return ret + + # ===================================================================== # --- network # ===================================================================== diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index e2220be49..ac63dc7cc 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -697,6 +697,16 @@ def test_cpu_stats(self): if name in ('ctx_switches', 'interrupts'): self.assertGreater(value, 0) + @unittest.skipUnless(hasattr(psutil, "cpu_freq"), + "platform not suported") + def test_cpu_freq(self): + ls = psutil.cpu_freq() + assert ls, ls + for nt in ls: + for name in nt._fields: + value = getattr(nt, name) + self.assertGreaterEqual(value, 0) + def test_os_constants(self): names = ["POSIX", "WINDOWS", "LINUX", "OSX", "FREEBSD", "OPENBSD", "NETBSD", "BSD", "SUNOS"] From d975e807aee0ac902d8105d584c22fb526ecd87d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 2 Dec 2016 14:40:53 +0100 Subject: [PATCH 018/922] debug print to figure out failure on travis --- psutil/__init__.py | 2 +- psutil/_pslinux.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index d3d7bda2b..77a972d67 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -181,7 +181,7 @@ "pid_exists", "pids", "process_iter", "wait_procs", # proc "virtual_memory", "swap_memory", # memory "cpu_times", "cpu_percent", "cpu_times_percent", "cpu_count", # cpu - "cpu_stats", + "cpu_stats", "cpu_freq", "net_io_counters", "net_connections", "net_if_addrs", # network "net_if_stats", "disk_io_counters", "disk_partitions", "disk_usage", # disk diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index c2247ae00..add06a11d 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -628,7 +628,9 @@ def cpu_freq(): # scaling_* files seem preferable to cpuinfo_*, see: # http://unix.stackexchange.com/a/87537/168884 ret = [] - ls = glob.glob("/sys/devices/system/cpu/cpufreq/policy[0-9]*") + # XXX + print(os.listdir("/sys/devices/system/cpu/cpufreq/")) + ls = glob.glob("/sys/devices/system/cpu/cpufreq/policy*") # Sort the list so that '10' comes after '2'. This should # ensure the CPU order is consistent with other CPU functions # having a 'percpu' argument and returning results for multiple From 6ba1ac4ebfcd8c95fca324b15606ab0ec1412d39 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 3 Dec 2016 18:05:07 +0100 Subject: [PATCH 019/922] #941: implement cpu_freq() for OSX --- psutil/_pslinux.py | 2 -- psutil/_psosx.py | 5 +++++ psutil/_psutil_osx.c | 31 +++++++++++++++++++++++++++++++ psutil/tests/test_osx.py | 15 +++++++++++++++ psutil/tests/test_system.py | 3 ++- 5 files changed, 53 insertions(+), 3 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index add06a11d..e073b06e0 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -628,8 +628,6 @@ def cpu_freq(): # scaling_* files seem preferable to cpuinfo_*, see: # http://unix.stackexchange.com/a/87537/168884 ret = [] - # XXX - print(os.listdir("/sys/devices/system/cpu/cpufreq/")) ls = glob.glob("/sys/devices/system/cpu/cpufreq/policy*") # Sort the list so that '10' comes after '2'. This should # ensure the CPU order is consistent with other CPU functions diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 2665080e0..0778b5fb7 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -165,6 +165,11 @@ def cpu_stats(): ctx_switches, interrupts, soft_interrupts, syscalls) +def cpu_freq(): + curr, min_, max_ = cext.cpu_freq() + return [_common.scpufreq(curr, min_, max_)] + + # ===================================================================== # --- disks # ===================================================================== diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index a1168c291..fb26dc9b4 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -808,6 +808,35 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { } +/* + * Retrieve CPU frequency. + */ +static PyObject * +psutil_cpu_freq(PyObject *self, PyObject *args) { + int64_t curr; + int64_t min; + int64_t max; + size_t size = sizeof(int64_t); + + if (sysctlbyname("hw.cpufrequency", &curr, &size, NULL, 0)) + goto error; + if (sysctlbyname("hw.cpufrequency_min", &min, &size, NULL, 0)) + goto error; + if (sysctlbyname("hw.cpufrequency_max", &max, &size, NULL, 0)) + goto error; + + return Py_BuildValue( + "KKK", + curr / 1000 / 1000, + min / 1000 / 1000, + max / 1000 / 1000); + +error: + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} + + /* * Return a Python float indicating the system boot time expressed in * seconds since the epoch. @@ -1778,6 +1807,8 @@ PsutilMethods[] = { "Return system cpu times as a tuple (user, system, nice, idle, irc)"}, {"per_cpu_times", psutil_per_cpu_times, METH_VARARGS, "Return system per-cpu times as a list of tuples"}, + {"cpu_freq", psutil_cpu_freq, METH_VARARGS, + "Return cpu current frequency"}, {"boot_time", psutil_boot_time, METH_VARARGS, "Return the system boot time expressed in seconds since the epoch."}, {"disk_partitions", psutil_disk_partitions, METH_VARARGS, diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 7b61bc74a..02fa430b7 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -111,6 +111,8 @@ def test_process_create_time(self): @unittest.skipUnless(OSX, "OSX only") class TestSystemAPIs(unittest.TestCase): + # --- disk + def test_disks(self): # test psutil.disk_usage() and psutil.disk_partitions() # against "df -a" @@ -138,6 +140,8 @@ def df(path): if abs(usage.used - used) > 10 * 1024 * 1024: self.fail("psutil=%s, df=%s" % usage.used, used) + # --- cpu + def test_cpu_count_logical(self): num = sysctl("sysctl hw.logicalcpu") self.assertEqual(num, psutil.cpu_count(logical=True)) @@ -146,6 +150,15 @@ def test_cpu_count_physical(self): num = sysctl("sysctl hw.physicalcpu") self.assertEqual(num, psutil.cpu_count(logical=False)) + def test_cpu_freq(self): + freq = psutil.cpu_freq()[0] + self.assertEqual( + freq.curr * 1000 * 1000, sysctl("sysctl hw.cpufrequency")) + self.assertEqual( + freq.min * 1000 * 1000, sysctl("sysctl hw.cpufrequency_min")) + self.assertEqual( + freq.max * 1000 * 1000, sysctl("sysctl hw.cpufrequency_max")) + # --- virtual mem def test_vmem_total(self): @@ -206,6 +219,8 @@ def test_swapmem_sout(self): # self.assertEqual(psutil_smem.used, human2bytes(used)) # self.assertEqual(psutil_smem.free, human2bytes(free)) + # --- network + def test_net_if_stats(self): for name, stats in psutil.net_if_stats().items(): try: diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index ac63dc7cc..d1b81838b 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -701,7 +701,8 @@ def test_cpu_stats(self): "platform not suported") def test_cpu_freq(self): ls = psutil.cpu_freq() - assert ls, ls + if not TRAVIS: + assert ls, ls for nt in ls: for name in nt._fields: value = getattr(nt, name) From c5cc270ed2b8953de81a4f4a29ce48808b9abafb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 3 Dec 2016 22:00:00 +0100 Subject: [PATCH 020/922] update OSX notes --- psutil/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/psutil/__init__.py b/psutil/__init__.py index 77a972d67..6994c476f 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1855,6 +1855,11 @@ def cpu_freq(): functions having a 'percpu' argument and returning results for multiple CPUs (cpu_times(), cpu_percent(), cpu_times_percent()). Values are expressed in Mhz. + + Notes about OSX: + - it is not possible to get per-cpu freq + - reported freq never changes: + https://arstechnica.com/civis/viewtopic.php?f=19&t=465002 """ return _psplatform.cpu_freq() From 2bf5f9762cb3be6045b342c970b6199f64c7dd72 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 6 Dec 2016 19:04:44 +0100 Subject: [PATCH 021/922] update doc --- docs/index.rst | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index bd2773da6..ba5dac938 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1291,8 +1291,8 @@ Process class some platform (Linux, OSX, Windows), also provides additional metrics (USS, PSS and swap). The additional metrics provide a better representation of "effective" - process memory consumption (in case of USS) as explained in detail - `here `__. + process memory consumption (in case of USS) as explained in detail in this + `blog post `__. It does so by passing through the whole process address. As such it usually requires higher user privileges than :meth:`memory_info` and is considerably slower. @@ -1938,12 +1938,27 @@ Q&A === * Q: What Windows versions are supported? -* A: From Windows **Vista** onwards. Latest binary (wheel / exe) release - supporting Windows **2000**, **XP** and **2003 server** which can installed - via pip without a compiler being installed is +* A: From Windows **Vista** onwards, both 32 and 64 bit versions. + Latest binary (wheel / exe) release which supports Windows **2000**, **XP** + and **2003 server** is `psutil 3.4.2 `__. - More recent psutil versions may still be compiled from sources and work - (more or less) but they are no longer being tested or maintained. + On such old systems psutil is no longer tested or maintained, but it can + still be compiled from sources (you'll need `Visual Studio <(https://github.com/giampaolo/psutil/blob/master/INSTALL.rst#windows>`__) + and it should "work" (more or less). + +---- + +* Q: Why do I get :class:`AccessDenied` for certain processes? +* A: This may happen when you query processess owned by another user, + especially on `OSX `__ and + Windows. + Unfortunately there's not much you can do about this except running the + Python process with higher privileges. + On Unix you may run the the Python process as root or use the SUID bit + (this is the trick used by tools such as ``ps`` and ``netstat``). + On Windows you may run the Python process as NT AUTHORITY\\SYSTEM or install + the Python script as a Windows service (this is the trick used by tools + such as ProcessHacker). Timeline From e4a41c162ea362277306c767222ca0c91bc4a6d2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Dec 2016 16:13:44 +0100 Subject: [PATCH 022/922] update doc style --- docs/_themes/pydoctheme/static/pydoctheme.css | 10 ++ docs/index.rst | 120 +++++++++--------- 2 files changed, 70 insertions(+), 60 deletions(-) diff --git a/docs/_themes/pydoctheme/static/pydoctheme.css b/docs/_themes/pydoctheme/static/pydoctheme.css index 4196e5582..c6f19ab79 100644 --- a/docs/_themes/pydoctheme/static/pydoctheme.css +++ b/docs/_themes/pydoctheme/static/pydoctheme.css @@ -185,3 +185,13 @@ div.body h3 { div.body h2 { padding-left:10px; } + +.data { + margin-top: 4px !important; + margin-bottom: 4px !important; +} + +.data dd { + margin-top: 0px !important; + margin-bottom: 0px !important; +} diff --git a/docs/index.rst b/docs/index.rst index ba5dac938..f617413ea 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1756,14 +1756,14 @@ Constants .. _const-oses: .. data:: POSIX - WINDOWS - LINUX - OSX - FREEBSD - NETBSD - OPENBSD - BSD - SUNOS +.. data:: WINDOWS +.. data:: LINUX +.. data:: OSX +.. data:: FREEBSD +.. data:: NETBSD +.. data:: OPENBSD +.. data:: BSD +.. data:: SUNOS ``bool`` constants which define what platform you're on. E.g. if on Windows, *WINDOWS* constant will be ``True``, all others will be ``False``. @@ -1784,18 +1784,18 @@ Constants .. _const-pstatus: .. data:: STATUS_RUNNING - STATUS_SLEEPING - STATUS_DISK_SLEEP - STATUS_STOPPED - STATUS_TRACING_STOP - STATUS_ZOMBIE - STATUS_DEAD - STATUS_WAKE_KILL - STATUS_WAKING - STATUS_IDLE (OSX, FreeBSD) - STATUS_LOCKED (FreeBSD) - STATUS_WAITING (FreeBSD) - STATUS_SUSPENDED (NetBSD) +.. data:: STATUS_SLEEPING +.. data:: STATUS_DISK_SLEEP +.. data:: STATUS_STOPPED +.. data:: STATUS_TRACING_STOP +.. data:: STATUS_ZOMBIE +.. data:: STATUS_DEAD +.. data:: STATUS_WAKE_KILL +.. data:: STATUS_WAKING +.. data:: STATUS_IDLE (OSX, FreeBSD) +.. data:: STATUS_LOCKED (FreeBSD) +.. data:: STATUS_WAITING (FreeBSD) +.. data:: STATUS_SUSPENDED (NetBSD) A set of strings representing the status of a process. Returned by :meth:`psutil.Process.status()`. @@ -1804,31 +1804,31 @@ Constants .. _const-conn: .. data:: CONN_ESTABLISHED - CONN_SYN_SENT - CONN_SYN_RECV - CONN_FIN_WAIT1 - CONN_FIN_WAIT2 - CONN_TIME_WAIT - CONN_CLOSE - CONN_CLOSE_WAIT - CONN_LAST_ACK - CONN_LISTEN - CONN_CLOSING - CONN_NONE - CONN_DELETE_TCB (Windows) - CONN_IDLE (Solaris) - CONN_BOUND (Solaris) +.. data:: CONN_SYN_SENT +.. data:: CONN_SYN_RECV +.. data:: CONN_FIN_WAIT1 +.. data:: CONN_FIN_WAIT2 +.. data:: CONN_TIME_WAIT +.. data:: CONN_CLOSE +.. data:: CONN_CLOSE_WAIT +.. data:: CONN_LAST_ACK +.. data:: CONN_LISTEN +.. data:: CONN_CLOSING +.. data:: CONN_NONE +.. data:: CONN_DELETE_TCB (Windows) +.. data:: CONN_IDLE (Solaris) +.. data:: CONN_BOUND (Solaris) A set of strings representing the status of a TCP connection. Returned by :meth:`psutil.Process.connections()` (`status` field). .. _const-prio: .. data:: ABOVE_NORMAL_PRIORITY_CLASS - BELOW_NORMAL_PRIORITY_CLASS - HIGH_PRIORITY_CLASS - IDLE_PRIORITY_CLASS - NORMAL_PRIORITY_CLASS - REALTIME_PRIORITY_CLASS +.. data:: BELOW_NORMAL_PRIORITY_CLASS +.. data:: HIGH_PRIORITY_CLASS +.. data:: IDLE_PRIORITY_CLASS +.. data:: NORMAL_PRIORITY_CLASS +.. data:: REALTIME_PRIORITY_CLASS A set of integers representing the priority of a process on Windows (see `MSDN documentation `__). @@ -1844,9 +1844,9 @@ Constants .. _const-ioprio: .. data:: IOPRIO_CLASS_NONE - IOPRIO_CLASS_RT - IOPRIO_CLASS_BE - IOPRIO_CLASS_IDLE +.. data:: IOPRIO_CLASS_RT +.. data:: IOPRIO_CLASS_BE +.. data:: IOPRIO_CLASS_IDLE A set of integers representing the I/O priority of a process on Linux. They can be used in conjunction with :meth:`psutil.Process.ionice()` to get or set @@ -1872,22 +1872,22 @@ Constants .. _const-rlimit: .. data:: RLIM_INFINITY - RLIMIT_AS - RLIMIT_CORE - RLIMIT_CPU - RLIMIT_DATA - RLIMIT_FSIZE - RLIMIT_LOCKS - RLIMIT_MEMLOCK - RLIMIT_MSGQUEUE - RLIMIT_NICE - RLIMIT_NOFILE - RLIMIT_NPROC - RLIMIT_RSS - RLIMIT_RTPRIO - RLIMIT_RTTIME - RLIMIT_SIGPENDING - RLIMIT_STACK +.. data:: RLIMIT_AS +.. data:: RLIMIT_CORE +.. data:: RLIMIT_CPU +.. data:: RLIMIT_DATA +.. data:: RLIMIT_FSIZE +.. data:: RLIMIT_LOCKS +.. data:: RLIMIT_MEMLOCK +.. data:: RLIMIT_MSGQUEUE +.. data:: RLIMIT_NICE +.. data:: RLIMIT_NOFILE +.. data:: RLIMIT_NPROC +.. data:: RLIMIT_RSS +.. data:: RLIMIT_RTPRIO +.. data:: RLIMIT_RTTIME +.. data:: RLIMIT_SIGPENDING +.. data:: RLIMIT_STACK Constants used for getting and setting process resource limits to be used in conjunction with :meth:`psutil.Process.rlimit()`. See @@ -1905,8 +1905,8 @@ Constants .. _const-duplex: .. data:: NIC_DUPLEX_FULL - NIC_DUPLEX_HALF - NIC_DUPLEX_UNKNOWN +.. data:: NIC_DUPLEX_HALF +.. data:: NIC_DUPLEX_UNKNOWN Constants which identifies whether a NIC (network interface card) has full or half mode speed. NIC_DUPLEX_FULL means the NIC is able to send and receive From f4de10369bf82c176eca2e1e622e3a1449412866 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 8 Dec 2016 12:14:24 +0100 Subject: [PATCH 023/922] source dist: include doc --- MANIFEST.in | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 945322928..ece2f0024 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,7 @@ include INSTALL* include LICENSE* include make.bar include Makefile -recursive-include psutil *.py *.c *.h +recursive-include psutil *.py *.c *.h *README* + +recursive-exclude docs/_build * +recursive-include docs *.rst *.js *.html *.py *.bat *Makefile* *README* From fe9b1a9b58ebb3787c321d5188a1e4334398eb40 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 8 Dec 2016 12:20:48 +0100 Subject: [PATCH 024/922] update MANIFEST.in --- MANIFEST.in | 11 +++++++++-- Makefile | 4 ++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index ece2f0024..ef711a091 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,17 @@ +include *.bat include *.rst +include .coveragerc include CREDITS* +include IDEAS include INSTALL* include LICENSE* include make.bar include Makefile -recursive-include psutil *.py *.c *.h *README* +include tox.ini + +recursive-include psutil *.py *.c *.h +recursive-include scripts *.py +recursive-include *README* recursive-exclude docs/_build * -recursive-include docs *.rst *.js *.html *.py *.bat *Makefile* *README* +recursive-include docs *.conf *.rst *.js *.html *.css *.py *.bat *Makefile* *README* diff --git a/Makefile b/Makefile index c91d3b961..32807b0bc 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ ARGS = # List of nice-to-have dev libs. DEPS = argparse \ + check-manifest \ coverage \ flake8 \ futures \ @@ -171,6 +172,9 @@ pyflakes: flake8: @git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8 +check-manifest: + $(PYTHON) -m check_manifest -v + # =================================================================== # GIT # =================================================================== From cf672a9e92dd0c5889332525eae3cd94af8e2d47 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 8 Dec 2016 12:32:25 +0100 Subject: [PATCH 025/922] update MANIFEST --- MANIFEST.in | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index ef711a091..d41bdc420 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,9 +9,10 @@ include make.bar include Makefile include tox.ini -recursive-include psutil *.py *.c *.h +recursive-include psutil *.py *.c *.h *.rst recursive-include scripts *.py -recursive-include *README* +recursive-include README* +recursive-include docs *.conf *.rst *.js *.html *.css *.py *.bat *Makefile* README* recursive-exclude docs/_build * -recursive-include docs *.conf *.rst *.js *.html *.css *.py *.bat *Makefile* *README* +recursive-exclude .ci * From 07c9a4850202eadf5c920ee2644c7d669851945f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 8 Dec 2016 12:50:01 +0100 Subject: [PATCH 026/922] update Makefile --- MANIFEST.in | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index d41bdc420..b0c156457 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,7 +5,7 @@ include CREDITS* include IDEAS include INSTALL* include LICENSE* -include make.bar +include HISTORY* include Makefile include tox.ini diff --git a/Makefile b/Makefile index 32807b0bc..13ed0c00c 100644 --- a/Makefile +++ b/Makefile @@ -173,7 +173,7 @@ flake8: @git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8 check-manifest: - $(PYTHON) -m check_manifest -v + $(PYTHON) -m check_manifest -v $(ARGS) # =================================================================== # GIT From d0d3b182f7e4d1e988bad696880bb884c54a31f6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 12 Dec 2016 00:18:43 +0100 Subject: [PATCH 027/922] update docstring --- psutil/tests/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 883d92850..b119a788c 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -506,6 +506,7 @@ def chdir(dirname): def create_exe(outpath, c_code=None): + """Creates an executable file in the given location.""" assert not os.path.exists(outpath), outpath if which("gcc"): if c_code is None: @@ -526,6 +527,9 @@ def create_exe(outpath, c_code=None): safe_rmpath(f.name) else: # fallback - use python's executable + if c_code is not None: + raise ValueError( + "can't specify c_code arg as gcc is not installed") shutil.copyfile(sys.executable, outpath) if POSIX: st = os.stat(outpath) From 4ce68ef999e1f8e3f37830e8123a56439652ccab Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 12 Dec 2016 02:56:09 +0100 Subject: [PATCH 028/922] #804: try to test failure on Debian --- psutil/tests/test_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index b01760839..1a95bc199 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -302,7 +302,7 @@ def test_create_time(self): @unittest.skipIf(TRAVIS, 'not reliable on TRAVIS') def test_terminal(self): terminal = psutil.Process().terminal() - if sys.stdin.isatty(): + if sys.stdin.isatty() or sys.stdout.isatty(): tty = os.path.realpath(sh('tty')) self.assertEqual(terminal, tty) else: From efb7728e510cd3eb4bd8cc524a787e410e7c0fb0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 16 Dec 2016 00:59:50 +0100 Subject: [PATCH 029/922] minor DEVGUIDE update --- DEVGUIDE.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index 95bea79a3..93dfa6903 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -1,6 +1,6 @@ -===== -Setup -===== +======================= +Setup and running tests +======================= If you plan on hacking on psutil this is what you're supposed to do first: From d5878ab9392666a3d0e20260f955621075649cf7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 16 Dec 2016 01:06:00 +0100 Subject: [PATCH 030/922] cpu_affinity() test: use addCleanup to reset initial value --- psutil/tests/test_process.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 1a95bc199..50e0cc746 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -845,6 +845,7 @@ def test_cwd_2(self): def test_cpu_affinity(self): p = psutil.Process() initial = p.cpu_affinity() + self.addCleanup(p.cpu_affinity, initial) if hasattr(os, "sched_getaffinity"): self.assertEqual(initial, list(os.sched_getaffinity(p.pid))) self.assertEqual(len(initial), len(set(initial))) From e7d0efad3fa44de4310925cf3063132545edc3bd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 19 Dec 2016 01:29:27 +0100 Subject: [PATCH 031/922] update doc --- docs/conf.py | 2 +- docs/index.rst | 30 ++++++++++++++---------------- psutil/__init__.py | 8 ++++++++ 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 2267b5d10..f0a206db7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -46,7 +46,7 @@ def get_version(): # ones. extensions = ['sphinx.ext.autodoc', 'sphinx.ext.coverage', - 'sphinx.ext.pngmath', + 'sphinx.ext.imgmath', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx'] diff --git a/docs/index.rst b/docs/index.rst index f617413ea..665ec7c52 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1118,9 +1118,9 @@ Process class .. method:: cpu_percent(interval=None) - Return a float representing the process CPU utilization as a percentage. - The returned value refers to the utilization of a single CPU, i.e. it is - not evenly split between the number of available CPU cores. + Return a float representing the process CPU utilization as a percentage + which can also be ``> 100.0`` in case of threads running on multiple + CPUs. When *interval* is > ``0.0`` compares process times to system CPU times elapsed before and after the interval (blocking). When interval is ``0.0`` or ``None`` compares process times to system CPU times elapsed since last @@ -1132,30 +1132,28 @@ Process class >>> import psutil >>> p = psutil.Process() - >>> >>> # blocking >>> p.cpu_percent(interval=1) 2.0 >>> # non-blocking (percentage since last call) >>> p.cpu_percent(interval=None) 2.9 - >>> .. note:: - a percentage > 100 is legitimate as it can result from a process with - multiple threads running on different CPU cores. + the returned value can be > 100.0 in case of a process running multiple + threads on different CPU cores. .. note:: - the returned value is explicitly **not** split evenly between all CPUs - cores (differently from :func:`psutil.cpu_percent()`). - This means that a busy loop process running on a system with 2 CPU - cores will be reported as having 100% CPU utilization instead of 50%. - This was done in order to be consistent with UNIX's "top" utility + the returned value is explicitly *not* split evenly between all available + logical CPUs (differently from :func:`psutil.cpu_percent()`). + This means that a busy loop process running on a system with 2 logical + CPUs will be reported as having 100% CPU utilization instead of 50%. + This was done in order to be consistent with UNIX's ``top`` utility and also to make it easier to identify processes hogging CPU resources - (independently from the number of CPU cores). - It must be noted that in the example above taskmgr.exe on Windows will - report 50% usage instead. - To emulate Windows's taskmgr.exe behavior you can do: + independently from the number of CPUs. + It must be noted that ``taskmgr.exe`` on Windows does not behave like + this (it would report 50% usage instead). + To emulate Windows ``taskmgr.exe`` behavior you can do: ``p.cpu_percent() / psutil.cpu_count()``. .. warning:: diff --git a/psutil/__init__.py b/psutil/__init__.py index 3269c8574..79817b1d1 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -981,6 +981,14 @@ def cpu_percent(self, interval=None): In this case is recommended for accuracy that this function be called with at least 0.1 seconds between calls. + A value > 100.0 can be returned in case of processes running + multiple threads on different CPU cores. + + The returned value is explicitly *not* split evenly between + all available logical CPUs. This means that a busy loop process + running on a system with 2 logical CPUs will be reported as + having 100% CPU utilization instead of 50%. + Examples: >>> import psutil From d43b1752f41339427c7f81fe6b838b401456aa77 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 19 Dec 2016 02:27:50 +0100 Subject: [PATCH 032/922] update doc --- README.rst | 2 +- docs/index.rst | 111 ++++++++++++++++++++++++++++++------------------- 2 files changed, 69 insertions(+), 44 deletions(-) diff --git a/README.rst b/README.rst index 91d41cdb0..dbfb7274b 100644 --- a/README.rst +++ b/README.rst @@ -71,7 +71,7 @@ Projects using psutil ===================== At the time of writing there are over -`4000 projects `__ +`4200 projects `__ on github which depend from psutil. Here's some I find particularly interesting: diff --git a/docs/index.rst b/docs/index.rst index 665ec7c52..031360d23 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -453,7 +453,7 @@ Network else ``None``. On some platforms (e.g. Linux) the availability of this field changes depending on process privileges (root is needed). - The *kind* parameter is a string which filters for connections that fit the + The *kind* parameter is a string which filters for connections matching the following criteria: .. table:: @@ -461,27 +461,27 @@ Network +----------------+-----------------------------------------------------+ | **Kind value** | **Connections using** | +================+=====================================================+ - | "inet" | IPv4 and IPv6 | + | ``"inet"`` | IPv4 and IPv6 | +----------------+-----------------------------------------------------+ - | "inet4" | IPv4 | + | ``"inet4"`` | IPv4 | +----------------+-----------------------------------------------------+ - | "inet6" | IPv6 | + | ``"inet6"`` | IPv6 | +----------------+-----------------------------------------------------+ - | "tcp" | TCP | + | ``"tcp"`` | TCP | +----------------+-----------------------------------------------------+ - | "tcp4" | TCP over IPv4 | + | ``"tcp4"`` | TCP over IPv4 | +----------------+-----------------------------------------------------+ - | "tcp6" | TCP over IPv6 | + | ``"tcp6"`` | TCP over IPv6 | +----------------+-----------------------------------------------------+ - | "udp" | UDP | + | ``"udp"`` | UDP | +----------------+-----------------------------------------------------+ - | "udp4" | UDP over IPv4 | + | ``"udp4"`` | UDP over IPv4 | +----------------+-----------------------------------------------------+ - | "udp6" | UDP over IPv6 | + | ``"udp6"`` | UDP over IPv6 | +----------------+-----------------------------------------------------+ - | "unix" | UNIX socket (both UDP and TCP protocols) | + | ``"unix"`` | UNIX socket (both UDP and TCP protocols) | +----------------+-----------------------------------------------------+ - | "all" | the sum of all the possible families and protocols | + | ``"all"`` | the sum of all the possible families and protocols | +----------------+-----------------------------------------------------+ On OSX this function requires root privileges. @@ -632,12 +632,16 @@ Functions .. function:: pids() Return a list of current running PIDs. To iterate over all processes - :func:`process_iter()` should be preferred. + and avoid race conditions :func:`process_iter()` should be preferred. + + >>> import psutil + >>> psutil.pids() + [1, 2, 3, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, ..., 32498] .. function:: pid_exists(pid) Check whether the given PID exists in the current process list. This is - faster than doing ``"pid in psutil.pids()"`` and should be preferred. + faster than doing ``pid in psutil.pids()`` and should be preferred. .. function:: process_iter() @@ -678,18 +682,18 @@ Functions - give them some time to terminate - send SIGKILL to those ones which are still alive - Example:: + Example which terminates and waits all the children of this process:: import psutil def on_terminate(proc): print("process {} terminated with exit code {}".format(proc, proc.returncode)) - procs = [...] # a list of Process instances + procs = psutil.Process().children() for p in procs: p.terminate() - gone, alive = psutil.wait_procs(procs, timeout=3, callback=on_terminate) - for p in alive: + gone, still_alive = psutil.wait_procs(procs, timeout=3, callback=on_terminate) + for p in still_alive: p.kill() Exceptions @@ -702,8 +706,8 @@ Exceptions .. class:: NoSuchProcess(pid, name=None, msg=None) Raised by :class:`Process` class methods when no process with the given - pid* is found in the current process list or when a process no longer - exists. "name" is the name the process had before disappearing + *pid* is found in the current process list or when a process no longer + exists. *name* is the name the process had before disappearing and gets set only if :meth:`Process.name()` was previously called. .. class:: ZombieProcess(pid, name=None, ppid=None, msg=None) @@ -858,7 +862,7 @@ Process class .. method:: ppid() - The process parent pid. On Windows the return value is cached after first + The process parent PID. On Windows the return value is cached after first call. Not on POSIX because `ppid may change `__ if process becomes a zombie. @@ -875,16 +879,28 @@ Process class On some systems this may also be an empty string. The return value is cached after first call. + >>> import psutil + >>> psutil.Process().exe() + '/usr/bin/python2.7' + .. method:: cmdline() - The command line this process has been called with. The return value is not - cached because the cmdline of a process may change. + The command line this process has been called with as a list of strings. + The return value is not cached because the cmdline of a process may change. + + >>> import psutil + >>> psutil.Process().cmdline() + ['python', 'manage.py', 'runserver'] .. method:: environ() The environment variables of the process as a dict. Note: this might not reflect changes made after the process started. + >>> import psutil + >>> psutil.Process().environ() + {'LC_NUMERIC': 'it_IT.UTF-8', 'QT_QPA_PLATFORMTHEME': 'appmenu-qt5', 'IM_CONFIG_PHASE': '1', 'XDG_GREETER_DATA_DIR': '/var/lib/lightdm-data/giampaolo', 'GNOME_DESKTOP_SESSION_ID': 'this-is-deprecated', 'XDG_CURRENT_DESKTOP': 'Unity', 'UPSTART_EVENTS': 'started starting', 'GNOME_KEYRING_PID': '', 'XDG_VTNR': '7', 'QT_IM_MODULE': 'ibus', 'LOGNAME': 'giampaolo', 'USER': 'giampaolo', 'PATH': '/home/giampaolo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/giampaolo/svn/sysconf/bin', 'LC_PAPER': 'it_IT.UTF-8', 'GNOME_KEYRING_CONTROL': '', 'GTK_IM_MODULE': 'ibus', 'DISPLAY': ':0', 'LANG': 'en_US.UTF-8', 'LESS_TERMCAP_se': '\x1b[0m', 'TERM': 'xterm-256color', 'SHELL': '/bin/bash', 'XDG_SESSION_PATH': '/org/freedesktop/DisplayManager/Session0', 'XAUTHORITY': '/home/giampaolo/.Xauthority', 'LANGUAGE': 'en_US', 'COMPIZ_CONFIG_PROFILE': 'ubuntu', 'LC_MONETARY': 'it_IT.UTF-8', 'QT_LINUX_ACCESSIBILITY_ALWAYS_ON': '1', 'LESS_TERMCAP_me': '\x1b[0m', 'LESS_TERMCAP_md': '\x1b[01;38;5;74m', 'LESS_TERMCAP_mb': '\x1b[01;31m', 'HISTSIZE': '100000', 'UPSTART_INSTANCE': '', 'CLUTTER_IM_MODULE': 'xim', 'WINDOWID': '58786407', 'EDITOR': 'vim', 'SESSIONTYPE': 'gnome-session', 'XMODIFIERS': '@im=ibus', 'GPG_AGENT_INFO': '/home/giampaolo/.gnupg/S.gpg-agent:0:1', 'HOME': '/home/giampaolo', 'HISTFILESIZE': '100000', 'QT4_IM_MODULE': 'xim', 'GTK2_MODULES': 'overlay-scrollbar', 'XDG_SESSION_DESKTOP': 'ubuntu', 'SHLVL': '1', 'XDG_RUNTIME_DIR': '/run/user/1000', 'INSTANCE': 'Unity', 'LC_ADDRESS': 'it_IT.UTF-8', 'SSH_AUTH_SOCK': '/run/user/1000/keyring/ssh', 'VTE_VERSION': '4205', 'GDMSESSION': 'ubuntu', 'MANDATORY_PATH': '/usr/share/gconf/ubuntu.mandatory.path', 'VISUAL': 'vim', 'DESKTOP_SESSION': 'ubuntu', 'QT_ACCESSIBILITY': '1', 'XDG_SEAT_PATH': '/org/freedesktop/DisplayManager/Seat0', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'XDG_SESSION_ID': 'c2', 'DBUS_SESSION_BUS_ADDRESS': 'unix:abstract=/tmp/dbus-9GAJpvnt8r', '_': '/usr/bin/python', 'DEFAULTS_PATH': '/usr/share/gconf/ubuntu.default.path', 'LC_IDENTIFICATION': 'it_IT.UTF-8', 'LESS_TERMCAP_ue': '\x1b[0m', 'UPSTART_SESSION': 'unix:abstract=/com/ubuntu/upstart-session/1000/1294', 'XDG_CONFIG_DIRS': '/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg', 'GTK_MODULES': 'gail:atk-bridge:unity-gtk-module', 'XDG_SESSION_TYPE': 'x11', 'PYTHONSTARTUP': '/home/giampaolo/.pythonstart', 'LC_NAME': 'it_IT.UTF-8', 'OLDPWD': '/home/giampaolo/svn/curio_giampaolo/tests', 'GDM_LANG': 'en_US', 'LC_TELEPHONE': 'it_IT.UTF-8', 'HISTCONTROL': 'ignoredups:erasedups', 'LC_MEASUREMENT': 'it_IT.UTF-8', 'PWD': '/home/giampaolo/svn/curio_giampaolo', 'JOB': 'gnome-session', 'LESS_TERMCAP_us': '\x1b[04;38;5;146m', 'UPSTART_JOB': 'unity-settings-daemon', 'LC_TIME': 'it_IT.UTF-8', 'LESS_TERMCAP_so': '\x1b[38;5;246m', 'PAGER': 'less', 'XDG_DATA_DIRS': '/usr/share/ubuntu:/usr/share/gnome:/usr/local/share/:/usr/share/:/var/lib/snapd/desktop', 'XDG_SEAT': 'seat0'} + Availability: Linux, OSX, Windows .. versionadded:: 4.0.0 @@ -1100,7 +1116,7 @@ Process class Return threads opened by process as a list of namedtuples including thread id and thread CPU times (user/system). On OpenBSD this method requires - root access. + root privileges. .. method:: cpu_times() @@ -1119,8 +1135,8 @@ Process class .. method:: cpu_percent(interval=None) Return a float representing the process CPU utilization as a percentage - which can also be ``> 100.0`` in case of threads running on multiple - CPUs. + which can also be ``> 100.0`` in case of a process running multiple threads + on different CPUs. When *interval* is > ``0.0`` compares process times to system CPU times elapsed before and after the interval (blocking). When interval is ``0.0`` or ``None`` compares process times to system CPU times elapsed since last @@ -1145,10 +1161,10 @@ Process class .. note:: the returned value is explicitly *not* split evenly between all available - logical CPUs (differently from :func:`psutil.cpu_percent()`). + CPUs (differently from :func:`psutil.cpu_percent()`). This means that a busy loop process running on a system with 2 logical CPUs will be reported as having 100% CPU utilization instead of 50%. - This was done in order to be consistent with UNIX's ``top`` utility + This was done in order to be consistent with ``top`` UNIX utility and also to make it easier to identify processes hogging CPU resources independently from the number of CPUs. It must be noted that ``taskmgr.exe`` on Windows does not behave like @@ -1388,7 +1404,13 @@ Process class pmmap_grouped(path='[heap]', rss=32768, size=139264, pss=32768, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=32768, referenced=32768, anonymous=32768, swap=0), pmmap_grouped(path='[stack]', rss=2465792, size=2494464, pss=2465792, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=2465792, referenced=2277376, anonymous=2465792, swap=0), ...] - >>> + >>> p.memory_maps(grouped=False) + [pmmap_ext(addr='00400000-006ea000', perms='r-xp', path='/usr/bin/python2.7', rss=2293760, size=3055616, pss=1157120, shared_clean=2273280, shared_dirty=0, private_clean=20480, private_dirty=0, referenced=2293760, anonymous=0, swap=0), + pmmap_ext(addr='008e9000-008eb000', perms='r--p', path='/usr/bin/python2.7', rss=8192, size=8192, pss=6144, shared_clean=4096, shared_dirty=0, private_clean=0, private_dirty=4096, referenced=8192, anonymous=4096, swap=0), + pmmap_ext(addr='008eb000-00962000', perms='rw-p', path='/usr/bin/python2.7', rss=417792, size=487424, pss=317440, shared_clean=200704, shared_dirty=0, private_clean=16384, private_dirty=200704, referenced=417792, anonymous=200704, swap=0), + pmmap_ext(addr='00962000-00985000', perms='rw-p', path='[anon]', rss=139264, size=143360, pss=139264, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=139264, referenced=139264, anonymous=139264, swap=0), + pmmap_ext(addr='02829000-02ccf000', perms='rw-p', path='[heap]', rss=4743168, size=4874240, pss=4743168, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=4743168, referenced=4718592, anonymous=4743168, swap=0), + ...] Availability: All platforms except OpenBSD and NetBSD. @@ -1397,7 +1419,7 @@ Process class Return the children of this process as a list of :Class:`Process` objects, preemptively checking whether PID has been reused. If recursive is `True` return all the parent descendants. - Example assuming *A == this process*: + Pseudo code example assuming *A == this process*: :: A ─┐ @@ -1440,8 +1462,7 @@ Process class >>> f = open('file.ext', 'w') >>> p = psutil.Process() >>> p.open_files() - [popenfile(path='/home/giampaolo/svn/psutil/setup.py', fd=3, position=0, mode='r', flags=32768), - popenfile(path='/var/log/monitd', fd=4, position=235542, mode='a', flags=33793)] + [popenfile(path='/home/giampaolo/svn/psutil/file.ext', fd=3, position=0, mode='w', flags=32769)] .. warning:: on Windows this is not fully reliable as due to some limitations of the @@ -1500,27 +1521,27 @@ Process class +----------------+-----------------------------------------------------+ | **Kind value** | **Connections using** | +================+=====================================================+ - | "inet" | IPv4 and IPv6 | + | ``"inet"`` | IPv4 and IPv6 | +----------------+-----------------------------------------------------+ - | "inet4" | IPv4 | + | ``"inet4"`` | IPv4 | +----------------+-----------------------------------------------------+ - | "inet6" | IPv6 | + | ``"inet6"`` | IPv6 | +----------------+-----------------------------------------------------+ - | "tcp" | TCP | + | ``"tcp"`` | TCP | +----------------+-----------------------------------------------------+ - | "tcp4" | TCP over IPv4 | + | ``"tcp4"`` | TCP over IPv4 | +----------------+-----------------------------------------------------+ - | "tcp6" | TCP over IPv6 | + | ``"tcp6"`` | TCP over IPv6 | +----------------+-----------------------------------------------------+ - | "udp" | UDP | + | ``"udp"`` | UDP | +----------------+-----------------------------------------------------+ - | "udp4" | UDP over IPv4 | + | ``"udp4"`` | UDP over IPv4 | +----------------+-----------------------------------------------------+ - | "udp6" | UDP over IPv6 | + | ``"udp6"`` | UDP over IPv6 | +----------------+-----------------------------------------------------+ - | "unix" | UNIX socket (both UDP and TCP protocols) | + | ``"unix"`` | UNIX socket (both UDP and TCP protocols) | +----------------+-----------------------------------------------------+ - | "all" | the sum of all the possible families and protocols | + | ``"all"`` | the sum of all the possible families and protocols | +----------------+-----------------------------------------------------+ Example: @@ -1600,6 +1621,10 @@ Process class either return immediately or raise :class:`TimeoutExpired`. To wait for multiple processes use :func:`psutil.wait_procs()`. + >>> import psutil + >>> p = psutil.Process(9891) + >>> p.terminate() + >>> p.wait() Popen class ----------- From 405c5feea6df792060c415d7c0423bf7f277ce23 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 20 Dec 2016 01:18:54 +0100 Subject: [PATCH 033/922] #609: fix compilation issue on solaris 10 --- HISTORY.rst | 1 + README.rst | 2 +- docs/index.rst | 20 +++++++++++--------- psutil/_psutil_posix.c | 36 +++++++++++++++++++++++------------- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f93e09745..fdd59e765 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,7 @@ **Bug fixes** +- 609_: [SunOS] psutil does not compile on Solaris 10. - 936_: [Windows] fix compilation error on VS 2013 (patch by Max Bélanger). - 940_: [Linux] cpu_percent() and cpu_times_percent() was calculated incorrectly as "iowait", "guest" and "guest_nice" times were not properly diff --git a/README.rst b/README.rst index dbfb7274b..d413f1e8b 100644 --- a/README.rst +++ b/README.rst @@ -71,7 +71,7 @@ Projects using psutil ===================== At the time of writing there are over -`4200 projects `__ +`4200 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: diff --git a/docs/index.rst b/docs/index.rst index 031360d23..ef46f03f3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1948,15 +1948,6 @@ Constants >>> if psutil.version_info >= (4, 5): ... pass - -Development guide -================= - -If you plan on hacking on psutil (e.g. want to add a new feature or fix a bug) -take a look at the -`development guide `_. - - Q&A === @@ -1971,6 +1962,11 @@ Q&A ---- +* Q: What SunOS versions are supported? +* A: From Solaris 10 onwards. + +---- + * Q: Why do I get :class:`AccessDenied` for certain processes? * A: This may happen when you query processess owned by another user, especially on `OSX `__ and @@ -1983,6 +1979,12 @@ Q&A the Python script as a Windows service (this is the trick used by tools such as ProcessHacker). +Development guide +================= + +If you plan on hacking on psutil (e.g. want to add a new feature or fix a bug) +take a look at the +`development guide `_. Timeline ======== diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index 2d9630ace..fa8fccbc9 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -16,25 +16,25 @@ #include #ifdef PSUTIL_SUNOS10 -#include "arch/solaris/v10/ifaddrs.h" + #include "arch/solaris/v10/ifaddrs.h" #else -#include + #include #endif #ifdef __linux -#include -#include -#endif // end linux + #include + #include +#endif #if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__) -#include -#include -#include + #include + #include + #include #endif #if defined(__sun) -#include -#include + #include + #include #endif @@ -264,8 +264,11 @@ psutil_net_if_mtu(PyObject *self, PyObject *args) { char *nic_name; int sock = 0; int ret; - int mtu; +#ifdef PSUTIL_SUNOS10 + struct lifreq lifr; +#else struct ifreq ifr; +#endif if (! PyArg_ParseTuple(args, "s", &nic_name)) return NULL; @@ -274,14 +277,21 @@ psutil_net_if_mtu(PyObject *self, PyObject *args) { if (sock == -1) goto error; +#ifdef PSUTIL_SUNOS10 + strncpy(lifr.lifr_name, nic_name, sizeof(lifr.lifr_name)); +#else strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); +#endif ret = ioctl(sock, SIOCGIFMTU, &ifr); if (ret == -1) goto error; close(sock); - mtu = ifr.ifr_mtu; - return Py_BuildValue("i", mtu); +#ifdef PSUTIL_SUNOS10 + return Py_BuildValue("i", lifr.lifr_mtu); +#else + return Py_BuildValue("i", ifr.ifr_mtu); +#endif error: if (sock != 0) From f4121d4678108e62f4e6c4ec8f798420ee21eaef Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 20 Dec 2016 02:04:48 +0100 Subject: [PATCH 034/922] minor setup.py refactoring --- setup.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 938cfef9c..210ab724f 100755 --- a/setup.py +++ b/setup.py @@ -254,7 +254,7 @@ def on_exit(): def main(): - setup_args = dict( + setup( name='psutil', version=VERSION, description=__doc__.replace('\n', '').strip(), @@ -271,6 +271,7 @@ def main(): platforms='Platform Independent', license='BSD', packages=['psutil', 'psutil.tests'], + ext_modules=extensions, # see: python setup.py register --list-classifiers classifiers=[ 'Development Status :: 5 - Production/Stable', @@ -317,9 +318,6 @@ def main(): 'Topic :: Utilities', ], ) - if extensions is not None: - setup_args["ext_modules"] = extensions - setup(**setup_args) if __name__ == '__main__': From 8b8405e249a005a29acdf0522c5e44dbcd8e382c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 20 Dec 2016 03:11:20 +0100 Subject: [PATCH 035/922] setup.py: C macros were not passed to _psutil_posix.c --- setup.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index 210ab724f..4e3b0c6b3 100755 --- a/setup.py +++ b/setup.py @@ -85,18 +85,6 @@ def write(self, s): VERSION = get_version() macros.append(('PSUTIL_VERSION', int(VERSION.replace('.', '')))) - -# POSIX -if POSIX: - posix_extension = Extension( - 'psutil._psutil_posix', - sources=['psutil/_psutil_posix.c']) - if SUNOS: - posix_extension.libraries.append('socket') - if platform.release() == '5.10': - posix_extension.sources.append('psutil/arch/solaris/v10/ifaddrs.c') - posix_extension.define_macros.append(('PSUTIL_SUNOS10', 1)) - # Windows if WINDOWS: def get_winver(): @@ -139,7 +127,6 @@ def get_winver(): # extra_compile_args=["/Z7"], # extra_link_args=["/DEBUG"] ) - extensions = [ext] # OS X elif OSX: @@ -155,7 +142,6 @@ def get_winver(): extra_link_args=[ '-framework', 'CoreFoundation', '-framework', 'IOKit' ]) - extensions = [ext, posix_extension] # FreeBSD elif FREEBSD: @@ -170,7 +156,6 @@ def get_winver(): ], define_macros=macros, libraries=["devstat"]) - extensions = [ext, posix_extension] # OpenBSD elif OPENBSD: @@ -184,7 +169,6 @@ def get_winver(): ], define_macros=macros, libraries=["kvm"]) - extensions = [ext, posix_extension] # NetBSD elif NETBSD: @@ -199,7 +183,6 @@ def get_winver(): ], define_macros=macros, libraries=["kvm"]) - extensions = [ext, posix_extension] # Linux elif LINUX: @@ -229,15 +212,15 @@ def on_exit(): else: return None - macros.append(("PSUTIL_LINUX", 1)) ETHTOOL_MACRO = get_ethtool_macro() + + macros.append(("PSUTIL_LINUX", 1)) if ETHTOOL_MACRO is not None: macros.append(ETHTOOL_MACRO) ext = Extension( 'psutil._psutil_linux', sources=['psutil/_psutil_linux.c'], define_macros=macros) - extensions = [ext, posix_extension] # Solaris elif SUNOS: @@ -247,11 +230,26 @@ def on_exit(): sources=['psutil/_psutil_sunos.c'], define_macros=macros, libraries=['kstat', 'nsl', 'socket']) - extensions = [ext, posix_extension] else: sys.exit('platform %s is not supported' % sys.platform) +# POSIX +if POSIX: + posix_extension = Extension( + 'psutil._psutil_posix', + define_macros=macros, + sources=['psutil/_psutil_posix.c']) + if SUNOS: + posix_extension.libraries.append('socket') + if platform.release() == '5.10': + posix_extension.sources.append('psutil/arch/solaris/v10/ifaddrs.c') + posix_extension.define_macros.append(('PSUTIL_SUNOS10', 1)) + + extensions = [ext, posix_extension] +else: + extensions = [ext] + def main(): setup( From 9782dc0203056c9da1457caa761409c443b21446 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 20 Dec 2016 03:48:47 +0100 Subject: [PATCH 036/922] refactor C macros --- psutil/_psutil_bsd.c | 90 ++++++++++++++++++++-------------------- psutil/_psutil_posix.c | 28 ++++++------- psutil/arch/bsd/netbsd.c | 4 +- 3 files changed, 59 insertions(+), 63 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 33448ccf1..cd5ec3d84 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -15,8 +15,8 @@ * - psutil.Process.memory_maps() */ -#if defined(__NetBSD__) -#define _KMEMUSER +#if defined(PSUTIL_NETBSD) + #define _KMEMUSER #endif #include @@ -61,17 +61,17 @@ #include "_psutil_common.h" -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD #include "arch/bsd/freebsd.h" #include "arch/bsd/freebsd_socks.h" -#elif __OpenBSD__ +#elif PSUTIL_OPENBSD #include "arch/bsd/openbsd.h" -#elif __NetBSD__ +#elif PSUTIL_NETBSD #include "arch/bsd/netbsd.h" #include "arch/bsd/netbsd_socks.h" #endif -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD #include #include // get io counters #include // process open files, shared libs (kinfo_getvmmap) @@ -82,7 +82,7 @@ #endif #endif -#ifdef __OpenBSD__ +#ifdef PSUTIL_OPENBSD #include #include // for VREG #define _KERNEL // for DTYPE_VNODE @@ -91,7 +91,7 @@ #include // for CPUSTATES & CP_* #endif -#if defined(__NetBSD__) +#if defined(PSUTIL_NETBSD) #include #include // for VREG #include // for CPUSTATES & CP_* @@ -104,13 +104,13 @@ // convert a timeval struct to a double #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD // convert a bintime struct to milliseconds #define PSUTIL_BT2MSEC(bt) (bt.sec * 1000 + (((uint64_t) 1000000000 * \ (uint32_t) (bt.frac >> 32) ) >> 32 ) / 1000000) #endif -#if defined(__OpenBSD__) || defined (__NetBSD__) +#if defined(PSUTIL_OPENBSD) || defined (PSUTIL_NETBSD) #define PSUTIL_KPT2DOUBLE(t) (t ## _sec + t ## _usec / 1000000.0) #endif @@ -146,9 +146,9 @@ psutil_pids(PyObject *self, PyObject *args) { if (num_processes > 0) { orig_address = proclist; // save so we can free it after we're done for (idx = 0; idx < num_processes; idx++) { -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD py_pid = Py_BuildValue("i", proclist->ki_pid); -#elif defined(__OpenBSD__) || defined(__NetBSD__) +#elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) py_pid = Py_BuildValue("i", proclist->p_pid); #endif if (!py_pid) @@ -215,9 +215,9 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { return NULL; // Process -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD sprintf(str, "%s", kp.ki_comm); -#elif defined(__OpenBSD__) || defined(__NetBSD__) +#elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) sprintf(str, "%s", kp.p_comm); #endif #if PY_MAJOR_VERSION >= 3 @@ -233,7 +233,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { } // Calculate memory. -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD rss = (long)kp.ki_rssize * pagesize; vms = (long)kp.ki_size; memtext = (long)kp.ki_tsize * pagesize; @@ -241,12 +241,12 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { memstack = (long)kp.ki_ssize * pagesize; #else rss = (long)kp.p_vm_rssize * pagesize; - #ifdef __OpenBSD__ + #ifdef PSUTIL_OPENBSD // VMS, this is how ps determines it on OpenBSD: // http://anoncvs.spacehopper.org/openbsd-src/tree/bin/ps/print.c#n461 // vms vms = (long)(kp.p_vm_dsize + kp.p_vm_ssize + kp.p_vm_tsize) * pagesize; - #elif __NetBSD__ + #elif PSUTIL_NETBSD // VMS, this is how top determines it on NetBSD: // ftp://ftp.iij.ad.jp/pub/NetBSD/NetBSD-release-6/src/external/bsd/ // top/dist/machine/m_netbsd.c @@ -260,7 +260,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { // Return a single big tuple with all process info. py_retlist = Py_BuildValue( "(lillllllidllllddddlllllO)", -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD // (long)kp.ki_ppid, // (long) ppid (int)kp.ki_stat, // (int) status @@ -292,7 +292,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { memtext, // (long) mem text memdata, // (long) mem data memstack, // (long) mem stack -#elif defined(__OpenBSD__) || defined(__NetBSD__) +#elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) // (long)kp.p_ppid, // (long) ppid (int)kp.p_stat, // (int) status @@ -352,9 +352,9 @@ psutil_proc_name(PyObject *self, PyObject *args) { if (psutil_kinfo_proc(pid, &kp) == -1) return NULL; -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD sprintf(str, "%s", kp.ki_comm); -#elif defined(__OpenBSD__) || defined(__NetBSD__) +#elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) sprintf(str, "%s", kp.p_comm); #endif @@ -412,7 +412,7 @@ psutil_cpu_count_logical(PyObject *self, PyObject *args) { */ static PyObject * psutil_cpu_times(PyObject *self, PyObject *args) { -#if defined(__NetBSD__) +#if defined(PSUTIL_NETBSD) u_int64_t cpu_time[CPUSTATES]; #else long cpu_time[CPUSTATES]; @@ -420,9 +420,9 @@ psutil_cpu_times(PyObject *self, PyObject *args) { size_t size = sizeof(cpu_time); int ret; -#if defined(__FreeBSD__) || defined(__NetBSD__) +#if defined(PSUTIL_FREEBSD) || defined(PSUTIL_NETBSD) ret = sysctlbyname("kern.cp_time", &cpu_time, &size, NULL, 0); -#elif __OpenBSD__ +#elif PSUTIL_OPENBSD int mib[] = {CTL_KERN, KERN_CPTIME}; ret = sysctl(mib, 2, &cpu_time, &size, NULL, 0); #endif @@ -447,7 +447,7 @@ psutil_cpu_times(PyObject *self, PyObject *args) { * utility has the same problem see: * https://github.com/giampaolo/psutil/issues/595 */ -#if (defined(__FreeBSD_version) && __FreeBSD_version >= 800000) || __OpenBSD__ || defined(__NetBSD__) +#if (defined(__FreeBSD_version) && __FreeBSD_version >= 800000) || PSUTIL_OPENBSD || defined(PSUTIL_NETBSD) static PyObject * psutil_proc_open_files(PyObject *self, PyObject *args) { long pid; @@ -474,17 +474,17 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { for (i = 0; i < cnt; i++) { kif = &freep[i]; -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD if ((kif->kf_type == KF_TYPE_VNODE) && (kif->kf_vnode_type == KF_VTYPE_VREG)) { py_tuple = Py_BuildValue("(si)", kif->kf_path, kif->kf_fd); -#elif defined(__OpenBSD__) +#elif defined(PSUTIL_OPENBSD) if ((kif->f_type == DTYPE_VNODE) && (kif->v_type == VREG)) { py_tuple = Py_BuildValue("(si)", "", kif->fd_fd); -#elif defined(__NetBSD__) +#elif defined(PSUTIL_NETBSD) if ((kif->ki_ftype == DTYPE_VNODE) && (kif->ki_vtype == VREG)) { @@ -521,7 +521,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { long len; uint64_t flags; char opts[200]; -#if defined(__NetBSD__) +#if defined(PSUTIL_NETBSD) struct statvfs *fs = NULL; #else struct statfs *fs = NULL; @@ -534,7 +534,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { // get the number of mount points Py_BEGIN_ALLOW_THREADS -#if defined(__NetBSD__) +#if defined(PSUTIL_NETBSD) num = getvfsstat(NULL, 0, MNT_NOWAIT); #else num = getfsstat(NULL, 0, MNT_NOWAIT); @@ -553,7 +553,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { } Py_BEGIN_ALLOW_THREADS -#if defined(__NetBSD__) +#if defined(PSUTIL_NETBSD) num = getvfsstat(fs, len, MNT_NOWAIT); #else num = getfsstat(fs, len, MNT_NOWAIT); @@ -567,7 +567,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { for (i = 0; i < num; i++) { py_tuple = NULL; opts[0] = 0; -#if defined(__NetBSD__) +#if defined(PSUTIL_NETBSD) flags = fs[i].f_flag; #else flags = fs[i].f_flags; @@ -590,7 +590,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strlcat(opts, ",noatime", sizeof(opts)); if (flags & MNT_SOFTDEP) strlcat(opts, ",softdep", sizeof(opts)); -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD if (flags & MNT_UNION) strlcat(opts, ",union", sizeof(opts)); if (flags & MNT_SUIDDIR) @@ -611,7 +611,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strlcat(opts, ",noclusterw", sizeof(opts)); if (flags & MNT_NFS4ACLS) strlcat(opts, ",nfs4acls", sizeof(opts)); -#elif __NetBSD__ +#elif PSUTIL_NETBSD if (flags & MNT_NODEV) strlcat(opts, ",nodev", sizeof(opts)); if (flags & MNT_UNION) @@ -769,7 +769,7 @@ psutil_users(PyObject *self, PyObject *args) { if (py_retlist == NULL) return NULL; -#if (defined(__FreeBSD_version) && (__FreeBSD_version < 900000)) || __OpenBSD__ +#if (defined(__FreeBSD_version) && (__FreeBSD_version < 900000)) || PSUTIL_OPENBSD struct utmp ut; FILE *fp; @@ -849,7 +849,7 @@ PsutilMethods[] = { "Return multiple info about a process"}, {"proc_name", psutil_proc_name, METH_VARARGS, "Return process name"}, -#if !defined(__NetBSD__) +#if !defined(PSUTIL_NETBSD) {"proc_connections", psutil_proc_connections, METH_VARARGS, "Return connections opened by process"}, #endif @@ -857,25 +857,25 @@ PsutilMethods[] = { "Return process cmdline as a list of cmdline arguments"}, {"proc_threads", psutil_proc_threads, METH_VARARGS, "Return process threads"}, -#if defined(__FreeBSD__) || defined(__OpenBSD__) +#if defined(PSUTIL_FREEBSD) || defined(PSUTIL_OPENBSD) {"proc_cwd", psutil_proc_cwd, METH_VARARGS, "Return process current working directory."}, #endif -#if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 || __OpenBSD__ || defined(__NetBSD__) +#if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 || PSUTIL_OPENBSD || defined(PSUTIL_NETBSD) {"proc_num_fds", psutil_proc_num_fds, METH_VARARGS, "Return the number of file descriptors opened by this process"}, #endif -#if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 || __OpenBSD__ || defined(__NetBSD__) +#if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 || PSUTIL_OPENBSD || defined(PSUTIL_NETBSD) {"proc_open_files", psutil_proc_open_files, METH_VARARGS, "Return files opened by process as a list of (path, fd) tuples"}, #endif -#if defined(__FreeBSD__) || defined(__NetBSD__) +#if defined(PSUTIL_FREEBSD) || defined(PSUTIL_NETBSD) {"proc_exe", psutil_proc_exe, METH_VARARGS, "Return process pathname executable"}, {"proc_num_threads", psutil_proc_num_threads, METH_VARARGS, "Return number of threads used by process"}, -#if defined(__FreeBSD__) +#if defined(PSUTIL_FREEBSD) {"proc_memory_maps", psutil_proc_memory_maps, METH_VARARGS, "Return a list of tuples for every process's memory map"}, {"proc_cpu_affinity_get", psutil_proc_cpu_affinity_get, METH_VARARGS, @@ -914,7 +914,7 @@ PsutilMethods[] = { "Return currently connected users as a list of tuples"}, {"cpu_stats", psutil_cpu_stats, METH_VARARGS, "Return CPU statistics"}, -#if defined(__FreeBSD__) || defined(__NetBSD__) +#if defined(PSUTIL_FREEBSD) || defined(PSUTIL_NETBSD) {"net_connections", psutil_net_connections, METH_VARARGS, "Return system-wide open connections."}, #endif @@ -976,7 +976,7 @@ void init_psutil_bsd(void) PyModule_AddIntConstant(module, "version", PSUTIL_VERSION); // process status constants -#ifdef __FreeBSD__ +#ifdef PSUTIL_FREEBSD PyModule_AddIntConstant(module, "SIDL", SIDL); PyModule_AddIntConstant(module, "SRUN", SRUN); PyModule_AddIntConstant(module, "SSLEEP", SSLEEP); @@ -984,7 +984,7 @@ void init_psutil_bsd(void) PyModule_AddIntConstant(module, "SZOMB", SZOMB); PyModule_AddIntConstant(module, "SWAIT", SWAIT); PyModule_AddIntConstant(module, "SLOCK", SLOCK); -#elif __OpenBSD__ +#elif PSUTIL_OPENBSD PyModule_AddIntConstant(module, "SIDL", SIDL); PyModule_AddIntConstant(module, "SRUN", SRUN); PyModule_AddIntConstant(module, "SSLEEP", SSLEEP); @@ -992,7 +992,7 @@ void init_psutil_bsd(void) PyModule_AddIntConstant(module, "SZOMB", SZOMB); // unused PyModule_AddIntConstant(module, "SDEAD", SDEAD); PyModule_AddIntConstant(module, "SONPROC", SONPROC); -#elif defined(__NetBSD__) +#elif defined(PSUTIL_NETBSD) PyModule_AddIntConstant(module, "SIDL", LSIDL); PyModule_AddIntConstant(module, "SRUN", LSRUN); PyModule_AddIntConstant(module, "SSLEEP", LSSLEEP); diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index fa8fccbc9..b1d7180b7 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -21,18 +21,14 @@ #include #endif -#ifdef __linux +#if defined(PSUTIL_LINUX) #include #include -#endif - -#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__) +#elif defined(PSUTIL_BSD) || defined(PSUTIL_OSX) #include #include #include -#endif - -#if defined(__sun) +#elif defined(PSUTIL_SUNOS) #include #include #endif @@ -50,7 +46,7 @@ psutil_posix_getpriority(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; -#if defined(__APPLE__) +#ifdef PSUTIL_OSX priority = getpriority(PRIO_PROCESS, (id_t)pid); #else priority = getpriority(PRIO_PROCESS, pid); @@ -73,7 +69,7 @@ psutil_posix_setpriority(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "li", &pid, &priority)) return NULL; -#if defined(__APPLE__) +#ifdef PSUTIL_OSX retval = setpriority(PRIO_PROCESS, (id_t)pid, priority); #else retval = setpriority(PRIO_PROCESS, pid, priority); @@ -122,14 +118,13 @@ psutil_convert_ipaddr(struct sockaddr *addr, int family) { return Py_BuildValue("s", buf); } } -#ifdef __linux +#ifdef PSUTIL_LINUX else if (family == AF_PACKET) { struct sockaddr_ll *lladdr = (struct sockaddr_ll *)addr; len = lladdr->sll_halen; data = (const char *)lladdr->sll_addr; } -#endif -#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__) +#elif defined(PSUTIL_BSD) || defined(PSUTIL_OSX) else if (addr->sa_family == AF_LINK) { // Note: prior to Python 3.4 socket module does not expose // AF_LINK so we'll do. @@ -342,7 +337,7 @@ psutil_net_if_flags(PyObject *self, PyObject *args) { /* * net_if_stats() OSX/BSD implementation. */ -#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__) +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) #include #include @@ -379,7 +374,7 @@ int psutil_get_nic_speed(int ifm_active) { case(IFM_1000_SX): // 1000BaseSX - multi-mode fiber case(IFM_1000_LX): // 1000baseLX - single-mode fiber case(IFM_1000_CX): // 1000baseCX - 150ohm STP -#if defined(IFM_1000_TX) && !defined(__OpenBSD__) +#if defined(IFM_1000_TX) && !defined(PSUTIL_OPENBSD) // FreeBSD 4 and others (but NOT OpenBSD) -> #define IFM_1000_T in net/if_media.h case(IFM_1000_TX): #endif @@ -552,7 +547,7 @@ PsutilMethods[] = { "Retrieve NIC MTU"}, {"net_if_flags", psutil_net_if_flags, METH_VARARGS, "Retrieve NIC flags"}, -#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__) +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) {"net_if_duplex_speed", psutil_net_if_duplex_speed, METH_VARARGS, "Return NIC stats."}, #endif @@ -577,6 +572,7 @@ psutil_posix_traverse(PyObject *m, visitproc visit, void *arg) { return 0; } + static int psutil_posix_clear(PyObject *m) { Py_CLEAR(GETSTATE(m)->error); @@ -611,7 +607,7 @@ void init_psutil_posix(void) PyObject *module = Py_InitModule("_psutil_posix", PsutilMethods); #endif -#if defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__APPLE__) || defined(__sun) || defined(__NetBSD__) +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) || defined(PSUTIL_SUNOS) PyModule_AddIntConstant(module, "AF_LINK", AF_LINK); #endif diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index 2cf2ef224..d5c3e3b9e 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -7,8 +7,8 @@ * Platform-specific module methods for NetBSD. */ -#if defined(__NetBSD__) -#define _KMEMUSER +#if defined(PSUTIL_NETBSD) + #define _KMEMUSER #endif #include From cf95befdf44aeed8cac17f394ac5bdd00267a54c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 20 Dec 2016 04:27:51 +0100 Subject: [PATCH 037/922] refactor C macros --- psutil/_psutil_bsd.c | 39 +++++++++++++++++---------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index cd5ec3d84..afe3834e2 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -64,14 +64,7 @@ #ifdef PSUTIL_FREEBSD #include "arch/bsd/freebsd.h" #include "arch/bsd/freebsd_socks.h" -#elif PSUTIL_OPENBSD - #include "arch/bsd/openbsd.h" -#elif PSUTIL_NETBSD - #include "arch/bsd/netbsd.h" - #include "arch/bsd/netbsd_socks.h" -#endif -#ifdef PSUTIL_FREEBSD #include #include // get io counters #include // process open files, shared libs (kinfo_getvmmap) @@ -80,27 +73,29 @@ #else #include #endif -#endif +#elif PSUTIL_OPENBSD + #include "arch/bsd/openbsd.h" -#ifdef PSUTIL_OPENBSD #include #include // for VREG #define _KERNEL // for DTYPE_VNODE #include #undef _KERNEL #include // for CPUSTATES & CP_* -#endif +#elif PSUTIL_NETBSD + #include "arch/bsd/netbsd.h" + #include "arch/bsd/netbsd_socks.h" -#if defined(PSUTIL_NETBSD) #include #include // for VREG #include // for CPUSTATES & CP_* #ifndef DTYPE_VNODE - #define DTYPE_VNODE 1 + #define DTYPE_VNODE 1 #endif #endif + // convert a timeval struct to a double #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) @@ -412,7 +407,7 @@ psutil_cpu_count_logical(PyObject *self, PyObject *args) { */ static PyObject * psutil_cpu_times(PyObject *self, PyObject *args) { -#if defined(PSUTIL_NETBSD) +#ifdef PSUTIL_NETBSD u_int64_t cpu_time[CPUSTATES]; #else long cpu_time[CPUSTATES]; @@ -479,12 +474,12 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { (kif->kf_vnode_type == KF_VTYPE_VREG)) { py_tuple = Py_BuildValue("(si)", kif->kf_path, kif->kf_fd); -#elif defined(PSUTIL_OPENBSD) +#elif PSUTIL_OPENBSD if ((kif->f_type == DTYPE_VNODE) && (kif->v_type == VREG)) { py_tuple = Py_BuildValue("(si)", "", kif->fd_fd); -#elif defined(PSUTIL_NETBSD) +#elif PSUTIL_NETBSD if ((kif->ki_ftype == DTYPE_VNODE) && (kif->ki_vtype == VREG)) { @@ -521,7 +516,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { long len; uint64_t flags; char opts[200]; -#if defined(PSUTIL_NETBSD) +#ifdef PSUTIL_NETBSD struct statvfs *fs = NULL; #else struct statfs *fs = NULL; @@ -534,7 +529,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { // get the number of mount points Py_BEGIN_ALLOW_THREADS -#if defined(PSUTIL_NETBSD) +#ifdef PSUTIL_NETBSD num = getvfsstat(NULL, 0, MNT_NOWAIT); #else num = getfsstat(NULL, 0, MNT_NOWAIT); @@ -553,7 +548,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { } Py_BEGIN_ALLOW_THREADS -#if defined(PSUTIL_NETBSD) +#ifdef PSUTIL_NETBSD num = getvfsstat(fs, len, MNT_NOWAIT); #else num = getfsstat(fs, len, MNT_NOWAIT); @@ -567,7 +562,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { for (i = 0; i < num; i++) { py_tuple = NULL; opts[0] = 0; -#if defined(PSUTIL_NETBSD) +#ifdef PSUTIL_NETBSD flags = fs[i].f_flag; #else flags = fs[i].f_flags; @@ -618,17 +613,17 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strlcat(opts, ",union", sizeof(opts)); if (flags & MNT_NOCOREDUMP) strlcat(opts, ",nocoredump", sizeof(opts)); -#if defined(MNT_RELATIME) +#ifdef MNT_RELATIME if (flags & MNT_RELATIME) strlcat(opts, ",relatime", sizeof(opts)); #endif if (flags & MNT_IGNORE) strlcat(opts, ",ignore", sizeof(opts)); -#if defined(MNT_DISCARD) +#ifdef MNT_DISCARD if (flags & MNT_DISCARD) strlcat(opts, ",discard", sizeof(opts)); #endif -#if defined(MNT_EXTATTR) +#ifdef MNT_EXTATTR if (flags & MNT_EXTATTR) strlcat(opts, ",extattr", sizeof(opts)); #endif From c9a417a6ba06aa9d504c24ec6104bdea58e9ab66 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 20 Dec 2016 19:26:11 +0100 Subject: [PATCH 038/922] #609: fix compilation issue on SunOS 10 --- psutil/_psutil_posix.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index b1d7180b7..7aa4b553b 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -274,10 +274,11 @@ psutil_net_if_mtu(PyObject *self, PyObject *args) { #ifdef PSUTIL_SUNOS10 strncpy(lifr.lifr_name, nic_name, sizeof(lifr.lifr_name)); + ret = ioctl(sock, SIOCGIFMTU, &lifr); #else strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); -#endif ret = ioctl(sock, SIOCGIFMTU, &ifr); +#endif if (ret == -1) goto error; close(sock); From 039ba9079120fa40c22918fe8d02465f22a0dd4a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 20 Dec 2016 20:22:32 +0100 Subject: [PATCH 039/922] fix #944: [OpenBSD] psutil.pids() was omitting PID 0 --- HISTORY.rst | 1 + psutil/_psbsd.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index fdd59e765..f21c4c20b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -18,6 +18,7 @@ - 940_: [Linux] cpu_percent() and cpu_times_percent() was calculated incorrectly as "iowait", "guest" and "guest_nice" times were not properly taken into account. +- 944_: [OpenBSD] psutil.pids() was omitting PID 0. 5.0.0 diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index a33015048..acf6c7c3c 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -17,6 +17,7 @@ from . import _psutil_posix as cext_posix from ._common import conn_tmap from ._common import FREEBSD +from ._common import memoize from ._common import memoize_when_activated from ._common import NETBSD from ._common import OPENBSD @@ -420,7 +421,26 @@ def users(): # ===================================================================== -pids = cext.pids +@memoize +def _pid_0_exists(): + try: + Process(0).name() + except NoSuchProcess: + return False + except AccessDenied: + return True + else: + return True + + +def pids(): + ret = cext.pids() + if OPENBSD and (0 not in ret) and _pid_0_exists(): + # On OpenBSD the kernel does not return PID 0 (neither does + # ps) but it's actually querable (Process(0) will succeed). + ret.insert(0, 0) + return ret + if OPENBSD or NETBSD: def pid_exists(pid): From 4924513541e23bf121f14e0fdf0c35facc9406a8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 20 Dec 2016 20:40:29 +0100 Subject: [PATCH 040/922] update doc --- docs/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index ef46f03f3..afa99c52a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -156,13 +156,14 @@ CPU in Python 3.4). If *logical* is ``False`` return the number of physical cores only (hyper thread CPUs are excluded). Return ``None`` if undetermined. + On OpenBSD and NetBSD ``psutil.cpu_count(logical=False)`` always return + ``None``. Example on a system having 2 physical hyper-thread CPU cores: >>> import psutil >>> psutil.cpu_count() 4 >>> psutil.cpu_count(logical=False) 2 - >>> .. function:: cpu_stats() From 7c2483b84912abaa6f3b7742ddd6aad3fbebbc3b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 21 Dec 2016 02:33:25 +0100 Subject: [PATCH 041/922] pre release --- HISTORY.rst | 2 +- docs/index.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index f21c4c20b..25223322d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.0.1 ===== -*XXXX-XX-XX* +*2016-12-21* **Enhancements** diff --git a/docs/index.rst b/docs/index.rst index afa99c52a..85b3e488c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1990,6 +1990,7 @@ take a look at the Timeline ======== +- 2016-12-21: `5.0.1 `__ - `what's new `__ - 2016-11-06: `5.5.0 `__ - `what's new `__ - 2016-10-26: `4.4.2 `__ - `what's new `__ - 2016-10-25: `4.4.1 `__ - `what's new `__ From 5834173c6a3e51ab954d2c4a17df6e48d9d524e5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 21 Dec 2016 02:34:12 +0100 Subject: [PATCH 042/922] pre release --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 85b3e488c..3aed80943 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1991,7 +1991,7 @@ Timeline ======== - 2016-12-21: `5.0.1 `__ - `what's new `__ -- 2016-11-06: `5.5.0 `__ - `what's new `__ +- 2016-11-06: `5.0.0 `__ - `what's new `__ - 2016-10-26: `4.4.2 `__ - `what's new `__ - 2016-10-25: `4.4.1 `__ - `what's new `__ - 2016-10-23: `4.4.0 `__ - `what's new `__ From 5ad419071e08f311a559f385e1c6e48df6176c65 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 21 Dec 2016 03:04:04 +0100 Subject: [PATCH 043/922] add make doc command --- Makefile | 6 ++++++ psutil/__init__.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 13ed0c00c..9224a0bbc 100644 --- a/Makefile +++ b/Makefile @@ -253,3 +253,9 @@ bench-oneshot: install # same as above but using perf module (supposed to be more precise) bench-oneshot-2: install $(PYTHON) scripts/internal/bench_oneshot_2.py + +# generate a doc.zip file and manually upload it to PYPI. +doc: + cd docs && make html && cd _build/html/ && zip doc.zip -r . + mv docs/_build/html/doc.zip . + echo "done; now manually upload doc.zip from here: https://pypi.python.org/pypi?:action=pkg_edit&name=psutil" diff --git a/psutil/__init__.py b/psutil/__init__.py index 79817b1d1..4abb57669 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -189,7 +189,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.0.1" +__version__ = "5.0.2" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK _TOTAL_PHYMEM = None From 0be629eace44cb65a1455970dedc9e8c4dac3b5e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 4 Jan 2017 19:12:57 +0100 Subject: [PATCH 044/922] fix #948: cannot install psutil with PYTHONOPTIMIZE=2 --- HISTORY.rst | 8 ++++++++ psutil/__init__.py | 5 +++-- setup.py | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 25223322d..94feab8e3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,13 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.0.2 +===== + +**Bug fixes** + +- 948_: cannot install psutil with PYTHONOPTIMIZE=2. + + 5.0.1 ===== diff --git a/psutil/__init__.py b/psutil/__init__.py index 4abb57669..40403392e 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -143,8 +143,8 @@ from ._pssunos import CONN_BOUND # NOQA from ._pssunos import CONN_IDLE # NOQA - # This is public API and it will be retrieved from _pssunos.py - # via sys.modules. + # This is public writable API which is read from _pslinux.py and + # _pssunos.py via sys.modules. PROCFS_PATH = "/proc" else: # pragma: no cover @@ -222,6 +222,7 @@ # --- exceptions # ===================================================================== + class Error(Exception): """Base exception class. All other psutil exceptions inherit from this one. diff --git a/setup.py b/setup.py index 4e3b0c6b3..04f6436a0 100755 --- a/setup.py +++ b/setup.py @@ -255,7 +255,7 @@ def main(): setup( name='psutil', version=VERSION, - description=__doc__.replace('\n', '').strip(), + description=__doc__ or ''.replace('\n', '').strip(), long_description=get_description(), keywords=[ 'ps', 'top', 'kill', 'free', 'lsof', 'netstat', 'nice', 'tty', From 5bf37636dbaa38a335eaa9df02aa950cfbf4848b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 4 Jan 2017 19:14:39 +0100 Subject: [PATCH 045/922] minor refactoring --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 04f6436a0..426b7eb78 100755 --- a/setup.py +++ b/setup.py @@ -255,7 +255,7 @@ def main(): setup( name='psutil', version=VERSION, - description=__doc__ or ''.replace('\n', '').strip(), + description=__doc__ .replace('\n', '').strip() if __doc__ else '', long_description=get_description(), keywords=[ 'ps', 'top', 'kill', 'free', 'lsof', 'netstat', 'nice', 'tty', From 250e3a51c13840501e01a7000b476742a220ea46 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 9 Jan 2017 16:49:27 +0100 Subject: [PATCH 046/922] #687: [Linux] pid_exists() no longer returns True if passed a process thread ID --- HISTORY.rst | 4 ++++ docs/index.rst | 8 +++++--- psutil/_pslinux.py | 28 ++++++++++++++++++++++++++-- psutil/tests/test_linux.py | 19 +++++++++++++++++++ psutil/tests/test_process.py | 10 ++-------- 5 files changed, 56 insertions(+), 13 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 94feab8e3..92413ec1e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,8 +3,12 @@ 5.0.2 ===== +*XXXX-XX-XX* + **Bug fixes** +- 687_: [Linux] pid_exists() no longer returns True if passed a process thread + ID. - 948_: cannot install psutil with PYTHONOPTIMIZE=2. diff --git a/docs/index.rst b/docs/index.rst index 3aed80943..022c6a6ba 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -738,10 +738,12 @@ Process class .. class:: Process(pid=None) - Represents an OS process with the given *pid*. If *pid* is omitted current - process *pid* (`os.getpid() `__) - is used. + Represents an OS process with the given *pid*. + If *pid* is omitted current process *pid* + (`os.getpid() `__) is used. Raise :class:`NoSuchProcess` if *pid* does not exist. + On Linux *pid* can also refer to a thread ID (the *id* field returned by + :meth:`threads` method). When accessing methods of this class always be prepared to catch :class:`NoSuchProcess`, :class:`ZombieProcess` and :class:`AccessDenied` exceptions. diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 91fdae4f8..fe2f459dd 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1070,8 +1070,32 @@ def pids(): def pid_exists(pid): - """Check For the existence of a unix pid.""" - return _psposix.pid_exists(pid) + """Check for the existence of a unix PID.""" + if not _psposix.pid_exists(pid): + return False + else: + # Linux's apparently does not distinguish between PIDs and TIDs + # (thread IDs). + # listdir("/proc") won't show any TID (only PIDs) but + # os.stat("/proc/{tid}") will succeed if {tid} exists. + # os.kill() can also be passed a TID. This is quite confusing. + # In here we want to enforce this distinction and support PIDs + # only, see: + # https://github.com/giampaolo/psutil/issues/687 + try: + # Note: already checked that this is faster than using a + # regular expr. Also (a lot) faster than doing + # 'return pid in pids()' + with open_binary("%s/%s/status" % (get_procfs_path(), pid)) as f: + for line in f: + if line.startswith(b"Tgid:"): + tgid = int(line.split()[1]) + # If tgid and pid are the same then we're + # dealing with a process PID. + return tgid == pid + raise ValueError("'Tgid' line not found") + except (EnvironmentError, ValueError): + return pid in pids() def wrap_exceptions(fun): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 871cb74eb..028a41d9c 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -40,6 +40,7 @@ from psutil.tests import sh from psutil.tests import skip_on_not_implemented from psutil.tests import TESTFN +from psutil.tests import ThreadTask from psutil.tests import TRAVIS from psutil.tests import unittest from psutil.tests import which @@ -1004,6 +1005,24 @@ def open_mock(name, *args, **kwargs): importlib.reload(psutil._pslinux) importlib.reload(psutil) + def test_issue_687(self): + # In case of thread ID: + # - pid_exists() is supposed to return False + # - Process(tid) is supposed to work + # - pids() should not return the TID + # See: https://github.com/giampaolo/psutil/issues/687 + t = ThreadTask() + t.start() + try: + p = psutil.Process() + tid = p.threads()[1].id + assert not psutil.pid_exists(tid), tid + pt = psutil.Process(tid) + pt.as_dict() + self.assertNotIn(tid, psutil.pids()) + finally: + t.stop() + # ===================================================================== # test process diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 50e0cc746..d25f44749 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -501,10 +501,8 @@ def test_num_threads(self): try: step2 = p.num_threads() self.assertEqual(step2, step1 + 1) - thread.stop() finally: - if thread._running: - thread.stop() + thread.stop() @unittest.skipUnless(WINDOWS, 'WINDOWS only') def test_num_handles(self): @@ -524,7 +522,6 @@ def test_threads(self): thread = ThreadTask() thread.start() - try: step2 = p.threads() self.assertEqual(len(step2), len(step1) + 1) @@ -536,11 +533,8 @@ def test_threads(self): self.assertEqual(athread.id, athread[0]) self.assertEqual(athread.user_time, athread[1]) self.assertEqual(athread.system_time, athread[2]) - # test num threads - thread.stop() finally: - if thread._running: - thread.stop() + thread.stop() @retry_before_failing() # see: https://travis-ci.org/giampaolo/psutil/jobs/111842553 From 454bd62e89433189929f06ccff047ba9de9a3ce4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 9 Jan 2017 17:28:12 +0100 Subject: [PATCH 047/922] fix #873: fix typo in INSTALL --- INSTALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.rst b/INSTALL.rst index b1b40d683..fcbc3736a 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -62,7 +62,7 @@ versions. OSX === -Install `XcodeTools `__ +Install `Xcode `__ first, then: :: From 6e60cc93c68f243dcaf3c895fcd8a9fabcedba4b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 10 Jan 2017 14:51:19 +0100 Subject: [PATCH 048/922] ignore me --- psutil/__init__.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 40403392e..f8ce48e6a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -454,6 +454,11 @@ def __hash__(self): self._hash = hash(self._ident) return self._hash + @property + def pid(self): + """The process PID.""" + return self._pid + # --- utility methods @contextlib.contextmanager @@ -602,11 +607,6 @@ def is_running(self): # --- actual API - @property - def pid(self): - """The process PID.""" - return self._pid - @memoize_when_activated def ppid(self): """The process parent PID. @@ -1195,6 +1195,8 @@ def connections(self, kind='inet'): """ return self._proc.connections(kind) + # --- signals + if POSIX: def _send_signal(self, sig): assert not self.pid < 0, self.pid From 16a3af5d64bf21c692d3db737b482fe4c8711140 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 15 Jan 2017 18:18:44 +0100 Subject: [PATCH 049/922] update doc + add linux specific test --- INSTALL.rst | 13 +++++++++++-- docs/index.rst | 5 +++-- psutil/tests/test_linux.py | 3 +++ setup.py | 1 + 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index fcbc3736a..d731bd3de 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -5,12 +5,13 @@ pip is the easiest way to install psutil. It is shipped by default with Python 2.7.9+ and 3.4+. If you're using an older Python version `install pip `__ first. -If you GIT cloned psutil source code you can also install pip with:: +If you GIT cloned psutil source code you can also install pip and/or upgrade +it to latest version with:: make install-pip Unless you're on Windows, in order to install psutil with pip you'll also need -a C compiler installed. +a C compiler installed (e.g. gcc). pip will retrieve psutil source code or binaries from `PYPI `__ repository. @@ -137,6 +138,14 @@ Install: pkg install gcc python -m pip install psutil +Install from sources +==================== + + git clone https://github.com/giampaolo/psutil.git + cd psutil + python setup.py install + + Dev Guide ========= diff --git a/docs/index.rst b/docs/index.rst index 022c6a6ba..270124b77 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,11 +39,12 @@ The psutil documentation you're reading is distributed as a single HTML page. Install ------- -On Windows, or on UNIX if you have a C compiler installed, the easiest way to -install psutil is via ``pip``:: +The easiest way to install psutil is via ``pip``:: pip install psutil +On UNIX this requires a C compiler (e.g. gcc) installed. On Windows pip will +automatically retrieve a pre-compiled wheel version. Alternatively, see more detailed `install `_ instructions. diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 028a41d9c..814cabd6b 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1079,6 +1079,9 @@ def test_compare_stat_and_status_files(self): self.assertAlmostEqual( p.num_ctx_switches().involuntary, invol, delta=2) + elif line.startswith('Cpus_allowed_list'): + min_, max_ = map(int, line.split()[1].split('-')) + self.assertEqual(p.cpu_affinity(), range(min_, max_ + 1)) def test_memory_full_info(self): src = textwrap.dedent(""" diff --git a/setup.py b/setup.py index 426b7eb78..80521a487 100755 --- a/setup.py +++ b/setup.py @@ -301,6 +301,7 @@ def main(): 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Programming Language :: Python', From 6efeb550ae81e643dff9c271c5d2174e5a23411b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 15 Jan 2017 19:00:52 +0100 Subject: [PATCH 050/922] make.bat: att build_exe and build_wheel cmds --- psutil/tests/test_linux.py | 3 ++- scripts/internal/winmake.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 814cabd6b..b365b39ca 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1081,7 +1081,8 @@ def test_compare_stat_and_status_files(self): delta=2) elif line.startswith('Cpus_allowed_list'): min_, max_ = map(int, line.split()[1].split('-')) - self.assertEqual(p.cpu_affinity(), range(min_, max_ + 1)) + self.assertEqual( + p.cpu_affinity(), list(range(min_, max_ + 1))) def test_memory_full_info(self): src = textwrap.dedent(""" diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index bbe73e0df..de8c02e5a 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -170,6 +170,20 @@ def build(): sh("%s setup.py build_ext -i" % PYTHON) +@cmd +def build_exe(): + """Create exe file.""" + build() + sh("%s setup.py bdist_wininst -i" % PYTHON) + + +@cmd +def build_wheel(): + """Create wheel file.""" + build() + sh("%s setup.py bdist_wheel -i" % PYTHON) + + @cmd def install_pip(): """Install pip""" From 4121b020586123459d64cf33225bcd45a3878f79 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 15 Jan 2017 19:01:40 +0100 Subject: [PATCH 051/922] make.bat: fix invalid cmdline --- scripts/internal/winmake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index de8c02e5a..e2c1f086e 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -174,14 +174,14 @@ def build(): def build_exe(): """Create exe file.""" build() - sh("%s setup.py bdist_wininst -i" % PYTHON) + sh("%s setup.py bdist_wininst" % PYTHON) @cmd def build_wheel(): """Create wheel file.""" build() - sh("%s setup.py bdist_wheel -i" % PYTHON) + sh("%s setup.py bdist_wheel" % PYTHON) @cmd From 62719fa390fe58dad822a8e0c416580b5dba000e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 15 Jan 2017 19:30:04 +0100 Subject: [PATCH 052/922] CI integration: add python 3.6 --- .travis.yml | 1 + Makefile | 2 +- appveyor.yml | 11 +++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 17206c58a..48c84a7b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,6 +8,7 @@ matrix: - python: 3.3 - python: 3.4 - python: 3.5 + - python: 3.6 - "pypy" # XXX - commented because OSX builds are deadly slow # - language: generic diff --git a/Makefile b/Makefile index 9224a0bbc..95db068cf 100644 --- a/Makefile +++ b/Makefile @@ -210,7 +210,7 @@ win-download-exes: # Upload exes/wheels in dist/* directory to PYPI. win-upload-exes: $(PYTHON) -m twine upload dist/*.exe - $(PYTHON) -m twine upload dist/*.wheel + $(PYTHON) -m twine upload dist/*.whl # All the necessary steps before making a release. pre-release: diff --git a/appveyor.yml b/appveyor.yml index 927d9cb3d..4428f7767 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,6 +30,10 @@ environment: PYTHON_VERSION: "3.5.x" PYTHON_ARCH: "32" + - PYTHON: "C:\\Python36" + PYTHON_VERSION: "3.6.x" + PYTHON_ARCH: "32" + # 64 bits - PYTHON: "C:\\Python27-x64" @@ -51,6 +55,13 @@ environment: VS_VER: "2015" INSTANCENAME: "SQL2012SP1" + - PYTHON: "C:\\Python36-x64" + PYTHON_VERSION: "3.6.x" + PYTHON_ARCH: "64" + ARCH: x86_64 + VS_VER: "2015" + INSTANCENAME: "SQL2012SP1" + # Also build on a Python version not pre-installed by Appveyor. # See: https://github.com/ogrisel/python-appveyor-demo/issues/10 From ad7b993d5fa3c60ceeb6012be585a431b37c3603 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 15 Jan 2017 19:49:48 +0100 Subject: [PATCH 053/922] refactor /proc/pid/status tests and put them in their own class --- psutil/tests/test_linux.py | 116 ++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 47 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index b365b39ca..c839d7e2b 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1037,53 +1037,6 @@ def setUp(self): tearDown = setUp - def test_compare_stat_and_status_files(self): - # /proc/pid/stat and /proc/pid/status have many values in common. - # Whenever possible, psutil uses /proc/pid/stat (it's faster). - # For all those cases we check that the value found in - # /proc/pid/stat (by psutil) matches the one found in - # /proc/pid/status. - p = psutil.Process() - with psutil._psplatform.open_text('/proc/%s/status' % p.pid) as f: - for line in f: - line = line.strip() - if line.startswith('Name:'): - name = line.split()[1] - # Name is truncated to 15 chars - self.assertEqual(p.name()[:15], name[:15]) - elif line.startswith('State:'): - status = line[line.find('(') + 1:line.rfind(')')] - status = status.replace(' ', '-') - self.assertEqual(p.status(), status) - elif line.startswith('PPid:'): - ppid = int(line.split()[1]) - self.assertEqual(p.ppid(), ppid) - # The ones below internally are determined by reading - # 'status' file but we use a re to extract the info - # so it makes sense to check them. - elif line.startswith('Threads:'): - num_threads = int(line.split()[1]) - self.assertEqual(p.num_threads(), num_threads) - elif line.startswith('Uid:'): - uids = tuple(map(int, line.split()[1:4])) - self.assertEqual(tuple(p.uids()), uids) - elif line.startswith('Gid:'): - gids = tuple(map(int, line.split()[1:4])) - self.assertEqual(tuple(p.gids()), gids) - elif line.startswith('voluntary_ctxt_switches:'): - vol = int(line.split()[1]) - self.assertAlmostEqual( - p.num_ctx_switches().voluntary, vol, delta=2) - elif line.startswith('nonvoluntary_ctxt_switches:'): - invol = int(line.split()[1]) - self.assertAlmostEqual( - p.num_ctx_switches().involuntary, invol, - delta=2) - elif line.startswith('Cpus_allowed_list'): - min_, max_ = map(int, line.split()[1].split('-')) - self.assertEqual( - p.cpu_affinity(), list(range(min_, max_ + 1))) - def test_memory_full_info(self): src = textwrap.dedent(""" import time @@ -1255,5 +1208,74 @@ def test_exe_mocked(self): self.assertRaises(psutil.ZombieProcess, psutil.Process().exe) +@unittest.skipUnless(LINUX, "LINUX only") +class TestProcessAgainstStatus(unittest.TestCase): + """/proc/pid/stat and /proc/pid/status have many values in common. + Whenever possible, psutil uses /proc/pid/stat (it's faster). + For all those cases we check that the value found in + /proc/pid/stat (by psutil) matches the one found in + /proc/pid/status. + """ + + @classmethod + def setUpClass(cls): + cls.proc = psutil.Process() + + def read_status_file(self, linestart): + with psutil._psplatform.open_text( + '/proc/%s/status' % self.proc.pid) as f: + for line in f: + line = line.strip() + if line.startswith(linestart): + value = line.partition('\t')[2] + try: + return int(value) + except ValueError: + return value + else: + raise ValueError("can't find %r" % linestart) + + def test_name(self): + value = self.read_status_file("Name:") + self.assertEqual(self.proc.name(), value) + + def test_status(self): + value = self.read_status_file("State:") + value = value[value.find('(') + 1:value.rfind(')')] + value = value.replace(' ', '-') + self.assertEqual(self.proc.status(), value) + + def test_ppid(self): + value = self.read_status_file("PPid:") + self.assertEqual(self.proc.ppid(), value) + + def test_num_threads(self): + value = self.read_status_file("Threads:") + self.assertEqual(self.proc.num_threads(), value) + + def test_uids(self): + value = self.read_status_file("Uid:") + value = tuple(map(int, value.split()[1:4])) + self.assertEqual(self.proc.uids(), value) + + def test_gids(self): + value = self.read_status_file("Gid:") + value = tuple(map(int, value.split()[1:4])) + self.assertEqual(self.proc.gids(), value) + + @retry_before_failing() + def test_ctx_switches(self): + value = self.read_status_file("voluntary_ctxt_switches:") + self.assertEqual(self.proc.num_ctx_switches().voluntary, value) + value = self.read_status_file("nonvoluntary_ctxt_switches:") + self.assertEqual(self.proc.num_ctx_switches().involuntary, value) + + def test_cpu_affinity(self): + value = self.read_status_file("Cpus_allowed_list:") + min_, max_ = map(int, value.split('-')) + self.assertEqual( + self.proc.cpu_affinity(), list(range(min_, max_ + 1))) + + if __name__ == '__main__': run_test_module_by_name(__file__) From db5776bf08127a90d4044b1133d53e738e9c144e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 15 Jan 2017 19:53:54 +0100 Subject: [PATCH 054/922] comment out unreliable linux test --- psutil/tests/test_linux.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index c839d7e2b..37352ecf2 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -581,7 +581,8 @@ def test_net_if_stats(self): except RuntimeError: pass else: - self.assertEqual(stats.isup, 'RUNNING' in out, msg=out) + # Not always reliable. + # self.assertEqual(stats.isup, 'RUNNING' in out, msg=out) self.assertEqual(stats.mtu, int(re.findall('MTU:(\d+)', out)[0])) @@ -1264,7 +1265,7 @@ def test_gids(self): self.assertEqual(self.proc.gids(), value) @retry_before_failing() - def test_ctx_switches(self): + def test_num_ctx_switches(self): value = self.read_status_file("voluntary_ctxt_switches:") self.assertEqual(self.proc.num_ctx_switches().voluntary, value) value = self.read_status_file("nonvoluntary_ctxt_switches:") From e1a9376eb5fd5debd5f824688f43867693ec9604 Mon Sep 17 00:00:00 2001 From: Pierre Fersing Date: Thu, 19 Jan 2017 17:08:14 +0100 Subject: [PATCH 055/922] Fix Process cpu_percent on Windows --- CREDITS | 5 +++++ HISTORY.rst | 2 ++ psutil/__init__.py | 9 ++------- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CREDITS b/CREDITS index ccc4515f2..031548aeb 100644 --- a/CREDITS +++ b/CREDITS @@ -420,3 +420,8 @@ I: 919 N: Max Bélanger W: https://github.com/maxbelanger I: 936 + +N: Pierre Fersing +C: France +E: pierre.fersing@bleemeo.com +I: 950 diff --git a/HISTORY.rst b/HISTORY.rst index 92413ec1e..c6db2aa0c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,8 @@ - 687_: [Linux] pid_exists() no longer returns True if passed a process thread ID. - 948_: cannot install psutil with PYTHONOPTIMIZE=2. +- 950_: [Windows] Process.cpu_percent() was calculated incorrectly and showed + higher number than real usage. 5.0.1 diff --git a/psutil/__init__.py b/psutil/__init__.py index f8ce48e6a..7472ca8e5 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1007,13 +1007,8 @@ def cpu_percent(self, interval=None): raise ValueError("interval is not positive (got %r)" % interval) num_cpus = cpu_count() or 1 - if POSIX: - def timer(): - return _timer() * num_cpus - else: - def timer(): - t = cpu_times() - return sum((t.user, t.system)) + def timer(): + return _timer() * num_cpus if blocking: st1 = timer() From 17cbdf8b5e6730373a3b8eb9b0d4a8e477a04fee Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 20 Jan 2017 18:28:11 +0100 Subject: [PATCH 056/922] #941: cpu frequency - windows implementation --- psutil/__init__.py | 4 +- psutil/_psutil_windows.c | 67 +++++++++++++++++++++++++++++++ psutil/_pswindows.py | 6 +++ psutil/tests/test_memory_leaks.py | 5 +++ setup.py | 2 +- 5 files changed, 82 insertions(+), 2 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 4fed9fea1..a8aa84e20 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -181,7 +181,7 @@ "pid_exists", "pids", "process_iter", "wait_procs", # proc "virtual_memory", "swap_memory", # memory "cpu_times", "cpu_percent", "cpu_times_percent", "cpu_count", # cpu - "cpu_stats", "cpu_freq", + "cpu_stats", # "cpu_freq", "net_io_counters", "net_connections", "net_if_addrs", # network "net_if_stats", "disk_io_counters", "disk_partitions", "disk_usage", # disk @@ -1869,6 +1869,8 @@ def cpu_freq(): """ return _psplatform.cpu_freq() + __all__.append("cpu_freq") + # ===================================================================== # --- system memory related functions diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 4d939aff6..4caace7d4 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -24,6 +24,7 @@ #include #include #include +#include // Link with Iphlpapi.lib #pragma comment(lib, "IPHLPAPI.lib") @@ -145,6 +146,16 @@ typedef struct _MIB_UDP6TABLE_OWNER_PID { } MIB_UDP6TABLE_OWNER_PID, *PMIB_UDP6TABLE_OWNER_PID; #endif +typedef struct _PROCESSOR_POWER_INFORMATION { + ULONG Number; + ULONG MaxMhz; + ULONG CurrentMhz; + ULONG MhzLimit; + ULONG MaxIdleState; + ULONG CurrentIdleState; +} PROCESSOR_POWER_INFORMATION, *PPROCESSOR_POWER_INFORMATION; + + PIP_ADAPTER_ADDRESSES psutil_get_nic_addresses() { // allocate a 15 KB buffer to start with @@ -3391,6 +3402,60 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { } +/* + * Return CPU frequency. + */ +static PyObject * +psutil_cpu_freq(PyObject *self, PyObject *args) { + PROCESSOR_POWER_INFORMATION *ppi; + NTSTATUS ret; + size_t size; + LPBYTE pBuffer = NULL; + ULONG current; + ULONG max; + unsigned int num_cpus; + SYSTEM_INFO system_info; + system_info.dwNumberOfProcessors = 0; + + // Get the number of CPUs. + GetSystemInfo(&system_info); + if (system_info.dwNumberOfProcessors == 0) + num_cpus = 1; + else + num_cpus = system_info.dwNumberOfProcessors; + + // Allocate size. + size = num_cpus * sizeof(PROCESSOR_POWER_INFORMATION); + pBuffer = (BYTE*)LocalAlloc(LPTR, size); + if (! pBuffer) { + PyErr_SetFromWindowsErr(0); + return NULL; + } + + // Syscall. + ret = CallNtPowerInformation( + ProcessorInformation, NULL, 0, pBuffer, size); + if (ret != 0) { + PyErr_SetString(PyExc_RuntimeError, + "CallNtPowerInformation syscall failed"); + goto error; + } + + // Results. + ppi = (PROCESSOR_POWER_INFORMATION *)pBuffer; + max = ppi->MaxMhz; + current = ppi->CurrentMhz; + LocalFree(pBuffer); + + return Py_BuildValue("kk", current, max); + +error: + if (pBuffer != NULL) + LocalFree(pBuffer); + return NULL; +} + + // ------------------------ Python init --------------------------- static PyMethodDef @@ -3495,6 +3560,8 @@ PsutilMethods[] = { "Return NICs stats."}, {"cpu_stats", psutil_cpu_stats, METH_VARARGS, "Return NICs stats."}, + {"cpu_freq", psutil_cpu_freq, METH_VARARGS, + "Return CPU frequency."}, // --- windows services {"winservice_enumerate", psutil_winservice_enumerate, METH_VARARGS, diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index cb816f73a..8c9625352 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -299,6 +299,12 @@ def cpu_stats(): syscalls) +def cpu_freq(): + curr, max_ = cext.cpu_freq() + min_ = 0 + return [_common.scpufreq(curr, min_, max_)] + + # ===================================================================== # --- network # ===================================================================== diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 46186e411..f1a951f01 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -458,6 +458,11 @@ def test_per_cpu_times(self): def test_cpu_stats(self): self.execute(psutil.cpu_stats) + @skip_if_linux() + @unittest.skipUnless(hasattr(psutil, "cpu_freq"), "platform not supported") + def test_cpu_freq(self): + self.execute(psutil.cpu_freq) + # --- mem def test_virtual_memory(self): diff --git a/setup.py b/setup.py index 80521a487..01543bee8 100755 --- a/setup.py +++ b/setup.py @@ -122,7 +122,7 @@ def get_winver(): define_macros=macros, libraries=[ "psapi", "kernel32", "advapi32", "shell32", "netapi32", - "iphlpapi", "wtsapi32", "ws2_32", + "iphlpapi", "wtsapi32", "ws2_32", "PowrProf", ], # extra_compile_args=["/Z7"], # extra_link_args=["/DEBUG"] From 5ddd83f7fe0c9a6d515cdc43ba247e24077dca6f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 20 Jan 2017 18:34:27 +0100 Subject: [PATCH 057/922] add windows specific test --- psutil/tests/test_windows.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 802242b55..2b466382b 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -85,6 +85,12 @@ def test_cpu_count(self): num_cpus = int(os.environ['NUMBER_OF_PROCESSORS']) self.assertEqual(num_cpus, psutil.cpu_count()) + def test_cpu_freq(self): + w = wmi.WMI() + proc = w.Win32_Processor()[0] + self.assertEqual(proc.CurrentClockSpeed, psutil.cpu_freq()[0].curr) + self.assertEqual(proc.MaxClockSpeed, psutil.cpu_freq()[0].max) + def test_total_phymem(self): w = wmi.WMI().Win32_ComputerSystem()[0] self.assertEqual(int(w.TotalPhysicalMemory), From 35e9fdaacb5c4f31edc201ec0e8960ce24a7827e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 20 Jan 2017 19:16:20 +0100 Subject: [PATCH 058/922] #941: add doc --- README.rst | 3 +++ docs/index.rst | 26 ++++++++++++++++++++++++++ psutil/__init__.py | 36 +++++++++++++++++++++++------------- psutil/_common.py | 2 +- psutil/_psosx.py | 5 +++++ psutil/_pswindows.py | 7 +++++-- psutil/tests/test_osx.py | 2 +- psutil/tests/test_system.py | 18 +++++++++++++----- 8 files changed, 77 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index d413f1e8b..67e5bc923 100644 --- a/README.rst +++ b/README.rst @@ -122,6 +122,9 @@ CPU >>> >>> psutil.cpu_stats() scpustats(ctx_switches=20455687, interrupts=6598984, soft_interrupts=2134212, syscalls=0) + >>> + >>> psutil.cpu_freq() + scpufreq(current=931.42925, min=800.0, max=3500.0) Memory ====== diff --git a/docs/index.rst b/docs/index.rst index 270124b77..71c0b9c48 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -191,6 +191,32 @@ CPU .. versionadded:: 4.1.0 +.. function:: cpu_freq(percpu=False) + + Return CPU frequency as a nameduple including *current*, *min* and *max* + frequencies expressed in Mhz. + If *percpu* is ``True`` and the system supports per-cpu frequency + retrieval (Linux only) a list of frequencies is returned for each CPU, + if not, a list with a single element is returned. + + Example (Linux): + + .. code-block:: python + + >>> import psutil + >>> psutil.cpu_freq() + scpufreq(current=931.42925, min=800.0, max=3500.0) + >>> psutil.cpu_freq(percpu=True) + [scpufreq(current=2394.945, min=800.0, max=3500.0), + scpufreq(current=2236.812, min=800.0, max=3500.0), + scpufreq(current=1703.609, min=800.0, max=3500.0), + scpufreq(current=1754.289, min=800.0, max=3500.0)] + + Availability: Linux, OSX, Windows + + .. versionadded:: 5.1.0 + + Memory ------ diff --git a/psutil/__init__.py b/psutil/__init__.py index a8aa84e20..77830230c 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1854,20 +1854,30 @@ def cpu_stats(): if hasattr(_psplatform, "cpu_freq"): - def cpu_freq(): - """Return CPU frequencies as a list of nameduples including - current, min and max CPU frequency. - The CPUs order is supposed to be consistent with other CPU - functions having a 'percpu' argument and returning results for - multiple CPUs (cpu_times(), cpu_percent(), cpu_times_percent()). - Values are expressed in Mhz. - - Notes about OSX: - - it is not possible to get per-cpu freq - - reported freq never changes: - https://arstechnica.com/civis/viewtopic.php?f=19&t=465002 + def cpu_freq(percpu=False): + """Return CPU frequency as a nameduple including current, + min and max frequency expressed in Mhz. + + If percpu is True and the system supports per-cpu frequency + retrieval (Linux only) a list of frequencies is returned for + each CPU. If not a list with one element is returned. """ - return _psplatform.cpu_freq() + ret = _psplatform.cpu_freq() + if percpu: + return ret + else: + num_cpus = len(ret) + if num_cpus == 1: + return ret[0] + currs, mins, maxs = [], [], [] + for cpu in ret: + currs.append(cpu.current) + mins.append(cpu.min) + maxs.append(cpu.max) + return _common.scpufreq( + sum(currs) / num_cpus, + sum(mins) / num_cpus, + sum(maxs) / num_cpus) __all__.append("cpu_freq") diff --git a/psutil/_common.py b/psutil/_common.py index a8ae27ac1..68134820f 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -157,7 +157,7 @@ class NicDuplex(enum.IntEnum): scpustats = namedtuple( 'scpustats', ['ctx_switches', 'interrupts', 'soft_interrupts', 'syscalls']) # psutil.cpu_freq() -scpufreq = namedtuple('scpufreq', ['curr', 'min', 'max']) +scpufreq = namedtuple('scpufreq', ['current', 'min', 'max']) # --- for Process methods diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 0778b5fb7..f7adb43ac 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -166,6 +166,11 @@ def cpu_stats(): def cpu_freq(): + """Return CPU frequency. + On OSX per-cpu frequency is not supported. + Also, the returned frequency never changes, see: + https://arstechnica.com/civis/viewtopic.php?f=19&t=465002 + """ curr, min_, max_ = cext.cpu_freq() return [_common.scpufreq(curr, min_, max_)] diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 8c9625352..da8552e13 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -300,9 +300,12 @@ def cpu_stats(): def cpu_freq(): + """Return CPU frequency. + On Windows per-cpu frequency is not supported. + """ curr, max_ = cext.cpu_freq() - min_ = 0 - return [_common.scpufreq(curr, min_, max_)] + min_ = 0.0 + return [_common.scpufreq(float(curr), min_, float(max_))] # ===================================================================== diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 02fa430b7..6e7a58917 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -153,7 +153,7 @@ def test_cpu_count_physical(self): def test_cpu_freq(self): freq = psutil.cpu_freq()[0] self.assertEqual( - freq.curr * 1000 * 1000, sysctl("sysctl hw.cpufrequency")) + freq.current * 1000 * 1000, sysctl("sysctl hw.cpufrequency")) self.assertEqual( freq.min * 1000 * 1000, sysctl("sysctl hw.cpufrequency_min")) self.assertEqual( diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index d1b81838b..4cbdb056e 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -700,13 +700,21 @@ def test_cpu_stats(self): @unittest.skipUnless(hasattr(psutil, "cpu_freq"), "platform not suported") def test_cpu_freq(self): - ls = psutil.cpu_freq() + def check_ls(ls): + for nt in ls: + self.assertLessEqual(nt.current, nt.max) + for name in nt._fields: + value = getattr(nt, name) + self.assertGreaterEqual(value, 0) + + ls = psutil.cpu_freq(percpu=True) if not TRAVIS: assert ls, ls - for nt in ls: - for name in nt._fields: - value = getattr(nt, name) - self.assertGreaterEqual(value, 0) + + check_ls([psutil.cpu_freq(percpu=False)]) + + if LINUX: + self.assertEqual(len(ls), psutil.cpu_count()) def test_os_constants(self): names = ["POSIX", "WINDOWS", "LINUX", "OSX", "FREEBSD", "OPENBSD", From 7ae1bb6341c99081f53e558e312d76a71cbb8abe Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 23 Jan 2017 21:07:53 +0100 Subject: [PATCH 059/922] winmake uninstall: make it a bit smarter --- scripts/internal/winmake.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index e2c1f086e..8ce51ed02 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -225,11 +225,12 @@ def install(): @cmd def uninstall(): """Uninstall psutil""" - clean() try: import psutil except ImportError: + clean() return + clean() install_pip() sh("%s -m pip uninstall -y psutil" % PYTHON) @@ -244,6 +245,7 @@ def uninstall(): try: import psutil # NOQA except ImportError: + clean() return sh("%s -m pip uninstall -y psutil" % PYTHON) finally: From 8ca1127ea85a043c7032bda2258fadada06474de Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 23 Jan 2017 21:33:34 +0100 Subject: [PATCH 060/922] fix win test --- psutil/tests/test_windows.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 2b466382b..07f1d7966 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -88,8 +88,8 @@ def test_cpu_count(self): def test_cpu_freq(self): w = wmi.WMI() proc = w.Win32_Processor()[0] - self.assertEqual(proc.CurrentClockSpeed, psutil.cpu_freq()[0].curr) - self.assertEqual(proc.MaxClockSpeed, psutil.cpu_freq()[0].max) + self.assertEqual(proc.CurrentClockSpeed, psutil.cpu_freq().current) + self.assertEqual(proc.MaxClockSpeed, psutil.cpu_freq().max) def test_total_phymem(self): w = wmi.WMI().Win32_ComputerSystem()[0] From 5debf21696ac9f94926614b817df0f7d9d81db95 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 23 Jan 2017 22:37:49 +0100 Subject: [PATCH 061/922] update doc --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 71c0b9c48..906a1e165 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -198,6 +198,7 @@ CPU If *percpu* is ``True`` and the system supports per-cpu frequency retrieval (Linux only) a list of frequencies is returned for each CPU, if not, a list with a single element is returned. + If *min* and *max* cannot be determined they are set to ``0``. Example (Linux): From 40eaade5c718c021b5b2d50e09626facf2feec75 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 23 Jan 2017 23:24:34 +0100 Subject: [PATCH 062/922] add cpu_distribution script --- docs/index.rst | 5 +- scripts/cpu_distribution.py | 102 ++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 2 deletions(-) create mode 100755 scripts/cpu_distribution.py diff --git a/docs/index.rst b/docs/index.rst index 7efddb242..f9ecf42f9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1245,9 +1245,10 @@ Process class The returned number should be ``<=`` :func:`psutil.cpu_count()` and ``<= len(psutil.cpu_percent(percpu=True))``. It may be used in conjunction with ``psutil.cpu_percent(percpu=True)`` to - observe the system workload distributed across multiple CPUs. + observe the system workload distributed across multiple CPUs as shown by + `cpu_workload.py `__ example script. - Availability: Linux + Availability: Linux, FreeBSD .. versionadded:: 5.1.0 diff --git a/scripts/cpu_distribution.py b/scripts/cpu_distribution.py new file mode 100755 index 000000000..31cdbb863 --- /dev/null +++ b/scripts/cpu_distribution.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Shows CPU workload split across different CPUs. + +$ python scripts/cpu_workload.py +CPU 0 CPU 1 CPU 2 CPU 3 CPU 4 CPU 5 CPU 6 CPU 7 +19.8 20.6 18.2 15.8 6.9 17.3 5.0 20.4 +gvfsd pytho kwork chrom unity kwork kwork kwork +chrom chrom indic ibus- whoop nfsd (sd-p gvfsd +ibus- cat at-sp chrom Modem nfsd4 light upsta +ibus- iprt- ibus- nacl_ cfg80 kwork nfsd bluet +chrom irqba gpg-a chrom ext4- biose nfsd dio/n +chrom acpid bamfd nvidi kwork scsi_ sshd rpc.m +upsta rsysl dbus- nfsd biose scsi_ ext4- polki +rtkit avahi upowe Netwo scsi_ biose UVM T irq/9 +light rpcbi snapd cron ipv6_ biose kwork dbus- +agett kvm-i avahi kwork biose biose scsi_ syste +nfsd syste rpc.i biose biose kbloc kthro UVM g +nfsd kwork kwork biose vmsta kwork crypt kaudi +nfsd scsi_ charg biose md ksoft kwork kwork +memca biose ksmd ecryp ksoft watch migra nvme +therm biose kcomp kswap migra cpuhp watch biose +syste biose kdevt khuge watch cpuhp biose +led_w devfr kwork write cpuhp biose +rpcio oom_r ksoft kwork syste biose +kwork kwork watch migra acpi_ +biose ksoft cpuhp watch watch +biose migra cpuhp kinte +biose watch rcu_s netns +biose cpuhp kthre kwork +cpuhp ksoft +watch migra +rcu_b cpuhp +kwork +""" + +from __future__ import print_function +import collections +import os +import sys +import time + +import psutil + + +if not hasattr(psutil.Process, "cpu_num"): + sys.exit("platform not supported") + + +def clean_screen(): + if psutil.POSIX: + os.system('clear') + else: + os.system('cls') + + +def main(): + total = psutil.cpu_count() + while True: + # header + clean_screen() + cpus_percent = psutil.cpu_percent(percpu=True) + for i in range(total): + print("CPU %-6i" % i, end="") + print() + for percent in cpus_percent: + print("%-10s" % percent, end="") + print() + + # processes + procs = collections.defaultdict(list) + for p in psutil.process_iter(): + try: + name = p.name()[:5] + cpunum = p.cpu_num() + except psutil.Error: + continue + else: + procs[cpunum].append(name) + + end_marker = [[] for x in range(total)] + while True: + for num in range(total): + try: + pname = procs[num].pop() + except IndexError: + pname = "" + print("%-10s" % pname[:10], end="") + print() + if procs.values() == end_marker: + break + + time.sleep(1) + + +if __name__ == '__main__': + main() From ca9cb0134635bffd83f6e79c9f45f023259d15be Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 13:31:39 +0100 Subject: [PATCH 063/922] #357: implement proc cpu_num() on SunOS --- docs/index.rst | 5 +- psutil/__init__.py | 1 + psutil/_pssunos.py | 4 ++ psutil/_psutil_sunos.c | 82 +++++++++++++++++++++++++++++++ psutil/tests/test_memory_leaks.py | 6 +++ 5 files changed, 95 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index f9ecf42f9..5c25c785d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1242,13 +1242,12 @@ Process class .. method:: cpu_num() Return what CPU this process is currently running on. - The returned number should be ``<=`` :func:`psutil.cpu_count()` and - ``<= len(psutil.cpu_percent(percpu=True))``. + The returned number should be ``<=`` :func:`psutil.cpu_count()`. It may be used in conjunction with ``psutil.cpu_percent(percpu=True)`` to observe the system workload distributed across multiple CPUs as shown by `cpu_workload.py `__ example script. - Availability: Linux, FreeBSD + Availability: Linux, FreeBSD, SunOS .. versionadded:: 5.1.0 diff --git a/psutil/__init__.py b/psutil/__init__.py index 68fdee8fd..b24b98229 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -837,6 +837,7 @@ def cpu_affinity(self, cpus=None): else: self._proc.cpu_affinity_set(list(set(cpus))) + # Linux, FreeBSD, SunOS if hasattr(_psplatform.Process, "cpu_num"): def cpu_num(self): diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index a62e0bf55..e6796bf92 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -468,6 +468,10 @@ def cpu_times(self): raise return _common.pcputimes(*times) + @wrap_exceptions + def cpu_num(self): + return cext.proc_cpu_num(self.pid, self._procfs_path) + @wrap_exceptions def terminal(self): procfs_path = self._procfs_path diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index e98ff7f28..48767add9 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -168,6 +168,86 @@ psutil_proc_cpu_times(PyObject *self, PyObject *args) { } +/* + * Return what CPU the process is running on. + */ +static PyObject * +psutil_proc_cpu_num(PyObject *self, PyObject *args) { + int fd = NULL; + int pid; + char path[1000]; + struct prheader header; + struct lwpsinfo *lwp; + char *lpsinfo = NULL; + char *ptr = NULL; + int nent; + int size; + int proc_num; + size_t nbytes; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + + sprintf(path, "%s/%i/lpsinfo", procfs_path, pid); + fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); + return NULL; + } + + // read header + nbytes = pread(fd, &header, sizeof(header), 0); + if (nbytes == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + if (nbytes != sizeof(header)) { + PyErr_SetString( + PyExc_RuntimeError, "read() file structure size mismatch"); + goto error; + } + + // malloc + nent = header.pr_nent; + size = header.pr_entsize * nent; + ptr = lpsinfo = malloc(size); + if (lpsinfo == NULL) { + PyErr_NoMemory(); + goto error; + } + + // read the rest + nbytes = pread(fd, lpsinfo, size, sizeof(header)); + if (nbytes == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + if (nbytes != size) { + PyErr_SetString( + PyExc_RuntimeError, "read() file structure size mismatch"); + goto error; + } + + // done + lwp = (lwpsinfo_t *)ptr; + proc_num = lwp->pr_onpro; + close(fd); + free(ptr); + free(lpsinfo); + return Py_BuildValue("i", proc_num); + +error: + if (fd != NULL) + close(fd); + if (ptr != NULL) + free(ptr); + if (lpsinfo != NULL) + free(lpsinfo); + return NULL; +} + + /* * Return process uids/gids as a Python tuple. */ @@ -1340,6 +1420,8 @@ PsutilMethods[] = { "Return process memory mappings"}, {"proc_num_ctx_switches", psutil_proc_num_ctx_switches, METH_VARARGS, "Return the number of context switches performed by process"}, + {"proc_cpu_num", psutil_proc_cpu_num, METH_VARARGS, + "Return what CPU the process is on"}, // --- system-related functions {"swap_mem", psutil_swap_mem, METH_VARARGS, diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index f1a951f01..6f724339d 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -266,6 +266,12 @@ def test_threads(self): def test_cpu_times(self): self.execute(self.proc.cpu_times) + @skip_if_linux() + @unittest.skipUnless(hasattr(psutil.Process, "cpu_num"), + "platform not supported") + def test_cpu_num(self): + self.execute(self.proc.cpu_num) + @skip_if_linux() def test_memory_info(self): self.execute(self.proc.memory_info) From b91d2fb0f9a630f3096f11612c380d106e89bb78 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 14:25:46 +0100 Subject: [PATCH 064/922] #357: implement cpu_num() on SunOS --- docs/index.rst | 5 +- psutil/__init__.py | 1 + psutil/_pssunos.py | 4 ++ psutil/_psutil_sunos.c | 82 +++++++++++++++++++++++++++++++ psutil/tests/test_memory_leaks.py | 6 +++ 5 files changed, 95 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index f9ecf42f9..5c25c785d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1242,13 +1242,12 @@ Process class .. method:: cpu_num() Return what CPU this process is currently running on. - The returned number should be ``<=`` :func:`psutil.cpu_count()` and - ``<= len(psutil.cpu_percent(percpu=True))``. + The returned number should be ``<=`` :func:`psutil.cpu_count()`. It may be used in conjunction with ``psutil.cpu_percent(percpu=True)`` to observe the system workload distributed across multiple CPUs as shown by `cpu_workload.py `__ example script. - Availability: Linux, FreeBSD + Availability: Linux, FreeBSD, SunOS .. versionadded:: 5.1.0 diff --git a/psutil/__init__.py b/psutil/__init__.py index 68fdee8fd..b24b98229 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -837,6 +837,7 @@ def cpu_affinity(self, cpus=None): else: self._proc.cpu_affinity_set(list(set(cpus))) + # Linux, FreeBSD, SunOS if hasattr(_psplatform.Process, "cpu_num"): def cpu_num(self): diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index a62e0bf55..e6796bf92 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -468,6 +468,10 @@ def cpu_times(self): raise return _common.pcputimes(*times) + @wrap_exceptions + def cpu_num(self): + return cext.proc_cpu_num(self.pid, self._procfs_path) + @wrap_exceptions def terminal(self): procfs_path = self._procfs_path diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index e98ff7f28..48767add9 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -168,6 +168,86 @@ psutil_proc_cpu_times(PyObject *self, PyObject *args) { } +/* + * Return what CPU the process is running on. + */ +static PyObject * +psutil_proc_cpu_num(PyObject *self, PyObject *args) { + int fd = NULL; + int pid; + char path[1000]; + struct prheader header; + struct lwpsinfo *lwp; + char *lpsinfo = NULL; + char *ptr = NULL; + int nent; + int size; + int proc_num; + size_t nbytes; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + + sprintf(path, "%s/%i/lpsinfo", procfs_path, pid); + fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); + return NULL; + } + + // read header + nbytes = pread(fd, &header, sizeof(header), 0); + if (nbytes == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + if (nbytes != sizeof(header)) { + PyErr_SetString( + PyExc_RuntimeError, "read() file structure size mismatch"); + goto error; + } + + // malloc + nent = header.pr_nent; + size = header.pr_entsize * nent; + ptr = lpsinfo = malloc(size); + if (lpsinfo == NULL) { + PyErr_NoMemory(); + goto error; + } + + // read the rest + nbytes = pread(fd, lpsinfo, size, sizeof(header)); + if (nbytes == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + if (nbytes != size) { + PyErr_SetString( + PyExc_RuntimeError, "read() file structure size mismatch"); + goto error; + } + + // done + lwp = (lwpsinfo_t *)ptr; + proc_num = lwp->pr_onpro; + close(fd); + free(ptr); + free(lpsinfo); + return Py_BuildValue("i", proc_num); + +error: + if (fd != NULL) + close(fd); + if (ptr != NULL) + free(ptr); + if (lpsinfo != NULL) + free(lpsinfo); + return NULL; +} + + /* * Return process uids/gids as a Python tuple. */ @@ -1340,6 +1420,8 @@ PsutilMethods[] = { "Return process memory mappings"}, {"proc_num_ctx_switches", psutil_proc_num_ctx_switches, METH_VARARGS, "Return the number of context switches performed by process"}, + {"proc_cpu_num", psutil_proc_cpu_num, METH_VARARGS, + "Return what CPU the process is on"}, // --- system-related functions {"swap_mem", psutil_swap_mem, METH_VARARGS, diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index f1a951f01..6f724339d 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -266,6 +266,12 @@ def test_threads(self): def test_cpu_times(self): self.execute(self.proc.cpu_times) + @skip_if_linux() + @unittest.skipUnless(hasattr(psutil.Process, "cpu_num"), + "platform not supported") + def test_cpu_num(self): + self.execute(self.proc.cpu_num) + @skip_if_linux() def test_memory_info(self): self.execute(self.proc.memory_info) From 9d64cad90cfd8ed0ff6924200b7a47aeea520632 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 13:44:05 +0100 Subject: [PATCH 065/922] procinfo.py: show cpu num --- scripts/procinfo.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/procinfo.py b/scripts/procinfo.py index 8dc34c459..d8625560f 100755 --- a/scripts/procinfo.py +++ b/scripts/procinfo.py @@ -190,6 +190,8 @@ def run(pid, verbose=False): print_('cpu-times', str_ntuple(pinfo['cpu_times'])) if hasattr(proc, "cpu_affinity"): print_("cpu-affinity", pinfo["cpu_affinity"]) + if hasattr(proc, "cpu_num"): + print_("cpu-num", pinfo["cpu_num"]) print_('memory', str_ntuple(pinfo['memory_info'], bytes2human=True)) print_('memory %', round(pinfo['memory_percent'], 2)) From 763232217d976e7f7dc06cbf2b473095734004b0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 14:14:01 +0100 Subject: [PATCH 066/922] sunos: specific test for cpu_count() --- psutil/tests/test_sunos.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/psutil/tests/test_sunos.py b/psutil/tests/test_sunos.py index 9694b22b5..0e7704443 100755 --- a/psutil/tests/test_sunos.py +++ b/psutil/tests/test_sunos.py @@ -36,6 +36,10 @@ def test_swap_memory(self): self.assertEqual(psutil_swap.used, used) self.assertEqual(psutil_swap.free, free) + def test_cpu_count(self): + out = sh("/usr/sbin/psrinfo") + self.assertEqual(psutil.cpu_count(), len(out.split('\n'))) + if __name__ == '__main__': run_test_module_by_name(__file__) From 908d62a5d6f88d2f4a440b0d6e77ab50b2f42f70 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 15:12:42 +0100 Subject: [PATCH 067/922] sunos: specific test for cpu_count() --- psutil/tests/test_sunos.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/psutil/tests/test_sunos.py b/psutil/tests/test_sunos.py index 9694b22b5..0e7704443 100755 --- a/psutil/tests/test_sunos.py +++ b/psutil/tests/test_sunos.py @@ -36,6 +36,10 @@ def test_swap_memory(self): self.assertEqual(psutil_swap.used, used) self.assertEqual(psutil_swap.free, free) + def test_cpu_count(self): + out = sh("/usr/sbin/psrinfo") + self.assertEqual(psutil.cpu_count(), len(out.split('\n'))) + if __name__ == '__main__': run_test_module_by_name(__file__) From 5e980102a40772a940259b9aa21b195ee49e728f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 15:36:08 +0100 Subject: [PATCH 068/922] update doc --- docs/index.rst | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 5c25c785d..0131b56a5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -850,35 +850,37 @@ Process class +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ | Linux | Windows | OSX | BSD | SunOS | +==============================+===============================+==============================+==============================+==========================+ - | :meth:`~Process.cpu_percent` | :meth:`~Process.cpu_percent` | :meth:`~Process.cpu_percent` | :meth:`~Process.cpu_percent` | :meth:`name` | + | :meth:`cpu_num` | :meth:`~Process.cpu_percent` | :meth:`~Process.cpu_percent` | :meth:`cpu_num` | :meth:`name` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`~Process.cpu_times` | :meth:`~Process.cpu_times` | :meth:`~Process.cpu_times` | :meth:`~Process.cpu_times` | :meth:`cmdline` | + | :meth:`~Process.cpu_percent` | :meth:`~Process.cpu_times` | :meth:`~Process.cpu_times` | :meth:`~Process.cpu_percent` | :meth:`cmdline` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`create_time` | :meth:`io_counters()` | :meth:`memory_info` | :meth:`create_time` | :meth:`create_time` | + | :meth:`~Process.cpu_times` | :meth:`io_counters()` | :meth:`memory_info` | :meth:`~Process.cpu_times` | :meth:`create_time` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`name` | :meth:`ionice` | :meth:`memory_percent` | :meth:`gids` | | + | :meth:`create_time` | :meth:`ionice` | :meth:`memory_percent` | :meth:`create_time` | | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`ppid` | :meth:`memory_info` | :meth:`num_ctx_switches` | :meth:`io_counters` | :meth:`memory_info` | + | :meth:`name` | :meth:`memory_info` | :meth:`num_ctx_switches` | :meth:`gids` | :meth:`memory_info` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`status` | :meth:`nice` | :meth:`num_threads` | :meth:`name` | :meth:`memory_percent` | + | :meth:`ppid` | :meth:`nice` | :meth:`num_threads` | :meth:`io_counters` | :meth:`memory_percent` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`terminal` | :meth:`memory_maps` | | :meth:`memory_info` | :meth:`nice` | + | :meth:`status` | :meth:`memory_maps` | | :meth:`name` | :meth:`nice` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | | :meth:`num_ctx_switches` | :meth:`create_time` | :meth:`memory_percent` | :meth:`num_threads` | + | :meth:`terminal` | :meth:`num_ctx_switches` | :meth:`create_time` | :meth:`memory_info` | :meth:`num_threads` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`gids` | :meth:`num_handles` | :meth:`gids` | :meth:`num_ctx_switches` | :meth:`ppid` | + | | :meth:`num_handles` | :meth:`gids` | :meth:`memory_percent` | :meth:`ppid` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`num_ctx_switches` | :meth:`num_threads` | :meth:`name` | :meth:`ppid` | :meth:`status` | + | :meth:`gids` | :meth:`num_threads` | :meth:`name` | :meth:`num_ctx_switches` | :meth:`status` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`num_threads` | :meth:`username` | :meth:`ppid` | :meth:`status` | :meth:`terminal` | + | :meth:`num_ctx_switches` | :meth:`username` | :meth:`ppid` | :meth:`ppid` | :meth:`terminal` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`uids` | | :meth:`status` | :meth:`terminal` | | + | :meth:`num_threads` | | :meth:`status` | :meth:`status` | | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`username` | | :meth:`terminal` | :meth:`uids` | :meth:`gids` | + | :meth:`uids` | | :meth:`terminal` | :meth:`terminal` | :meth:`gids` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | | | :meth:`uids` | :meth:`username` | :meth:`uids` | + | :meth:`username` | | :meth:`uids` | :meth:`uids` | :meth:`uids` | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`memory_full_info` | | :meth:`username` | | :meth:`username` | + | | | :meth:`username` | :meth:`username` | :meth:`username` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ + | :meth:`memory_full_info` | | | | | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ | :meth:`memory_maps` | | | | | +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ From 903d7b7635847296478404bafe2f74445c6a5379 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 15:40:49 +0100 Subject: [PATCH 069/922] update HISTORY / README --- HISTORY.rst | 6 +++++- README.rst | 2 ++ psutil/__init__.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c6db2aa0c..2806c2b0a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,10 +1,14 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -5.0.2 +5.1.0 ===== *XXXX-XX-XX* +**Enhancements** + +- 357_: added Process.cpu_num() (what CPU the process is on). + **Bug fixes** - 687_: [Linux] pid_exists() no longer returns True if passed a process thread diff --git a/README.rst b/README.rst index 67e5bc923..6d9a1d93e 100644 --- a/README.rst +++ b/README.rst @@ -249,6 +249,8 @@ Process management >>> p.cpu_affinity() [0, 1, 2, 3] >>> p.cpu_affinity([0]) # set + >>> p.cpu_num() + 2 >>> >>> p.memory_info() pmem(rss=10915840, vms=67608576, shared=3313664, text=2310144, lib=0, data=7262208, dirty=0) diff --git a/psutil/__init__.py b/psutil/__init__.py index b24b98229..d0e018e23 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -189,7 +189,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.0.2" +__version__ = "5.1.0" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK _TOTAL_PHYMEM = None From de2bce900d5c4be75091ffc50ab99abb1d77f387 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 15:42:50 +0100 Subject: [PATCH 070/922] update HISTORY --- HISTORY.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2806c2b0a..755a6d7fd 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,7 +7,8 @@ **Enhancements** -- 357_: added Process.cpu_num() (what CPU the process is on). +- 357_: added psutil.Process.cpu_num() (what CPU a process is on). +- 941_: added psutil.cpu_freq() (CPU frequency). **Bug fixes** From 1fd9d80c374382d010ee7e44944bcd0bde4bbb41 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 15:56:32 +0100 Subject: [PATCH 071/922] add linux specifc test for cpu_count() --- IDEAS | 2 -- psutil/tests/test_linux.py | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/IDEAS b/IDEAS index 015b5fffe..247b8b386 100644 --- a/IDEAS +++ b/IDEAS @@ -79,8 +79,6 @@ FEATURES - Number of system threads. - Windows: http://msdn.microsoft.com/en-us/library/windows/desktop/ms684824(v=vs.85).aspx -- #357: what CPU a process is on. - - Doc / wiki which compares similarities between UNIX cli tools and psutil. Example: ``` diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 37352ecf2..2a4445920 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -462,6 +462,14 @@ def test_cpu_times(self): else: self.assertNotIn('guest_nice', fields) + @unittest.skipUnless(os.path.exists("/sys/devices/system/cpu/possible"), + "/sys/devices/system/cpu/possible does not exist") + def test_cpu_count(self): + with open("/sys/devices/system/cpu/possible", "rb") as f: + data = f.read().strip() + highest = int(data.split('-')[1]) + 1 + self.assertEqual(psutil.cpu_count(), highest) + @unittest.skipUnless(os.path.exists("/sys/devices/system/cpu/online"), "/sys/devices/system/cpu/online does not exist") def test_cpu_count_logical_w_sysdev_cpu_online(self): From 232b6910b0c14ff2c91da44f39bc20158b68e999 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 15:58:23 +0100 Subject: [PATCH 072/922] remove useless test --- psutil/tests/test_linux.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 2a4445920..37352ecf2 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -462,14 +462,6 @@ def test_cpu_times(self): else: self.assertNotIn('guest_nice', fields) - @unittest.skipUnless(os.path.exists("/sys/devices/system/cpu/possible"), - "/sys/devices/system/cpu/possible does not exist") - def test_cpu_count(self): - with open("/sys/devices/system/cpu/possible", "rb") as f: - data = f.read().strip() - highest = int(data.split('-')[1]) + 1 - self.assertEqual(psutil.cpu_count(), highest) - @unittest.skipUnless(os.path.exists("/sys/devices/system/cpu/online"), "/sys/devices/system/cpu/online does not exist") def test_cpu_count_logical_w_sysdev_cpu_online(self): From ff1547b98bbda585e65654bed7c938d8060478ef Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 16:07:03 +0100 Subject: [PATCH 073/922] make temperatures() available only if /sys/class/hwmon exists --- psutil/__init__.py | 4 ++- psutil/_pslinux.py | 88 ++++++++++++++++++++++------------------------ 2 files changed, 45 insertions(+), 47 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 7d6b660c0..b4a60d329 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -185,7 +185,7 @@ "net_io_counters", "net_connections", "net_if_addrs", # network "net_if_stats", "disk_io_counters", "disk_partitions", "disk_usage", # disk - "users", "boot_time", # others + "users", "boot_time", # "temperatures" # others ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" @@ -2217,6 +2217,8 @@ def to_fahrenheit(n): ret.append(_common.shwtemp(name, label, current, high, critical)) return ret + __all__.append("temperatures") + # ===================================================================== # --- Windows services diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index bc72a5025..0206d8932 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1095,54 +1095,50 @@ def boot_time(): "line 'btime' not found in %s/stat" % get_procfs_path()) -def temperatures(): - """Return hardware (CPU and others) temperatures as a list - of named tuples including name, label, current, max and - critical temperatures. - - Implementation notes: - - /sys/class/hwmon looks like the most recent interface to - retrieve this info, and this implementation relies on it - only (old distros will probably use something else) - - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon - - /sys/class/thermal/thermal_zone* is another one but it's more - difficult to parse - """ - def cat(fname, replace=_DEFAULT): - try: - f = open(fname) - except IOError: - if replace != _DEFAULT: - return replace +if os.path.exists('/sys/class/hwmon'): + def temperatures(): + """Return hardware (CPU and others) temperatures as a list + of named tuples including name, label, current, max and + critical temperatures. + + Implementation notes: + - /sys/class/hwmon looks like the most recent interface to + retrieve this info, and this implementation relies on it + only (old distros will probably use something else) + - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon + - /sys/class/thermal/thermal_zone* is another one but it's more + difficult to parse + """ + def cat(fname, replace=_DEFAULT): + try: + f = open(fname) + except IOError: + if replace != _DEFAULT: + return replace + else: + raise else: - raise - else: - with f: - return f.read().strip() - - path = '/sys/class/hwmon' - if not os.path.exists(path): - raise NotImplementedError( - "%s hwmon fs does not exist on this platform" % path) - - ret = [] - basenames = sorted(set( - [x.split('_')[0] for x in - glob.glob('/sys/class/hwmon/hwmon*/temp*_*')])) - for base in basenames: - name = cat(os.path.join(os.path.dirname(base), 'name')) - label = cat(base + '_label', replace='') - current = int(cat(base + '_input')) / 1000.0 - high = cat(base + '_max', replace=None) - if high is not None: - high = int(high) / 1000.0 - critical = cat(base + '_crit', replace=None) - if critical is not None: - critical = int(critical) / 1000.0 - - ret.append((name, label, current, high, critical)) + with f: + return f.read().strip() - return ret + ret = [] + basenames = sorted(set( + [x.split('_')[0] for x in + glob.glob('/sys/class/hwmon/hwmon*/temp*_*')])) + for base in basenames: + name = cat(os.path.join(os.path.dirname(base), 'name')) + label = cat(base + '_label', replace='') + current = int(cat(base + '_input')) / 1000.0 + high = cat(base + '_max', replace=None) + critical = cat(base + '_crit', replace=None) + if high is not None: + high = int(high) / 1000.0 + if critical is not None: + critical = int(critical) / 1000.0 + + ret.append((name, label, current, high, critical)) + + return ret # ===================================================================== From c69f2591f07341681cc56dbf93cf08cbb3cba48a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 16:39:24 +0100 Subject: [PATCH 074/922] rename function --- psutil/__init__.py | 57 ++++++++++++++++++++----------------- psutil/_common.py | 2 +- psutil/_pslinux.py | 2 +- psutil/tests/test_system.py | 12 ++++++++ 4 files changed, 45 insertions(+), 28 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index b4a60d329..afd30e902 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -185,7 +185,7 @@ "net_io_counters", "net_connections", "net_if_addrs", # network "net_if_stats", "disk_io_counters", "disk_partitions", "disk_usage", # disk - "users", "boot_time", # "temperatures" # others + "users", "boot_time", # "sensors_temperatures" # others ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" @@ -2161,33 +2161,13 @@ def net_if_stats(): # ===================================================================== -# --- other system related functions +# --- sensors # ===================================================================== -def boot_time(): - """Return the system boot time expressed in seconds since the epoch.""" - # Note: we are not caching this because it is subject to - # system clock updates. - return _psplatform.boot_time() - - -def users(): - """Return users currently connected on the system as a list of - namedtuples including the following fields. +if hasattr(_psplatform, "sensors_temperatures"): - - user: the name of the user - - terminal: the tty or pseudo-tty associated with the user, if any. - - host: the host name associated with the entry, if any. - - started: the creation time as a floating point number expressed in - seconds since the epoch. - """ - return _psplatform.users() - - -if hasattr(_psplatform, "temperatures"): - - def temperatures(fahrenheit=False): + def sensors_temperatures(fahrenheit=False): """Return hardware temperatures as a list of named tuples. Each entry represents a "sensor" monitoring a certain hardware resource. @@ -2202,7 +2182,7 @@ def to_fahrenheit(n): return (n * 9 / 5) + 32 ret = [] - for rawtuple in _psplatform.temperatures(): + for rawtuple in _psplatform.sensors_temperatures(): name, label, current, high, critical = rawtuple if fahrenheit: current = to_fahrenheit(current) @@ -2217,7 +2197,32 @@ def to_fahrenheit(n): ret.append(_common.shwtemp(name, label, current, high, critical)) return ret - __all__.append("temperatures") + __all__.append("sensors_temperatures") + + +# ===================================================================== +# --- other system related functions +# ===================================================================== + + +def boot_time(): + """Return the system boot time expressed in seconds since the epoch.""" + # Note: we are not caching this because it is subject to + # system clock updates. + return _psplatform.boot_time() + + +def users(): + """Return users currently connected on the system as a list of + namedtuples including the following fields. + + - user: the name of the user + - terminal: the tty or pseudo-tty associated with the user, if any. + - host: the host name associated with the entry, if any. + - started: the creation time as a floating point number expressed in + seconds since the epoch. + """ + return _psplatform.users() # ===================================================================== diff --git a/psutil/_common.py b/psutil/_common.py index fd8bc1860..8866ef199 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -158,7 +158,7 @@ class NicDuplex(enum.IntEnum): 'scpustats', ['ctx_switches', 'interrupts', 'soft_interrupts', 'syscalls']) # psutil.cpu_freq() scpufreq = namedtuple('scpufreq', ['current', 'min', 'max']) -# psutil.temperatures() +# psutil.sensors_temperatures() shwtemp = namedtuple( 'shwtemp', ['name', 'label', 'current', 'high', 'critical']) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 0206d8932..28e4a77c3 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1096,7 +1096,7 @@ def boot_time(): if os.path.exists('/sys/class/hwmon'): - def temperatures(): + def sensors_temps(): """Return hardware (CPU and others) temperatures as a list of named tuples including name, label, current, max and critical temperatures. diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 4cbdb056e..0ee4b6b68 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -753,6 +753,18 @@ def test_os_constants(self): for name in names: self.assertIs(getattr(psutil, name), False, msg=name) + @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), + "platform not suported") + def test_sensors_temperatures(self): + ls = psutil.sensors_temperatures() + for entry in ls: + if entry.current is not None: + self.assertGreaterEqual(entry.current, 0) + if entry.high is not None: + self.assertGreaterEqual(entry.high, 0) + if entry.critical is not None: + self.assertGreaterEqual(entry.critical, 0) + if __name__ == '__main__': run_test_module_by_name(__file__) From 1389d031809f82226fc139f32e6bc938d5372a38 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 16:50:23 +0100 Subject: [PATCH 075/922] memleaks test: add test to grant coverage of all Process methods --- psutil/tests/test_memory_leaks.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 6f724339d..7b60409ef 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -179,6 +179,19 @@ class TestProcessObjectLeaks(TestMemLeak): proc = thisproc + def test_coverage(self): + skip = set(( + "pid", "as_dict", "children", "cpu_affinity", "cpu_percent", + "ionice", "is_running", "kill", "memory_info_ex", "memory_percent", + "nice", "oneshot", "parent", "rlimit", "send_signal", "suspend", + "suspend", "terminate", "wait")) + for name in dir(psutil.Process): + if name.startswith('_'): + continue + if name in skip: + continue + self.assertTrue(hasattr(self, "test_" + name), msg=name) + @skip_if_linux() def test_name(self): self.execute(self.proc.name) @@ -258,6 +271,10 @@ def test_num_handles(self): def test_num_fds(self): self.execute(self.proc.num_fds) + @skip_if_linux() + def test_num_ctx_switches(self): + self.execute(self.proc.num_ctx_switches) + @skip_if_linux() def test_threads(self): self.execute(self.proc.threads) From f61e0024d73bf8a65deb44225bcd197daee21a66 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 16:58:30 +0100 Subject: [PATCH 076/922] memleaks test: add test to grant coverage of all psutil system functions --- psutil/tests/test_memory_leaks.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 7b60409ef..44be1ec58 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -184,7 +184,7 @@ def test_coverage(self): "pid", "as_dict", "children", "cpu_affinity", "cpu_percent", "ionice", "is_running", "kill", "memory_info_ex", "memory_percent", "nice", "oneshot", "parent", "rlimit", "send_signal", "suspend", - "suspend", "terminate", "wait")) + "terminate", "wait")) for name in dir(psutil.Process): if name.startswith('_'): continue @@ -460,6 +460,17 @@ def call(): class TestModuleFunctionsLeaks(TestMemLeak): """Test leaks of psutil module functions.""" + def test_coverage(self): + skip = set(( + "version_info", "__version__", "process_iter", "wait_procs", + "cpu_percent", "cpu_times_percent", "cpu_count")) + for name in psutil.__all__: + if not name.islower(): + continue + if name in skip: + continue + self.assertTrue(hasattr(self, "test_" + name), msg=name) + # --- cpu @skip_if_linux() @@ -518,6 +529,12 @@ def test_disk_partitions(self): def test_disk_io_counters(self): self.execute(psutil.disk_io_counters) + # --- proc + + @skip_if_linux() + def test_pids(self): + self.execute(psutil.pids) + # --- net @skip_if_linux() From de07c1c88c8650dd1c5db9fecfa79d93e247e9fa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 17:03:12 +0100 Subject: [PATCH 077/922] fix test; add debug print for failing test on travis --- psutil/__init__.py | 3 +++ psutil/tests/test_misc.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/psutil/__init__.py b/psutil/__init__.py index d0e018e23..97f1df976 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1876,6 +1876,9 @@ def cpu_freq(percpu=False): each CPU. If not a list with one element is returned. """ ret = _psplatform.cpu_freq() + # XXX + from pprint import pprint as pp + pp(ret) if percpu: return ret else: diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 6db776ee1..0b696f8cb 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -466,6 +466,9 @@ def test_pidof(self): def test_winservices(self): self.assert_stdout('winservices.py') + def test_cpu_distribution(self): + self.assert_syntax('cpu_distribution.py') + # =================================================================== # --- Unit tests for test utilities. From 167e3938c15f3b1dcc7b20685f73df3d3077f276 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 17:14:36 +0100 Subject: [PATCH 078/922] fiz ZeroDivionError on cpu_freq() --- psutil/__init__.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 97f1df976..5b248476b 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1876,24 +1876,33 @@ def cpu_freq(percpu=False): each CPU. If not a list with one element is returned. """ ret = _psplatform.cpu_freq() - # XXX - from pprint import pprint as pp - pp(ret) if percpu: return ret else: - num_cpus = len(ret) - if num_cpus == 1: + num_cpus = float(len(ret)) + if num_cpus == 0: + return [] + elif num_cpus == 1: return ret[0] - currs, mins, maxs = [], [], [] - for cpu in ret: - currs.append(cpu.current) - mins.append(cpu.min) - maxs.append(cpu.max) - return _common.scpufreq( - sum(currs) / num_cpus, - sum(mins) / num_cpus, - sum(maxs) / num_cpus) + else: + currs, mins, maxs = [], [], [] + for cpu in ret: + currs.append(cpu.current) + mins.append(cpu.min) + maxs.append(cpu.max) + try: + current = sum(currs) / num_cpus, + except ZeroDivisionError: + current = 0.0 + try: + min_ = sum(mins) / num_cpus, + except ZeroDivisionError: + min_ = 0.0 + try: + max_ = sum(maxs) / num_cpus, + except ZeroDivisionError: + max_ = 0.0 + return _common.scpufreq(current, min_, max_) __all__.append("cpu_freq") From 8e81948e711e693536d98c229158cbda2c080bca Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 17:20:38 +0100 Subject: [PATCH 079/922] fix TypeError --- psutil/__init__.py | 14 +++++++------- psutil/tests/test_system.py | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 5b248476b..d67097d5b 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1885,21 +1885,21 @@ def cpu_freq(percpu=False): elif num_cpus == 1: return ret[0] else: - currs, mins, maxs = [], [], [] + currs, mins, maxs = 0.0, 0.0, 0.0 for cpu in ret: - currs.append(cpu.current) - mins.append(cpu.min) - maxs.append(cpu.max) + currs += cpu.current + mins += cpu.min + maxs += cpu.max try: - current = sum(currs) / num_cpus, + current = currs / num_cpus except ZeroDivisionError: current = 0.0 try: - min_ = sum(mins) / num_cpus, + min_ = mins / num_cpus except ZeroDivisionError: min_ = 0.0 try: - max_ = sum(maxs) / num_cpus, + max_ = maxs / num_cpus except ZeroDivisionError: max_ = 0.0 return _common.scpufreq(current, min_, max_) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 4cbdb056e..3b97bdcdc 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -705,6 +705,7 @@ def check_ls(ls): self.assertLessEqual(nt.current, nt.max) for name in nt._fields: value = getattr(nt, name) + self.assertIsInstance(value, (int, long, float)) self.assertGreaterEqual(value, 0) ls = psutil.cpu_freq(percpu=True) From b64b87943b2e1d5955db90a94de611bf2c4bad2a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 17:40:51 +0100 Subject: [PATCH 080/922] #371: temperatures: change returned data type from list to dict --- psutil/__init__.py | 41 ++++++++++++++++++++++++----------------- psutil/_common.py | 2 +- psutil/_pslinux.py | 15 +++++++++------ 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 31f79f460..797d3b8d5 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2191,23 +2191,30 @@ def sensors_temperatures(fahrenheit=False): is not configured in order to provide these metrics. """ def to_fahrenheit(n): - return (n * 9 / 5) + 32 - - ret = [] - for rawtuple in _psplatform.sensors_temperatures(): - name, label, current, high, critical = rawtuple - if fahrenheit: - current = to_fahrenheit(current) - if high is not None: - high = to_fahrenheit(high) - if critical is not None: - critical = to_fahrenheit(critical) - if high and not critical: - critical = high - elif critical and not high: - high = critical - ret.append(_common.shwtemp(name, label, current, high, critical)) - return ret + return (float(n) * 9 / 5) + 32 + + ret = collections.defaultdict(list) + rawdict = _psplatform.sensors_temperatures() + + for name, values in rawdict.items(): + while values: + label, current, high, critical = values.pop(0) + if fahrenheit: + current = to_fahrenheit(current) + if high is not None: + high = to_fahrenheit(high) + if critical is not None: + critical = to_fahrenheit(critical) + + if high and not critical: + critical = high + elif critical and not high: + high = critical + + ret[name].append( + _common.shwtemp(label, current, high, critical)) + + return dict(ret) __all__.append("sensors_temperatures") diff --git a/psutil/_common.py b/psutil/_common.py index 8866ef199..0d965ac12 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -160,7 +160,7 @@ class NicDuplex(enum.IntEnum): scpufreq = namedtuple('scpufreq', ['current', 'min', 'max']) # psutil.sensors_temperatures() shwtemp = namedtuple( - 'shwtemp', ['name', 'label', 'current', 'high', 'critical']) + 'shwtemp', ['label', 'current', 'high', 'critical']) # --- for Process methods diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 28e4a77c3..bb3d171b5 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -7,6 +7,7 @@ from __future__ import division import base64 +import collections import errno import functools import glob @@ -1096,7 +1097,8 @@ def boot_time(): if os.path.exists('/sys/class/hwmon'): - def sensors_temps(): + + def sensors_temperatures(): """Return hardware (CPU and others) temperatures as a list of named tuples including name, label, current, max and critical temperatures. @@ -1121,22 +1123,23 @@ def cat(fname, replace=_DEFAULT): with f: return f.read().strip() - ret = [] + ret = collections.defaultdict(list) basenames = sorted(set( [x.split('_')[0] for x in glob.glob('/sys/class/hwmon/hwmon*/temp*_*')])) for base in basenames: name = cat(os.path.join(os.path.dirname(base), 'name')) label = cat(base + '_label', replace='') - current = int(cat(base + '_input')) / 1000.0 + current = float(cat(base + '_input')) / 1000.0 high = cat(base + '_max', replace=None) critical = cat(base + '_crit', replace=None) + if high is not None: - high = int(high) / 1000.0 + high = float(high) / 1000.0 if critical is not None: - critical = int(critical) / 1000.0 + critical = float(critical) / 1000.0 - ret.append((name, label, current, high, critical)) + ret[name].append((label, current, high, critical)) return ret From a17a6d6c0de3a9804452b13881eb544fc726b2d6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 17:51:37 +0100 Subject: [PATCH 081/922] #371 add sensors.py example script --- psutil/tests/test_misc.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 0b696f8cb..e534e0061 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -388,7 +388,7 @@ def assert_syntax(self, exe, args=None): src = f.read() ast.parse(src) - def test_check_presence(self): + def test_coverage(self): # make sure all example scripts have a test method defined meths = dir(self) for name in os.listdir(SCRIPTS_DIR): @@ -469,6 +469,11 @@ def test_winservices(self): def test_cpu_distribution(self): self.assert_syntax('cpu_distribution.py') + @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), + "platform not supported") + def test_sensors(self): + self.assert_stdout('sensors.py') + # =================================================================== # --- Unit tests for test utilities. From 62b92a1d569f40ca06ff45cb8b00f58b233cfbe6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 21:13:35 +0100 Subject: [PATCH 082/922] #371: update doc --- README.rst | 15 +++++++++++++++ docs/index.rst | 35 +++++++++++++++++++++++++++++++++++ psutil/__init__.py | 15 ++++++--------- psutil/_pslinux.py | 6 +++--- 4 files changed, 59 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 6d9a1d93e..f43b079ed 100644 --- a/README.rst +++ b/README.rst @@ -181,6 +181,21 @@ Network {'eth0': snicstats(isup=True, duplex=, speed=100, mtu=1500), 'lo': snicstats(isup=True, duplex=, speed=0, mtu=65536)} +Sensors (Linux only) +==================== + +.. code-block:: python + + >>> import psutil + >>> psutil.sensors_temperatures() + {'acpitz': [shwtemp(label='', current=47.0, high=103.0, critical=103.0)], + 'asus': [shwtemp(label='', current=47.0, high=None, critical=None)], + 'coretemp': [shwtemp(label='Physical id 0', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 0', current=45.0, high=100.0, critical=100.0), + shwtemp(label='Core 1', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 2', current=45.0, high=100.0, critical=100.0), + shwtemp(label='Core 3', current=47.0, high=100.0, critical=100.0)]} + Other system info ================= diff --git a/docs/index.rst b/docs/index.rst index 0131b56a5..8cd8875f0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -617,6 +617,41 @@ Network .. versionadded:: 3.0.0 +Sensors +------- + +.. function:: sensors_temperatures(fahrenheit=False) + + Return hardware temperatures. Each entry is a namedtuple representing a + certain hardware sensor (it may be a CPU, an hard disk or something + else, depending on the OS and its configuration). + All temperatures are expressed in celsius unless *fahrenheit* is set to + ``True``. Example:: + + >>> import psutil + >>> psutil.sensors_temperatures() + {'acpitz': [shwtemp(label='', current=47.0, high=103.0, critical=103.0)], + 'asus': [shwtemp(label='', current=47.0, high=None, critical=None)], + 'coretemp': [shwtemp(label='Physical id 0', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 0', current=45.0, high=100.0, critical=100.0), + shwtemp(label='Core 1', current=52.0, high=100.0, critical=100.0), + shwtemp(label='Core 2', current=45.0, high=100.0, critical=100.0), + shwtemp(label='Core 3', current=47.0, high=100.0, critical=100.0)]} + + See also `sensors.py `__ + for an example application. + + .. warning:: + + This API is experimental. Backwards incompatible changes may occur if + deemed necessary. + + Availability: Linux + + .. versionadded:: 5.1.0 + + + Other system info ----------------- diff --git a/psutil/__init__.py b/psutil/__init__.py index 797d3b8d5..a57eacc9a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2180,15 +2180,12 @@ def net_if_stats(): if hasattr(_psplatform, "sensors_temperatures"): def sensors_temperatures(fahrenheit=False): - """Return hardware temperatures as a list of named tuples. - Each entry represents a "sensor" monitoring a certain hardware - resource. - The hardware resource may be a CPU, an hard disk or something - else, depending on the OS and its configuration. - All temperatures are expressed in celsius unless 'fahrenheit' - parameter is specified. - This function may raise NotImplementedError in case the OS - is not configured in order to provide these metrics. + """Return hardware temperatures. Each entry is a namedtuple + representing a certain hardware sensor (it may be a CPU, an + hard disk or something else, depending on the OS and its + configuration). + All temperatures are expressed in celsius unless *fahrenheit* + is set to True. """ def to_fahrenheit(n): return (float(n) * 9 / 5) + 32 diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index bb3d171b5..aea4aa526 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1099,9 +1099,9 @@ def boot_time(): if os.path.exists('/sys/class/hwmon'): def sensors_temperatures(): - """Return hardware (CPU and others) temperatures as a list - of named tuples including name, label, current, max and - critical temperatures. + """Return hardware (CPU and others) temperatures as a dict + including hardware name, label, current, max and critical + temperatures. Implementation notes: - /sys/class/hwmon looks like the most recent interface to From 77a8df098f354b990832b33f324577fa24ad16e6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 22:09:50 +0100 Subject: [PATCH 083/922] refactoring --- psutil/__init__.py | 3 ++- psutil/_pslinux.py | 20 ++++---------------- psutil/tests/test_system.py | 21 +++++++++++++-------- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index a57eacc9a..0417ed7f4 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -185,7 +185,8 @@ "net_io_counters", "net_connections", "net_if_addrs", # network "net_if_stats", "disk_io_counters", "disk_partitions", "disk_usage", # disk - "users", "boot_time", # "sensors_temperatures" # others + # "sensors_temperatures", # sensors + "users", "boot_time", # others ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index aea4aa526..2de7dbcb4 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -285,7 +285,7 @@ def cat(fname, fallback=_DEFAULT, binary=True): """Return file content.""" try: with open_binary(fname) if binary else open_text(fname) as f: - return f.read() + return f.read().strip() except IOError: if fallback != _DEFAULT: return fallback @@ -1111,28 +1111,16 @@ def sensors_temperatures(): - /sys/class/thermal/thermal_zone* is another one but it's more difficult to parse """ - def cat(fname, replace=_DEFAULT): - try: - f = open(fname) - except IOError: - if replace != _DEFAULT: - return replace - else: - raise - else: - with f: - return f.read().strip() - ret = collections.defaultdict(list) basenames = sorted(set( [x.split('_')[0] for x in glob.glob('/sys/class/hwmon/hwmon*/temp*_*')])) for base in basenames: name = cat(os.path.join(os.path.dirname(base), 'name')) - label = cat(base + '_label', replace='') + label = cat(base + '_label', fallback='') current = float(cat(base + '_input')) / 1000.0 - high = cat(base + '_max', replace=None) - critical = cat(base + '_crit', replace=None) + high = cat(base + '_max', fallback=None) + critical = cat(base + '_crit', fallback=None) if high is not None: high = float(high) / 1000.0 diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index de2ddef0f..f93317915 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -29,6 +29,7 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._compat import long +from psutil._compat import unicode from psutil.tests import AF_INET6 from psutil.tests import APPVEYOR from psutil.tests import check_net_address @@ -757,14 +758,18 @@ def test_os_constants(self): @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), "platform not suported") def test_sensors_temperatures(self): - ls = psutil.sensors_temperatures() - for entry in ls: - if entry.current is not None: - self.assertGreaterEqual(entry.current, 0) - if entry.high is not None: - self.assertGreaterEqual(entry.high, 0) - if entry.critical is not None: - self.assertGreaterEqual(entry.critical, 0) + temps = psutil.sensors_temperatures() + assert temps, temps + for name, entries in temps.items(): + self.assertIsInstance(name, (str, unicode)) + for entry in entries: + self.assertIsInstance(entry.label, (str, unicode)) + if entry.current is not None: + self.assertGreaterEqual(entry.current, 0) + if entry.high is not None: + self.assertGreaterEqual(entry.high, 0) + if entry.critical is not None: + self.assertGreaterEqual(entry.critical, 0) if __name__ == '__main__': From e6e24354854c18f38a4671cc63c98374d6082404 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 22:14:04 +0100 Subject: [PATCH 084/922] #371 add sensors.py example script --- psutil/_pslinux.py | 9 ++++---- psutil/tests/test_system.py | 1 - scripts/sensors.py | 46 +++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 5 deletions(-) create mode 100755 scripts/sensors.py diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 2de7dbcb4..e4ded9d40 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -284,13 +284,14 @@ def set_scputimes_ntuple(procfs_path): def cat(fname, fallback=_DEFAULT, binary=True): """Return file content.""" try: - with open_binary(fname) if binary else open_text(fname) as f: - return f.read().strip() + f = open_binary(fname) if binary else open_text(fname) except IOError: if fallback != _DEFAULT: return fallback - else: - raise + raise + else: + with f: + return f.read().strip() try: diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index f93317915..31d4c3fb2 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -759,7 +759,6 @@ def test_os_constants(self): "platform not suported") def test_sensors_temperatures(self): temps = psutil.sensors_temperatures() - assert temps, temps for name, entries in temps.items(): self.assertIsInstance(name, (str, unicode)) for entry in entries: diff --git a/scripts/sensors.py b/scripts/sensors.py new file mode 100755 index 000000000..4b14180ef --- /dev/null +++ b/scripts/sensors.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +A clone of 'sensors' utility on Linux printing hardware temperatures. + +$ python scripts/sensors.py +asus + asus 47.0 °C (high = None °C, critical = None °C) + +acpitz + acpitz 47.0 °C (high = 103.0 °C, critical = 103.0 °C) + +coretemp + Physical id 0 54.0 °C (high = 100.0 °C, critical = 100.0 °C) + Core 0 47.0 °C (high = 100.0 °C, critical = 100.0 °C) + Core 1 48.0 °C (high = 100.0 °C, critical = 100.0 °C) + Core 2 47.0 °C (high = 100.0 °C, critical = 100.0 °C) + Core 3 54.0 °C (high = 100.0 °C, critical = 100.0 °C) +""" + +from __future__ import print_function +import sys + +import psutil + + +def main(): + if not hasattr(psutil, "sensors_temperatures"): + sys.exit("platform not supported") + temps = psutil.sensors_temperatures() + for name, entries in temps.items(): + print(name) + for entry in entries: + print(" %-20s %s °C (high = %s °C, critical = %s °C)" % ( + entry.label or name, entry.current, entry.high, + entry.critical)) + print() + + +if __name__ == '__main__': + main() From ea919d5bf74fb776a4535c2d6bb40967e77803d4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 22:15:29 +0100 Subject: [PATCH 085/922] modify test --- psutil/tests/test_misc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index e534e0061..8697bcff7 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -469,10 +469,11 @@ def test_winservices(self): def test_cpu_distribution(self): self.assert_syntax('cpu_distribution.py') - @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), - "platform not supported") def test_sensors(self): - self.assert_stdout('sensors.py') + if hasattr(psutil, "sensors_temperatures"): + self.assert_stdout('sensors.py') + else: + self.assert_syntax('sensors.py') # =================================================================== From eb4b66c9553ebf6137e28f87d6729c11375e1144 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 22:20:06 +0100 Subject: [PATCH 086/922] small refactoring --- psutil/_pslinux.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index ad95e39a6..85647bb22 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -282,13 +282,14 @@ def set_scputimes_ntuple(procfs_path): def cat(fname, fallback=_DEFAULT, binary=True): """Return file content.""" try: - with open_binary(fname) if binary else open_text(fname) as f: - return f.read() + f = open_binary(fname) if binary else open_text(fname) except IOError: if fallback != _DEFAULT: return fallback - else: - raise + raise + else: + with f: + return f.read().strip() try: From 6943870e4501db28d4dffb134a20b5af08a30d56 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 22:36:12 +0100 Subject: [PATCH 087/922] move function up --- psutil/_pslinux.py | 69 +++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index e4ded9d40..807706eb7 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1062,41 +1062,10 @@ def disk_partitions(all=False): # ===================================================================== -# --- other system functions +# --- sensors # ===================================================================== -def users(): - """Return currently connected users as a list of namedtuples.""" - retlist = [] - rawlist = cext.users() - for item in rawlist: - user, tty, hostname, tstamp, user_process = item - # note: the underlying C function includes entries about - # system boot, run level and others. We might want - # to use them in the future. - if not user_process: - continue - if hostname == ':0.0' or hostname == ':0': - hostname = 'localhost' - nt = _common.suser(user, tty or None, hostname, tstamp) - retlist.append(nt) - return retlist - - -def boot_time(): - """Return the system boot time expressed in seconds since the epoch.""" - global BOOT_TIME - with open_binary('%s/stat' % get_procfs_path()) as f: - for line in f: - if line.startswith(b'btime'): - ret = float(line.strip().split()[1]) - BOOT_TIME = ret - return ret - raise RuntimeError( - "line 'btime' not found in %s/stat" % get_procfs_path()) - - if os.path.exists('/sys/class/hwmon'): def sensors_temperatures(): @@ -1133,6 +1102,42 @@ def sensors_temperatures(): return ret +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + for item in rawlist: + user, tty, hostname, tstamp, user_process = item + # note: the underlying C function includes entries about + # system boot, run level and others. We might want + # to use them in the future. + if not user_process: + continue + if hostname == ':0.0' or hostname == ':0': + hostname = 'localhost' + nt = _common.suser(user, tty or None, hostname, tstamp) + retlist.append(nt) + return retlist + + +def boot_time(): + """Return the system boot time expressed in seconds since the epoch.""" + global BOOT_TIME + with open_binary('%s/stat' % get_procfs_path()) as f: + for line in f: + if line.startswith(b'btime'): + ret = float(line.strip().split()[1]) + BOOT_TIME = ret + return ret + raise RuntimeError( + "line 'btime' not found in %s/stat" % get_procfs_path()) + + # ===================================================================== # --- processes # ===================================================================== From df9250cdee1c241324e71e7f68e1602eaf82d2c9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 24 Jan 2017 22:38:26 +0100 Subject: [PATCH 088/922] change variable name --- psutil/_pslinux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 807706eb7..833cad012 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1086,7 +1086,7 @@ def sensors_temperatures(): [x.split('_')[0] for x in glob.glob('/sys/class/hwmon/hwmon*/temp*_*')])) for base in basenames: - name = cat(os.path.join(os.path.dirname(base), 'name')) + unit_name = cat(os.path.join(os.path.dirname(base), 'name')) label = cat(base + '_label', fallback='') current = float(cat(base + '_input')) / 1000.0 high = cat(base + '_max', fallback=None) @@ -1097,7 +1097,7 @@ def sensors_temperatures(): if critical is not None: critical = float(critical) / 1000.0 - ret[name].append((label, current, high, critical)) + ret[unit_name].append((label, current, high, critical)) return ret From 13114eefc8da22a7b18a8e7f0b877044ac0e1897 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Jan 2017 15:00:24 +0100 Subject: [PATCH 089/922] #955: sensors_batter() linux impl --- psutil/__init__.py | 17 +++++++++++++++++ psutil/_common.py | 2 ++ psutil/_pslinux.py | 19 +++++++++++++++++++ psutil/tests/test_memory_leaks.py | 8 ++++++++ psutil/tests/test_system.py | 8 ++++++++ 5 files changed, 54 insertions(+) diff --git a/psutil/__init__.py b/psutil/__init__.py index d67097d5b..7e3217df5 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -185,6 +185,7 @@ "net_io_counters", "net_connections", "net_if_addrs", # network "net_if_stats", "disk_io_counters", "disk_partitions", "disk_usage", # disk + # "sensors_battery", # sensors "users", "boot_time", # others ] __all__.extend(_psplatform.__extra__all__) @@ -2172,6 +2173,22 @@ def net_if_stats(): return _psplatform.net_if_stats() +# ===================================================================== +# --- sensors +# ===================================================================== + + +if hasattr(_psplatform, "sensors_battery"): + + def sensors_battery(): + """Return information about battery. If no battery can be found + returns None. + """ + return _psplatform.sensors_battery() + + __all__.append("sensors_battery") + + # ===================================================================== # --- other system related functions # ===================================================================== diff --git a/psutil/_common.py b/psutil/_common.py index 68134820f..4342d2545 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -158,6 +158,8 @@ class NicDuplex(enum.IntEnum): 'scpustats', ['ctx_switches', 'interrupts', 'soft_interrupts', 'syscalls']) # psutil.cpu_freq() scpufreq = namedtuple('scpufreq', ['current', 'min', 'max']) +# psutil.sensors_battery() +sbattery = namedtuple('sbattery', ['percent']) # --- for Process methods diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 85647bb22..1a5304a62 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1059,6 +1059,25 @@ def disk_partitions(all=False): return retlist +# ===================================================================== +# --- sensors +# ===================================================================== + + +def sensors_battery(): + root = "/sys/class/power_supply/BAT0/" + if not os.path.exists(root.rstrip('/')): + return None + + # TODO: figure out the algorithm to calculate residual time. + # energy_now = int(cat(root + "energy_now")) + # power_now = int(cat(root + "power_now")) + # energy_full = int(cat(root + "energy_full")) + # secsleft = 3600 * energy_now / power_now + percent = int(cat(root + "capacity")) + return _common.sbattery(percent) + + # ===================================================================== # --- other system functions # ===================================================================== diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 44be1ec58..f025530fd 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -556,6 +556,14 @@ def test_net_if_addrs(self): def test_net_if_stats(self): self.execute(psutil.net_if_stats) + # --- sensors + + @unittest.skipUnless(hasattr(psutil, "sensors_battery"), + "platform not supported") + @skip_if_linux() + def test_sensors_battery(self): + self.execute(psutil.sensors_battery()) + # --- others @skip_if_linux() diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 3b97bdcdc..58b7c9c05 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -754,6 +754,14 @@ def test_os_constants(self): for name in names: self.assertIs(getattr(psutil, name), False, msg=name) + @unittest.skipUnless(hasattr(psutil, "sensors_battery"), + "platform not supported") + def test_sensors_battery(self): + ret = psutil.sensors_battery() + if ret.percent is not None: + self.assertGreaterEqual(ret.percent, 0) + self.assertLessEqual(ret.percent, 100) + if __name__ == '__main__': run_test_module_by_name(__file__) From 109f873ea81b28c9f82142a556f75e68256dbaa5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Jan 2017 18:29:20 +0100 Subject: [PATCH 090/922] #955: sensors_batter() win impl --- docs/index.rst | 28 ++++++++++++++++++++++++++++ psutil/__init__.py | 2 ++ psutil/_common.py | 13 ++++++++++++- psutil/_pslinux.py | 2 +- psutil/_psutil_windows.c | 27 +++++++++++++++++++++++++++ psutil/_pswindows.py | 26 ++++++++++++++++++++++++++ psutil/tests/test_system.py | 5 +++++ 7 files changed, 101 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 0131b56a5..8db5b111d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -617,6 +617,25 @@ Network .. versionadded:: 3.0.0 +Sensors +------- + +.. function:: sensors_battery() + + Return a namedtuple with the following values: + + - **percent**: battery power left as a percentage. + - **secsleft**: number of seconds left before battery run out of power; this + may also be :data:`psutil.POWER_TIME_UNKNOWN ` + or :data:`psutil.POWER_TIME_UNLIMITED `. + + If no battery is installed this function will return ``None``. + + Availability: Linux, Windows + + .. versionadded:: 5.1.0 + + Other system info ----------------- @@ -1984,6 +2003,15 @@ Constants .. versionadded:: 3.0.0 +.. _const-power: +.. data:: POWER_TIME_UNKNOWN +.. data:: POWER_TIME_UNLIMITED + + This can be the value of *secsleft* field of + :func:`psutil.sensors_battery()`. + + .. versionadded:: 5.1.0 + .. _const-version-info: .. data:: version_info diff --git a/psutil/__init__.py b/psutil/__init__.py index 7e3217df5..745d66fd2 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -171,6 +171,8 @@ "NIC_DUPLEX_FULL", "NIC_DUPLEX_HALF", "NIC_DUPLEX_UNKNOWN", + "POWER_TIME_UNKNOWN", "POWER_TIME_UNLIMITED", + "BSD", "FREEBSD", "LINUX", "NETBSD", "OPENBSD", "OSX", "POSIX", "SUNOS", "WINDOWS", diff --git a/psutil/_common.py b/psutil/_common.py index 4342d2545..b28af9992 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -121,6 +121,17 @@ class NicDuplex(enum.IntEnum): globals().update(NicDuplex.__members__) +# sensors_battery() +if enum is None: + POWER_TIME_UNKNOWN = -1 + POWER_TIME_UNLIMITED = -2 +else: + class Power(enum.IntEnum): + POWER_TIME_UNKNOWN = -1 + POWER_TIME_UNLIMITED = -2 + + globals().update(NicDuplex.__members__) + # =================================================================== # --- namedtuples @@ -159,7 +170,7 @@ class NicDuplex(enum.IntEnum): # psutil.cpu_freq() scpufreq = namedtuple('scpufreq', ['current', 'min', 'max']) # psutil.sensors_battery() -sbattery = namedtuple('sbattery', ['percent']) +sbattery = namedtuple('sbattery', ['percent', 'secsleft']) # --- for Process methods diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 1a5304a62..0fba0fa6d 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1075,7 +1075,7 @@ def sensors_battery(): # energy_full = int(cat(root + "energy_full")) # secsleft = 3600 * energy_now / power_now percent = int(cat(root + "capacity")) - return _common.sbattery(percent) + return _common.sbattery(percent, 0) # ===================================================================== diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 4caace7d4..74edc6cd8 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -3456,6 +3456,31 @@ psutil_cpu_freq(PyObject *self, PyObject *args) { } +/* + * Return battery usage stats. + */ +static PyObject * +psutil_sensors_battery(PyObject *self, PyObject *args) { + SYSTEM_POWER_STATUS sps; + + if (GetSystemPowerStatus(&sps) == 0) { + PyErr_SetFromWindowsErr(0); + return NULL; + } + return Py_BuildValue( + "iiiI", + sps.ACLineStatus, // whether AC is connected: 0=no, 1=yes, 255=unknown + // status flag: + // 1, 2, 4 = high, low, critical + // 8 = charging + // 128 = no battery + sps.BatteryFlag, + sps.BatteryLifePercent, // percent + sps.BatteryLifeTime // remaining secs + ); +} + + // ------------------------ Python init --------------------------- static PyMethodDef @@ -3562,6 +3587,8 @@ PsutilMethods[] = { "Return NICs stats."}, {"cpu_freq", psutil_cpu_freq, METH_VARARGS, "Return CPU frequency."}, + {"sensors_battery", psutil_sensors_battery, METH_VARARGS, + "Return battery metrics usage."}, // --- windows services {"winservice_enumerate", psutil_winservice_enumerate, METH_VARARGS, diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index da8552e13..80faec919 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -361,6 +361,32 @@ def net_if_addrs(): return ret +# ===================================================================== +# --- sensors +# ===================================================================== + + +def sensors_battery(): + acline_status, flags, percent, secsleft = cext.sensors_battery() + power_connected = acline_status == 1 + no_battery = bool(flags & 128) + charging = bool(flags & 8) + + # print("acline_status=%s, flags=%s, percent=%s, secsleft=%s" % ( + # acline_status, flags, percent, secsleft)) + # print("power_connected=%s, no_battery=%s, charging=%s" % ( + # power_connected, no_battery, charging)) + + if no_battery: + return None + if power_connected or charging: + secsleft = _common.POWER_TIME_UNLIMITED + elif secsleft == -1: + secsleft = _common.POWER_TIME_UNKNOWN + + return _common.sbattery(percent, secsleft) + + # ===================================================================== # --- other system functions # ===================================================================== diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 58b7c9c05..906641e4e 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -758,9 +758,14 @@ def test_os_constants(self): "platform not supported") def test_sensors_battery(self): ret = psutil.sensors_battery() + if ret is None: + return # no battery if ret.percent is not None: self.assertGreaterEqual(ret.percent, 0) self.assertLessEqual(ret.percent, 100) + if ret.secsleft not in (psutil.POWER_TIME_UNKNOWN, + psutil.POWER_TIME_UNLIMITED): + self.assertGreaterEqual(ret.secsleft, 0) if __name__ == '__main__': From e5fa0038ba8ab1dc09cb3befe1a78ee1fc3eca0a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Jan 2017 18:35:39 +0100 Subject: [PATCH 091/922] debug travis failure --- psutil/tests/test_system.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 3b97bdcdc..e69b7acfc 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -712,6 +712,9 @@ def check_ls(ls): if not TRAVIS: assert ls, ls + # XXX + from pprint import pprint as pp + pp(psutil.cpu_freq(percpu=False)) check_ls([psutil.cpu_freq(percpu=False)]) if LINUX: From 783395e70b631201cf6a3c4f77ba0565f354300a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Jan 2017 18:42:57 +0100 Subject: [PATCH 092/922] fix travis failure --- psutil/tests/test_system.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index e69b7acfc..753e52691 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -709,12 +709,10 @@ def check_ls(ls): self.assertGreaterEqual(value, 0) ls = psutil.cpu_freq(percpu=True) - if not TRAVIS: - assert ls, ls + if TRAVIS and not ls: + return - # XXX - from pprint import pprint as pp - pp(psutil.cpu_freq(percpu=False)) + assert ls, ls check_ls([psutil.cpu_freq(percpu=False)]) if LINUX: From cec84ca48352a8f412c21f88054f8fac361f003e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Jan 2017 19:32:43 +0100 Subject: [PATCH 093/922] winmake: add pyreadline to list of deps --- scripts/internal/winmake.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 8ce51ed02..7215560d5 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -36,6 +36,7 @@ "perf", "pip", "pypiwin32", + "pyreadline", "setuptools", "unittest2", "wheel", From a120b1e94322b17bf5ae41bb885faf3d17b991cd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Jan 2017 19:52:56 +0100 Subject: [PATCH 094/922] #955: battery windows specific tests --- psutil/__init__.py | 2 ++ psutil/tests/test_windows.py | 47 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/psutil/__init__.py b/psutil/__init__.py index 745d66fd2..bfef05123 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -195,6 +195,8 @@ __version__ = "5.1.0" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK +POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED +POWER_TIME_UNKNOWN = _common.POWER_TIME_UNKNOWN _TOTAL_PHYMEM = None _timer = getattr(time, 'monotonic', time.time) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 07f1d7966..669adad0b 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -158,6 +158,53 @@ def test_net_if_stats(self): "no common entries in %s, %s" % (ps_names, wmi_names)) +# =================================================================== +# sensors_battery() +# =================================================================== + + +@unittest.skipUnless(WINDOWS, "WINDOWS only") +class TestSensorsBattery(unittest.TestCase): + + def test_percent(self): + w = wmi.WMI() + battery_psutil = psutil.sensors_battery() + battery_wmi = w.query('select * from Win32_Battery')[0] + if battery_psutil is None: + self.assertNot(battery_wmi.EstimatedChargeRemaining) + else: + self.assertAlmostEqual( + battery_psutil.percent, battery_wmi.EstimatedChargeRemaining, + delta=1) + + def test_emulate_no_battery(self): + with mock.patch("psutil._pswindows.cext.sensors_battery", + return_value=(0, 128, 0, 0)) as m: + self.assertIsNone(psutil.sensors_battery()) + assert m.called + + def test_emulate_power_connected(self): + with mock.patch("psutil._pswindows.cext.sensors_battery", + return_value=(1, 0, 0, 0)) as m: + self.assertEqual(psutil.sensors_battery().secsleft, + psutil.POWER_TIME_UNLIMITED) + assert m.called + + def test_emulate_power_charging(self): + with mock.patch("psutil._pswindows.cext.sensors_battery", + return_value=(0, 8, 0, 0)) as m: + self.assertEqual(psutil.sensors_battery().secsleft, + psutil.POWER_TIME_UNLIMITED) + assert m.called + + def test_emulate_secs_left_unknown(self): + with mock.patch("psutil._pswindows.cext.sensors_battery", + return_value=(0, 0, 0, -1)) as m: + self.assertEqual(psutil.sensors_battery().secsleft, + psutil.POWER_TIME_UNKNOWN) + assert m.called + + # =================================================================== # Process APIs # =================================================================== From e2d8f58a1833686024b013de707c4786abce1279 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Jan 2017 20:46:08 +0100 Subject: [PATCH 095/922] #955: sensors_battery() / linux: implement secsleft --- docs/index.rst | 13 ++++++++++++- psutil/_pslinux.py | 10 ++++------ psutil/tests/test_linux.py | 6 ++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 8db5b111d..52424b39f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -629,7 +629,18 @@ Sensors may also be :data:`psutil.POWER_TIME_UNKNOWN ` or :data:`psutil.POWER_TIME_UNLIMITED `. - If no battery is installed this function will return ``None``. + If no battery is installed this function will return ``None``. Example:: + + >>> def secs2hours(secs): + ... m, s = divmod(secs, 60) + ... h, m = divmod(m, 60) + ... return "%d:%02d:%02d" % (h, m, s) + ... + >>> batt = psutil.sensors_battery() + >>> batt + sbattery(percent=93, secsleft=16628) + >>> print("charge = %s%%, time left = %s" % (batt.percent, secs2hours(batt.secsleft))) + charge = 93%, time left = 4:37:08 Availability: Linux, Windows diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 0fba0fa6d..fff77aab7 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1069,13 +1069,11 @@ def sensors_battery(): if not os.path.exists(root.rstrip('/')): return None - # TODO: figure out the algorithm to calculate residual time. - # energy_now = int(cat(root + "energy_now")) - # power_now = int(cat(root + "power_now")) - # energy_full = int(cat(root + "energy_full")) - # secsleft = 3600 * energy_now / power_now + energy_now = int(cat(root + "energy_now")) + power_now = int(cat(root + "power_now")) percent = int(cat(root + "capacity")) - return _common.sbattery(percent, 0) + secsleft = int(energy_now / power_now * 3600) + return _common.sbattery(percent, secsleft) # ===================================================================== diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 37352ecf2..54d5e251b 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1024,6 +1024,12 @@ def test_issue_687(self): finally: t.stop() + def test_sensors_battery_percent(self): + out = sh("acpi -b") + acpi_value = int(out.split(",")[1].strip().replace('%', '')) + psutil_value = psutil.sensors_battery().percent + self.assertAlmostEqual(acpi_value, psutil_value, delta=1) + # ===================================================================== # test process From 022cf0a05d34f4274269d4f8002ee95b9f3e32d2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Jan 2017 23:26:13 +0100 Subject: [PATCH 096/922] #955: sensors_batter() freebsd impl --- psutil/_psbsd.py | 14 ++++++++++++++ psutil/_psutil_bsd.c | 4 ++++ psutil/arch/bsd/freebsd.c | 22 ++++++++++++++++++++++ psutil/arch/bsd/freebsd.h | 3 +++ psutil/tests/test_bsd.py | 16 ++++++++++++++++ 5 files changed, 59 insertions(+) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 022f57583..20f9cbcb3 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -395,6 +395,20 @@ def net_connections(kind): return list(ret) +# ===================================================================== +# --- sensors +# ===================================================================== + + +def sensors_battery(): + percent, minsleft = cext.sensors_battery() + if minsleft == -1: + secsleft = _common.POWER_TIME_UNLIMITED + else: + secsleft = minsleft * 60 + return _common.sbattery(percent, secsleft) + + # ===================================================================== # --- other system functions # ===================================================================== diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index adcedf79c..de748dccb 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -933,6 +933,10 @@ PsutilMethods[] = { #if defined(PSUTIL_FREEBSD) || defined(PSUTIL_NETBSD) {"net_connections", psutil_net_connections, METH_VARARGS, "Return system-wide open connections."}, +#endif +#if defined(PSUTIL_FREEBSD) + {"sensors_battery", psutil_sensors_battery, METH_VARARGS, + "Return battery information."}, #endif {NULL, NULL, 0, NULL} }; diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index 456a50aa4..c0286c866 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -994,3 +994,25 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } + + +/* + * Return battery information. + */ +PyObject * +psutil_sensors_battery(PyObject *self, PyObject *args) { + int percent; + int minsleft; + size_t size = sizeof(percent); + + if (sysctlbyname("hw.acpi.battery.life", &percent, &size, NULL, 0)) + goto error; + // -1 if power is connected + if (sysctlbyname("hw.acpi.battery.time", &minsleft, &size, NULL, 0)) + goto error; + return Py_BuildValue("ii", percent, minsleft); + +error: + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} diff --git a/psutil/arch/bsd/freebsd.h b/psutil/arch/bsd/freebsd.h index e15706c66..0df66eccb 100644 --- a/psutil/arch/bsd/freebsd.h +++ b/psutil/arch/bsd/freebsd.h @@ -27,3 +27,6 @@ PyObject* psutil_proc_threads(PyObject* self, PyObject* args); PyObject* psutil_swap_mem(PyObject* self, PyObject* args); PyObject* psutil_virtual_mem(PyObject* self, PyObject* args); PyObject* psutil_cpu_stats(PyObject* self, PyObject* args); +#if defined(PSUTIL_FREEBSD) +PyObject* psutil_sensors_battery(PyObject* self, PyObject* args); +#endif diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 244672e6a..479237e55 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -357,6 +357,22 @@ def test_boot_time(self): btime = int(s) self.assertEqual(btime, psutil.boot_time()) + @unittest.skipUnless(psutil.sensors_battery(), "no battery") + def test_sensors_battery(self): + def secs2hours(secs): + m, s = divmod(secs, 60) + h, m = divmod(m, 60) + return "%d:%02d" % (h, m) + + out = sh("acpiconf -i 0") + fields = dict([(x.split('\t')[0], x.split('\t')[-1]) + for x in out.split("\n")]) + metrics = psutil.sensors_battery() + percent = int(fields['Remaining capacity:'].replace('%', '')) + remaining_time = fields['Remaining time:'] + self.assertEqual(metrics.percent, percent) + self.assertEqual(secs2hours(metrics.secsleft), remaining_time) + # ===================================================================== # --- OpenBSD From 78f8a41eb902c144dd31a9e4bafe63c25fea3c76 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Jan 2017 23:42:17 +0100 Subject: [PATCH 097/922] update doc --- docs/index.rst | 22 +++++++++++++--------- psutil/_pslinux.py | 5 ++++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 52424b39f..af44c516b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -622,19 +622,22 @@ Sensors .. function:: sensors_battery() - Return a namedtuple with the following values: + Return battery status information as a namedtuple including the following + values: - **percent**: battery power left as a percentage. - - **secsleft**: number of seconds left before battery run out of power; this - may also be :data:`psutil.POWER_TIME_UNKNOWN ` - or :data:`psutil.POWER_TIME_UNLIMITED `. + - **secsleft**: (rough approximation) number of seconds left before the + battery run out of power; this may be set to + :data:`psutil.POWER_TIME_UNKNOWN ` + or :data:`psutil.POWER_TIME_UNLIMITED ` in + case the remaining time cannot be determined or is unlimited. If no battery is installed this function will return ``None``. Example:: >>> def secs2hours(secs): - ... m, s = divmod(secs, 60) - ... h, m = divmod(m, 60) - ... return "%d:%02d:%02d" % (h, m, s) + ... mm, ss = divmod(secs, 60) + ... hh, mm = divmod(m, 60) + ... return "%d:%02d:%02d" % (hh, mm, ss) ... >>> batt = psutil.sensors_battery() >>> batt @@ -2018,8 +2021,9 @@ Constants .. data:: POWER_TIME_UNKNOWN .. data:: POWER_TIME_UNLIMITED - This can be the value of *secsleft* field of - :func:`psutil.sensors_battery()`. + Whether the remaining time of the battery cannot be determined or is + unlimited. + May be assigned to :func:`psutil.sensors_battery()`'s *secsleft* field. .. versionadded:: 5.1.0 diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index fff77aab7..5be7f7597 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1072,7 +1072,10 @@ def sensors_battery(): energy_now = int(cat(root + "energy_now")) power_now = int(cat(root + "power_now")) percent = int(cat(root + "capacity")) - secsleft = int(energy_now / power_now * 3600) + try: + secsleft = int(energy_now / power_now * 3600) + except ZeroDivisionError: + secsleft = _common.POWER_TIME_UNKNOWN return _common.sbattery(percent, secsleft) From 90b261aef4c94e364596f07711565ea250f3a10e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jan 2017 01:12:45 +0100 Subject: [PATCH 098/922] #955: add power_plugged info --- docs/index.rst | 16 +++++++---- psutil/_common.py | 6 ++-- psutil/_pslinux.py | 27 +++++++++++------- psutil/_pswindows.py | 3 +- psutil/tests/test_linux.py | 57 ++++++++++++++++++++++++++++++++++++- psutil/tests/test_system.py | 9 ++++-- 6 files changed, 95 insertions(+), 23 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index af44c516b..f9ee7ee4d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -627,10 +627,11 @@ Sensors - **percent**: battery power left as a percentage. - **secsleft**: (rough approximation) number of seconds left before the - battery run out of power; this may be set to - :data:`psutil.POWER_TIME_UNKNOWN ` - or :data:`psutil.POWER_TIME_UNLIMITED ` in - case the remaining time cannot be determined or is unlimited. + battery run out of power. If the AC power cable is connected this will be + set to :data:`psutil.POWER_TIME_UNLIMITED `. + If it can't be determined it will be set to + :data:`psutil.POWER_TIME_UNKNOWN `. + - **power_plugged**: ``True`` if the AC power cable is connected. If no battery is installed this function will return ``None``. Example:: @@ -641,10 +642,15 @@ Sensors ... >>> batt = psutil.sensors_battery() >>> batt - sbattery(percent=93, secsleft=16628) + sbattery(percent=93, secsleft=16628, power_plugged=False) >>> print("charge = %s%%, time left = %s" % (batt.percent, secs2hours(batt.secsleft))) charge = 93%, time left = 4:37:08 + .. warning:: + + This API is experimental. Backward incompatible changes may occur if + deemed necessary. + Availability: Linux, Windows .. versionadded:: 5.1.0 diff --git a/psutil/_common.py b/psutil/_common.py index b28af9992..89dc16177 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -126,11 +126,11 @@ class NicDuplex(enum.IntEnum): POWER_TIME_UNKNOWN = -1 POWER_TIME_UNLIMITED = -2 else: - class Power(enum.IntEnum): + class BatteryTime(enum.IntEnum): POWER_TIME_UNKNOWN = -1 POWER_TIME_UNLIMITED = -2 - globals().update(NicDuplex.__members__) + globals().update(BatteryTime.__members__) # =================================================================== @@ -170,7 +170,7 @@ class Power(enum.IntEnum): # psutil.cpu_freq() scpufreq = namedtuple('scpufreq', ['current', 'min', 'max']) # psutil.sensors_battery() -sbattery = namedtuple('sbattery', ['percent', 'secsleft']) +sbattery = namedtuple('sbattery', ['percent', 'secsleft', 'power_plugged']) # --- for Process methods diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 5be7f7597..fec47be57 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1065,18 +1065,25 @@ def disk_partitions(all=False): def sensors_battery(): - root = "/sys/class/power_supply/BAT0/" - if not os.path.exists(root.rstrip('/')): + root = "/sys/class/power_supply/BAT0" + if not os.path.exists(root): return None - energy_now = int(cat(root + "energy_now")) - power_now = int(cat(root + "power_now")) - percent = int(cat(root + "capacity")) - try: - secsleft = int(energy_now / power_now * 3600) - except ZeroDivisionError: - secsleft = _common.POWER_TIME_UNKNOWN - return _common.sbattery(percent, secsleft) + power_plugged = \ + cat("/sys/class/power_supply/AC0/online", fallback=b"0") == b"1" + energy_now = int(cat(root + "/energy_now")) + power_now = int(cat(root + "/power_now")) + percent = int(cat(root + "/capacity")) + + if power_plugged: + secsleft = _common.POWER_TIME_UNLIMITED + else: + try: + secsleft = int(energy_now / power_now * 3600) + except ZeroDivisionError: + secsleft = _common.POWER_TIME_UNKNOWN + + return _common.sbattery(percent, secsleft, power_plugged) # ===================================================================== diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 80faec919..51e2efabc 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -384,7 +384,8 @@ def sensors_battery(): elif secsleft == -1: secsleft = _common.POWER_TIME_UNKNOWN - return _common.sbattery(percent, secsleft) + # TODO: implement power_plugged + return _common.sbattery(percent, secsleft, False) # ===================================================================== diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 54d5e251b..a00bdae67 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1024,12 +1024,67 @@ def test_issue_687(self): finally: t.stop() - def test_sensors_battery_percent(self): + +class TestSensorsBattery(unittest.TestCase): + + def test_percent(self): out = sh("acpi -b") acpi_value = int(out.split(",")[1].strip().replace('%', '')) psutil_value = psutil.sensors_battery().percent self.assertAlmostEqual(acpi_value, psutil_value, delta=1) + def test_power_plugged(self): + out = sh("acpi -b") + plugged = "Charging" in out.split('\n')[0] + self.assertEqual(psutil.sensors_battery().power_plugged, plugged) + + def test_emulate_power_plugged(self): + # Pretend the AC power cable is connected. + def open_mock(name, *args, **kwargs): + if name.startswith("/sys/class/power_supply/AC0/online"): + return io.BytesIO(b"1") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: + self.assertEqual(psutil.sensors_battery().power_plugged, True) + self.assertEqual( + psutil.sensors_battery().secsleft, psutil.POWER_TIME_UNLIMITED) + assert m.called + + def test_emulate_power_not_plugged(self): + # Pretend the AC power cable is not connected. + def open_mock(name, *args, **kwargs): + if name.startswith("/sys/class/power_supply/AC0/online"): + return io.BytesIO(b"0") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: + self.assertEqual(psutil.sensors_battery().power_plugged, False) + self.assertGreaterEqual(psutil.sensors_battery().secsleft, 0) + assert m.called + + def test_emulate_power_undetermined(self): + # Pretend we can't know whether the AC power cable not + # connected (assert fallback to False). + def open_mock(name, *args, **kwargs): + if name.startswith("/sys/class/power_supply/AC0/online"): + raise IOError(errno.ENOENT, "") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: + self.assertEqual(psutil.sensors_battery().power_plugged, False) + self.assertGreaterEqual(psutil.sensors_battery().secsleft, 0) + assert m.called + # ===================================================================== # test process diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index cf33d1408..36b1b11c4 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -761,12 +761,15 @@ def test_sensors_battery(self): ret = psutil.sensors_battery() if ret is None: return # no battery - if ret.percent is not None: - self.assertGreaterEqual(ret.percent, 0) - self.assertLessEqual(ret.percent, 100) + self.assertGreaterEqual(ret.percent, 0) + self.assertLessEqual(ret.percent, 100) if ret.secsleft not in (psutil.POWER_TIME_UNKNOWN, psutil.POWER_TIME_UNLIMITED): self.assertGreaterEqual(ret.secsleft, 0) + else: + if ret.secsleft == psutil.POWER_TIME_UNLIMITED: + self.assertTrue(ret.power_plugged) + self.assertIsInstance(ret.power_plugged, bool) if __name__ == '__main__': From bfc5e528574e14485d61ddad7e9a57850b9fb93e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jan 2017 01:33:15 +0100 Subject: [PATCH 099/922] #955: freebsd / battery: implement power_plugged field --- psutil/_psbsd.py | 9 ++++++--- psutil/arch/bsd/freebsd.c | 6 ++++-- psutil/tests/test_bsd.py | 18 +++++++++++++++++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 20f9cbcb3..ea16fc61e 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -401,12 +401,15 @@ def net_connections(kind): def sensors_battery(): - percent, minsleft = cext.sensors_battery() - if minsleft == -1: + percent, minsleft, power_plugged = cext.sensors_battery() + power_plugged = power_plugged == 1 + if power_plugged: secsleft = _common.POWER_TIME_UNLIMITED + elif minsleft == -1: + secsleft = _common.POWER_TIME_UNKNOWN else: secsleft = minsleft * 60 - return _common.sbattery(percent, secsleft) + return _common.sbattery(percent, secsleft, power_plugged) # ===================================================================== diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index c0286c866..0bec81d87 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -1003,14 +1003,16 @@ PyObject * psutil_sensors_battery(PyObject *self, PyObject *args) { int percent; int minsleft; + int power_plugged; size_t size = sizeof(percent); if (sysctlbyname("hw.acpi.battery.life", &percent, &size, NULL, 0)) goto error; - // -1 if power is connected if (sysctlbyname("hw.acpi.battery.time", &minsleft, &size, NULL, 0)) goto error; - return Py_BuildValue("ii", percent, minsleft); + if (sysctlbyname("hw.acpi.acline", &power_plugged, &size, NULL, 0)) + goto error; + return Py_BuildValue("iii", percent, minsleft, power_plugged); error: PyErr_SetFromErrno(PyExc_OSError); diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 479237e55..ff46ab334 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -357,6 +357,8 @@ def test_boot_time(self): btime = int(s) self.assertEqual(btime, psutil.boot_time()) + # --- sensors_battery + @unittest.skipUnless(psutil.sensors_battery(), "no battery") def test_sensors_battery(self): def secs2hours(secs): @@ -371,7 +373,21 @@ def secs2hours(secs): percent = int(fields['Remaining capacity:'].replace('%', '')) remaining_time = fields['Remaining time:'] self.assertEqual(metrics.percent, percent) - self.assertEqual(secs2hours(metrics.secsleft), remaining_time) + if remaining_time == 'unknown': + self.assertEqual(metrics.secsleft, psutil.POWER_TIME_UNLIMITED) + else: + self.assertEqual(secs2hours(metrics.secsleft), remaining_time) + + def test_sensors_battery_against_sysctl(self): + self.assertEqual(psutil.sensors_battery().percent, + sysctl("hw.acpi.battery.life")) + self.assertEqual(psutil.sensors_battery().power_plugged, + sysctl("hw.acpi.acline") == 1) + secsleft = psutil.sensors_battery().secsleft + if secsleft < 0: + self.assertEqual(sysctl("hw.acpi.battery.time"), -1) + else: + self.assertEqual(secsleft, sysctl("hw.acpi.battery.time") * 60) # ===================================================================== From 2033bf7409c6599dd36e3241c4d299f8eefa2c09 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jan 2017 01:49:17 +0100 Subject: [PATCH 100/922] #955: win / bttery - implement power plugged --- psutil/_pswindows.py | 15 ++++++--------- psutil/tests/test_windows.py | 11 +++++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 51e2efabc..dd83c9294 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -367,25 +367,22 @@ def net_if_addrs(): def sensors_battery(): + # For constants meaning see: + # https://msdn.microsoft.com/en-us/library/windows/desktop/ + # aa373232(v=vs.85).aspx acline_status, flags, percent, secsleft = cext.sensors_battery() - power_connected = acline_status == 1 + power_plugged = acline_status == 1 no_battery = bool(flags & 128) charging = bool(flags & 8) - # print("acline_status=%s, flags=%s, percent=%s, secsleft=%s" % ( - # acline_status, flags, percent, secsleft)) - # print("power_connected=%s, no_battery=%s, charging=%s" % ( - # power_connected, no_battery, charging)) - if no_battery: return None - if power_connected or charging: + if power_plugged or charging: secsleft = _common.POWER_TIME_UNLIMITED elif secsleft == -1: secsleft = _common.POWER_TIME_UNKNOWN - # TODO: implement power_plugged - return _common.sbattery(percent, secsleft, False) + return _common.sbattery(percent, secsleft, power_plugged) # ===================================================================== diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 669adad0b..aca8afbb0 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -172,10 +172,13 @@ def test_percent(self): battery_wmi = w.query('select * from Win32_Battery')[0] if battery_psutil is None: self.assertNot(battery_wmi.EstimatedChargeRemaining) - else: - self.assertAlmostEqual( - battery_psutil.percent, battery_wmi.EstimatedChargeRemaining, - delta=1) + return + + self.assertAlmostEqual( + battery_psutil.percent, battery_wmi.EstimatedChargeRemaining, + delta=1) + self.assertEqual( + battery_psutil.power_plugged, battery_wmi.BatteryStatus == 1) def test_emulate_no_battery(self): with mock.patch("psutil._pswindows.cext.sensors_battery", From e462e7b3b3966a90e0430584fa43ffdc71078f90 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jan 2017 02:08:43 +0100 Subject: [PATCH 101/922] update doc --- README.rst | 7 +++++++ docs/index.rst | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 6d9a1d93e..1bc3244c3 100644 --- a/README.rst +++ b/README.rst @@ -181,6 +181,13 @@ Network {'eth0': snicstats(isup=True, duplex=, speed=100, mtu=1500), 'lo': snicstats(isup=True, duplex=, speed=0, mtu=65536)} +Sensors +======= + + >>> psutil.sensors_battery() + sbattery(percent=93, secsleft=16628, power_plugged=False) + + Other system info ================= diff --git a/docs/index.rst b/docs/index.rst index f9ee7ee4d..1dfe9dda4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -626,8 +626,8 @@ Sensors values: - **percent**: battery power left as a percentage. - - **secsleft**: (rough approximation) number of seconds left before the - battery run out of power. If the AC power cable is connected this will be + - **secsleft**: a rough approximation of how many seconds are left before the + battery runs out of power. If the AC power cable is connected this will be set to :data:`psutil.POWER_TIME_UNLIMITED `. If it can't be determined it will be set to :data:`psutil.POWER_TIME_UNKNOWN `. @@ -651,7 +651,7 @@ Sensors This API is experimental. Backward incompatible changes may occur if deemed necessary. - Availability: Linux, Windows + Availability: Linux, Windows, FreeBSD .. versionadded:: 5.1.0 From 32cec3e427621b77fa2f267ffe416f89cd120536 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jan 2017 03:46:35 +0100 Subject: [PATCH 102/922] #955: add battery.py example script --- docs/index.rst | 9 ++++++--- scripts/battery.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) create mode 100755 scripts/battery.py diff --git a/docs/index.rst b/docs/index.rst index 1dfe9dda4..7f400f151 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -637,15 +637,18 @@ Sensors >>> def secs2hours(secs): ... mm, ss = divmod(secs, 60) - ... hh, mm = divmod(m, 60) + ... hh, mm = divmod(mm, 60) ... return "%d:%02d:%02d" % (hh, mm, ss) ... - >>> batt = psutil.sensors_battery() - >>> batt + >>> battery = psutil.sensors_battery() + >>> battery sbattery(percent=93, secsleft=16628, power_plugged=False) >>> print("charge = %s%%, time left = %s" % (batt.percent, secs2hours(batt.secsleft))) charge = 93%, time left = 4:37:08 + See also `battery.py `__ + for an example application. + .. warning:: This API is experimental. Backward incompatible changes may occur if diff --git a/scripts/battery.py b/scripts/battery.py new file mode 100755 index 000000000..67d706e36 --- /dev/null +++ b/scripts/battery.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Show battery information. + +$ python battery.py +charge: 49% +left: 2:11:31 +status: discharging +plugged in: no +""" + +from __future__ import print_function +import sys + +import psutil + + +def secs2hours(secs): + mm, ss = divmod(secs, 60) + hh, mm = divmod(mm, 60) + return "%d:%02d:%02d" % (hh, mm, ss) + + +def main(): + if not hasattr(psutil, "sensors_battery"): + return sys.exit("platform not supported") + batt = psutil.sensors_battery() + if batt is None: + return sys.exit("no battery is installed") + + print("charge: %s%%" % batt.percent) + if batt.power_plugged: + print("status: %s" % ( + "charging" if batt.percent < 100 else "fully charged")) + print("plugged in: yes") + else: + print("left: %s" % secs2hours(batt.secsleft)) + print("status: %s" % "discharging") + print("plugged in: no") + + +if __name__ == '__main__': + main() From 2f1e3e5b8d66f1d347b042e0509d82484d8fd8fb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jan 2017 04:16:16 +0100 Subject: [PATCH 103/922] update doc; fix tests --- docs/index.rst | 13 ++++++++----- psutil/__init__.py | 9 ++++++++- psutil/tests/test_linux.py | 6 ++++++ psutil/tests/test_misc.py | 6 ++++++ scripts/battery.py | 2 +- 5 files changed, 29 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 7f400f151..e552b44df 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -623,18 +623,21 @@ Sensors .. function:: sensors_battery() Return battery status information as a namedtuple including the following - values: + values. If no battery is installed returns ``None``. - **percent**: battery power left as a percentage. - **secsleft**: a rough approximation of how many seconds are left before the - battery runs out of power. If the AC power cable is connected this will be - set to :data:`psutil.POWER_TIME_UNLIMITED `. - If it can't be determined it will be set to + battery runs out of power. + If the AC power cable is connected this is set to + :data:`psutil.POWER_TIME_UNLIMITED `. + If it can't be determined it is set to :data:`psutil.POWER_TIME_UNKNOWN `. - **power_plugged**: ``True`` if the AC power cable is connected. - If no battery is installed this function will return ``None``. Example:: + Example:: + >>> import psutil + >>> >>> def secs2hours(secs): ... mm, ss = divmod(secs, 60) ... hh, mm = divmod(mm, 60) diff --git a/psutil/__init__.py b/psutil/__init__.py index bfef05123..7b4818d39 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2182,11 +2182,18 @@ def net_if_stats(): # ===================================================================== +# Linux, Windows, FreeBSD if hasattr(_psplatform, "sensors_battery"): def sensors_battery(): - """Return information about battery. If no battery can be found + """Return battery information. If no battery is installed returns None. + + - percent: battery power left as a percentage. + - secsleft: a rough approximation of how many seconds are left + before the battery runs out of power. + May be POWER_TIME_UNLIMITED or POWER_TIME_UNLIMITED. + - power_plugged: True if the AC power cable is connected. """ return _psplatform.sensors_battery() diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index a00bdae67..3dc81f5e7 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1025,14 +1025,20 @@ def test_issue_687(self): t.stop() +@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipUnless(hasattr(psutil, "sensors_battery") and + psutil.sensors_battery() is not None, + "no battery") class TestSensorsBattery(unittest.TestCase): + @unittest.skipUnless(which("acpi"), "acpi utility not available") def test_percent(self): out = sh("acpi -b") acpi_value = int(out.split(",")[1].strip().replace('%', '')) psutil_value = psutil.sensors_battery().percent self.assertAlmostEqual(acpi_value, psutil_value, delta=1) + @unittest.skipUnless(which("acpi"), "acpi utility not available") def test_power_plugged(self): out = sh("acpi -b") plugged = "Charging" in out.split('\n')[0] diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 0b696f8cb..fa777ef48 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -469,6 +469,12 @@ def test_winservices(self): def test_cpu_distribution(self): self.assert_syntax('cpu_distribution.py') + @unittest.skipUnless(hasattr(psutil, "sensors_battery") and + psutil.sensors_battery() is not None, + "no battery") + def test_battery(self): + self.assert_stdout('battery.py') + # =================================================================== # --- Unit tests for test utilities. diff --git a/scripts/battery.py b/scripts/battery.py index 67d706e36..eb8b16bb5 100755 --- a/scripts/battery.py +++ b/scripts/battery.py @@ -8,7 +8,7 @@ Show battery information. $ python battery.py -charge: 49% +charge: 74% left: 2:11:31 status: discharging plugged in: no From 549ae3d423278fb8eba2a2fca0238682bde57f91 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jan 2017 04:59:10 +0100 Subject: [PATCH 104/922] update doc --- README.rst | 14 ++++++-------- docs/index.rst | 4 ++-- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 6d9a1d93e..3e70b79d7 100644 --- a/README.rst +++ b/README.rst @@ -374,11 +374,15 @@ Windows services 'username': 'NT AUTHORITY\\LocalService'} ====== -Donate +Author ====== +psutil was created and is maintained by +`Giampaolo Rodola' `_. A lot of time and effort went into making psutil as it is right now. -If you feel psutil is useful to you or your business and want to support its future development please consider donating me (`Giampaolo Rodola' `_) some money. +If you feel psutil is useful to you or your business and want to support its +future development please consider donating me +(`Giampaolo `_) some money. I only ask for a small donation, but of course I appreciate any amount. .. image:: http://www.paypal.com/en_US/i/btn/x-click-but04.gif @@ -386,9 +390,3 @@ I only ask for a small donation, but of course I appreciate any amount. :alt: Donate via PayPal Don't want to donate money? Then maybe you could `write me a recommendation on Linkedin `_. - -============ -Mailing list -============ - -http://groups.google.com/group/psutil/ diff --git a/docs/index.rst b/docs/index.rst index 0131b56a5..5ccc11b31 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -399,7 +399,7 @@ Disks .. warning:: on some systems such as Linux, on a very busy or long-lived system these numbers may wrap (restart from zero), see - `issues #802 `__. + `issue #802 `__. Applications should be prepared to deal with that. .. versionchanged:: @@ -1834,7 +1834,7 @@ Constants .. data:: SUNOS ``bool`` constants which define what platform you're on. E.g. if on Windows, - *WINDOWS* constant will be ``True``, all others will be ``False``. + :const:`WINDOWS` constant will be ``True``, all others will be ``False``. .. versionadded:: 4.0.0 From 2c6f48cc4d1d8b2b3da179a9059d5df13aed1608 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jan 2017 05:00:31 +0100 Subject: [PATCH 105/922] update doc --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index 3e70b79d7..bd8f62b1e 100644 --- a/README.rst +++ b/README.rst @@ -383,7 +383,6 @@ A lot of time and effort went into making psutil as it is right now. If you feel psutil is useful to you or your business and want to support its future development please consider donating me (`Giampaolo `_) some money. -I only ask for a small donation, but of course I appreciate any amount. .. image:: http://www.paypal.com/en_US/i/btn/x-click-but04.gif :target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8 From 2cd73cafee4548e2b73d0797ca8a1f7b06ccddae Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jan 2017 05:12:07 +0100 Subject: [PATCH 106/922] BSD: expose sensors_batter() for FreeBSD only --- psutil/_psbsd.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index ea16fc61e..fb141cf27 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -400,16 +400,18 @@ def net_connections(kind): # ===================================================================== -def sensors_battery(): - percent, minsleft, power_plugged = cext.sensors_battery() - power_plugged = power_plugged == 1 - if power_plugged: - secsleft = _common.POWER_TIME_UNLIMITED - elif minsleft == -1: - secsleft = _common.POWER_TIME_UNKNOWN - else: - secsleft = minsleft * 60 - return _common.sbattery(percent, secsleft, power_plugged) +if FREEBSD: + + def sensors_battery(): + percent, minsleft, power_plugged = cext.sensors_battery() + power_plugged = power_plugged == 1 + if power_plugged: + secsleft = _common.POWER_TIME_UNLIMITED + elif minsleft == -1: + secsleft = _common.POWER_TIME_UNKNOWN + else: + secsleft = minsleft * 60 + return _common.sbattery(percent, secsleft, power_plugged) # ===================================================================== From c0728d6c4f75ddd669ee19ece3f49ee455b33a2e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 27 Jan 2017 06:05:49 +0100 Subject: [PATCH 107/922] small refactoring --- psutil/_pslinux.py | 3 ++- psutil/tests/test_system.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index fec47be57..8db33fe89 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -61,6 +61,7 @@ # --- constants # ===================================================================== +POWER_SUPPLY_PATH = "/sys/class/power_supply" HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) HAS_PRLIMIT = hasattr(cext, "linux_prlimit") @@ -1065,7 +1066,7 @@ def disk_partitions(all=False): def sensors_battery(): - root = "/sys/class/power_supply/BAT0" + root = os.path.join(POWER_SUPPLY_PATH, "BAT0") if not os.path.exists(root): return None diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 36b1b11c4..48c81763d 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -755,7 +755,7 @@ def test_os_constants(self): for name in names: self.assertIs(getattr(psutil, name), False, msg=name) - @unittest.skipUnless(hasattr(psutil, "sensors_battery"), + @unittest.skipUnless(LINUX or WINDOWS or FREEBSD, "platform not supported") def test_sensors_battery(self): ret = psutil.sensors_battery() From 42ff4afe8ba81e16efe802466c63dcc3de8ee30f Mon Sep 17 00:00:00 2001 From: Thiago Borges Abdnur Date: Mon, 30 Jan 2017 10:03:27 -0200 Subject: [PATCH 108/922] fix exception pickling --- psutil/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/__init__.py b/psutil/__init__.py index d67097d5b..d07838b68 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -229,6 +229,7 @@ class Error(Exception): """ def __init__(self, msg=""): + Exception.__init__(self, msg) self.msg = msg def __repr__(self): From 53be3ab5de8212e0164030bfd44f835b63ea82c0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Jan 2017 13:21:30 +0100 Subject: [PATCH 109/922] #959: update HISTORY / CREDITS --- CREDITS | 4 ++++ HISTORY.rst | 1 + 2 files changed, 5 insertions(+) diff --git a/CREDITS b/CREDITS index 031548aeb..a353b3da1 100644 --- a/CREDITS +++ b/CREDITS @@ -425,3 +425,7 @@ N: Pierre Fersing C: France E: pierre.fersing@bleemeo.com I: 950 + +N: Thiago Borges Abdnur +W: https://github.com/bolaum +I: 959 diff --git a/HISTORY.rst b/HISTORY.rst index 755a6d7fd..72ed0c861 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ - 948_: cannot install psutil with PYTHONOPTIMIZE=2. - 950_: [Windows] Process.cpu_percent() was calculated incorrectly and showed higher number than real usage. +- 959_: psutil exception objects could not be pickled. 5.0.1 From 8b57f42f0453804ae6d3bab7ef34d3a49f5cb6f1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Jan 2017 14:22:48 +0100 Subject: [PATCH 110/922] #956: cpu_affinity([]) can now be used as an alias to set affinity against all eligible CPUs. --- HISTORY.rst | 2 ++ README.rst | 4 ++-- appveyor.yml | 3 ++- docs/index.rst | 31 +++++++++++++++++-------------- psutil/__init__.py | 7 +++++++ psutil/_pslinux.py | 21 ++++++++++++++++++--- psutil/tests/test_linux.py | 5 +++++ psutil/tests/test_process.py | 14 +++++++++++--- 8 files changed, 64 insertions(+), 23 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 72ed0c861..510c3de1e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ - 357_: added psutil.Process.cpu_num() (what CPU a process is on). - 941_: added psutil.cpu_freq() (CPU frequency). +- 956_: cpu_affinity([]) can now be used as an alias to set affinity against + all eligible CPUs. **Bug fixes** diff --git a/README.rst b/README.rst index bd8f62b1e..20c2f9b31 100644 --- a/README.rst +++ b/README.rst @@ -248,9 +248,9 @@ Process management 12.1 >>> p.cpu_affinity() [0, 1, 2, 3] - >>> p.cpu_affinity([0]) # set + >>> p.cpu_affinity([0, 1]) # set >>> p.cpu_num() - 2 + 1 >>> >>> p.memory_info() pmem(rss=10915840, vms=67608576, shared=3313664, text=2310144, lib=0, data=7262208, dirty=0) diff --git a/appveyor.yml b/appveyor.yml index 4428f7767..b569a7ade 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,3 +1,5 @@ +# Build: 0 (bump this up by 1 to force an appveyor run) + os: Visual Studio 2015 environment: @@ -124,4 +126,3 @@ only_commits: psutil/tests/test_windows.py scripts/* setup.py - diff --git a/docs/index.rst b/docs/index.rst index 5ccc11b31..645df8df7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1214,32 +1214,35 @@ Process class Get or set process current `CPU affinity `__. - CPU affinity consists in telling the OS to run a certain process on a - limited set of CPUs only. + CPU affinity consists in telling the OS to run a process on a limited set + of CPUs only. On Linux this is done via the ``taskset`` command. - The number of eligible CPUs can be obtained with - ``list(range(psutil.cpu_count()))``. - ``ValueError`` will be raised on set in case an invalid CPU number is - specified. + If no argument is passed it returns the current CPU affinity as a list + of integers. + If passed it must be a list of integers specifying the new CPUs affinity. + If an empty list is passed all eligible CPUs are assumed (and set); + on Linux this may not necessarily mean all available CPUs as in + ``list(range(psutil.cpu_count()))``). >>> import psutil >>> psutil.cpu_count() 4 >>> p = psutil.Process() - >>> p.cpu_affinity() # get + >>> # get + >>> p.cpu_affinity() [0, 1, 2, 3] - >>> p.cpu_affinity([0]) # set; from now on, process will run on CPU #0 only + >>> # set; from now on, process will run on CPU #0 and #1 only + >>> p.cpu_affinity([0, 1]) >>> p.cpu_affinity() - [0] - >>> - >>> # reset affinity against all CPUs - >>> all_cpus = list(range(psutil.cpu_count())) - >>> p.cpu_affinity(all_cpus) - >>> + [0, 1] + >>> # reset affinity against all eligible CPUs + >>> p.cpu_affinity([]) Availability: Linux, Windows, FreeBSD .. versionchanged:: 2.2.0 added support for FreeBSD + .. versionchanged:: 5.1.0 an empty list can be passed to set affinity + against all eligible CPUs. .. method:: cpu_num() diff --git a/psutil/__init__.py b/psutil/__init__.py index d07838b68..647af0a68 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -828,6 +828,8 @@ def cpu_affinity(self, cpus=None): """Get or set process CPU affinity. If specified 'cpus' must be a list of CPUs for which you want to set the affinity (e.g. [0, 1]). + If an empty list is passed, all egible CPUs are assumed + (and set). (Windows, Linux and BSD only). """ # Automatically remove duplicates both on get and @@ -836,6 +838,11 @@ def cpu_affinity(self, cpus=None): if cpus is None: return list(set(self._proc.cpu_affinity_get())) else: + if not cpus: + if hasattr(self._proc, "_get_eligible_cpus"): + cpus = self._proc._get_eligible_cpus() + else: + cpus = tuple(range(len(cpu_times(percpu=True)))) self._proc.cpu_affinity_set(list(set(cpus))) # Linux, FreeBSD, SunOS diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 85647bb22..cdfd3375f 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1521,18 +1521,33 @@ def nice_set(self, value): def cpu_affinity_get(self): return cext.proc_cpu_affinity_get(self.pid) + def _get_eligible_cpus( + self, _re=re.compile("Cpus_allowed_list:\t(\d+)-(\d+)")): + # See: https://github.com/giampaolo/psutil/issues/956 + data = self._read_status_file() + match = _re.findall(data) + if match: + return tuple(range(int(match[0][0]), int(match[0][1]) + 1)) + else: + return tuple(range(len(per_cpu_times()))) + @wrap_exceptions def cpu_affinity_set(self, cpus): try: cext.proc_cpu_affinity_set(self.pid, cpus) except (OSError, ValueError) as err: if isinstance(err, ValueError) or err.errno == errno.EINVAL: - allcpus = tuple(range(len(per_cpu_times()))) + eligible_cpus = self._get_eligible_cpus() + all_cpus = tuple(range(len(per_cpu_times()))) for cpu in cpus: - if cpu not in allcpus: + if cpu not in all_cpus: raise ValueError( "invalid CPU number %r; choose between %s" % ( - cpu, allcpus)) + cpu, eligible_cpus)) + if cpu not in eligible_cpus: + raise ValueError( + "CPU number %r is not eligible; choose " + "between %s" % (cpu, eligible_cpus)) raise # only starting from kernel 2.6.13 diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 37352ecf2..0f0b09c30 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1277,6 +1277,11 @@ def test_cpu_affinity(self): self.assertEqual( self.proc.cpu_affinity(), list(range(min_, max_ + 1))) + def test_cpu_affinity_eligible_cpus(self): + with mock.patch("psutil._pslinux.per_cpu_times") as m: + self.proc._proc._get_eligible_cpus() + assert not m.called + if __name__ == '__main__': run_test_module_by_name(__file__) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 747504735..161d5e5e1 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -849,10 +849,13 @@ def test_cwd_2(self): def test_cpu_affinity(self): p = psutil.Process() initial = p.cpu_affinity() + assert initial, initial self.addCleanup(p.cpu_affinity, initial) + if hasattr(os, "sched_getaffinity"): self.assertEqual(initial, list(os.sched_getaffinity(p.pid))) self.assertEqual(len(initial), len(set(initial))) + all_cpus = list(range(len(psutil.cpu_percent(percpu=True)))) # setting on travis doesn't seem to work (always return all # CPUs on get): @@ -867,9 +870,14 @@ def test_cpu_affinity(self): if hasattr(p, "num_cpu"): self.assertEqual(p.cpu_affinity()[0], p.num_cpu()) - # - p.cpu_affinity(all_cpus) - self.assertEqual(p.cpu_affinity(), all_cpus) + # [] is an alias for "all eligible CPUs"; on Linux this may + # not be equal to all available CPUs, see: + # https://github.com/giampaolo/psutil/issues/956 + p.cpu_affinity([]) + if LINUX: + self.assertEqual(p.cpu_affinity(), p._proc._get_eligible_cpus()) + else: + self.assertEqual(p.cpu_affinity(), all_cpus) if hasattr(os, "sched_getaffinity"): self.assertEqual(p.cpu_affinity(), list(os.sched_getaffinity(p.pid))) From b04cf1f4186b1181a7aec23c0f0dbaca68f896e0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Jan 2017 14:31:44 +0100 Subject: [PATCH 111/922] update HISTORY --- HISTORY.rst | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 510c3de1e..782b147c4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2463,3 +2463,204 @@ DeprecationWarning. .. _997: https://github.com/giampaolo/psutil/issues/997 .. _998: https://github.com/giampaolo/psutil/issues/998 .. _999: https://github.com/giampaolo/psutil/issues/999 +.. _1000: https://github.com/giampaolo/psutil/issues/1000 +.. _1001: https://github.com/giampaolo/psutil/issues/1001 +.. _1002: https://github.com/giampaolo/psutil/issues/1002 +.. _1003: https://github.com/giampaolo/psutil/issues/1003 +.. _1004: https://github.com/giampaolo/psutil/issues/1004 +.. _1005: https://github.com/giampaolo/psutil/issues/1005 +.. _1006: https://github.com/giampaolo/psutil/issues/1006 +.. _1007: https://github.com/giampaolo/psutil/issues/1007 +.. _1008: https://github.com/giampaolo/psutil/issues/1008 +.. _1009: https://github.com/giampaolo/psutil/issues/1009 +.. _1010: https://github.com/giampaolo/psutil/issues/1010 +.. _1011: https://github.com/giampaolo/psutil/issues/1011 +.. _1012: https://github.com/giampaolo/psutil/issues/1012 +.. _1013: https://github.com/giampaolo/psutil/issues/1013 +.. _1014: https://github.com/giampaolo/psutil/issues/1014 +.. _1015: https://github.com/giampaolo/psutil/issues/1015 +.. _1016: https://github.com/giampaolo/psutil/issues/1016 +.. _1017: https://github.com/giampaolo/psutil/issues/1017 +.. _1018: https://github.com/giampaolo/psutil/issues/1018 +.. _1019: https://github.com/giampaolo/psutil/issues/1019 +.. _1020: https://github.com/giampaolo/psutil/issues/1020 +.. _1021: https://github.com/giampaolo/psutil/issues/1021 +.. _1022: https://github.com/giampaolo/psutil/issues/1022 +.. _1023: https://github.com/giampaolo/psutil/issues/1023 +.. _1024: https://github.com/giampaolo/psutil/issues/1024 +.. _1025: https://github.com/giampaolo/psutil/issues/1025 +.. _1026: https://github.com/giampaolo/psutil/issues/1026 +.. _1027: https://github.com/giampaolo/psutil/issues/1027 +.. _1028: https://github.com/giampaolo/psutil/issues/1028 +.. _1029: https://github.com/giampaolo/psutil/issues/1029 +.. _1030: https://github.com/giampaolo/psutil/issues/1030 +.. _1031: https://github.com/giampaolo/psutil/issues/1031 +.. _1032: https://github.com/giampaolo/psutil/issues/1032 +.. _1033: https://github.com/giampaolo/psutil/issues/1033 +.. _1034: https://github.com/giampaolo/psutil/issues/1034 +.. _1035: https://github.com/giampaolo/psutil/issues/1035 +.. _1036: https://github.com/giampaolo/psutil/issues/1036 +.. _1037: https://github.com/giampaolo/psutil/issues/1037 +.. _1038: https://github.com/giampaolo/psutil/issues/1038 +.. _1039: https://github.com/giampaolo/psutil/issues/1039 +.. _1040: https://github.com/giampaolo/psutil/issues/1040 +.. _1041: https://github.com/giampaolo/psutil/issues/1041 +.. _1042: https://github.com/giampaolo/psutil/issues/1042 +.. _1043: https://github.com/giampaolo/psutil/issues/1043 +.. _1044: https://github.com/giampaolo/psutil/issues/1044 +.. _1045: https://github.com/giampaolo/psutil/issues/1045 +.. _1046: https://github.com/giampaolo/psutil/issues/1046 +.. _1047: https://github.com/giampaolo/psutil/issues/1047 +.. _1048: https://github.com/giampaolo/psutil/issues/1048 +.. _1049: https://github.com/giampaolo/psutil/issues/1049 +.. _1050: https://github.com/giampaolo/psutil/issues/1050 +.. _1051: https://github.com/giampaolo/psutil/issues/1051 +.. _1052: https://github.com/giampaolo/psutil/issues/1052 +.. _1053: https://github.com/giampaolo/psutil/issues/1053 +.. _1054: https://github.com/giampaolo/psutil/issues/1054 +.. _1055: https://github.com/giampaolo/psutil/issues/1055 +.. _1056: https://github.com/giampaolo/psutil/issues/1056 +.. _1057: https://github.com/giampaolo/psutil/issues/1057 +.. _1058: https://github.com/giampaolo/psutil/issues/1058 +.. _1059: https://github.com/giampaolo/psutil/issues/1059 +.. _1060: https://github.com/giampaolo/psutil/issues/1060 +.. _1061: https://github.com/giampaolo/psutil/issues/1061 +.. _1062: https://github.com/giampaolo/psutil/issues/1062 +.. _1063: https://github.com/giampaolo/psutil/issues/1063 +.. _1064: https://github.com/giampaolo/psutil/issues/1064 +.. _1065: https://github.com/giampaolo/psutil/issues/1065 +.. _1066: https://github.com/giampaolo/psutil/issues/1066 +.. _1067: https://github.com/giampaolo/psutil/issues/1067 +.. _1068: https://github.com/giampaolo/psutil/issues/1068 +.. _1069: https://github.com/giampaolo/psutil/issues/1069 +.. _1070: https://github.com/giampaolo/psutil/issues/1070 +.. _1071: https://github.com/giampaolo/psutil/issues/1071 +.. _1072: https://github.com/giampaolo/psutil/issues/1072 +.. _1073: https://github.com/giampaolo/psutil/issues/1073 +.. _1074: https://github.com/giampaolo/psutil/issues/1074 +.. _1075: https://github.com/giampaolo/psutil/issues/1075 +.. _1076: https://github.com/giampaolo/psutil/issues/1076 +.. _1077: https://github.com/giampaolo/psutil/issues/1077 +.. _1078: https://github.com/giampaolo/psutil/issues/1078 +.. _1079: https://github.com/giampaolo/psutil/issues/1079 +.. _1080: https://github.com/giampaolo/psutil/issues/1080 +.. _1081: https://github.com/giampaolo/psutil/issues/1081 +.. _1082: https://github.com/giampaolo/psutil/issues/1082 +.. _1083: https://github.com/giampaolo/psutil/issues/1083 +.. _1084: https://github.com/giampaolo/psutil/issues/1084 +.. _1085: https://github.com/giampaolo/psutil/issues/1085 +.. _1086: https://github.com/giampaolo/psutil/issues/1086 +.. _1087: https://github.com/giampaolo/psutil/issues/1087 +.. _1088: https://github.com/giampaolo/psutil/issues/1088 +.. _1089: https://github.com/giampaolo/psutil/issues/1089 +.. _1090: https://github.com/giampaolo/psutil/issues/1090 +.. _1091: https://github.com/giampaolo/psutil/issues/1091 +.. _1092: https://github.com/giampaolo/psutil/issues/1092 +.. _1093: https://github.com/giampaolo/psutil/issues/1093 +.. _1094: https://github.com/giampaolo/psutil/issues/1094 +.. _1095: https://github.com/giampaolo/psutil/issues/1095 +.. _1096: https://github.com/giampaolo/psutil/issues/1096 +.. _1097: https://github.com/giampaolo/psutil/issues/1097 +.. _1098: https://github.com/giampaolo/psutil/issues/1098 +.. _1099: https://github.com/giampaolo/psutil/issues/1099 +.. _1100: https://github.com/giampaolo/psutil/issues/1100 +.. _1101: https://github.com/giampaolo/psutil/issues/1101 +.. _1102: https://github.com/giampaolo/psutil/issues/1102 +.. _1103: https://github.com/giampaolo/psutil/issues/1103 +.. _1104: https://github.com/giampaolo/psutil/issues/1104 +.. _1105: https://github.com/giampaolo/psutil/issues/1105 +.. _1106: https://github.com/giampaolo/psutil/issues/1106 +.. _1107: https://github.com/giampaolo/psutil/issues/1107 +.. _1108: https://github.com/giampaolo/psutil/issues/1108 +.. _1109: https://github.com/giampaolo/psutil/issues/1109 +.. _1110: https://github.com/giampaolo/psutil/issues/1110 +.. _1111: https://github.com/giampaolo/psutil/issues/1111 +.. _1112: https://github.com/giampaolo/psutil/issues/1112 +.. _1113: https://github.com/giampaolo/psutil/issues/1113 +.. _1114: https://github.com/giampaolo/psutil/issues/1114 +.. _1115: https://github.com/giampaolo/psutil/issues/1115 +.. _1116: https://github.com/giampaolo/psutil/issues/1116 +.. _1117: https://github.com/giampaolo/psutil/issues/1117 +.. _1118: https://github.com/giampaolo/psutil/issues/1118 +.. _1119: https://github.com/giampaolo/psutil/issues/1119 +.. _1120: https://github.com/giampaolo/psutil/issues/1120 +.. _1121: https://github.com/giampaolo/psutil/issues/1121 +.. _1122: https://github.com/giampaolo/psutil/issues/1122 +.. _1123: https://github.com/giampaolo/psutil/issues/1123 +.. _1124: https://github.com/giampaolo/psutil/issues/1124 +.. _1125: https://github.com/giampaolo/psutil/issues/1125 +.. _1126: https://github.com/giampaolo/psutil/issues/1126 +.. _1127: https://github.com/giampaolo/psutil/issues/1127 +.. _1128: https://github.com/giampaolo/psutil/issues/1128 +.. _1129: https://github.com/giampaolo/psutil/issues/1129 +.. _1130: https://github.com/giampaolo/psutil/issues/1130 +.. _1131: https://github.com/giampaolo/psutil/issues/1131 +.. _1132: https://github.com/giampaolo/psutil/issues/1132 +.. _1133: https://github.com/giampaolo/psutil/issues/1133 +.. _1134: https://github.com/giampaolo/psutil/issues/1134 +.. _1135: https://github.com/giampaolo/psutil/issues/1135 +.. _1136: https://github.com/giampaolo/psutil/issues/1136 +.. _1137: https://github.com/giampaolo/psutil/issues/1137 +.. _1138: https://github.com/giampaolo/psutil/issues/1138 +.. _1139: https://github.com/giampaolo/psutil/issues/1139 +.. _1140: https://github.com/giampaolo/psutil/issues/1140 +.. _1141: https://github.com/giampaolo/psutil/issues/1141 +.. _1142: https://github.com/giampaolo/psutil/issues/1142 +.. _1143: https://github.com/giampaolo/psutil/issues/1143 +.. _1144: https://github.com/giampaolo/psutil/issues/1144 +.. _1145: https://github.com/giampaolo/psutil/issues/1145 +.. _1146: https://github.com/giampaolo/psutil/issues/1146 +.. _1147: https://github.com/giampaolo/psutil/issues/1147 +.. _1148: https://github.com/giampaolo/psutil/issues/1148 +.. _1149: https://github.com/giampaolo/psutil/issues/1149 +.. _1150: https://github.com/giampaolo/psutil/issues/1150 +.. _1151: https://github.com/giampaolo/psutil/issues/1151 +.. _1152: https://github.com/giampaolo/psutil/issues/1152 +.. _1153: https://github.com/giampaolo/psutil/issues/1153 +.. _1154: https://github.com/giampaolo/psutil/issues/1154 +.. _1155: https://github.com/giampaolo/psutil/issues/1155 +.. _1156: https://github.com/giampaolo/psutil/issues/1156 +.. _1157: https://github.com/giampaolo/psutil/issues/1157 +.. _1158: https://github.com/giampaolo/psutil/issues/1158 +.. _1159: https://github.com/giampaolo/psutil/issues/1159 +.. _1160: https://github.com/giampaolo/psutil/issues/1160 +.. _1161: https://github.com/giampaolo/psutil/issues/1161 +.. _1162: https://github.com/giampaolo/psutil/issues/1162 +.. _1163: https://github.com/giampaolo/psutil/issues/1163 +.. _1164: https://github.com/giampaolo/psutil/issues/1164 +.. _1165: https://github.com/giampaolo/psutil/issues/1165 +.. _1166: https://github.com/giampaolo/psutil/issues/1166 +.. _1167: https://github.com/giampaolo/psutil/issues/1167 +.. _1168: https://github.com/giampaolo/psutil/issues/1168 +.. _1169: https://github.com/giampaolo/psutil/issues/1169 +.. _1170: https://github.com/giampaolo/psutil/issues/1170 +.. _1171: https://github.com/giampaolo/psutil/issues/1171 +.. _1172: https://github.com/giampaolo/psutil/issues/1172 +.. _1173: https://github.com/giampaolo/psutil/issues/1173 +.. _1174: https://github.com/giampaolo/psutil/issues/1174 +.. _1175: https://github.com/giampaolo/psutil/issues/1175 +.. _1176: https://github.com/giampaolo/psutil/issues/1176 +.. _1177: https://github.com/giampaolo/psutil/issues/1177 +.. _1178: https://github.com/giampaolo/psutil/issues/1178 +.. _1179: https://github.com/giampaolo/psutil/issues/1179 +.. _1180: https://github.com/giampaolo/psutil/issues/1180 +.. _1181: https://github.com/giampaolo/psutil/issues/1181 +.. _1182: https://github.com/giampaolo/psutil/issues/1182 +.. _1183: https://github.com/giampaolo/psutil/issues/1183 +.. _1184: https://github.com/giampaolo/psutil/issues/1184 +.. _1185: https://github.com/giampaolo/psutil/issues/1185 +.. _1186: https://github.com/giampaolo/psutil/issues/1186 +.. _1187: https://github.com/giampaolo/psutil/issues/1187 +.. _1188: https://github.com/giampaolo/psutil/issues/1188 +.. _1189: https://github.com/giampaolo/psutil/issues/1189 +.. _1190: https://github.com/giampaolo/psutil/issues/1190 +.. _1191: https://github.com/giampaolo/psutil/issues/1191 +.. _1192: https://github.com/giampaolo/psutil/issues/1192 +.. _1193: https://github.com/giampaolo/psutil/issues/1193 +.. _1194: https://github.com/giampaolo/psutil/issues/1194 +.. _1195: https://github.com/giampaolo/psutil/issues/1195 +.. _1196: https://github.com/giampaolo/psutil/issues/1196 +.. _1197: https://github.com/giampaolo/psutil/issues/1197 +.. _1198: https://github.com/giampaolo/psutil/issues/1198 +.. _1199: https://github.com/giampaolo/psutil/issues/1199 +.. _1200: https://github.com/giampaolo/psutil/issues/1200 From 89a729e63db9a10e4e45b15b2c5994f3e9e295b9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Jan 2017 14:35:53 +0100 Subject: [PATCH 112/922] fix py3 type issue --- psutil/_pslinux.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index cdfd3375f..a9fbd8bc6 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1522,14 +1522,14 @@ def cpu_affinity_get(self): return cext.proc_cpu_affinity_get(self.pid) def _get_eligible_cpus( - self, _re=re.compile("Cpus_allowed_list:\t(\d+)-(\d+)")): + self, _re=re.compile(b"Cpus_allowed_list:\t(\d+)-(\d+)")): # See: https://github.com/giampaolo/psutil/issues/956 data = self._read_status_file() match = _re.findall(data) if match: - return tuple(range(int(match[0][0]), int(match[0][1]) + 1)) + return list(range(int(match[0][0]), int(match[0][1]) + 1)) else: - return tuple(range(len(per_cpu_times()))) + return list(range(len(per_cpu_times()))) @wrap_exceptions def cpu_affinity_set(self, cpus): From d717b45eb5559dceead926de62a2bf42ca690fbb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Jan 2017 15:09:32 +0100 Subject: [PATCH 113/922] fix osx test --- psutil/tests/test_osx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 6e7a58917..69d6c8408 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -151,7 +151,7 @@ def test_cpu_count_physical(self): self.assertEqual(num, psutil.cpu_count(logical=False)) def test_cpu_freq(self): - freq = psutil.cpu_freq()[0] + freq = psutil.cpu_freq() self.assertEqual( freq.current * 1000 * 1000, sysctl("sysctl hw.cpufrequency")) self.assertEqual( From 8291347dd0f89131800983e6412b534911df695c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Jan 2017 17:17:24 +0100 Subject: [PATCH 114/922] try to fix occasional windows failure --- psutil/tests/test_process.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 161d5e5e1..089809b64 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1149,6 +1149,7 @@ def test_num_ctx_switches(self): self.fail("num ctx switches still the same after 50.000 iterations") def test_parent_ppid(self): + reap_children(recursive=True) this_parent = os.getpid() sproc = get_test_subprocess() p = psutil.Process(sproc.pid) From 88e96ffaafb8422121cd98775a408cd13bdbd572 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 31 Jan 2017 15:39:53 +0100 Subject: [PATCH 115/922] #fix 960 / Popen.wait: return negative exit code if process is killed by a signal --- HISTORY.rst | 2 ++ psutil/_psposix.py | 2 +- psutil/tests/test_process.py | 12 ++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 782b147c4..abd2d456e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,8 @@ - 950_: [Windows] Process.cpu_percent() was calculated incorrectly and showed higher number than real usage. - 959_: psutil exception objects could not be pickled. +- 960_: Popen.wait() did not return the correct negative exit status if process + is ``kill()``ed by a signal. 5.0.1 diff --git a/psutil/_psposix.py b/psutil/_psposix.py index acbc7855b..6debdb327 100644 --- a/psutil/_psposix.py +++ b/psutil/_psposix.py @@ -110,7 +110,7 @@ def waitcall(): # process exited due to a signal; return the integer of # that signal if os.WIFSIGNALED(status): - return os.WTERMSIG(status) + return -os.WTERMSIG(status) # process exited using exit(2) system call; return the # integer exit(2) system call has been called with elif os.WIFEXITED(status): diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 089809b64..bdbb72a9d 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -104,7 +104,7 @@ def test_kill(self): sig = p.wait() self.assertFalse(psutil.pid_exists(test_pid)) if POSIX: - self.assertEqual(sig, signal.SIGKILL) + self.assertEqual(sig, -signal.SIGKILL) def test_terminate(self): sproc = get_test_subprocess() @@ -114,7 +114,7 @@ def test_terminate(self): sig = p.wait() self.assertFalse(psutil.pid_exists(test_pid)) if POSIX: - self.assertEqual(sig, signal.SIGTERM) + self.assertEqual(sig, -signal.SIGTERM) def test_send_signal(self): sig = signal.SIGKILL if POSIX else signal.SIGTERM @@ -124,7 +124,7 @@ def test_send_signal(self): exit_sig = p.wait() self.assertFalse(psutil.pid_exists(p.pid)) if POSIX: - self.assertEqual(exit_sig, sig) + self.assertEqual(exit_sig, -sig) # sproc = get_test_subprocess() p = psutil.Process(sproc.pid) @@ -155,7 +155,7 @@ def test_wait(self): p.kill() code = p.wait() if POSIX: - self.assertEqual(code, signal.SIGKILL) + self.assertEqual(code, -signal.SIGKILL) else: self.assertEqual(code, 0) self.assertFalse(p.is_running()) @@ -165,7 +165,7 @@ def test_wait(self): p.terminate() code = p.wait() if POSIX: - self.assertEqual(code, signal.SIGTERM) + self.assertEqual(code, -signal.SIGTERM) else: self.assertEqual(code, 0) self.assertFalse(p.is_running()) @@ -231,7 +231,7 @@ def test_wait_timeout_0(self): else: break if POSIX: - self.assertEqual(code, signal.SIGKILL) + self.assertEqual(code, -signal.SIGKILL) else: self.assertEqual(code, 0) self.assertFalse(p.is_running()) From db7a18a25facf8651844b84df946a7cf7997a346 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 31 Jan 2017 16:38:58 +0100 Subject: [PATCH 116/922] fix test --- psutil/tests/test_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 753e52691..18a5ae2d7 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -111,7 +111,7 @@ def test(procs, callback): gone, alive = test(procs, callback) self.assertIn(sproc3.pid, [x.pid for x in gone]) if POSIX: - self.assertEqual(gone.pop().returncode, signal.SIGTERM) + self.assertEqual(gone.pop().returncode, -signal.SIGTERM) else: self.assertEqual(gone.pop().returncode, 1) self.assertEqual(l, [sproc3.pid]) From 48ec7777fb03afeed826bceb71d72652ac15373e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 31 Jan 2017 16:58:01 +0100 Subject: [PATCH 117/922] fix windows test failure --- psutil/tests/test_process.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 747504735..fdb9f03d2 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1799,8 +1799,11 @@ def cwd(self, ret, proc): try: st = os.stat(ret) except OSError as err: + if WINDOWS and err.errno in \ + psutil._psplatform.ACCESS_DENIED_SET: + pass # directory has been removed in mean time - if err.errno != errno.ENOENT: + elif err.errno != errno.ENOENT: raise else: self.assertTrue(stat.S_ISDIR(st.st_mode)) From 69a273209a41f3c51bdc22d12c9f833ccc68dd55 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 15:03:27 +0100 Subject: [PATCH 118/922] fix #961 / Windows / WindowsService.description(): catch ERROR_MUI_FILE_NOT_FOUND and return an empty string --- HISTORY.rst | 2 ++ psutil/arch/windows/services.c | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 755a6d7fd..52cb04d05 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,8 @@ - 948_: cannot install psutil with PYTHONOPTIMIZE=2. - 950_: [Windows] Process.cpu_percent() was calculated incorrectly and showed higher number than real usage. +- 961_: [Windows] WindowsService.description() may fail with + ERROR_MUI_FILE_NOT_FOUND. 5.0.1 diff --git a/psutil/arch/windows/services.c b/psutil/arch/windows/services.c index 7923ddc27..8e7fff336 100644 --- a/psutil/arch/windows/services.c +++ b/psutil/arch/windows/services.c @@ -376,6 +376,12 @@ psutil_winservice_query_descr(PyObject *self, PyObject *args) { bytesNeeded = 0; QueryServiceConfig2(hService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, &bytesNeeded); + if (GetLastError() == ERROR_MUI_FILE_NOT_FOUND) { + // Also services.msc fails in the same manner, so we return an + // empty string. + CloseServiceHandle(hService); + return Py_BuildValue("s", ""); + } if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { PyErr_SetFromWindowsErr(0); goto error; From f49c29bc22ce323c8965d342b15e824d91d59227 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 15:21:13 +0100 Subject: [PATCH 119/922] win: fix unicode decode error --- scripts/winservices.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/winservices.py b/scripts/winservices.py index fed6a734e..1a65adcef 100755 --- a/scripts/winservices.py +++ b/scripts/winservices.py @@ -45,7 +45,7 @@ def main(): for service in psutil.win_service_iter(): info = service.as_dict() - print("%s (%s)" % (info['name'], info['display_name'])) + print("%r (%r)" % (info['name'], info['display_name'])) print("status: %s, start: %s, username: %s, pid: %s" % ( info['status'], info['start_type'], info['username'], info['pid'])) print("binpath: %s" % info['binpath']) From b524439056484e81a17407dc31cfcea1ff81715b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 15:33:24 +0100 Subject: [PATCH 120/922] fix #961: [Windows] WindowsService.description() may fail with ERROR_MUI_FILE_NOT_FOUND. --- HISTORY.rst | 2 ++ psutil/arch/windows/services.c | 6 ++++++ psutil/tests/test_process.py | 5 ++++- scripts/winservices.py | 2 +- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index abd2d456e..fb7cddefe 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,8 @@ - 959_: psutil exception objects could not be pickled. - 960_: Popen.wait() did not return the correct negative exit status if process is ``kill()``ed by a signal. +- 961_: [Windows] WindowsService.description() may fail with + ERROR_MUI_FILE_NOT_FOUND. 5.0.1 diff --git a/psutil/arch/windows/services.c b/psutil/arch/windows/services.c index 7923ddc27..cb85afb52 100644 --- a/psutil/arch/windows/services.c +++ b/psutil/arch/windows/services.c @@ -308,6 +308,12 @@ psutil_winservice_query_status(PyObject *self, PyObject *args) { // right size. QueryServiceStatusEx(hService, SC_STATUS_PROCESS_INFO, NULL, 0, &bytesNeeded); + if (GetLastError() == ERROR_MUI_FILE_NOT_FOUND) { + // Also services.msc fails in the same manner, so we return an + // empty string. + CloseServiceHandle(hService); + return Py_BuildValue("s", ""); + } if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { PyErr_SetFromWindowsErr(0); goto error; diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index bdbb72a9d..db86290b0 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1809,7 +1809,10 @@ def cwd(self, ret, proc): st = os.stat(ret) except OSError as err: # directory has been removed in mean time - if err.errno != errno.ENOENT: + if WINDOWS and err.errno in \ + psutil._psplatform.ACCESS_DENIED_SET: + pass + elif err.errno != errno.ENOENT: raise else: self.assertTrue(stat.S_ISDIR(st.st_mode)) diff --git a/scripts/winservices.py b/scripts/winservices.py index fed6a734e..1a65adcef 100755 --- a/scripts/winservices.py +++ b/scripts/winservices.py @@ -45,7 +45,7 @@ def main(): for service in psutil.win_service_iter(): info = service.as_dict() - print("%s (%s)" % (info['name'], info['display_name'])) + print("%r (%r)" % (info['name'], info['display_name'])) print("status: %s, start: %s, username: %s, pid: %s" % ( info['status'], info['start_type'], info['username'], info['pid'])) print("binpath: %s" % info['binpath']) From b934815fbb8b4aadbed43d807dc8bc3caec13df6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 15:37:54 +0100 Subject: [PATCH 121/922] fix win service description (again) --- psutil/arch/windows/services.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/psutil/arch/windows/services.c b/psutil/arch/windows/services.c index cb85afb52..26e582255 100644 --- a/psutil/arch/windows/services.c +++ b/psutil/arch/windows/services.c @@ -382,6 +382,12 @@ psutil_winservice_query_descr(PyObject *self, PyObject *args) { bytesNeeded = 0; QueryServiceConfig2(hService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, &bytesNeeded); + if (GetLastError() == ERROR_MUI_FILE_NOT_FOUND) { + // Also services.msc fails in the same manner, so we return an + // empty string. + CloseServiceHandle(hService); + return Py_BuildValue("s", ""); + } if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { PyErr_SetFromWindowsErr(0); goto error; From cba54bcfd8c0e618d087e5079f33b6be7d518a37 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 16:29:02 +0100 Subject: [PATCH 122/922] update HISTORY / README --- HISTORY.rst | 1 + README.rst | 4 ++-- docs/index.rst | 2 +- scripts/{sensors.py => temperatures.py} | 0 4 files changed, 4 insertions(+), 3 deletions(-) rename scripts/{sensors.py => temperatures.py} (100%) diff --git a/HISTORY.rst b/HISTORY.rst index fb7cddefe..2aef89bd2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,7 @@ **Enhancements** - 357_: added psutil.Process.cpu_num() (what CPU a process is on). +- 371_: added psutil.sensors_temperatures() (Linux only). - 941_: added psutil.cpu_freq() (CPU frequency). - 956_: cpu_affinity([]) can now be used as an alias to set affinity against all eligible CPUs. diff --git a/README.rst b/README.rst index 00fbc87e2..f890fcb38 100644 --- a/README.rst +++ b/README.rst @@ -181,8 +181,8 @@ Network {'eth0': snicstats(isup=True, duplex=, speed=100, mtu=1500), 'lo': snicstats(isup=True, duplex=, speed=0, mtu=65536)} -Sensors (Linux only) -==================== +Sensors +======= .. code-block:: python diff --git a/docs/index.rst b/docs/index.rst index 87ebf7347..e5431e62f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -638,7 +638,7 @@ Sensors shwtemp(label='Core 2', current=45.0, high=100.0, critical=100.0), shwtemp(label='Core 3', current=47.0, high=100.0, critical=100.0)]} - See also `sensors.py `__ + See also `temperatures.py `__ for an example application. .. warning:: diff --git a/scripts/sensors.py b/scripts/temperatures.py similarity index 100% rename from scripts/sensors.py rename to scripts/temperatures.py From ed0975dec40434b0e40bf8681325ffdaa8c5858d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 16:31:23 +0100 Subject: [PATCH 123/922] fix memleaks test --- psutil/tests/test_memory_leaks.py | 6 ++++++ psutil/tests/test_misc.py | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 44be1ec58..b6ebe9b62 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -567,6 +567,12 @@ def test_boot_time(self): def test_users(self): self.execute(psutil.users) + @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), + "platform not supported") + @skip_if_linux() + def test_sensors_temperatures(self): + self.execute(psutil.users) + if WINDOWS: # --- win services diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 8697bcff7..491ab63ac 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -469,11 +469,11 @@ def test_winservices(self): def test_cpu_distribution(self): self.assert_syntax('cpu_distribution.py') - def test_sensors(self): + def test_temperatures(self): if hasattr(psutil, "sensors_temperatures"): - self.assert_stdout('sensors.py') + self.assert_stdout('temperatures.py') else: - self.assert_syntax('sensors.py') + self.assert_syntax('temperatures.py') # =================================================================== From 1800329c0dc931bc66b6b504d3f4cc32e62d29f8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 17:10:59 +0100 Subject: [PATCH 124/922] update README / HISTORY --- HISTORY.rst | 50 +------------------------------------------------- README.rst | 14 ++++++++------ 2 files changed, 9 insertions(+), 55 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2aef89bd2..1254e5af4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ - 357_: added psutil.Process.cpu_num() (what CPU a process is on). - 371_: added psutil.sensors_temperatures() (Linux only). - 941_: added psutil.cpu_freq() (CPU frequency). +- 955_: added psutil.sensors_battery() (Linux, Windows, only). - 956_: cpu_affinity([]) can now be used as an alias to set affinity against all eligible CPUs. @@ -26,7 +27,6 @@ - 961_: [Windows] WindowsService.description() may fail with ERROR_MUI_FILE_NOT_FOUND. - 5.0.1 ===== @@ -47,7 +47,6 @@ taken into account. - 944_: [OpenBSD] psutil.pids() was omitting PID 0. - 5.0.0 ===== @@ -65,7 +64,6 @@ raising an exception. - 933_: [Windows] memory leak in cpu_stats() and WindowsService.description(). - 4.4.2 ===== @@ -75,7 +73,6 @@ - 931_: psutil no longer compiles on Solaris. - 4.4.1 ===== @@ -85,7 +82,6 @@ - 927_: ``Popen.__del__`` may cause maximum recursion depth error. - 4.4.0 ===== @@ -122,7 +118,6 @@ OSError with no exception set if process is gone. - 916_: [OSX] fix many compilation warnings. - 4.3.1 ===== @@ -147,7 +142,6 @@ unit (ms instead of sec). - 870_: [Windows] Handle leak inside psutil_get_process_data. - 4.3.0 ===== @@ -171,7 +165,6 @@ - 816_: [Windows] fixed net_io_counter() values wrapping after 4.3GB in Windows Vista (NT 6.0) and above using 64bit values from newer win APIs. - 4.2.0 ===== @@ -194,7 +187,6 @@ - 813_: Process.as_dict() should ignore extraneous attribute names which gets attached to the Process instance. - 4.1.0 ===== @@ -222,7 +214,6 @@ - 788_: [NetBSD] virtual_memory()'s buffers and shared values were set to 0. - 790_: [OSX] psutil won't compile on OSX 10.4. - 4.0.0 ===== @@ -266,7 +257,6 @@ broken on 2.4 kernels. - 770_: [NetBSD] disk_io_counters() metrics didn't update. - 3.4.2 ===== @@ -282,7 +272,6 @@ - 724_: [FreeBSD] psutil.virtual_memory().total is incorrect. - 730_: [FreeBSD] psutil.virtual_memory() crashes. - 3.4.1 ===== @@ -306,7 +295,6 @@ due to missing /proc/vmstat. - 724_: [FreeBSD] virtual_memory().total is slightly incorrect. - 3.3.0 ===== @@ -322,7 +310,6 @@ - 692_: [UNIX] Process.name() is no longer cached as it may change. - 3.2.2 ===== @@ -341,7 +328,6 @@ - 688_: [Windows] compilation fails with MSVC 2015, Python 3.5. (patch by Mike Sarahan) - 3.2.1 ===== @@ -351,7 +337,6 @@ - 677_: [Linux] can't install psutil due to bug in setup.py. - 3.2.0 ===== @@ -391,7 +376,6 @@ - 675_: [Linux] net_connections(); UnicodeDecodeError may occur when listing UNIX sockets. - 3.1.1 ===== @@ -403,7 +387,6 @@ - 645_: [Linux] psutil.cpu_times_percent() may produce negative results. - 656_: 'from psutil import *' does not work. - 3.1.0 ===== @@ -436,7 +419,6 @@ - 653_: [Windows] Add inet_ntop function for Windows XP to support IPv6. - 641_: [Windows] Replace deprecated string functions with safe equivalents. - 3.0.1 ===== @@ -449,7 +431,6 @@ - 635_: [UNIX] crash on module import if 'enum' package is installed on python < 3.4. - 3.0.0 ===== @@ -497,7 +478,6 @@ - 628_: [Linux] Process.name() truncates process name in case it contains spaces or parentheses. - 2.2.1 ===== @@ -508,7 +488,6 @@ - 496_: [Linux] fix "ValueError: ambiguos inode with multiple PIDs references" (patch by Bruno Binet) - 2.2.0 ===== @@ -539,7 +518,6 @@ - 571_: [Linux] Process.open_files() might swallow AccessDenied exceptions and return an incomplete list of open files. - 2.1.3 ===== @@ -547,7 +525,6 @@ - 536_: [Linux]: fix "undefined symbol: CPU_ALLOC" compilation error. - 2.1.2 ===== @@ -578,7 +555,6 @@ (< 2.6.5) (patch by Yaolong Huang) - 533_: [Linux] Process.memory_maps() may raise TypeError on old Linux distros. - 2.1.1 ===== @@ -591,7 +567,6 @@ - 460_: [Windows] net_io_counters() wraps after 4G. - 491_: [Linux] psutil.net_connections() exceptions. (patch by Alexander Grothe) - 2.1.0 ===== @@ -607,7 +582,6 @@ Roudsari) - 489_: [Linux] psutil.disk_partitions() return an empty list. - 2.0.0 ===== @@ -776,7 +750,6 @@ DeprecationWarning. - Process instances' "retcode" attribute returned by psutil.wait_procs() has been renamed to "returncode" for consistency with subprocess.Popen. - 1.2.1 ===== @@ -789,7 +762,6 @@ DeprecationWarning. - 425_: [Solaris] crash on import due to failure at determining BOOT_TIME. - 443_: [Linux] can't set CPU affinity on systems with more than 64 cores. - 1.2.0 ===== @@ -807,7 +779,6 @@ DeprecationWarning. - 348_: [Windows XP/Vista] fix "ImportError: DLL load failed" occurring on module import. - 1.1.3 ===== @@ -818,7 +789,6 @@ DeprecationWarning. - 442_: [Linux] psutil won't compile on certain version of Linux because of missing prlimit(2) syscall. - 1.1.2 ===== @@ -829,7 +799,6 @@ DeprecationWarning. - 442_: [Linux] psutil won't compile on Debian 6.0 because of missing prlimit(2) syscall. - 1.1.1 ===== @@ -840,7 +809,6 @@ DeprecationWarning. - 442_: [Linux] psutil won't compile on kernels < 2.6.36 due to missing prlimit(2) syscall. - 1.1.0 ===== @@ -871,7 +839,6 @@ DeprecationWarning. - 408_: turn STATUS_* and CONN_* constants into plain Python strings. - 1.0.1 ===== @@ -881,7 +848,6 @@ DeprecationWarning. - 405_: network_io_counters(pernic=True) no longer works as intended in 1.0.0. - 1.0.0 ===== @@ -911,7 +877,6 @@ DeprecationWarning. renamed to 'laddr' and 'raddr'. - psutil.network_io_counters() renamed to psutil.net_io_counters(). - 0.7.1 ===== @@ -925,7 +890,6 @@ DeprecationWarning. - 372_: [BSD] different process methods raise NoSuchProcess instead of AccessDenied. - 0.7.0 ===== @@ -989,7 +953,6 @@ DeprecationWarning. will raise NotImplementedError instead of RuntimeError. - psutil.error module is deprecated and scheduled for removal. - 0.6.1 ===== @@ -1010,7 +973,6 @@ DeprecationWarning. - process exe can now return an empty string instead of raising AccessDenied. - process exe is no longer resolved in case it's a symlink. - 0.6.0 ===== @@ -1100,7 +1062,6 @@ DeprecationWarning. - [Windows and BSD] psutil.virtmem_usage() now returns information about swap memory instead of virtual memory. - 0.5.1 ===== @@ -1116,7 +1077,6 @@ DeprecationWarning. - 292_: [Linux] race condition in process files/threads/connections. - 294_: [Windows] Process CPU affinity is only able to set CPU #0. - 0.5.0 ===== @@ -1187,7 +1147,6 @@ DeprecationWarning. - psutil.STATUS_* constants can now be compared by using their string representation. - 0.4.1 ===== @@ -1202,7 +1161,6 @@ DeprecationWarning. - 236_: [Windows] memory/handle leak in Process's get_memory_info(), suspend() and resume() methods. - 0.4.0 ===== @@ -1243,7 +1201,6 @@ DeprecationWarning. line in /proc/meminfo. - 226_: [FreeBSD] crash at import time on FreeBSD 7 and minor. - 0.3.0 ===== @@ -1273,7 +1230,6 @@ DeprecationWarning. - 180_: [Windows] Process's get_num_threads() and get_threads() methods can raise NoSuchProcess exception while process still exists. - 0.2.1 ===== @@ -1316,7 +1272,6 @@ DeprecationWarning. - Process "uid" and "gid" properties are deprecated in favor of "uids" and "gids" properties. - 0.2.0 ===== @@ -1378,7 +1333,6 @@ DeprecationWarning. - psutil.Process.get_cpu_percent() and psutil.cpu_percent() no longer returns immediately by default (see issue 123). - 0.1.3 ===== @@ -1409,7 +1363,6 @@ DeprecationWarning. - 77_: NoSuchProcess wasn't raised on Process.create_time if kill() was used first. - 0.1.2 ===== @@ -1433,7 +1386,6 @@ DeprecationWarning. - 40_: test_get_cpu_times() failing on FreeBSD and OS X. - 42_: [Windows] get_memory_percent() raises AccessDenied. - 0.1.1 ===== diff --git a/README.rst b/README.rst index ddcdbbf68..17bab67a9 100644 --- a/README.rst +++ b/README.rst @@ -125,6 +125,7 @@ CPU >>> >>> psutil.cpu_freq() scpufreq(current=931.42925, min=800.0, max=3500.0) + >>> Memory ====== @@ -180,6 +181,7 @@ Network >>> psutil.net_if_stats() {'eth0': snicstats(isup=True, duplex=, speed=100, mtu=1500), 'lo': snicstats(isup=True, duplex=, speed=0, mtu=65536)} + >>> Sensors ======= @@ -198,6 +200,7 @@ Sensors >>> >>> psutil.sensors_battery() sbattery(percent=93, secsleft=16628, power_plugged=False) + >>> Other system info ================= @@ -219,12 +222,11 @@ Process management >>> import psutil >>> psutil.pids() - [1, 2, 3, 4, 5, 6, 7, 46, 48, 50, 51, 178, 182, 222, 223, 224, - 268, 1215, 1216, 1220, 1221, 1243, 1244, 1301, 1601, 2237, 2355, - 2637, 2774, 3932, 4176, 4177, 4185, 4187, 4189, 4225, 4243, 4245, - 4263, 4282, 4306, 4311, 4312, 4313, 4314, 4337, 4339, 4357, 4358, - 4363, 4383, 4395, 4408, 4433, 4443, 4445, 4446, 5167, 5234, 5235, - 5252, 5318, 5424, 5644, 6987, 7054, 7055, 7071] + [1, 2, 3, 4, 5, 6, 7, 46, 48, 50, 51, 178, 182, 222, 223, 224, 268, 1215, 1216, 1220, 1221, + 1243, 1244, 1301, 1601, 2237, 2355, 2637, 2774, 3932, 4176, 4177, 4185, 4187, 4189, 4225, + 4243, 4245, 4263, 4282, 4306, 4311, 4312, 4313, 4314, 4337, 4339, 4357, 4358, 4363, 4383, + 4395, 4408, 4433, 4443, 4445, 4446, 5167, 5234, 5235, 5252, 5318, 5424, 5644, 6987, 7054, + 7055, 7071] >>> >>> p = psutil.Process(7055) >>> p.name() From b8901419275d1111e13165f62528e89938cd228e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 17:26:53 +0100 Subject: [PATCH 125/922] update IDEAS --- IDEAS | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/IDEAS b/IDEAS index 247b8b386..f565b991f 100644 --- a/IDEAS +++ b/IDEAS @@ -5,6 +5,7 @@ A collection of ideas and notes about stuff to implement in future versions. "#NNN" occurrences refer to bug tracker issues at: https://github.com/giampaolo/psutil/issues + PLATFORMS ========= @@ -15,15 +16,11 @@ PLATFORMS - HP-UX -APIS -==== - -- cpu_info() (#550) - - FEATURES ======== +- #371: sensors_temperatures() at least for OSX. + - #669: Windows / net_if_addrs(): return broadcast addr. - #550: CPU info (frequency, architecture, threads per core, cores per socket, @@ -50,9 +47,6 @@ FEATURES - (Linux) locked files via /proc/locks: https://www.centos.org/docs/5/html/5.2/Deployment_Guide/s2-proc-locks.html -- #371: CPU temperature (apparently OSX and Linux only; on Linux it requires - lm-sensors lib). - - #269: NIC rx/tx queue. This should probably go into net_if_stats(). Figure out on what platforms this is supported: Linux: yes @@ -153,8 +147,6 @@ FEATURES - #550: number of threads per core. -- Have psutil.Process().cpu_affinity([]) be an alias for "all CPUs"? - BUGFIXES ======== From 673029e1cab61b215e5836039f2c7adfabb0d7a4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 17:28:40 +0100 Subject: [PATCH 126/922] update doc --- docs/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index ec35d7695..84829161f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -195,6 +195,8 @@ CPU Return CPU frequency as a nameduple including *current*, *min* and *max* frequencies expressed in Mhz. + On Linux **current** frequency reports the real-time value, on all other + platforms it represents the nominal "fixed" value. If *percpu* is ``True`` and the system supports per-cpu frequency retrieval (Linux only) a list of frequencies is returned for each CPU, if not, a list with a single element is returned. From 80d34325fe2e91990e635c6594052f7f42bc88e8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 18:07:19 +0100 Subject: [PATCH 127/922] make sensors_temperatures() always available and return {} if appropriate; also fix tests --- psutil/_pslinux.py | 69 ++++++++++++++++++------------------ psutil/tests/test_misc.py | 3 +- psutil/tests/test_windows.py | 10 ++++-- 3 files changed, 43 insertions(+), 39 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 0d5f6ecb3..2e0023f50 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1067,40 +1067,39 @@ def disk_partitions(all=False): # ===================================================================== -if os.path.exists('/sys/class/hwmon'): - - def sensors_temperatures(): - """Return hardware (CPU and others) temperatures as a dict - including hardware name, label, current, max and critical - temperatures. - - Implementation notes: - - /sys/class/hwmon looks like the most recent interface to - retrieve this info, and this implementation relies on it - only (old distros will probably use something else) - - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon - - /sys/class/thermal/thermal_zone* is another one but it's more - difficult to parse - """ - ret = collections.defaultdict(list) - basenames = sorted(set( - [x.split('_')[0] for x in - glob.glob('/sys/class/hwmon/hwmon*/temp*_*')])) - for base in basenames: - unit_name = cat(os.path.join(os.path.dirname(base), 'name')) - label = cat(base + '_label', fallback='') - current = float(cat(base + '_input')) / 1000.0 - high = cat(base + '_max', fallback=None) - critical = cat(base + '_crit', fallback=None) - - if high is not None: - high = float(high) / 1000.0 - if critical is not None: - critical = float(critical) / 1000.0 - - ret[unit_name].append((label, current, high, critical)) +def sensors_temperatures(): + """Return hardware (CPU and others) temperatures as a dict + including hardware name, label, current, max and critical + temperatures. + + Implementation notes: + - /sys/class/hwmon looks like the most recent interface to + retrieve this info, and this implementation relies on it + only (old distros will probably use something else) + - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon + - /sys/class/thermal/thermal_zone* is another one but it's more + difficult to parse + """ + ret = collections.defaultdict(list) + # Will return an empty dict if path does not exist. + basenames = sorted(set( + [x.split('_')[0] for x in + glob.glob('/sys/class/hwmon/hwmon*/temp*_*')])) + for base in basenames: + unit_name = cat(os.path.join(os.path.dirname(base), 'name')) + label = cat(base + '_label', fallback='') + current = float(cat(base + '_input')) / 1000.0 + high = cat(base + '_max', fallback=None) + critical = cat(base + '_crit', fallback=None) + + if high is not None: + high = float(high) / 1000.0 + if critical is not None: + critical = float(critical) / 1000.0 + + ret[unit_name].append((label, current, high, critical)) - return ret + return ret def sensors_battery(): @@ -1108,8 +1107,8 @@ def sensors_battery(): if not os.path.exists(root): return None - power_plugged = \ - cat("/sys/class/power_supply/AC0/online", fallback=b"0") == b"1" + power_plugged = cat(os.path.join(POWER_SUPPLY_PATH, "AC0/online"), + fallback=b"0") == b"1" energy_now = int(cat(root + "/energy_now")) power_now = int(cat(root + "/power_now")) percent = int(cat(root + "/capacity")) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 1d7d71ce1..358ac0747 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -476,7 +476,8 @@ def test_temperatures(self): self.assert_syntax('temperatures.py') def test_battery(self): - if hasattr(psutil, "sensors_battery"): + if hasattr(psutil, "sensors_battery") and \ + psutil.sensors_battery() is not None: self.assert_stdout('battery.py') else: self.assert_syntax('battery.py') diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index aca8afbb0..39709e494 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -169,10 +169,14 @@ class TestSensorsBattery(unittest.TestCase): def test_percent(self): w = wmi.WMI() battery_psutil = psutil.sensors_battery() - battery_wmi = w.query('select * from Win32_Battery')[0] if battery_psutil is None: - self.assertNot(battery_wmi.EstimatedChargeRemaining) - return + with self.assertRaises(IndexError): + w.query('select * from Win32_Battery')[0] + else: + battery_wmi = w.query('select * from Win32_Battery')[0] + if battery_psutil is None: + self.assertNot(battery_wmi.EstimatedChargeRemaining) + return self.assertAlmostEqual( battery_psutil.percent, battery_wmi.EstimatedChargeRemaining, From 55ac1de8f1a322fc982d7deec75f04b06acf3bc5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 18:16:47 +0100 Subject: [PATCH 128/922] fix windows test --- psutil/tests/test_windows.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 39709e494..cf6825fe5 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -178,11 +178,11 @@ def test_percent(self): self.assertNot(battery_wmi.EstimatedChargeRemaining) return - self.assertAlmostEqual( - battery_psutil.percent, battery_wmi.EstimatedChargeRemaining, - delta=1) - self.assertEqual( - battery_psutil.power_plugged, battery_wmi.BatteryStatus == 1) + self.assertAlmostEqual( + battery_psutil.percent, battery_wmi.EstimatedChargeRemaining, + delta=1) + self.assertEqual( + battery_psutil.power_plugged, battery_wmi.BatteryStatus == 1) def test_emulate_no_battery(self): with mock.patch("psutil._pswindows.cext.sensors_battery", From 5dfb92a434ef97bdf39895e76fcfaee6400adf9b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 18:45:37 +0100 Subject: [PATCH 129/922] try to debug win failure --- psutil/tests/test_process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 74ae93300..648f8df33 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1149,7 +1149,6 @@ def test_num_ctx_switches(self): self.fail("num ctx switches still the same after 50.000 iterations") def test_parent_ppid(self): - reap_children(recursive=True) this_parent = os.getpid() sproc = get_test_subprocess() p = psutil.Process(sproc.pid) @@ -1160,7 +1159,8 @@ def test_parent_ppid(self): for p in psutil.process_iter(): if p.pid == sproc.pid: continue - self.assertNotEqual(p.ppid(), this_parent) + # XXX: sometimes this fails on Windows; not sure why. + self.assertNotEqual(p.ppid(), this_parent, msg=p) def test_children(self): p = psutil.Process() From 84e706afcfa5e24523d6768579d0a11979cd15d8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 19:24:30 +0100 Subject: [PATCH 130/922] pre release --- HISTORY.rst | 2 +- docs/index.rst | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1254e5af4..78d89996a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.1.0 ===== -*XXXX-XX-XX* +*2017-02-01* **Enhancements** diff --git a/docs/index.rst b/docs/index.rst index 84829161f..0d9989135 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2118,8 +2118,9 @@ take a look at the Timeline ======== -- 2016-12-21: `5.0.1 `__ - `what's new `__ -- 2016-11-06: `5.0.0 `__ - `what's new `__ +- 2017-02-01: `5.1.0 `__ - `what's new `__ +- 2016-12-21: `5.0.1 `__ - `what's new `__ +- 2016-11-06: `5.0.0 `__ - `what's new `__ - 2016-10-26: `4.4.2 `__ - `what's new `__ - 2016-10-25: `4.4.1 `__ - `what's new `__ - 2016-10-23: `4.4.0 `__ - `what's new `__ From cbd83bf3087ba83f27f0454ac6ee40084ca3d092 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 19:35:23 +0100 Subject: [PATCH 131/922] update readme --- HISTORY.rst | 1 + psutil/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 78d89996a..f867cbd77 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -21,6 +21,7 @@ - 948_: cannot install psutil with PYTHONOPTIMIZE=2. - 950_: [Windows] Process.cpu_percent() was calculated incorrectly and showed higher number than real usage. +- 951_: [Windows] the uploaded wheels for Python 3.6 64 bit didn't work. - 959_: psutil exception objects could not be pickled. - 960_: Popen.wait() did not return the correct negative exit status if process is ``kill()``ed by a signal. diff --git a/psutil/__init__.py b/psutil/__init__.py index 9054c615a..a879e0dc1 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -192,7 +192,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.1.0" +__version__ = "5.1.1" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From c2d77bf3536da1c862da32ca8025b60845079a48 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 20:11:35 +0100 Subject: [PATCH 132/922] fix #964: windows Process.username() / psutil.users() may return badly encoded chars on python 3 --- HISTORY.rst | 10 ++++++++++ psutil/_psutil_windows.c | 11 +++++++++++ 2 files changed, 21 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index f867cbd77..77e18b458 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.1.1 +===== + +*XXXX-XX-XX* + +**Bug fixes** + +- 964_: [Windows] Process.username() and psutil.users() may return badly + decoding character on Python 3. + 5.1.0 ===== diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 74edc6cd8..e1e537236 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1418,8 +1418,13 @@ psutil_proc_username(PyObject *self, PyObject *args) { memcpy(&fullName[domainNameSize + 1], name, nameSize); fullName[domainNameSize + 1 + nameSize] = '\0'; +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 + py_unicode = PyUnicode_DecodeLocaleAndSize( + fullName, _tcslen(fullName), "surrogateescape"); +#else py_unicode = PyUnicode_Decode( fullName, _tcslen(fullName), Py_FileSystemDefaultEncoding, "replace"); +#endif free(fullName); free(name); @@ -2660,9 +2665,15 @@ psutil_users(PyObject *self, PyObject *args) { station_info.ConnectTime.dwLowDateTime - 116444736000000000LL; unix_time /= 10000000; +#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 + py_buffer_user_encoded = PyUnicode_DecodeLocaleAndSize( + buffer_user, _tcslen(buffer_user), "surrogateescape"); +#else py_buffer_user_encoded = PyUnicode_Decode( buffer_user, _tcslen(buffer_user), Py_FileSystemDefaultEncoding, "replace"); +#endif + if (py_buffer_user_encoded == NULL) goto error; py_tuple = Py_BuildValue("OOd", py_buffer_user_encoded, py_address, From 53ccd9a56b20b25c627820b3ce66b86875e79e6d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 21:22:21 +0100 Subject: [PATCH 133/922] fix test failing on travis --- psutil/tests/test_misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 358ac0747..88d02f18a 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -469,6 +469,7 @@ def test_winservices(self): def test_cpu_distribution(self): self.assert_syntax('cpu_distribution.py') + @unittest.skipIf(TRAVIS, "unreliable on travis") def test_temperatures(self): if hasattr(psutil, "sensors_temperatures"): self.assert_stdout('temperatures.py') From 9060f675702ccc7d56a89d44e9a5167d7ad35a41 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 22:05:35 +0100 Subject: [PATCH 134/922] update README --- README.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 17bab67a9..36aa3947d 100644 --- a/README.rst +++ b/README.rst @@ -222,11 +222,10 @@ Process management >>> import psutil >>> psutil.pids() - [1, 2, 3, 4, 5, 6, 7, 46, 48, 50, 51, 178, 182, 222, 223, 224, 268, 1215, 1216, 1220, 1221, - 1243, 1244, 1301, 1601, 2237, 2355, 2637, 2774, 3932, 4176, 4177, 4185, 4187, 4189, 4225, - 4243, 4245, 4263, 4282, 4306, 4311, 4312, 4313, 4314, 4337, 4339, 4357, 4358, 4363, 4383, - 4395, 4408, 4433, 4443, 4445, 4446, 5167, 5234, 5235, 5252, 5318, 5424, 5644, 6987, 7054, - 7055, 7071] + [1, 2, 3, 4, 5, 6, 7, 46, 48, 50, 51, 178, 182, 222, 223, 224, 268, 1215, 1216, 1220, 1221, 1243, 1244, 1301, + 1601, 2237, 2355, 2637, 2774, 3932, 4176, 4177, 4185, 4187, 4189, 4225, 4243, 4245, 4263, 4282, 4306, 4311, + 4312, 4313, 4314, 4337, 4339, 4357, 4358, 4363, 4383, 4395, 4408, 4433, 4443, 4445, 4446, 5167, 5234, 5235, + 5252, 5318, 5424, 5644, 6987, 7054, 7055, 7071] >>> >>> p = psutil.Process(7055) >>> p.name() From d1e46249b769a44cc58c0da7b2c9749650aa5138 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 1 Feb 2017 22:06:07 +0100 Subject: [PATCH 135/922] update README --- README.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 36aa3947d..ddd27ee65 100644 --- a/README.rst +++ b/README.rst @@ -222,10 +222,10 @@ Process management >>> import psutil >>> psutil.pids() - [1, 2, 3, 4, 5, 6, 7, 46, 48, 50, 51, 178, 182, 222, 223, 224, 268, 1215, 1216, 1220, 1221, 1243, 1244, 1301, - 1601, 2237, 2355, 2637, 2774, 3932, 4176, 4177, 4185, 4187, 4189, 4225, 4243, 4245, 4263, 4282, 4306, 4311, - 4312, 4313, 4314, 4337, 4339, 4357, 4358, 4363, 4383, 4395, 4408, 4433, 4443, 4445, 4446, 5167, 5234, 5235, - 5252, 5318, 5424, 5644, 6987, 7054, 7055, 7071] + [1, 2, 3, 4, 5, 6, 7, 46, 48, 50, 51, 178, 182, 222, 223, 224, 268, 1215, 1216, 1220, 1221, 1243, 1244, + 1301, 1601, 2237, 2355, 2637, 2774, 3932, 4176, 4177, 4185, 4187, 4189, 4225, 4243, 4245, 4263, 4282, + 4306, 4311, 4312, 4313, 4314, 4337, 4339, 4357, 4358, 4363, 4383, 4395, 4408, 4433, 4443, 4445, 4446, + 5167, 5234, 5235, 5252, 5318, 5424, 5644, 6987, 7054, 7055, 7071] >>> >>> p = psutil.Process(7055) >>> p.name() From 547a6a6ff4be8bff9d14fca24b68a95a96743f1b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 2 Feb 2017 13:10:24 +0100 Subject: [PATCH 136/922] refactor test runner --- psutil/tests/runner.py | 34 ++++++++++++++++++++++------------ setup.py | 2 ++ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/psutil/tests/runner.py b/psutil/tests/runner.py index 1c282f685..88bcd6208 100755 --- a/psutil/tests/runner.py +++ b/psutil/tests/runner.py @@ -13,15 +13,25 @@ from psutil.tests import VERBOSITY -HERE = os.path.abspath(os.path.dirname(__file__)) -testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) - if x.endswith('.py') and x.startswith('test_') and not - x.startswith('test_memory_leaks')] -suite = unittest.TestSuite() -for tm in testmodules: - # ...so that "make test" will print the full test paths - tm = "psutil.tests.%s" % tm - suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) -result = unittest.TextTestRunner(verbosity=VERBOSITY).run(suite) -success = result.wasSuccessful() -sys.exit(0 if success else 1) +def get_suite(): + HERE = os.path.abspath(os.path.dirname(__file__)) + testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) + if x.endswith('.py') and x.startswith('test_') and not + x.startswith('test_memory_leaks')] + suite = unittest.TestSuite() + for tm in testmodules: + # ...so that "make test" will print the full test paths + tm = "psutil.tests.%s" % tm + suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) + return suite + + +def main(): + # run tests + result = unittest.TextTestRunner(verbosity=VERBOSITY).run(get_suite()) + success = result.wasSuccessful() + sys.exit(0 if success else 1) + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 01543bee8..47772da9e 100755 --- a/setup.py +++ b/setup.py @@ -270,6 +270,8 @@ def main(): license='BSD', packages=['psutil', 'psutil.tests'], ext_modules=extensions, + test_suite="psutil.tests.runner.get_suite", + tests_require=['ipaddress', 'mock', 'unittest2'], # see: python setup.py register --list-classifiers classifiers=[ 'Development Status :: 5 - Production/Stable', From 26af8bf2f1e883fb0a1f2bbebce828ac2205d7c1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 2 Feb 2017 13:30:03 +0100 Subject: [PATCH 137/922] add test for as_dict() --- psutil/tests/test_process.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 648f8df33..c0095e902 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1238,15 +1238,22 @@ def test_as_dict(self): if not isinstance(d['connections'], list): self.assertEqual(d['connections'], 'foo') + # Test ad_value is set on AccessDenied. + with mock.patch('psutil.Process.name', create=True, + side_effect=psutil.AccessDenied): + self.assertEqual( + p.as_dict(attrs=["name"], ad_value=1), {"name": 1}) + + # By default APIs raising NotImplementedError are + # supposed to be skipped. with mock.patch('psutil.Process.name', create=True, side_effect=NotImplementedError): - # By default APIs raising NotImplementedError are - # supposed to be skipped. d = p.as_dict() self.assertNotIn('name', list(d.keys())) # ...unless the user explicitly asked for some attr. with self.assertRaises(NotImplementedError): p.as_dict(attrs=["name"]) + # errors with self.assertRaises(TypeError): p.as_dict('name') From dc4faf1d7f50e1021f926c956361a6072c7db6f2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 2 Feb 2017 13:46:27 +0100 Subject: [PATCH 138/922] adjust as_dict() tests --- psutil/tests/test_process.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index c0095e902..5a0db615d 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1239,20 +1239,32 @@ def test_as_dict(self): self.assertEqual(d['connections'], 'foo') # Test ad_value is set on AccessDenied. - with mock.patch('psutil.Process.name', create=True, + with mock.patch('psutil.Process.nice', create=True, side_effect=psutil.AccessDenied): self.assertEqual( - p.as_dict(attrs=["name"], ad_value=1), {"name": 1}) + p.as_dict(attrs=["nice"], ad_value=1), {"nice": 1}) + + # Test that NoSuchProcess bubbles up. + with mock.patch('psutil.Process.nice', create=True, + side_effect=psutil.NoSuchProcess(p.pid, "name")): + self.assertRaises( + psutil.NoSuchProcess, p.as_dict, attrs=["nice"]) + + # Test that ZombieProcess is swallowed. + with mock.patch('psutil.Process.nice', create=True, + side_effect=psutil.ZombieProcess(p.pid, "name")): + self.assertEqual( + p.as_dict(attrs=["nice"], ad_value="foo"), {"nice": "foo"}) # By default APIs raising NotImplementedError are # supposed to be skipped. - with mock.patch('psutil.Process.name', create=True, + with mock.patch('psutil.Process.nice', create=True, side_effect=NotImplementedError): d = p.as_dict() - self.assertNotIn('name', list(d.keys())) + self.assertNotIn('nice', list(d.keys())) # ...unless the user explicitly asked for some attr. with self.assertRaises(NotImplementedError): - p.as_dict(attrs=["name"]) + p.as_dict(attrs=["nice"]) # errors with self.assertRaises(TypeError): From 892a55b4f8b4b5ae1d7a2ea37b10fb2089d952d6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 2 Feb 2017 13:48:41 +0100 Subject: [PATCH 139/922] fix fialing test on travis --- psutil/tests/test_bsd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index ff46ab334..9c1753d5e 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -359,7 +359,9 @@ def test_boot_time(self): # --- sensors_battery - @unittest.skipUnless(psutil.sensors_battery(), "no battery") + @unittest.skipUnless( + hasattr(psutil, "sensors_battery") and psutil.sensors_battery(), + "no battery") def test_sensors_battery(self): def secs2hours(secs): m, s = divmod(secs, 60) From 8815aac2673a623dc144474ed20706f9343a704d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 2 Feb 2017 14:07:16 +0100 Subject: [PATCH 140/922] fix #965: disk_io_counters() may miscalculate sector size and report the wrong number of bytes read and written --- HISTORY.rst | 2 ++ psutil/_pslinux.py | 12 +++++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 77e18b458..073f5b504 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ - 964_: [Windows] Process.username() and psutil.users() may return badly decoding character on Python 3. +- 965_: [Linux] disk_io_counters() may miscalculate sector size and report the + wrong number of bytes read and written. 5.1.0 ===== diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 2e0023f50..32c813706 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -243,9 +243,9 @@ def file_flags_to_mode(flags): return mode -def get_sector_size(): +def get_sector_size(partition): try: - with open(b"/sys/block/sda/queue/hw_sector_size") as f: + with open(b"/sys/block/%s/queue/hw_sector_size" % partition) as f: return int(f.read()) except (IOError, ValueError): # man iostat states that sectors are equivalent with blocks and @@ -254,9 +254,6 @@ def get_sector_size(): return 512 -SECTOR_SIZE = get_sector_size() - - @memoize def set_scputimes_ntuple(procfs_path): """Return a namedtuple of variable fields depending on the @@ -1027,8 +1024,9 @@ def get_partitions(): raise ValueError("not sure how to interpret line %r" % line) if name in partitions: - rbytes = rbytes * SECTOR_SIZE - wbytes = wbytes * SECTOR_SIZE + sector_size = get_sector_size(name) + rbytes = rbytes * sector_size + wbytes = wbytes * sector_size retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime, reads_merged, writes_merged, busy_time) return retdict From 11150d2588f47ec4b335ab2ec3591662fc6e88d7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 2 Feb 2017 14:13:27 +0100 Subject: [PATCH 141/922] fix broken test --- psutil/tests/test_linux.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index f28c70b87..2d659ad1a 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -51,7 +51,7 @@ SIOCGIFCONF = 0x8912 SIOCGIFHWADDR = 0x8927 if LINUX: - SECTOR_SIZE = psutil._psplatform.SECTOR_SIZE + SECTOR_SIZE = 512 # ===================================================================== @@ -987,7 +987,7 @@ def test_sector_size_mock(self): def open_mock(name, *args, **kwargs): if PY3 and isinstance(name, bytes): name = name.decode() - if name.startswith("/sys/block/sda/queue/hw_sector_size"): + if "hw_sector_size" in name: flag.append(None) raise IOError(errno.ENOENT, '') else: @@ -996,15 +996,9 @@ def open_mock(name, *args, **kwargs): flag = [] orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' - try: - with mock.patch(patch_point, side_effect=open_mock): - importlib.reload(psutil._pslinux) - importlib.reload(psutil) - self.assertEqual(flag, [None]) - self.assertEqual(psutil._pslinux.SECTOR_SIZE, 512) - finally: - importlib.reload(psutil._pslinux) - importlib.reload(psutil) + with mock.patch(patch_point, side_effect=open_mock): + psutil.disk_io_counters() + self.assertTrue(flag) def test_issue_687(self): # In case of thread ID: From 4157e6fe5a6f25400342f8c89b44456cc566ac9d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 2 Feb 2017 22:41:23 +0100 Subject: [PATCH 142/922] #966: sensors_battery() fails with no such file error --- HISTORY.rst | 2 ++ psutil/_pslinux.py | 48 ++++++++++++++++++++++++++++++++++++++++++---- scripts/battery.py | 2 +- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 073f5b504..1a21983b6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,8 @@ decoding character on Python 3. - 965_: [Linux] disk_io_counters() may miscalculate sector size and report the wrong number of bytes read and written. +- 966_: [Linux] sensors_battery() fails with no such file error. + 5.1.0 ===== diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 32c813706..a06ee73fb 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1101,16 +1101,56 @@ def sensors_temperatures(): def sensors_battery(): + """Return battery information. + Implementation note: it appears /sys/class/power_supply/BAT0/ + directory structure may vary and provide files with the same + meaning but under different names, see: + https://github.com/giampaolo/psutil/issues/966 + """ + null = object() + + def multi_cat(*paths): + """Read content of multiple files which may not exist. + # If none of them exist returns None. + """ + for path in paths: + ret = cat(path, fallback=null) + if ret != null: + return int(ret) + return None + root = os.path.join(POWER_SUPPLY_PATH, "BAT0") if not os.path.exists(root): return None + # Base metrics. + energy_now = multi_cat( + root + "/energy_now", + root + "/charge_now") + power_now = multi_cat( + root + "/power_now", + root + "/current_now") + energy_full = multi_cat( + root + "/energy_full", + root + "/charge_full") + if energy_now is None or power_now is None: + return None + + # Percent. If we have energy_full the percentage will be more + # accurate compared to reading /capacity file (float vs. int). + if energy_full is not None: + try: + percent = 100.0 * energy_now / energy_full + except ZeroDivisionError: + percent = 0.0 + else: + percent = int(cat(root + "/capacity"), fallback=null) + if percent == null: + return None + + # Secs left. power_plugged = cat(os.path.join(POWER_SUPPLY_PATH, "AC0/online"), fallback=b"0") == b"1" - energy_now = int(cat(root + "/energy_now")) - power_now = int(cat(root + "/power_now")) - percent = int(cat(root + "/capacity")) - if power_plugged: secsleft = _common.POWER_TIME_UNLIMITED else: diff --git a/scripts/battery.py b/scripts/battery.py index eb8b16bb5..23e0f669d 100755 --- a/scripts/battery.py +++ b/scripts/battery.py @@ -33,7 +33,7 @@ def main(): if batt is None: return sys.exit("no battery is installed") - print("charge: %s%%" % batt.percent) + print("charge: %s%%" % round(batt.percent, 2)) if batt.power_plugged: print("status: %s" % ( "charging" if batt.percent < 100 else "fully charged")) From 24964038e109799c890b71b0232a268fc3ed7358 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 2 Feb 2017 22:44:52 +0100 Subject: [PATCH 143/922] update HISTORY --- HISTORY.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 1a21983b6..a88682669 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,11 @@ *XXXX-XX-XX* + +**Enhancements** + +- 966_: [Linux] sensors_battery().percent is a float and is more precise. + **Bug fixes** - 964_: [Windows] Process.username() and psutil.users() may return badly From 1f30d13ced7faec37946b0eb7f8127b86fed4116 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 2 Feb 2017 22:49:30 +0100 Subject: [PATCH 144/922] #966: catch IOError: [Errno 19] No such device --- docs/index.rst | 3 ++- psutil/_pslinux.py | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 0d9989135..2c4c87ed8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -655,7 +655,8 @@ Sensors .. function:: sensors_battery() Return battery status information as a namedtuple including the following - values. If no battery is installed returns ``None``. + values. If no battery is installed or metrics can't be determined returns + ``None``. - **percent**: battery power left as a percentage. - **secsleft**: a rough approximation of how many seconds are left before the diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index a06ee73fb..2b724fd38 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -282,14 +282,12 @@ def set_scputimes_ntuple(procfs_path): def cat(fname, fallback=_DEFAULT, binary=True): """Return file content.""" try: - f = open_binary(fname) if binary else open_text(fname) + with open_binary(fname) if binary else open_text(fname) as f: + return f.read().strip() except IOError: if fallback != _DEFAULT: return fallback raise - else: - with f: - return f.read().strip() try: @@ -1110,8 +1108,8 @@ def sensors_battery(): null = object() def multi_cat(*paths): - """Read content of multiple files which may not exist. - # If none of them exist returns None. + """Attempt to read the content of multiple files which may + not exist. If none of them exist return None. """ for path in paths: ret = cat(path, fallback=null) From 0a02bdcf7b3fcfe9ab328148187ed9f4b9f93a00 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 2 Feb 2017 23:15:08 +0100 Subject: [PATCH 145/922] #966: sensors_battery().power_plugged may lie if AC0/online is not there; fallback on using /BAT0/status instead --- HISTORY.rst | 3 ++- docs/index.rst | 3 ++- psutil/_pslinux.py | 23 ++++++++++++++++++++--- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a88682669..b24f64443 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,7 +16,8 @@ decoding character on Python 3. - 965_: [Linux] disk_io_counters() may miscalculate sector size and report the wrong number of bytes read and written. -- 966_: [Linux] sensors_battery() fails with no such file error. +- 966_: [Linux] sensors_battery() may fail with "no such file error". +- 966_: [Linux] sensors_battery().power_plugged may lie. 5.1.0 diff --git a/docs/index.rst b/docs/index.rst index 2c4c87ed8..2e362272e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -665,7 +665,8 @@ Sensors :data:`psutil.POWER_TIME_UNLIMITED `. If it can't be determined it is set to :data:`psutil.POWER_TIME_UNKNOWN `. - - **power_plugged**: ``True`` if the AC power cable is connected. + - **power_plugged**: ``True`` if the AC power cable is connected, ``False`` + if not or ``None`` if it can't be determined. Example:: diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 2b724fd38..1d5005ebf 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1146,9 +1146,26 @@ def multi_cat(*paths): if percent == null: return None - # Secs left. - power_plugged = cat(os.path.join(POWER_SUPPLY_PATH, "AC0/online"), - fallback=b"0") == b"1" + # Is AC power cable plugged in? + if os.path.exists(os.path.join(POWER_SUPPLY_PATH, "AC0/online")): + power_plugged = cat( + os.path.join(POWER_SUPPLY_PATH, "AC0/online"), + fallback=b"0") == b"1" + elif os.path.exists(root + "/status"): + status = cat(root + "/status", fallback="").lower() + if status == "discharging": + power_plugged = False + elif status in ("charging", "full"): + power_plugged = True + else: + power_plugged = None + else: + power_plugged = None + + # Seconds left. + # Note to self: we may also calculate the charging ETA as per: + # https://github.com/thialfihar/dotfiles/blob/ + # 013937745fd9050c30146290e8f963d65c0179e6/bin/battery.py#L55 if power_plugged: secsleft = _common.POWER_TIME_UNLIMITED else: From 65037c09c73c8ea789965afc03cfff0f7f9d5e1e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 12:31:50 +0100 Subject: [PATCH 146/922] #966: add more sensors_battery() linux specific testst --- psutil/_pslinux.py | 2 +- psutil/tests/test_linux.py | 75 +++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 1d5005ebf..82cdb2d37 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1142,7 +1142,7 @@ def multi_cat(*paths): except ZeroDivisionError: percent = 0.0 else: - percent = int(cat(root + "/capacity"), fallback=null) + percent = int(cat(root + "/capacity", fallback=null)) if percent == null: return None diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 2d659ad1a..3e55d32ee 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1066,7 +1066,6 @@ def open_mock(name, *args, **kwargs): patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock) as m: self.assertEqual(psutil.sensors_battery().power_plugged, False) - self.assertGreaterEqual(psutil.sensors_battery().secsleft, 0) assert m.called def test_emulate_power_undetermined(self): @@ -1082,9 +1081,81 @@ def open_mock(name, *args, **kwargs): patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock) as m: self.assertEqual(psutil.sensors_battery().power_plugged, False) - self.assertGreaterEqual(psutil.sensors_battery().secsleft, 0) assert m.called + def test_emulate_no_base_files(self): + # Emulate a case where base metrics files are not present, + # in which case we're supposed to get None. + def open_mock(name, *args, **kwargs): + if name.startswith("/sys/class/power_supply/BAT0/energy_now") or \ + name.startswith("/sys/class/power_supply/BAT0/charge_now"): + raise IOError(errno.ENOENT, "") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: + self.assertIsNone(psutil.sensors_battery()) + assert m.called + + def test_emulate_energy_full_0(self): + # Emulate a case where energy_full files returns 0. + def open_mock(name, *args, **kwargs): + if name.startswith("/sys/class/power_supply/BAT0/energy_full"): + return io.BytesIO(b"0") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: + self.assertEqual(psutil.sensors_battery().percent, 0) + assert m.called + + def test_emulate_energy_full_not_avail(self): + # Emulate a case where energy_full file does not exist. + # Expected fallback on /capacity. + def open_mock(name, *args, **kwargs): + if name.startswith("/sys/class/power_supply/BAT0/energy_full"): + raise IOError(errno.ENOENT, "") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: + self.assertGreaterEqual(psutil.sensors_battery().percent, 0) + assert m.called + + def test_emulate_no_ac0_online(self): + # Emulate a case where /AC0/online file does not exist. + def path_exists_mock(name): + if name.startswith("/sys/class/power_supply/AC0/online"): + return False + else: + return orig_path_exists(name) + + orig_path_exists = os.path.exists + with mock.patch("psutil._pslinux.os.path.exists", + side_effect=path_exists_mock) as m: + psutil.sensors_battery() + assert m.called + + def test_emulate_no_power(self): + # Emulate a case where /AC0/online file nor /BAT0/status exist. + def path_exists_mock(name): + if name.startswith("/sys/class/power_supply/AC0/online") or \ + name.startswith("/sys/class/power_supply/BAT0/status"): + return False + else: + return orig_path_exists(name) + + orig_path_exists = os.path.exists + with mock.patch("psutil._pslinux.os.path.exists", + side_effect=path_exists_mock) as m: + self.assertIsNone(psutil.sensors_battery().power_plugged) + assert m.called # ===================================================================== # test process From 9ad0590f4d801cfbb0224da33127dc56dbd55130 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 12:50:20 +0100 Subject: [PATCH 147/922] pre release --- HISTORY.rst | 4 +--- docs/index.rst | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b24f64443..974623027 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,8 +3,7 @@ 5.1.1 ===== -*XXXX-XX-XX* - +*2017-02-03* **Enhancements** @@ -19,7 +18,6 @@ - 966_: [Linux] sensors_battery() may fail with "no such file error". - 966_: [Linux] sensors_battery().power_plugged may lie. - 5.1.0 ===== diff --git a/docs/index.rst b/docs/index.rst index 2e362272e..9b9ac0e02 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2120,7 +2120,8 @@ take a look at the Timeline ======== -- 2017-02-01: `5.1.0 `__ - `what's new `__ +- 2017-02-03: `5.1.1 `__ - `what's new `__ +- 2017-02-01: `5.1.0 `__ - `what's new `__ - 2016-12-21: `5.0.1 `__ - `what's new `__ - 2016-11-06: `5.0.0 `__ - `what's new `__ - 2016-10-26: `4.4.2 `__ - `what's new `__ From fef8ef01f016b2fbc7e89ebbf02b2da964ca466c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 13:08:07 +0100 Subject: [PATCH 148/922] be more cautious when converting to int() --- psutil/__init__.py | 2 +- psutil/_pslinux.py | 2 +- scripts/internal/download_exes.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index a879e0dc1..511fdacf2 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -192,7 +192,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.1.1" +__version__ = "5.1.2" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 82cdb2d37..1183052b3 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1114,7 +1114,7 @@ def multi_cat(*paths): for path in paths: ret = cat(path, fallback=null) if ret != null: - return int(ret) + return int(ret) if ret.isdigit() else ret return None root = os.path.join(POWER_SUPPLY_PATH, "BAT0") diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_exes.py index 2a40168c1..d8d2768b2 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_exes.py @@ -27,7 +27,7 @@ BASE_URL = 'https://ci.appveyor.com/api' -PY_VERSIONS = ['2.7', '3.3', '3.4', '3.5'] +PY_VERSIONS = ['2.7', '3.3', '3.4', '3.5', '3.6'] def exit(msg): From b83c45b56512301976f57edbaf6aae789791a149 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 13:30:42 +0100 Subject: [PATCH 149/922] update doc --- Makefile | 2 +- docs/index.rst | 91 +++++++++++++++++++++++++------------------------- 2 files changed, 47 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index 95db068cf..9f802ec18 100644 --- a/Makefile +++ b/Makefile @@ -258,4 +258,4 @@ bench-oneshot-2: install doc: cd docs && make html && cd _build/html/ && zip doc.zip -r . mv docs/_build/html/doc.zip . - echo "done; now manually upload doc.zip from here: https://pypi.python.org/pypi?:action=pkg_edit&name=psutil" + @echo "done; now manually upload doc.zip from here: https://pypi.python.org/pypi?:action=pkg_edit&name=psutil" diff --git a/docs/index.rst b/docs/index.rst index 9b9ac0e02..599091a12 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,16 +21,16 @@ About psutil (python system and process utilities) is a cross-platform library for retrieving information on running -**processes** and **system utilization** (CPU, memory, disks, network) in -**Python**. -It is useful mainly for **system monitoring**, **profiling** and **limiting -process resources** and **management of running processes**. +**processes** and **system utilization** (CPU, memory, disks, network, sensors) +in **Python**. +It is useful mainly for **system monitoring**, **profiling**, **limiting +process resources** and the **management of running processes**. It implements many functionalities offered by command line tools such as: *ps, top, lsof, netstat, ifconfig, who, df, kill, free, nice, ionice, iostat, iotop, uptime, pidof, tty, taskset, pmap*. It currently supports **Linux, Windows, OSX, Sun Solaris, FreeBSD, OpenBSD** and **NetBSD**, both **32-bit** and **64-bit** architectures, with Python -versions from **2.6 to 3.5** (users of Python 2.4 and 2.5 may use +versions from **2.6 to 3.6** (users of Python 2.4 and 2.5 may use `2.1.3 `__ version). `PyPy `__ is also known to work. @@ -44,7 +44,8 @@ The easiest way to install psutil is via ``pip``:: pip install psutil On UNIX this requires a C compiler (e.g. gcc) installed. On Windows pip will -automatically retrieve a pre-compiled wheel version. +automatically retrieve a pre-compiled wheel version from +`PYPI repository `__. Alternatively, see more detailed `install `_ instructions. @@ -58,7 +59,7 @@ CPU .. function:: cpu_times(percpu=False) - Return system CPU times as a namedtuple. + Return system CPU times as a named tuple. Every attribute represents the seconds the CPU has spent in the given mode. The attributes availability varies depending on the platform: @@ -69,13 +70,13 @@ CPU Platform-specific fields: - - **nice** *(UNIX)*: time spent by niced processes executing in user mode; - on Linux this also includes **guest_nice** time + - **nice** *(UNIX)*: time spent by niced (prioritized) processes executing in + user mode; on Linux this also includes **guest_nice** time - **iowait** *(Linux)*: time spent waiting for I/O to complete - **irq** *(Linux, BSD)*: time spent for servicing hardware interrupts - **softirq** *(Linux)*: time spent for servicing software interrupts - - **steal** *(Linux 2.6.11+)*: time spent by other operating systems when - running in a virtualized environment + - **steal** *(Linux 2.6.11+)*: time spent by other operating systems running + in a virtualized environment - **guest** *(Linux 2.6.24+)*: time spent running a virtual CPU for guest operating systems under the control of the Linux kernel - **guest_nice** *(Linux 3.2.0+)*: time spent running a niced guest @@ -86,7 +87,7 @@ CPU - **dpc** *(Windows)*: time spent servicing deferred procedure calls (DPCs); DPCs are interrupts that run at a lower priority than standard interrupts. - When *percpu* is ``True`` return a list of namedtuples for each logical CPU + When *percpu* is ``True`` return a list of named tuples for each logical CPU on the system. First element of the list refers to first CPU, second element to second CPU and so on. @@ -108,8 +109,8 @@ CPU since last call or module import, returning immediately. That means the first time this is called it will return a meaningless ``0.0`` value which you are supposed to ignore. - In this case is recommended for accuracy that this function be called with at - least ``0.1`` seconds between calls. + In this case it is recommended for accuracy that this function be called with + at least ``0.1`` seconds between calls. When *percpu* is ``True`` returns a list of floats representing the utilization as a percentage for each CPU. First element of the list refers to first CPU, second element to second CPU @@ -168,7 +169,7 @@ CPU .. function:: cpu_stats() - Return various CPU statistics as a namedtuple: + Return various CPU statistics as a named tuple: - **ctx_switches**: number of context switches (voluntary + involuntary) since boot. @@ -195,7 +196,7 @@ CPU Return CPU frequency as a nameduple including *current*, *min* and *max* frequencies expressed in Mhz. - On Linux **current** frequency reports the real-time value, on all other + On Linux *current* frequency reports the real-time value, on all other platforms it represents the nominal "fixed" value. If *percpu* is ``True`` and the system supports per-cpu frequency retrieval (Linux only) a list of frequencies is returned for each CPU, @@ -225,7 +226,7 @@ Memory .. function:: virtual_memory() - Return statistics about system memory usage as a namedtuple including the + Return statistics about system memory usage as a named tuple including the following fields, expressed in bytes. Main metrics: - **total**: total physical memory. @@ -281,7 +282,7 @@ Memory .. function:: swap_memory() - Return system swap memory statistics as a namedtuple including the following + Return system swap memory statistics as a named tuple including the following fields: * **total**: total swap memory in bytes @@ -306,7 +307,7 @@ Disks .. function:: disk_partitions(all=False) - Return all mounted disk partitions as a list of namedtuples including device, + Return all mounted disk partitions as a list of named tuples including device, mount point and filesystem type, similarly to "df" command on UNIX. If *all* parameter is ``False`` it tries to distinguish and return physical devices only (e.g. hard disks, cd-rom drives, USB keys) and ignore all others @@ -314,7 +315,7 @@ Disks `/dev/shm `__). Note that this may not be fully reliable on all systems (e.g. on BSD this parameter is ignored). - Namedtuple's **fstype** field is a string which varies depending on the + Named tuple's **fstype** field is a string which varies depending on the platform. On Linux it can be one of the values found in /proc/filesystems (e.g. ``'ext3'`` for an ext3 hard drive o ``'iso9660'`` for the CD-ROM drive). @@ -333,7 +334,7 @@ Disks .. function:: disk_usage(path) - Return disk usage statistics about the given *path* as a namedtuple including + Return disk usage statistics about the given *path* as a named tuple including **total**, **used** and **free** space expressed in bytes, plus the **percentage** usage. `OSError `__ is @@ -362,7 +363,7 @@ Disks .. function:: disk_io_counters(perdisk=False) - Return system-wide disk I/O statistics as a namedtuple including the + Return system-wide disk I/O statistics as a named tuple including the following fields: - **read_count**: number of reads @@ -385,7 +386,7 @@ Disks If *perdisk* is ``True`` return the same information for every physical disk installed on the system as a dictionary with partition names as the keys and - the namedtuple described above as the values. + the named tuple described above as the values. See `iotop.py `__ for an example application. @@ -416,7 +417,7 @@ Network .. function:: net_io_counters(pernic=False) - Return system-wide network I/O statistics as a namedtuple including the + Return system-wide network I/O statistics as a named tuple including the following attributes: - **bytes_sent**: number of bytes sent @@ -431,7 +432,7 @@ Network If *pernic* is ``True`` return the same information for every network interface installed on the system as a dictionary with network interface - names as the keys and the namedtuple described above as the values. + names as the keys and the named tuple described above as the values. >>> import psutil >>> psutil.net_io_counters() @@ -453,8 +454,8 @@ Network .. function:: net_connections(kind='inet') - Return system-wide socket connections as a list of namedtuples. - Every namedtuple provides 7 attributes: + Return system-wide socket connections as a list of named tuples. + Every named tuple provides 7 attributes: - **fd**: the socket file descriptor, if retrievable, else ``-1``. If the connection refers to the current process this may be passed to @@ -542,8 +543,8 @@ Network Return the addresses associated to each NIC (network interface card) installed on the system as a dictionary whose keys are the NIC names and - value is a list of namedtuples for each address assigned to the NIC. - Each namedtuple includes 5 fields: + value is a list of named tuples for each address assigned to the NIC. + Each named tuple includes 5 fields: - **family**: the address family, either `AF_INET `__, @@ -594,7 +595,7 @@ Network .. function:: net_if_stats() Return information about each NIC (network interface card) installed on the - system as a dictionary whose keys are the NIC names and value is a namedtuple + system as a dictionary whose keys are the NIC names and value is a named tuple with the following fields: - **isup**: a bool indicating whether the NIC is up and running. @@ -624,7 +625,7 @@ Sensors .. function:: sensors_temperatures(fahrenheit=False) - Return hardware temperatures. Each entry is a namedtuple representing a + Return hardware temperatures. Each entry is a named tuple representing a certain hardware sensor (it may be a CPU, an hard disk or something else, depending on the OS and its configuration). All temperatures are expressed in celsius unless *fahrenheit* is set to @@ -654,7 +655,7 @@ Sensors .. function:: sensors_battery() - Return battery status information as a namedtuple including the following + Return battery status information as a named tuple including the following values. If no battery is installed or metrics can't be determined returns ``None``. @@ -708,7 +709,7 @@ Other system info .. function:: users() - Return users currently connected on the system as a list of namedtuples + Return users currently connected on the system as a list of named tuples including the following fields: - **user**: the name of the user. @@ -1072,7 +1073,7 @@ Process class .. method:: uids() The real, effective and saved user ids of this process as a - namedtuple. This is the same as + named tuple. This is the same as `os.getresuid() `__ but can be used for any process PID. @@ -1081,7 +1082,7 @@ Process class .. method:: gids() The real, effective and saved group ids of this process as a - namedtuple. This is the same as + named tuple. This is the same as `os.getresgid() `__ but can be used for any process PID. @@ -1182,7 +1183,7 @@ Process class .. method:: io_counters() - Return process I/O statistics as a namedtuple including the number of read + Return process I/O statistics as a named tuple including the number of read and write operations performed by the process and the amount of bytes read and written. For Linux refer to `/proc filesysem documentation `__. @@ -1220,13 +1221,13 @@ Process class .. method:: threads() - Return threads opened by process as a list of namedtuples including thread + Return threads opened by process as a list of named tuples including thread id and thread CPU times (user/system). On OpenBSD this method requires root privileges. .. method:: cpu_times() - Return a `(user, system, children_user, children_system)` namedtuple + Return a `(user, system, children_user, children_system)` named tuple representing the accumulated process time, in seconds (see `explanation `__). On Windows and OSX only *user* and *system* are filled, the others are @@ -1331,7 +1332,7 @@ Process class .. method:: memory_info() - Return a namedtuple with variable fields depending on the platform + Return a named tuple with variable fields depending on the platform representing memory information about the process. The "portable" fields available on all plaforms are `rss` and `vms`. All numbers are expressed in bytes. @@ -1470,7 +1471,7 @@ Process class Compare process memory to total physical system memory and calculate process memory utilization as a percentage. *memtype* argument is a string that dictates what type of process memory - you want to compare against. You can choose between the namedtuple field + you want to compare against. You can choose between the named tuple field names returned by :meth:`memory_info` and :meth:`memory_full_info` (defaults to ``"rss"``). @@ -1478,7 +1479,7 @@ Process class .. method:: memory_maps(grouped=True) - Return process's mapped memory regions as a list of namedtuples whose + Return process's mapped memory regions as a list of named tuples whose fields are variable depending on the platform. This method is useful to obtain a detailed representation of process memory usage as explained @@ -1487,7 +1488,7 @@ Process class If *grouped* is ``True`` the mapped regions with the same *path* are grouped together and the different memory fields are summed. If *grouped* is ``False`` each mapped region is shown as a single entity and the - namedtuple will also include the mapped region's address space (*addr*) + named tuple will also include the mapped region's address space (*addr*) and permission set (*perms*). See `pmap.py `__ for an example application. @@ -1561,7 +1562,7 @@ Process class .. method:: open_files() - Return regular files opened by process as a list of namedtuples including + Return regular files opened by process as a list of named tuples including the following fields: - **path**: the absolute file name. @@ -1608,9 +1609,9 @@ Process class .. method:: connections(kind="inet") - Return socket connections opened by process as a list of namedtuples. + Return socket connections opened by process as a list of named tuples. To get system-wide connections use :func:`psutil.net_connections()`. - Every namedtuple provides 6 attributes: + Every named tuple provides 6 attributes: - **fd**: the socket file descriptor. This can be passed to `socket.fromfd() `__ From 0efe637c5e6370f142da2f6ba54122015ca7dcc3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 13:31:34 +0100 Subject: [PATCH 150/922] makefile: remove upload-doc (outdated) --- Makefile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Makefile b/Makefile index 9f802ec18..cad65e0c8 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,6 @@ DEPS = argparse \ requests \ setuptools \ sphinx \ - sphinx-pypi-upload \ twine \ unittest2 @@ -197,12 +196,6 @@ install-git-hooks: upload-src: clean $(PYTHON) setup.py sdist upload -# Build and upload doc on https://pythonhosted.org/psutil/. -# Requires "pip install sphinx-pypi-upload". -upload-doc: - cd docs && make html - $(PYTHON) setup.py upload_sphinx --upload-dir=docs/_build/html - # Download exes/wheels hosted on appveyor. win-download-exes: $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil From 7200a7c06af21aeb24adf28f4186f467ceb7314c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 13:41:40 +0100 Subject: [PATCH 151/922] download_exes.py; show file size --- scripts/internal/download_exes.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_exes.py index d8d2768b2..e607b87d6 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_exes.py @@ -85,16 +85,36 @@ def onerror(fun, path, excinfo): shutil.rmtree(path, onerror=onerror) +def bytes2human(n): + """ + >>> bytes2human(10000) + '9.8 K' + >>> bytes2human(100001221) + '95.4 M' + """ + symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') + prefix = {} + for i, s in enumerate(symbols): + prefix[s] = 1 << (i + 1) * 10 + for s in reversed(symbols): + if n >= prefix[s]: + value = float(n) / prefix[s] + return '%.2f %s' % (value, s) + return '%.2f B' % (n) + + def download_file(url): local_fname = url.split('/')[-1] local_fname = os.path.join('dist', local_fname) - print(local_fname) safe_makedirs('dist') r = requests.get(url, stream=True) + tot_bytes = 0 with open(local_fname, 'wb') as f: - for chunk in r.iter_content(chunk_size=1024): + for chunk in r.iter_content(chunk_size=16384): if chunk: # filter out keep-alive new chunks f.write(chunk) + tot_bytes += len(chunk) + print("downloaded %-45s %s" % (local_fname, bytes2human(tot_bytes))) return local_fname From 9eb8f85fab0784b2fc23d4a1522f9b70427a0101 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 13:45:06 +0100 Subject: [PATCH 152/922] update README --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index ddd27ee65..e3d09c576 100644 --- a/README.rst +++ b/README.rst @@ -132,6 +132,7 @@ Memory .. code-block:: python + >>> import psutil >>> psutil.virtual_memory() svmem(total=10367352832, available=6472179712, percent=37.6, used=8186245120, free=2181107712, active=4748992512, inactive=2758115328, buffers=790724608, cached=3500347392, shared=787554304) >>> psutil.swap_memory() @@ -143,6 +144,7 @@ Disks .. code-block:: python + >>> import psutil >>> psutil.disk_partitions() [sdiskpart(device='/dev/sda1', mountpoint='/', fstype='ext4', opts='rw,nosuid'), sdiskpart(device='/dev/sda2', mountpoint='/home', fstype='ext, opts='rw')] @@ -159,6 +161,7 @@ Network .. code-block:: python + >>> import psutil >>> psutil.net_io_counters(pernic=True) {'eth0': netio(bytes_sent=485291293, bytes_recv=6004858642, packets_sent=3251564, packets_recv=4787798, errin=0, errout=0, dropin=0, dropout=0), 'lo': netio(bytes_sent=2838627, bytes_recv=2838627, packets_sent=30567, packets_recv=30567, errin=0, errout=0, dropin=0, dropout=0)} @@ -207,6 +210,7 @@ Other system info .. code-block:: python + >>> import psutil >>> psutil.users() [user(name='giampaolo', terminal='pts/2', host='localhost', started=1340737536.0), user(name='giampaolo', terminal='pts/3', host='localhost', started=1340737792.0)] From 8e7760b3c76097c669ddb235fb076ce0d6dad36a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 14:10:58 +0100 Subject: [PATCH 153/922] win: small optimization --- psutil/_pswindows.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index dd83c9294..53f57a7a1 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -173,10 +173,6 @@ class Priority(enum.IntEnum): @lru_cache(maxsize=512) -def _win32_QueryDosDevice(s): - return cext.win32_QueryDosDevice(s) - - def convert_dos_path(s): # convert paths using native DOS format like: # "\Device\HarddiskVolume1\Windows\systemew\file.txt" @@ -184,7 +180,7 @@ def convert_dos_path(s): if PY3 and not isinstance(s, str): s = s.decode('utf8') rawdrive = '\\'.join(s.split('\\')[:3]) - driveletter = _win32_QueryDosDevice(rawdrive) + driveletter = cext.win32_QueryDosDevice(rawdrive) return os.path.join(driveletter, s[len(rawdrive):]) From bde7f7bac923f83cc675c526cca791b9abc9baac Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 18:31:55 +0100 Subject: [PATCH 154/922] fix #968 / Linux: disk_io_counters() is broken on python 3 --- HISTORY.rst | 9 +++++++++ psutil/_pslinux.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 974623027..817988995 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,14 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.1.2 +===== + +*2017-02-03* + +**Bug fixes** + +- 968_: [Linux] disk_io_counters() raises TypeError on python 3. + 5.1.1 ===== diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 1183052b3..902071c94 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -245,7 +245,7 @@ def file_flags_to_mode(flags): def get_sector_size(partition): try: - with open(b"/sys/block/%s/queue/hw_sector_size" % partition) as f: + with open("/sys/block/%s/queue/hw_sector_size" % partition, "rt") as f: return int(f.read()) except (IOError, ValueError): # man iostat states that sectors are equivalent with blocks and From 44ca122c1c7fc131ad7f6587203a5d8d97a4bb30 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 18:52:03 +0100 Subject: [PATCH 155/922] fix #970 [Linux] sensors_battery()'s name and label fields on Python 3 are bytes --- HISTORY.rst | 2 ++ psutil/_pslinux.py | 5 +++-- psutil/tests/test_system.py | 11 ++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 817988995..e55633486 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,8 @@ **Bug fixes** - 968_: [Linux] disk_io_counters() raises TypeError on python 3. +- 970_: [Linux] sensors_battery()'s name and label fields on Python 3 are bytes + instead of str. 5.1.1 ===== diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 902071c94..fdb48f969 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1082,8 +1082,9 @@ def sensors_temperatures(): [x.split('_')[0] for x in glob.glob('/sys/class/hwmon/hwmon*/temp*_*')])) for base in basenames: - unit_name = cat(os.path.join(os.path.dirname(base), 'name')) - label = cat(base + '_label', fallback='') + unit_name = cat(os.path.join(os.path.dirname(base), 'name'), + binary=False) + label = cat(base + '_label', fallback='', binary=False) current = float(cat(base + '_input')) / 1000.0 high = cat(base + '_max', fallback=None) critical = cat(base + '_crit', fallback=None) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 01648380d..1c143f97c 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -454,6 +454,10 @@ def test_disk_partitions(self): # AssertionError: Lists differ: [0, 1, 2, 3, 4, 5, 6, 7,... != [0] self.assertTrue(ls, msg=ls) for disk in ls: + self.assertIsInstance(disk.device, (str, unicode)) + self.assertIsInstance(disk.mountpoint, (str, unicode)) + self.assertIsInstance(disk.fstype, (str, unicode)) + self.assertIsInstance(disk.opts, (str, unicode)) if WINDOWS and 'cdrom' in disk.opts: continue if not POSIX: @@ -468,7 +472,6 @@ def test_disk_partitions(self): else: assert os.path.isdir(disk.mountpoint), disk assert disk.fstype, disk - self.assertIsInstance(disk.opts, str) # all = True ls = psutil.disk_partitions(all=True) @@ -512,6 +515,7 @@ def check(cons, families, types_): self.assertIn(conn.family, families, msg=conn) if conn.family != getattr(socket, 'AF_UNIX', object()): self.assertIn(conn.type, types_, msg=conn) + self.assertIsInstance(conn.status, (str, unicode)) from psutil._common import conn_tmap for kind, groups in conn_tmap.items(): @@ -547,6 +551,7 @@ def check_ntuple(nt): self.assertNotEqual(ret, []) for key in ret: self.assertTrue(key) + self.assertIsInstance(key, (str, unicode)) check_ntuple(ret[key]) def test_net_if_addrs(self): @@ -562,6 +567,7 @@ def test_net_if_addrs(self): families = set([socket.AF_INET, AF_INET6, psutil.AF_LINK]) for nic, addrs in nics.items(): + self.assertIsInstance(nic, (str, unicode)) self.assertEqual(len(set(addrs)), len(addrs)) for addr in addrs: self.assertIsInstance(addr.family, int) @@ -684,6 +690,9 @@ def test_users(self): self.assertNotEqual(users, []) for user in users: assert user.name, user + self.assertIsInstance(user.name, (str, unicode)) + self.assertIsInstance(user.terminal, (str, unicode, None)) + self.assertIsInstance(user.host, (str, unicode, None)) user.terminal user.host assert user.started > 0.0, user From ca01be5196ddb3879e924681934485f03f7f192b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 18:57:29 +0100 Subject: [PATCH 156/922] #966: sensors_battery().power_plugged may erroneously return None on Python 3 --- HISTORY.rst | 2 ++ psutil/_pslinux.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index e55633486..403a2d7de 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,8 @@ **Bug fixes** +- 966_: [Linux] sensors_battery().power_plugged may erroneously return None on + Python 3. - 968_: [Linux] disk_io_counters() raises TypeError on python 3. - 970_: [Linux] sensors_battery()'s name and label fields on Python 3 are bytes instead of str. diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index fdb48f969..beddb8b61 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1153,7 +1153,7 @@ def multi_cat(*paths): os.path.join(POWER_SUPPLY_PATH, "AC0/online"), fallback=b"0") == b"1" elif os.path.exists(root + "/status"): - status = cat(root + "/status", fallback="").lower() + status = cat(root + "/status", fallback="", binary=False).lower() if status == "discharging": power_plugged = False elif status in ("charging", "full"): From e7bf6e9d6cbdc6ca0e043db8b3559617381fbdf9 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 3 Feb 2017 13:50:48 -0500 Subject: [PATCH 157/922] fix TypeError and failing test on CentOS --- .git-pre-commit | 2 +- psutil/_pslinux.py | 4 ++-- psutil/tests/__init__.py | 2 +- psutil/tests/test_linux.py | 4 +++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.git-pre-commit b/.git-pre-commit index 071957d14..da932a1db 100755 --- a/.git-pre-commit +++ b/.git-pre-commit @@ -14,7 +14,7 @@ from __future__ import print_function import os import subprocess import sys - +sys.exit(0) def term_supports_colors(): try: diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index beddb8b61..8fc0bb6d0 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1143,8 +1143,8 @@ def multi_cat(*paths): except ZeroDivisionError: percent = 0.0 else: - percent = int(cat(root + "/capacity", fallback=null)) - if percent == null: + percent = int(cat(root + "/capacity", fallback=-1)) + if percent == -1: return None # Is AC power cable plugged in? diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index b119a788c..13c4cfca2 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -199,7 +199,7 @@ def get_test_subprocess(cmd=None, **kwds): kwds.setdefault("stdin", DEVNULL) kwds.setdefault("stdout", DEVNULL) if cmd is None: - assert not os.path.exists(_TESTFN) + safe_rmpath(_TESTFN) pyline = "from time import sleep;" pyline += "open(r'%s', 'w').close();" % _TESTFN pyline += "sleep(60)" diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 3e55d32ee..5db4c3098 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1119,13 +1119,15 @@ def test_emulate_energy_full_not_avail(self): def open_mock(name, *args, **kwargs): if name.startswith("/sys/class/power_supply/BAT0/energy_full"): raise IOError(errno.ENOENT, "") + elif name.startswith("/sys/class/power_supply/BAT0/capacity"): + return io.BytesIO(b"88") else: return orig_open(name, *args, **kwargs) orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock) as m: - self.assertGreaterEqual(psutil.sensors_battery().percent, 0) + self.assertEqual(psutil.sensors_battery().percent, 88) assert m.called def test_emulate_no_ac0_online(self): From 0db4ec677d0812a63bdc841558077c36a96412f3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 19:51:42 +0100 Subject: [PATCH 158/922] revert change commit by accident --- .git-pre-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.git-pre-commit b/.git-pre-commit index da932a1db..071957d14 100755 --- a/.git-pre-commit +++ b/.git-pre-commit @@ -14,7 +14,7 @@ from __future__ import print_function import os import subprocess import sys -sys.exit(0) + def term_supports_colors(): try: From bb3cc6035bccc04dea7f98ce5e0bba113cb677ac Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 20:11:42 +0100 Subject: [PATCH 159/922] pre-release --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 599091a12..b73088efe 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2121,6 +2121,7 @@ take a look at the Timeline ======== +- 2017-02-03: `5.1.2 `__ - `what's new `__ - 2017-02-03: `5.1.1 `__ - `what's new `__ - 2017-02-01: `5.1.0 `__ - `what's new `__ - 2016-12-21: `5.0.1 `__ - `what's new `__ From 8113eadf7861becfd52dffb634695e93970fdaee Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Feb 2017 21:01:00 +0100 Subject: [PATCH 160/922] add docstrings --- psutil/_psbsd.py | 8 ++++++++ psutil/_pslinux.py | 32 ++++++++++++++++++++++++-------- psutil/_psosx.py | 3 +++ psutil/_psposix.py | 3 +++ psutil/_pssunos.py | 6 +++++- psutil/_pswindows.py | 17 ++++++++++++++--- 6 files changed, 57 insertions(+), 12 deletions(-) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index fb141cf27..72ef71e8b 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -229,6 +229,7 @@ def per_cpu_times(): # crash at psutil import time. # Next calls will fail with NotImplementedError def per_cpu_times(): + """Return system CPU times as a namedtuple""" if cpu_count_logical() == 1: return [cpu_times()] if per_cpu_times.__called__: @@ -278,6 +279,7 @@ def cpu_count_physical(): def cpu_stats(): + """Return various CPU stats as a named tuple.""" if FREEBSD: # Note: the C ext is returning some metrics we are not exposing: # traps. @@ -353,6 +355,7 @@ def net_if_stats(): def net_connections(kind): + """System-wide network connections.""" if OPENBSD: ret = [] for pid in pids(): @@ -403,6 +406,7 @@ def net_connections(kind): if FREEBSD: def sensors_battery(): + """Return battery info.""" percent, minsleft, power_plugged = cext.sensors_battery() power_plugged = power_plugged == 1 if power_plugged: @@ -425,6 +429,7 @@ def boot_time(): def users(): + """Return currently connected users as a list of namedtuples.""" retlist = [] rawlist = cext.users() for item in rawlist: @@ -454,6 +459,7 @@ def _pid_0_exists(): def pids(): + """Returns a list of PIDs currently running on the system.""" ret = cext.pids() if OPENBSD and (0 not in ret) and _pid_0_exists(): # On OpenBSD the kernel does not return PID 0 (neither does @@ -464,6 +470,7 @@ def pids(): if OPENBSD or NETBSD: def pid_exists(pid): + """Return True if pid exists.""" exists = _psposix.pid_exists(pid) if not exists: # We do this because _psposix.pid_exists() lies in case of @@ -502,6 +509,7 @@ def wrapper(self, *args, **kwargs): @contextlib.contextmanager def wrap_exceptions_procfs(inst): + """Same as above, for routines relying on reading /proc fs.""" try: yield except EnvironmentError as err: diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 8fc0bb6d0..4df43ea37 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -210,6 +210,7 @@ def decode(s): def get_procfs_path(): + """Return updated psutil.PROCFS_PATH constant.""" return sys.modules['psutil'].PROCFS_PATH @@ -234,6 +235,9 @@ def readlink(path): def file_flags_to_mode(flags): + """Convert file's open() flags into a readable string. + Used by Process.open_files(). + """ modes_map = {os.O_RDONLY: 'r', os.O_WRONLY: 'w', os.O_RDWR: 'w+'} mode = modes_map[flags & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR)] if flags & os.O_APPEND: @@ -244,22 +248,25 @@ def file_flags_to_mode(flags): def get_sector_size(partition): + """Return the sector size of a partition. + Used by disk_io_counters(). + """ try: with open("/sys/block/%s/queue/hw_sector_size" % partition, "rt") as f: return int(f.read()) except (IOError, ValueError): # man iostat states that sectors are equivalent with blocks and - # have a size of 512 bytes since 2.4 kernels. This value is - # needed to calculate the amount of disk I/O in bytes. + # have a size of 512 bytes since 2.4 kernels. return 512 @memoize def set_scputimes_ntuple(procfs_path): - """Return a namedtuple of variable fields depending on the - CPU times available on this Linux kernel version which may be: + """Set a namedtuple of variable fields depending on the CPU times + available on this Linux kernel version which may be: (user, nice, system, idle, iowait, irq, softirq, [steal, [guest, [guest_nice]]]) + Used by cpu_times() function. """ global scputimes with open_binary('%s/stat' % procfs_path) as f: @@ -276,11 +283,14 @@ def set_scputimes_ntuple(procfs_path): # Linux >= 3.2.0 fields.append('guest_nice') scputimes = namedtuple('scputimes', fields) - return scputimes def cat(fname, fallback=_DEFAULT, binary=True): - """Return file content.""" + """Return file content. + fallback: the value returned in case the file does not exist or + cannot be read + binary: whether to open the file in binary or text mode. + """ try: with open_binary(fname) if binary else open_text(fname) as f: return f.read().strip() @@ -291,7 +301,7 @@ def cat(fname, fallback=_DEFAULT, binary=True): try: - scputimes = set_scputimes_ntuple("/proc") + set_scputimes_ntuple("/proc") except Exception: # Don't want to crash at import time. traceback.print_exc() @@ -469,6 +479,7 @@ def virtual_memory(): def swap_memory(): + """Return swap memory metrics.""" _, _, _, _, total, free, unit_multiplier = cext.linux_sysinfo() total *= unit_multiplier free *= unit_multiplier @@ -602,6 +613,7 @@ def cpu_count_physical(): def cpu_stats(): + """Return various CPU stats as a named tuple.""" with open_binary('%s/stat' % get_procfs_path()) as f: ctx_switches = None interrupts = None @@ -624,6 +636,10 @@ def cpu_stats(): if os.path.exists("/sys/devices/system/cpu/cpufreq"): def cpu_freq(): + """Return frequency metrics for all CPUs. + Contrarily to other OSes, Linux updates these values in + real-time. + """ # scaling_* files seem preferable to cpuinfo_*, see: # http://unix.stackexchange.com/a/87537/168884 ret = [] @@ -1031,7 +1047,7 @@ def get_partitions(): def disk_partitions(all=False): - """Return mounted disk partitions as a list of namedtuples""" + """Return mounted disk partitions as a list of namedtuples.""" fstypes = set() with open_text("%s/filesystems" % get_procfs_path()) as f: for line in f: diff --git a/psutil/_psosx.py b/psutil/_psosx.py index f7adb43ac..f22ef2fbd 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -185,6 +185,7 @@ def cpu_freq(): def disk_partitions(all=False): + """Return mounted disk partitions as a list of namedtuples.""" retlist = [] partitions = cext.disk_partitions() for partition in partitions: @@ -209,6 +210,7 @@ def disk_partitions(all=False): def net_connections(kind='inet'): + """System-wide network connections.""" # Note: on OSX this will fail with AccessDenied unless # the process is owned by root. ret = [] @@ -250,6 +252,7 @@ def boot_time(): def users(): + """Return currently connected users as a list of namedtuples.""" retlist = [] rawlist = cext.users() for item in rawlist: diff --git a/psutil/_psposix.py b/psutil/_psposix.py index 6debdb327..6ed7694a1 100644 --- a/psutil/_psposix.py +++ b/psutil/_psposix.py @@ -169,6 +169,9 @@ def disk_usage(path): @memoize def get_terminal_map(): + """Get a map of device-id -> path as a dict. + Used by Process.terminal() + """ ret = {} ls = glob.glob('/dev/tty*') + glob.glob('/dev/pts/*') for name in ls: diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index e6796bf92..41547a8f5 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -102,6 +102,7 @@ def get_procfs_path(): + """Return updated psutil.PROCFS_PATH constant.""" return sys.modules['psutil'].PROCFS_PATH @@ -111,7 +112,8 @@ def get_procfs_path(): def virtual_memory(): - # we could have done this with kstat, but imho this is good enough + """Report virtual memory metrics.""" + # we could have done this with kstat, but IMHO this is good enough total = os.sysconf('SC_PHYS_PAGES') * PAGE_SIZE # note: there's no difference on Solaris free = avail = os.sysconf('SC_AVPHYS_PAGES') * PAGE_SIZE @@ -121,6 +123,7 @@ def virtual_memory(): def swap_memory(): + """Report swap memory metrics.""" sin, sout = cext.swap_mem() # XXX # we are supposed to get total/free by doing so: @@ -184,6 +187,7 @@ def cpu_count_physical(): def cpu_stats(): + """Return various CPU stats as a named tuple.""" ctx_switches, interrupts, syscalls, traps = cext.cpu_stats() soft_interrupts = 0 return _common.scpustats(ctx_switches, interrupts, soft_interrupts, diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 53f57a7a1..271b4c7d7 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -174,9 +174,11 @@ class Priority(enum.IntEnum): @lru_cache(maxsize=512) def convert_dos_path(s): - # convert paths using native DOS format like: - # "\Device\HarddiskVolume1\Windows\systemew\file.txt" - # into: "C:\Windows\systemew\file.txt" + """Convert paths using native DOS format like: + "\Device\HarddiskVolume1\Windows\systemew\file.txt" + into: + "C:\Windows\systemew\file.txt" + """ if PY3 and not isinstance(s, str): s = s.decode('utf8') rawdrive = '\\'.join(s.split('\\')[:3]) @@ -185,6 +187,9 @@ def convert_dos_path(s): def py2_strencode(s, encoding=sys.getfilesystemencoding()): + """Encode a string in the given encoding. Falls back on returning + the string as is if it can't be encoded. + """ if PY3 or isinstance(s, str): return s else: @@ -333,6 +338,7 @@ def net_connections(kind, _pid=-1): def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" ret = cext.net_if_stats() for name, items in ret.items(): name = py2_strencode(name) @@ -344,11 +350,15 @@ def net_if_stats(): def net_io_counters(): + """Return network I/O statistics for every network interface + installed on the system as a dict of raw tuples. + """ ret = cext.net_io_counters() return dict([(py2_strencode(k), v) for k, v in ret.items()]) def net_if_addrs(): + """Return the addresses associated to each NIC.""" ret = [] for items in cext.net_if_addrs(): items = list(items) @@ -363,6 +373,7 @@ def net_if_addrs(): def sensors_battery(): + """Return battery information.""" # For constants meaning see: # https://msdn.microsoft.com/en-us/library/windows/desktop/ # aa373232(v=vs.85).aspx From 561f32a476e7e68dc8db46850b1a2e18d18f4385 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 5 Feb 2017 22:27:48 +0100 Subject: [PATCH 161/922] #971 / sensors_temperatures(): try to be nice to CentOS which has a different tree structure --- psutil/_pslinux.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 4df43ea37..557ec1243 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1093,10 +1093,13 @@ def sensors_temperatures(): difficult to parse """ ret = collections.defaultdict(list) - # Will return an empty dict if path does not exist. - basenames = sorted(set( - [x.split('_')[0] for x in - glob.glob('/sys/class/hwmon/hwmon*/temp*_*')])) + basenames = glob.glob('/sys/class/hwmon/hwmon*/temp*_*') + if not basenames: + # CentOS has an intermediate /device directory: + # https://github.com/giampaolo/psutil/issues/971 + basenames = glob.glob('/sys/class/hwmon/hwmon*/device/temp*_*') + + basenames = sorted(set([x.split('_')[0] for x in basenames])) for base in basenames: unit_name = cat(os.path.join(os.path.dirname(base), 'name'), binary=False) From d5ac3e11c5359a3ded7ce6ef49f6e2127d3bdb33 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 5 Feb 2017 22:34:37 +0100 Subject: [PATCH 162/922] fix linux test failures --- psutil/tests/test_linux.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 5db4c3098..b00b49650 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -467,8 +467,9 @@ def test_cpu_times(self): def test_cpu_count_logical_w_sysdev_cpu_online(self): with open("/sys/devices/system/cpu/online") as f: value = f.read().strip() - value = int(value.split('-')[1]) + 1 - self.assertEqual(psutil.cpu_count(), value) + if "-" in str(value): + value = int(value.split('-')[1]) + 1 + self.assertEqual(psutil.cpu_count(), value) @unittest.skipUnless(os.path.exists("/sys/devices/system/cpu"), "/sys/devices/system/cpu does not exist") @@ -1407,14 +1408,19 @@ def test_num_ctx_switches(self): def test_cpu_affinity(self): value = self.read_status_file("Cpus_allowed_list:") - min_, max_ = map(int, value.split('-')) - self.assertEqual( - self.proc.cpu_affinity(), list(range(min_, max_ + 1))) + if '-' in str(value): + min_, max_ = map(int, value.split('-')) + self.assertEqual( + self.proc.cpu_affinity(), list(range(min_, max_ + 1))) def test_cpu_affinity_eligible_cpus(self): + value = self.read_status_file("Cpus_allowed_list:") with mock.patch("psutil._pslinux.per_cpu_times") as m: self.proc._proc._get_eligible_cpus() - assert not m.called + if '-' in str(value): + assert not m.called + else: + assert m.called if __name__ == '__main__': From c912e3917d76c2aeb0eaf184964a32ecb04f8ba2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 5 Feb 2017 22:40:19 +0100 Subject: [PATCH 163/922] fix linux test failures --- psutil/_pslinux.py | 2 ++ psutil/tests/test_linux.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 557ec1243..4720e7621 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1167,6 +1167,8 @@ def multi_cat(*paths): return None # Is AC power cable plugged in? + # Note: AC0 is not always available. Sometimes (e.g. CentOS7) + # it's called "AC". if os.path.exists(os.path.join(POWER_SUPPLY_PATH, "AC0/online")): power_plugged = cat( os.path.join(POWER_SUPPLY_PATH, "AC0/online"), diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index b00b49650..2f8aa3c4c 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1060,6 +1060,8 @@ def test_emulate_power_not_plugged(self): def open_mock(name, *args, **kwargs): if name.startswith("/sys/class/power_supply/AC0/online"): return io.BytesIO(b"0") + elif name.startswith("/sys/class/power_supply/BAT0/status"): + return io.BytesIO(b"discharging") else: return orig_open(name, *args, **kwargs) From b1faa75d9108d3897d3f51176a663f17b7c076f0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 5 Feb 2017 22:41:14 +0100 Subject: [PATCH 164/922] fix linux test failures --- psutil/tests/test_linux.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 2f8aa3c4c..3a9f17283 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1077,13 +1077,15 @@ def test_emulate_power_undetermined(self): def open_mock(name, *args, **kwargs): if name.startswith("/sys/class/power_supply/AC0/online"): raise IOError(errno.ENOENT, "") + elif name.startswith("/sys/class/power_supply/BAT0/status"): + return io.BytesIO(b"???") else: return orig_open(name, *args, **kwargs) orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock) as m: - self.assertEqual(psutil.sensors_battery().power_plugged, False) + self.assertIsNone(psutil.sensors_battery().power_plugged) assert m.called def test_emulate_no_base_files(self): From ba2fdbd15f43bf2f82458d39310edd0bc2c57588 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 5 Feb 2017 22:54:48 +0100 Subject: [PATCH 165/922] battery: handle the case where AC0/online is AC/online --- psutil/_pslinux.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 4720e7621..80de7b809 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1167,12 +1167,13 @@ def multi_cat(*paths): return None # Is AC power cable plugged in? - # Note: AC0 is not always available. Sometimes (e.g. CentOS7) + # Note: AC0 is not always available and sometimes (e.g. CentOS7) # it's called "AC". - if os.path.exists(os.path.join(POWER_SUPPLY_PATH, "AC0/online")): - power_plugged = cat( - os.path.join(POWER_SUPPLY_PATH, "AC0/online"), - fallback=b"0") == b"1" + online = multi_cat( + os.path.join(POWER_SUPPLY_PATH, "AC0/online"), + os.path.join(POWER_SUPPLY_PATH, "AC/online")) + if online is not None: + power_plugged = online == 1 elif os.path.exists(root + "/status"): status = cat(root + "/status", fallback="", binary=False).lower() if status == "discharging": From 99bf6e2d01cb948f1f87346c43935d26913f4b7b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 6 Feb 2017 14:33:19 +0100 Subject: [PATCH 166/922] fix various failures occurring on CentOS --- psutil/_pslinux.py | 7 ++----- psutil/tests/test_linux.py | 18 ++++++++++-------- psutil/tests/test_misc.py | 3 ++- scripts/temperatures.py | 2 ++ 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 80de7b809..e019bbcf7 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1169,21 +1169,18 @@ def multi_cat(*paths): # Is AC power cable plugged in? # Note: AC0 is not always available and sometimes (e.g. CentOS7) # it's called "AC". + power_plugged = None online = multi_cat( os.path.join(POWER_SUPPLY_PATH, "AC0/online"), os.path.join(POWER_SUPPLY_PATH, "AC/online")) if online is not None: power_plugged = online == 1 - elif os.path.exists(root + "/status"): + else: status = cat(root + "/status", fallback="", binary=False).lower() if status == "discharging": power_plugged = False elif status in ("charging", "full"): power_plugged = True - else: - power_plugged = None - else: - power_plugged = None # Seconds left. # Note to self: we may also calculate the charging ETA as per: diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 3a9f17283..807175643 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1075,7 +1075,8 @@ def test_emulate_power_undetermined(self): # Pretend we can't know whether the AC power cable not # connected (assert fallback to False). def open_mock(name, *args, **kwargs): - if name.startswith("/sys/class/power_supply/AC0/online"): + if name.startswith("/sys/class/power_supply/AC0/online") or \ + name.startswith("/sys/class/power_supply/AC/online"): raise IOError(errno.ENOENT, "") elif name.startswith("/sys/class/power_supply/BAT0/status"): return io.BytesIO(b"???") @@ -1151,16 +1152,17 @@ def path_exists_mock(name): def test_emulate_no_power(self): # Emulate a case where /AC0/online file nor /BAT0/status exist. - def path_exists_mock(name): - if name.startswith("/sys/class/power_supply/AC0/online") or \ + def open_mock(name, *args, **kwargs): + if name.startswith("/sys/class/power_supply/AC/online") or \ + name.startswith("/sys/class/power_supply/AC0/online") or \ name.startswith("/sys/class/power_supply/BAT0/status"): - return False + raise IOError(errno.ENOENT, "") else: - return orig_path_exists(name) + return orig_open(name, *args, **kwargs) - orig_path_exists = os.path.exists - with mock.patch("psutil._pslinux.os.path.exists", - side_effect=path_exists_mock) as m: + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: self.assertIsNone(psutil.sensors_battery().power_plugged) assert m.called diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 88d02f18a..571c03a36 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -471,7 +471,8 @@ def test_cpu_distribution(self): @unittest.skipIf(TRAVIS, "unreliable on travis") def test_temperatures(self): - if hasattr(psutil, "sensors_temperatures"): + if hasattr(psutil, "sensors_temperatures") and \ + psutil.sensors_temperatures(): self.assert_stdout('temperatures.py') else: self.assert_syntax('temperatures.py') diff --git a/scripts/temperatures.py b/scripts/temperatures.py index 4b14180ef..15b9156b8 100755 --- a/scripts/temperatures.py +++ b/scripts/temperatures.py @@ -33,6 +33,8 @@ def main(): if not hasattr(psutil, "sensors_temperatures"): sys.exit("platform not supported") temps = psutil.sensors_temperatures() + if not temps: + sys.exit("can't read any temperature") for name, entries in temps.items(): print(name) for entry in entries: From 2683b824b193f61980149e0083419e50a2916a61 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 7 Feb 2017 11:36:49 +0100 Subject: [PATCH 167/922] disable test which occasionally files on appveyor --- psutil/__init__.py | 2 +- psutil/tests/test_process.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 511fdacf2..38822e25d 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -192,7 +192,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.1.2" +__version__ = "5.1.3" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 5a0db615d..1b8d3a62d 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1156,6 +1156,11 @@ def test_parent_ppid(self): self.assertEqual(p.parent().pid, this_parent) # no other process is supposed to have us as parent reap_children(recursive=True) + if APPVEYOR: + # Occasional failures, see: + # https://ci.appveyor.com/project/giampaolo/psutil/build/ + # job/0hs623nenj7w4m33 + return for p in psutil.process_iter(): if p.pid == sproc.pid: continue From 6dbce97db28232c4daee9e4ea83be6161964745f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 7 Feb 2017 21:13:54 +0100 Subject: [PATCH 168/922] pre-release --- HISTORY.rst | 9 +++++++++ docs/index.rst | 1 + 2 files changed, 10 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 403a2d7de..fd18260b6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,14 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +*2017-02-07* + +5.1.3 +===== + +**Bug fixes** + +- 971_: [Linux] sensors_temperatures() didn't work on CentOS 7. + 5.1.2 ===== diff --git a/docs/index.rst b/docs/index.rst index b73088efe..df3837390 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2121,6 +2121,7 @@ take a look at the Timeline ======== +- 2017-02-07: `5.1.3 `__ - `what's new `__ - 2017-02-03: `5.1.2 `__ - `what's new `__ - 2017-02-03: `5.1.1 `__ - `what's new `__ - 2017-02-01: `5.1.0 `__ - `what's new `__ From fe994a73a1771897b34562bd0317601a4ad649ea Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 7 Feb 2017 21:14:40 +0100 Subject: [PATCH 169/922] run appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index b569a7ade..896f550f3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,4 +1,4 @@ -# Build: 0 (bump this up by 1 to force an appveyor run) +# Build: 1 (bump this up by 1 to force an appveyor run) os: Visual Studio 2015 From e5a32988815641e0fee816ba4d2c6d779167f4c6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 7 Feb 2017 21:47:31 +0100 Subject: [PATCH 170/922] fix #973: cpu_percent() may raise ZeroDivisionError. --- HISTORY.rst | 1 + psutil/__init__.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fd18260b6..05bbd9363 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,7 @@ **Bug fixes** - 971_: [Linux] sensors_temperatures() didn't work on CentOS 7. +- 973_: cpu_percent() may raise ZeroDivisionError. 5.1.2 ===== diff --git a/psutil/__init__.py b/psutil/__init__.py index 38822e25d..7b539ef67 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1751,8 +1751,12 @@ def calculate(t1, t2): busy_delta = t2_busy - t1_busy all_delta = t2_all - t1_all - busy_perc = (busy_delta / all_delta) * 100 - return round(busy_perc, 1) + try: + busy_perc = (busy_delta / all_delta) * 100 + except ZeroDivisionError: + return 0.0 + else: + return round(busy_perc, 1) # system-wide usage if not percpu: From 137f704fb162023eae9510b8c67227d48beb9032 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 8 Feb 2017 15:24:18 +0100 Subject: [PATCH 171/922] update doc --- docs/index.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index df3837390..1a0438a11 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1185,7 +1185,7 @@ Process class Return process I/O statistics as a named tuple including the number of read and write operations performed by the process and the amount of bytes read - and written. For Linux refer to + and written (cumulative). For Linux you can refer to `/proc filesysem documentation `__. On BSD there's apparently no way to retrieve bytes counters, hence ``-1`` is returned for **read_bytes** and **write_bytes** fields. OSX is not @@ -1201,23 +1201,24 @@ Process class .. method:: num_ctx_switches() The number voluntary and involuntary context switches performed by - this process. + this process (cumulative). .. method:: num_fds() - The number of file descriptors used by this process. + The number of file descriptors currently opened by this process + (non cumulative). Availability: UNIX .. method:: num_handles() - The number of handles used by this process. + The number of handles currently used by this process (non cumulative). Availability: Windows .. method:: num_threads() - The number of threads used by this process. + The number of threads currently used by this process (non cumulative). .. method:: threads() From ac956ce299c58b751b5c51fb86aac481424b48aa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 9 Feb 2017 11:44:23 +0100 Subject: [PATCH 172/922] fix osx test --- psutil/tests/test_system.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 1c143f97c..bfe551b66 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -692,7 +692,8 @@ def test_users(self): assert user.name, user self.assertIsInstance(user.name, (str, unicode)) self.assertIsInstance(user.terminal, (str, unicode, None)) - self.assertIsInstance(user.host, (str, unicode, None)) + if user.host is not None: + self.assertIsInstance(user.host, (str, unicode, None)) user.terminal user.host assert user.started > 0.0, user From 9e5051c4d550834260aa9d1a9e407f34131d9cbd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 10 Feb 2017 12:57:14 +0100 Subject: [PATCH 173/922] C small refactoring --- psutil/_psutil_posix.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index 7aa4b553b..707c55a1c 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -28,6 +28,9 @@ #include #include #include + #include + #include + #include #elif defined(PSUTIL_SUNOS) #include #include @@ -340,10 +343,6 @@ psutil_net_if_flags(PyObject *self, PyObject *args) { */ #if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) -#include -#include -#include - int psutil_get_nic_speed(int ifm_active) { // Determine NIC speed. Taken from: // http://www.i-scream.org/libstatgrab/ From bf533cc13558479880c5ca32f8cb93aae998f4ad Mon Sep 17 00:00:00 2001 From: nicolargo Date: Sat, 11 Feb 2017 16:53:31 +0100 Subject: [PATCH 174/922] Add new sensors_fans method --- HISTORY.rst | 7 +++++++ docs/index.rst | 19 +++++++++++++++++++ psutil/__init__.py | 15 ++++++++++++++- psutil/_pslinux.py | 29 +++++++++++++++++++++++++++++ psutil/tests/test_linux.py | 13 +++++++++++++ 5 files changed, 82 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 05bbd9363..457003df9 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,13 @@ *2017-02-07* +5.1.4 +===== + +**Enhancements** + +- 971_: [Linux] Add sensors_fans method. + 5.1.3 ===== diff --git a/docs/index.rst b/docs/index.rst index 1a0438a11..d8e1af806 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -653,6 +653,25 @@ Sensors This API is experimental. Backward incompatible changes may occur if deemed necessary. +.. function:: sensors_fans() + + Return hardware fans speed. Each entry is a named tuple representing a + certain hardware sensor. + All speed is expressed in RPM (round per minut). Example:: + + >>> import psutil + >>> psutil.sensors_fans() + defaultdict(, {'dell_smm': [('Processor Fan', 3028)]}) + + Availability: Linux + + .. versionadded:: 5.1.4 + + .. warning:: + + This API is experimental. Backward incompatible changes may occur if + deemed necessary. + .. function:: sensors_battery() Return battery status information as a named tuple including the following diff --git a/psutil/__init__.py b/psutil/__init__.py index 7b539ef67..244e6fca6 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -187,7 +187,7 @@ "net_io_counters", "net_connections", "net_if_addrs", # network "net_if_stats", "disk_io_counters", "disk_partitions", "disk_usage", # disk - # "sensors_temperatures", "sensors_battery", # sensors + # "sensors_temperatures", "sensors_battery", "sensors_fans" # sensors "users", "boot_time", # others ] __all__.extend(_psplatform.__extra__all__) @@ -2234,6 +2234,19 @@ def to_fahrenheit(n): __all__.append("sensors_temperatures") +# Linux +if hasattr(_psplatform, "sensors_fans"): + + def sensors_fans(): + """Return fans speed. Each entry is a namedtuple + representing a certain hardware sensor. + All speed are expressed in RPM (rounds per minute). + """ + return _psplatform.sensors_fans() + + __all__.append("sensors_fans") + + # Linux, Windows, FreeBSD if hasattr(_psplatform, "sensors_battery"): diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index e019bbcf7..ce20e8a05 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1118,6 +1118,35 @@ def sensors_temperatures(): return ret +def sensors_fans(): + """Return hardware (CPU and others) fans as a dict + including hardware label, current speed. + + Implementation notes: + - /sys/class/hwmon looks like the most recent interface to + retrieve this info, and this implementation relies on it + only (old distros will probably use something else) + - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon + """ + ret = collections.defaultdict(list) + basenames = glob.glob('/sys/class/hwmon/hwmon*/fan*_*') + if not basenames: + # CentOS has an intermediate /device directory: + # https://github.com/giampaolo/psutil/issues/971 + basenames = glob.glob('/sys/class/hwmon/hwmon*/device/fan*_*') + + basenames = sorted(set([x.split('_')[0] for x in basenames])) + for base in basenames: + unit_name = cat(os.path.join(os.path.dirname(base), 'name'), + binary=False) + label = cat(base + '_label', fallback='', binary=False) + current = int(cat(base + '_input')) + + ret[unit_name].append((label, current)) + + return ret + + def sensors_battery(): """Return battery information. Implementation note: it appears /sys/class/power_supply/BAT0/ diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 807175643..e5fda9ad2 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1020,6 +1020,19 @@ def test_issue_687(self): t.stop() +@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipUnless(hasattr(psutil, "sensors_fans") and + psutil.sensors_fans() is not None, + "no fan") +class TestSensorsFans(unittest.TestCase): + + def test_current(self): + psutil_value = psutil.sensors_fans() + for fk in psutil_value: + # Fan speed should always be > 0 + self.assertTrue(psutil_value[fk][0][1] >= 0) + + @unittest.skipUnless(LINUX, "LINUX only") @unittest.skipUnless(hasattr(psutil, "sensors_battery") and psutil.sensors_battery() is not None, From 662989cf086dacd55f7cf485a28b2ed4df586b9a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 18:30:26 +0100 Subject: [PATCH 175/922] fix tests --- psutil/tests/test_memory_leaks.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 5adb2b5e3..a55008e52 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -562,7 +562,19 @@ def test_net_if_stats(self): "platform not supported") @skip_if_linux() def test_sensors_battery(self): - self.execute(psutil.sensors_battery()) + self.execute(psutil.sensors_battery) + + @skip_if_linux() + @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), + "platform not supported") + def test_sensors_temperatures(self): + self.execute(psutil.sensors_temperatures) + + @unittest.skipUnless(hasattr(psutil, "sensors_fans"), + "platform not supported") + @skip_if_linux() + def test_sensors_fans(self): + self.execute(psutil.sensors_fans) # --- others @@ -575,12 +587,6 @@ def test_boot_time(self): def test_users(self): self.execute(psutil.users) - @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), - "platform not supported") - @skip_if_linux() - def test_sensors_temperatures(self): - self.execute(psutil.users) - if WINDOWS: # --- win services From f1005a37a05652a230d6c36f3f22e774ae8c1274 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 18:45:21 +0100 Subject: [PATCH 176/922] #974 sensors_fans(): add example script, return dict and named tuple instead of dict + tuple; give CREDITS --- CREDITS | 4 ++++ HISTORY.rst | 4 ++-- README.rst | 17 +++++++++++------ docs/index.rst | 17 ++++++++++++----- psutil/__init__.py | 2 +- psutil/_common.py | 2 ++ psutil/_pslinux.py | 4 ++-- psutil/tests/test_misc.py | 7 +++++++ scripts/fans.py | 35 +++++++++++++++++++++++++++++++++++ 9 files changed, 76 insertions(+), 16 deletions(-) create mode 100755 scripts/fans.py diff --git a/CREDITS b/CREDITS index a353b3da1..75d86db5b 100644 --- a/CREDITS +++ b/CREDITS @@ -429,3 +429,7 @@ I: 950 N: Thiago Borges Abdnur W: https://github.com/bolaum I: 959 + +N: Nicolas Hennion +W: https://github.com/nicolargo +I: 974 diff --git a/HISTORY.rst b/HISTORY.rst index 457003df9..603bc1c02 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,12 +2,12 @@ *2017-02-07* -5.1.4 +5.2.0 ===== **Enhancements** -- 971_: [Linux] Add sensors_fans method. +- 971_: [Linux] Add psutil.sensors_fans() function. (patch by Nicolas Hennion) 5.1.3 ===== diff --git a/README.rst b/README.rst index e3d09c576..741fd4f87 100644 --- a/README.rst +++ b/README.rst @@ -41,12 +41,14 @@ Summary psutil (process and system utilities) is a cross-platform library for retrieving information on **running processes** and **system utilization** -(CPU, memory, disks, network) in Python. It is useful mainly for **system -monitoring**, **profiling and limiting process resources** and **management of -running processes**. It implements many functionalities offered by command line -tools such as: ps, top, lsof, netstat, ifconfig, who, df, kill, free, nice, -ionice, iostat, iotop, uptime, pidof, tty, taskset, pmap. It currently supports -**Linux, Windows, OSX, Sun Solaris, FreeBSD, OpenBSD** and **NetBSD**, +(CPU, memory, disks, networkm sensors) in Python. +It is useful mainly for **system monitoring**, **profiling and limiting process +resources** and **management of running processes**. +It implements many functionalities offered by command line tools such as: +ps, top, lsof, netstat, ifconfig, who, df, kill, free, nice, ionice, iostat, +iotop, uptime, pidof, tty, taskset, pmap. +It currently supports **Linux**, **Windows**, **OSX**, **Sun Solaris**, +**FreeBSD**, **OpenBSD** and **NetBSD**, both **32-bit** and **64-bit** architectures, with Python versions from **2.6 to 3.5** (users of Python 2.4 and 2.5 may use `2.1.3 `__ version). @@ -201,6 +203,9 @@ Sensors shwtemp(label='Core 2', current=45.0, high=100.0, critical=100.0), shwtemp(label='Core 3', current=47.0, high=100.0, critical=100.0)]} >>> + >>> psutil.sensors_fans() + {'asus': [sfan(label='cpu_fan', current=3200)]} + >>> >>> psutil.sensors_battery() sbattery(percent=93, secsleft=16628, power_plugged=False) >>> diff --git a/docs/index.rst b/docs/index.rst index d8e1af806..7186893bc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -626,8 +626,8 @@ Sensors .. function:: sensors_temperatures(fahrenheit=False) Return hardware temperatures. Each entry is a named tuple representing a - certain hardware sensor (it may be a CPU, an hard disk or something - else, depending on the OS and its configuration). + certain hardware temperature sensor (it may be a CPU, an hard disk or + something else, depending on the OS and its configuration). All temperatures are expressed in celsius unless *fahrenheit* is set to ``True``. Example:: @@ -656,16 +656,19 @@ Sensors .. function:: sensors_fans() Return hardware fans speed. Each entry is a named tuple representing a - certain hardware sensor. + certain hardware sensor fan. All speed is expressed in RPM (round per minut). Example:: >>> import psutil >>> psutil.sensors_fans() - defaultdict(, {'dell_smm': [('Processor Fan', 3028)]}) + {'asus': [sfan(label='cpu_fan', current=3200)]} + + See also `fans.py `__ + for an example application. Availability: Linux - .. versionadded:: 5.1.4 + .. versionadded:: 5.2.0 .. warning:: @@ -709,6 +712,10 @@ Sensors .. versionadded:: 5.1.0 + .. warning:: + + This API is experimental. Backward incompatible changes may occur if + deemed necessary. Other system info ----------------- diff --git a/psutil/__init__.py b/psutil/__init__.py index 244e6fca6..c72b7a7d2 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -192,7 +192,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.1.3" +__version__ = "5.2.0" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED diff --git a/psutil/_common.py b/psutil/_common.py index 3b68b6d3e..2497226af 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -174,6 +174,8 @@ class BatteryTime(enum.IntEnum): 'shwtemp', ['label', 'current', 'high', 'critical']) # psutil.sensors_battery() sbattery = namedtuple('sbattery', ['percent', 'secsleft', 'power_plugged']) +# psutil.sensors_battery() +sfan = namedtuple('sfan', ['label', 'current']) # --- for Process methods diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index ce20e8a05..52c67302c 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1142,9 +1142,9 @@ def sensors_fans(): label = cat(base + '_label', fallback='', binary=False) current = int(cat(base + '_input')) - ret[unit_name].append((label, current)) + ret[unit_name].append(_common.sfan(label, current)) - return ret + return dict(ret) def sensors_battery(): diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 571c03a36..615f18a15 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -477,6 +477,13 @@ def test_temperatures(self): else: self.assert_syntax('temperatures.py') + @unittest.skipIf(TRAVIS, "unreliable on travis") + def test_fans(self): + if hasattr(psutil, "sensors_fans") and psutil.sensors_fans(): + self.assert_stdout('fans.py') + else: + self.assert_syntax('fans.py') + def test_battery(self): if hasattr(psutil, "sensors_battery") and \ psutil.sensors_battery() is not None: diff --git a/scripts/fans.py b/scripts/fans.py new file mode 100755 index 000000000..e302aec5d --- /dev/null +++ b/scripts/fans.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Show fans information. + +$ python fans.py +asus + cpu_fan 3200 RPM +""" + +from __future__ import print_function +import sys + +import psutil + + +def main(): + if not hasattr(psutil, "sensors_fans"): + return sys.exit("platform not supported") + fans = psutil.sensors_fans() + if not fans: + return sys.exit("no fans detected") + for name, entries in fans.items(): + print(name) + for entry in entries: + print(" %-20s %s RPM" % (entry.label or name, entry.current)) + print() + + +if __name__ == '__main__': + main() From 98babf25006c3f688c46b06d39b27385a65b9313 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 18:55:08 +0100 Subject: [PATCH 177/922] #974: move sensors_fans() test --- docs/index.rst | 2 +- psutil/tests/test_linux.py | 33 +++++++++++++-------------------- psutil/tests/test_system.py | 25 ++++++++++++++++++------- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 7186893bc..cdc5e828a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -657,7 +657,7 @@ Sensors Return hardware fans speed. Each entry is a named tuple representing a certain hardware sensor fan. - All speed is expressed in RPM (round per minut). Example:: + Fan speed is expressed in RPM (round per minute). Example:: >>> import psutil >>> psutil.sensors_fans() diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index e5fda9ad2..35310b639 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -55,7 +55,7 @@ # ===================================================================== -# utils +# --- utils # ===================================================================== @@ -141,7 +141,7 @@ def get_free_version_info(): # ===================================================================== -# system virtual memory +# --- system virtual memory # ===================================================================== @@ -375,7 +375,7 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -# system swap memory +# --- system swap memory # ===================================================================== @@ -437,7 +437,7 @@ def test_no_vmstat_mocked(self): # ===================================================================== -# system CPU +# --- system CPU # ===================================================================== @@ -538,7 +538,7 @@ def test_cpu_count_physical_mocked(self): # ===================================================================== -# system CPU stats +# --- system CPU stats # ===================================================================== @@ -559,7 +559,7 @@ def test_interrupts(self): # ===================================================================== -# system network +# --- system network # ===================================================================== @@ -671,7 +671,7 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -# system disk +# --- system disk # ===================================================================== @@ -829,7 +829,7 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -# misc +# --- misc # ===================================================================== @@ -1020,17 +1020,9 @@ def test_issue_687(self): t.stop() -@unittest.skipUnless(LINUX, "LINUX only") -@unittest.skipUnless(hasattr(psutil, "sensors_fans") and - psutil.sensors_fans() is not None, - "no fan") -class TestSensorsFans(unittest.TestCase): - - def test_current(self): - psutil_value = psutil.sensors_fans() - for fk in psutil_value: - # Fan speed should always be > 0 - self.assertTrue(psutil_value[fk][0][1] >= 0) +# ===================================================================== +# --- sensors +# ===================================================================== @unittest.skipUnless(LINUX, "LINUX only") @@ -1179,8 +1171,9 @@ def open_mock(name, *args, **kwargs): self.assertIsNone(psutil.sensors_battery().power_plugged) assert m.called + # ===================================================================== -# test process +# --- test process # ===================================================================== diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index bfe551b66..013ae8e39 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -81,9 +81,9 @@ def test_process_iter(self): def test_wait_procs(self): def callback(p): - l.append(p.pid) + pids.append(p.pid) - l = [] + pids = [] sproc1 = get_test_subprocess() sproc2 = get_test_subprocess() sproc3 = get_test_subprocess() @@ -96,7 +96,7 @@ def callback(p): self.assertLess(time.time() - t, 0.5) self.assertEqual(gone, []) self.assertEqual(len(alive), 3) - self.assertEqual(l, []) + self.assertEqual(pids, []) for p in alive: self.assertFalse(hasattr(p, 'returncode')) @@ -115,7 +115,7 @@ def test(procs, callback): self.assertEqual(gone.pop().returncode, -signal.SIGTERM) else: self.assertEqual(gone.pop().returncode, 1) - self.assertEqual(l, [sproc3.pid]) + self.assertEqual(pids, [sproc3.pid]) for p in alive: self.assertFalse(hasattr(p, 'returncode')) @@ -130,7 +130,7 @@ def test(procs, callback): sproc1.terminate() sproc2.terminate() gone, alive = test(procs, callback) - self.assertEqual(set(l), set([sproc1.pid, sproc2.pid, sproc3.pid])) + self.assertEqual(set(pids), set([sproc1.pid, sproc2.pid, sproc3.pid])) for p in gone: self.assertTrue(hasattr(p, 'returncode')) @@ -767,7 +767,7 @@ def test_os_constants(self): self.assertIs(getattr(psutil, name), False, msg=name) @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), - "platform not suported") + "platform not supported") def test_sensors_temperatures(self): temps = psutil.sensors_temperatures() for name, entries in temps.items(): @@ -781,7 +781,7 @@ def test_sensors_temperatures(self): if entry.critical is not None: self.assertGreaterEqual(entry.critical, 0) - @unittest.skipUnless(LINUX or WINDOWS or FREEBSD, + @unittest.skipUnless(hasattr(psutil, "sensors_battery"), "platform not supported") def test_sensors_battery(self): ret = psutil.sensors_battery() @@ -797,6 +797,17 @@ def test_sensors_battery(self): self.assertTrue(ret.power_plugged) self.assertIsInstance(ret.power_plugged, bool) + @unittest.skipUnless(hasattr(psutil, "sensors_fans"), + "platform not supported") + def test_sensors_fans(self): + fans = psutil.sensors_fans() + for name, entries in fans.items(): + self.assertIsInstance(name, (str, unicode)) + for entry in entries: + self.assertIsInstance(entry.label, (str, unicode)) + self.assertIsInstance(entry.current, (int, long)) + self.assertGreaterEqual(entry.current, 0) + if __name__ == '__main__': run_test_module_by_name(__file__) From 66ea5055819ffdd031a299dd2037bfce77323627 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 19:18:17 +0100 Subject: [PATCH 178/922] add sensors.py example script --- docs/index.rst | 6 +-- scripts/battery.py | 2 +- scripts/sensors.py | 108 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 112 insertions(+), 4 deletions(-) create mode 100755 scripts/sensors.py diff --git a/docs/index.rst b/docs/index.rst index cdc5e828a..9e98c3fe1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -641,7 +641,7 @@ Sensors shwtemp(label='Core 2', current=45.0, high=100.0, critical=100.0), shwtemp(label='Core 3', current=47.0, high=100.0, critical=100.0)]} - See also `temperatures.py `__ + See also `temperatures.py `__ and `sensors.py `__ for an example application. Availability: Linux @@ -663,7 +663,7 @@ Sensors >>> psutil.sensors_fans() {'asus': [sfan(label='cpu_fan', current=3200)]} - See also `fans.py `__ + See also `fans.py `__ and `sensors.py `__ for an example application. Availability: Linux @@ -706,7 +706,7 @@ Sensors >>> print("charge = %s%%, time left = %s" % (batt.percent, secs2hours(batt.secsleft))) charge = 93%, time left = 4:37:08 - See also `battery.py `__ + See also `battery.py `__ and `sensors.py `__ for an example application. Availability: Linux, Windows, FreeBSD diff --git a/scripts/battery.py b/scripts/battery.py index 23e0f669d..abbad8785 100755 --- a/scripts/battery.py +++ b/scripts/battery.py @@ -7,7 +7,7 @@ """ Show battery information. -$ python battery.py +$ python scripts/battery.py charge: 74% left: 2:11:31 status: discharging diff --git a/scripts/sensors.py b/scripts/sensors.py new file mode 100755 index 000000000..99d415966 --- /dev/null +++ b/scripts/sensors.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +A clone of 'sensors' utility on Linux printing hardware temperatures. + +$ python scripts/sensors.py +SENSORS +======= + +asus + Temperatures: + asus 50.0 °C (high=None °C, critical=None °C) + Fans: + cpu_fan 3300 RPM + +acpitz + Temperatures: + acpitz 50.0 °C (high=108.0 °C, critical=108.0 °C) + +coretemp + Temperatures: + Physical id 0 51.0 °C (high=87.0 °C, critical=105.0 °C) + Core 0 49.0 °C (high=87.0 °C, critical=105.0 °C) + Core 1 51.0 °C (high=87.0 °C, critical=105.0 °C) + +BATTERY +======= + + charge: 87.51% + left: 1:12:28 + status: discharging + plugged in: no +""" + +from __future__ import print_function +import sys + +import psutil + + +def secs2hours(secs): + mm, ss = divmod(secs, 60) + hh, mm = divmod(mm, 60) + return "%d:%02d:%02d" % (hh, mm, ss) + + +def main(): + if hasattr(psutil, "sensors_temperatures"): + temps = psutil.sensors_temperatures() + else: + temps = {} + if hasattr(psutil, "sensors_fans"): + fans = psutil.sensors_fans() + else: + fans = {} + if hasattr(psutil, "sensors_battery"): + battery = psutil.sensors_battery() + else: + battery = None + + if not any((temps, fans, battery)): + return sys.exit("can't read any temperature, fans or battery info") + + if temps or fans: + print("SENSORS") + print("=======\n") + + names = set(temps.keys() + fans.keys()) + for name in names: + print(name) + # Temperatures. + if name in temps: + print(" Temperatures:") + for entry in temps[name]: + print(" %-20s %s °C (high=%s °C, critical=%s °C)" % ( + entry.label or name, entry.current, entry.high, + entry.critical)) + # Fans. + if name in fans: + print(" Fans:") + for entry in fans[name]: + print(" %-20s %s RPM" % ( + entry.label or name, entry.current)) + + print() + + # Battery + if battery: + print("BATTERY") + print("=======\n") + print(" charge: %s%%" % round(battery.percent, 2)) + if battery.power_plugged: + print(" status: %s" % ( + "charging" if battery.percent < 100 else "fully charged")) + print(" plugged in: yes") + else: + print(" left: %s" % secs2hours(battery.secsleft)) + print(" status: %s" % "discharging") + print(" plugged in: no") + + +if __name__ == '__main__': + main() From 22b22ccaf82ddcb98122cc7ba1d00dd759ef14ba Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 19:19:24 +0100 Subject: [PATCH 179/922] add test --- psutil/tests/test_misc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 615f18a15..ace60264c 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -491,6 +491,9 @@ def test_battery(self): else: self.assert_syntax('battery.py') + def test_sensors(self): + self.assert_stdout('sensors.py') + # =================================================================== # --- Unit tests for test utilities. From 5b6b133e56a489556f0d180019748aa30d29147e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 19:34:53 +0100 Subject: [PATCH 180/922] sensors.py: change output --- scripts/sensors.py | 38 +++++++++++--------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/scripts/sensors.py b/scripts/sensors.py index 99d415966..8409c6b85 100755 --- a/scripts/sensors.py +++ b/scripts/sensors.py @@ -9,32 +9,23 @@ A clone of 'sensors' utility on Linux printing hardware temperatures. $ python scripts/sensors.py -SENSORS -======= - asus Temperatures: - asus 50.0 °C (high=None °C, critical=None °C) + asus 57.0 °C (high=None °C, critical=None °C) Fans: - cpu_fan 3300 RPM - + cpu_fan 3500 RPM acpitz Temperatures: - acpitz 50.0 °C (high=108.0 °C, critical=108.0 °C) - + acpitz 57.0 °C (high=108.0 °C, critical=108.0 °C) coretemp Temperatures: - Physical id 0 51.0 °C (high=87.0 °C, critical=105.0 °C) - Core 0 49.0 °C (high=87.0 °C, critical=105.0 °C) - Core 1 51.0 °C (high=87.0 °C, critical=105.0 °C) - -BATTERY -======= - - charge: 87.51% - left: 1:12:28 - status: discharging - plugged in: no + Physical id 0 61.0 °C (high=87.0 °C, critical=105.0 °C) + Core 0 61.0 °C (high=87.0 °C, critical=105.0 °C) + Core 1 59.0 °C (high=87.0 °C, critical=105.0 °C) +Battery: + charge: 84.95% + status: charging + plugged in: yes """ from __future__ import print_function @@ -66,10 +57,6 @@ def main(): if not any((temps, fans, battery)): return sys.exit("can't read any temperature, fans or battery info") - if temps or fans: - print("SENSORS") - print("=======\n") - names = set(temps.keys() + fans.keys()) for name in names: print(name) @@ -87,12 +74,9 @@ def main(): print(" %-20s %s RPM" % ( entry.label or name, entry.current)) - print() - # Battery if battery: - print("BATTERY") - print("=======\n") + print("Battery:") print(" charge: %s%%" % round(battery.percent, 2)) if battery.power_plugged: print(" status: %s" % ( From 4e0e14932f42a9eee14b0a5ec988726039cb77b2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 19:52:14 +0100 Subject: [PATCH 181/922] update doc --- docs/index.rst | 12 ++++++++---- scripts/sensors.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 9e98c3fe1..31136c103 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -629,7 +629,9 @@ Sensors certain hardware temperature sensor (it may be a CPU, an hard disk or something else, depending on the OS and its configuration). All temperatures are expressed in celsius unless *fahrenheit* is set to - ``True``. Example:: + ``True``. + If sensors are not supported by the OS an empty dict is returned. + Example:: >>> import psutil >>> psutil.sensors_temperatures() @@ -657,7 +659,9 @@ Sensors Return hardware fans speed. Each entry is a named tuple representing a certain hardware sensor fan. - Fan speed is expressed in RPM (round per minute). Example:: + Fan speed is expressed in RPM (round per minute). + If sensors are not supported by the OS an empty dict is returned. + Example:: >>> import psutil >>> psutil.sensors_fans() @@ -678,8 +682,8 @@ Sensors .. function:: sensors_battery() Return battery status information as a named tuple including the following - values. If no battery is installed or metrics can't be determined returns - ``None``. + values. If no battery is installed or metrics can't be determined ``None`` + is returned. - **percent**: battery power left as a percentage. - **secsleft**: a rough approximation of how many seconds are left before the diff --git a/scripts/sensors.py b/scripts/sensors.py index 8409c6b85..4c055efac 100755 --- a/scripts/sensors.py +++ b/scripts/sensors.py @@ -74,7 +74,7 @@ def main(): print(" %-20s %s RPM" % ( entry.label or name, entry.current)) - # Battery + # Battery. if battery: print("Battery:") print(" charge: %s%%" % round(battery.percent, 2)) From 09f2bb3583fabb6766115f5a9f9efef2e7ec12f9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 19:53:35 +0100 Subject: [PATCH 182/922] fix appveyor test --- psutil/tests/test_misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index ace60264c..c6d3ce48c 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -491,6 +491,7 @@ def test_battery(self): else: self.assert_syntax('battery.py') + @unittest.skipIf(APPVEYOR, "unreliable on appveyor") def test_sensors(self): self.assert_stdout('sensors.py') From 24b6b482f7e416cdb606be9cc2cb0d8fb6df09d3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 20:17:55 +0100 Subject: [PATCH 183/922] fix failure on travis --- psutil/tests/test_misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index c6d3ce48c..84215d30c 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -491,7 +491,7 @@ def test_battery(self): else: self.assert_syntax('battery.py') - @unittest.skipIf(APPVEYOR, "unreliable on appveyor") + @unittest.skipIf(APPVEYOR or TRAVIS, "unreliable on CI") def test_sensors(self): self.assert_stdout('sensors.py') From 08629ac3b53986f2a10b603c93756ca8d3f12f47 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 21:39:35 +0100 Subject: [PATCH 184/922] add test for ppid() - add FAQ --- docs/index.rst | 7 +++++++ psutil/tests/test_process.py | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 31136c103..af5abd4bb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2142,6 +2142,13 @@ Q&A the Python script as a Windows service (this is the trick used by tools such as ProcessHacker). +---- + +* Q: What about load average? +* A: psutil does not expose any load average function as it's already available + in python as + `os.getloadavg `__ + Development guide ================= diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 1b8d3a62d..ee260d315 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1148,7 +1148,9 @@ def test_num_ctx_switches(self): return self.fail("num ctx switches still the same after 50.000 iterations") - def test_parent_ppid(self): + def test_ppid(self): + if hasattr(os, 'getppid'): + self.assertEqual(psutil.Process().ppid(), os.getppid()) this_parent = os.getpid() sproc = get_test_subprocess() p = psutil.Process(sproc.pid) From 172ba7dd266c63c8eba362edb4ce934f3fab514d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 21:54:24 +0100 Subject: [PATCH 185/922] add test for nice() --- psutil/tests/test_process.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index ee260d315..d982b3811 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -92,9 +92,12 @@ def tearDown(self): reap_children() def test_pid(self): - self.assertEqual(psutil.Process().pid, os.getpid()) + p = psutil.Process() + self.assertEqual(p.pid, os.getpid()) sproc = get_test_subprocess() self.assertEqual(psutil.Process(sproc.pid).pid, sproc.pid) + with self.assertRaises(AttributeError): + p.pid = 33 def test_kill(self): sproc = get_test_subprocess() @@ -792,11 +795,17 @@ def test_nice(self): finally: p.nice(psutil.NORMAL_PRIORITY_CLASS) else: + first_nice = p.nice() try: - first_nice = p.nice() + if hasattr(os, "getpriority"): + self.assertEqual( + os.getpriority(os.PRIO_PROCESS, os.getpid()), p.nice()) p.nice(1) self.assertEqual(p.nice(), 1) - # going back to previous nice value raises + if hasattr(os, "getpriority"): + self.assertEqual( + os.getpriority(os.PRIO_PROCESS, os.getpid()), p.nice()) + # XXX - going back to previous nice value raises # AccessDenied on OSX if not OSX: p.nice(0) From 397a758aee9e780e9a92819b89ed0875a81432db Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Feb 2017 22:00:05 +0100 Subject: [PATCH 186/922] add tests for uids() and gids() --- psutil/tests/test_process.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index d982b3811..4d1a88a3c 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -759,10 +759,11 @@ def test_uids(self): self.assertEqual(real, os.getuid()) # os.geteuid() refers to "effective" uid self.assertEqual(effective, os.geteuid()) - # no such thing as os.getsuid() ("saved" uid), but starting - # from python 2.7 we have os.getresuid()[2] + # No such thing as os.getsuid() ("saved" uid), but starting + # from python 2.7 we have os.getresuid() which returns all + # of them. if hasattr(os, "getresuid"): - self.assertEqual(saved, os.getresuid()[2]) + self.assertEqual(os.getresuid(), p.uids()) @unittest.skipUnless(POSIX, 'POSIX only') def test_gids(self): @@ -772,10 +773,11 @@ def test_gids(self): self.assertEqual(real, os.getgid()) # os.geteuid() refers to "effective" uid self.assertEqual(effective, os.getegid()) - # no such thing as os.getsgid() ("saved" gid), but starting - # from python 2.7 we have os.getresgid()[2] + # No such thing as os.getsgid() ("saved" gid), but starting + # from python 2.7 we have os.getresgid() which returns all + # of them. if hasattr(os, "getresuid"): - self.assertEqual(saved, os.getresgid()[2]) + self.assertEqual(os.getresgid(), p.gids()) def test_nice(self): p = psutil.Process() From 76707d0dfc3982419cf3c6a6c8a7198a6b07575f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 11:36:20 +0100 Subject: [PATCH 187/922] README: add psutil portings --- README.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.rst b/README.rst index 741fd4f87..32fde350e 100644 --- a/README.rst +++ b/README.rst @@ -83,6 +83,17 @@ Here's some I find particularly interesting: - https://github.com/Jahaja/psdash - https://github.com/ajenti/ajenti + +======== +Portings +======== + +- Go: https://github.com/shirou/gopsutil +- C: https://github.com/hamon-in/cpslib +- Node: https://github.com/christkv/node-psutil +- Rust: https://github.com/borntyping/rust-psutil +- Ruby: https://github.com/spacewander/posixpsutil + ============== Example usages ============== From 6cbf4afde230783df241d4c2b7657bc8eb883e58 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 11:59:20 +0100 Subject: [PATCH 188/922] add cpu_count() windows specific test --- psutil/tests/test_windows.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index cf6825fe5..fe8b5dac2 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -85,6 +85,11 @@ def test_cpu_count(self): num_cpus = int(os.environ['NUMBER_OF_PROCESSORS']) self.assertEqual(num_cpus, psutil.cpu_count()) + def test_cpu_count_2(self): + sys_value = win32api.GetSystemInfo()[5] + psutil_value = psutil.cpu_count() + self.assertEqual(sys_value, psutil_value) + def test_cpu_freq(self): w = wmi.WMI() proc = w.Win32_Processor()[0] From f2b9b3cdc6e12ed19b28cc1eeefacb047136c48e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 12:04:08 +0100 Subject: [PATCH 189/922] add username() windows specific test --- psutil/tests/test_windows.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index fe8b5dac2..b18c04d43 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -343,6 +343,11 @@ def test_compare_name_exe(self): else: self.assertEqual(a, b) + def test_username(self): + sys_value = win32api.GetUserName() + psutil_value = psutil.Process(self.pid).username() + self.assertEqual(sys_value, psutil_value.split('\\')[1]) + @unittest.skipUnless(WINDOWS, "WINDOWS only") class TestProcessWMI(unittest.TestCase): From c7bb0ed3b6f0142f42adc7c2bffb114483bb264e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 12:09:01 +0100 Subject: [PATCH 190/922] add cmdline() windows specific test --- psutil/tests/test_windows.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index b18c04d43..135486567 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -11,6 +11,7 @@ import glob import os import platform +import re import signal import subprocess import sys @@ -345,9 +346,14 @@ def test_compare_name_exe(self): def test_username(self): sys_value = win32api.GetUserName() - psutil_value = psutil.Process(self.pid).username() + psutil_value = psutil.Process().username() self.assertEqual(sys_value, psutil_value.split('\\')[1]) + def test_cmdline(self): + sys_value = re.sub(' +', ' ', win32api.GetCommandLine()) + psutil_value = ' '.join(psutil.Process().cmdline()) + self.assertEqual(sys_value, psutil_value) + @unittest.skipUnless(WINDOWS, "WINDOWS only") class TestProcessWMI(unittest.TestCase): From 75c022e2cb240e27c73f92b6c131f28b3a29be90 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 12:16:54 +0100 Subject: [PATCH 191/922] add disk_usage() windows specific test --- psutil/tests/test_windows.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 135486567..5d72b2b3a 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -153,6 +153,17 @@ def test_disks(self): else: self.fail("can't find partition %s" % repr(ps_part)) + def test_disk_usage(self): + for disk in psutil.disk_partitions(): + sys_value = win32api.GetDiskFreeSpaceEx(disk.mountpoint) + psutil_value = psutil.disk_usage(disk.mountpoint) + self.assertAlmostEqual(sys_value[0], psutil_value.free, + delta=1024 * 1024) + self.assertAlmostEqual(sys_value[1], psutil_value.total, + delta=1024 * 1024) + self.assertEqual(psutil_value.used, + psutil_value.total - psutil_value.free) + def test_net_if_stats(self): ps_names = set(cext.net_if_stats()) wmi_adapters = wmi.WMI().Win32_NetworkAdapter() From fae3b9a6074468e145c5bd1a7dbfd7db8605e475 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 12:23:23 +0100 Subject: [PATCH 192/922] add disk_partitions() windows specific test --- psutil/tests/test_windows.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 5d72b2b3a..68eb75bb4 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -164,6 +164,13 @@ def test_disk_usage(self): self.assertEqual(psutil_value.used, psutil_value.total - psutil_value.free) + def test_disk_partitions(self): + sys_value = [ + x + '\\' for x in win32api.GetLogicalDriveStrings().split("\\\x00") + if x and not x.startswith('A:')] + psutil_value = [x.mountpoint for x in psutil.disk_partitions(all=True)] + self.assertEqual(sys_value, psutil_value) + def test_net_if_stats(self): ps_names = set(cext.net_if_stats()) wmi_adapters = wmi.WMI().Win32_NetworkAdapter() From e0d48ad6b8b8b17a0077ea7d65406063151092b0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 12:32:29 +0100 Subject: [PATCH 193/922] add sensors_battery() windows specific test --- psutil/tests/test_windows.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 68eb75bb4..4514c61fd 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -208,6 +208,12 @@ def test_percent(self): self.assertEqual( battery_psutil.power_plugged, battery_wmi.BatteryStatus == 1) + def test_battery_present(self): + if win32api.GetPwrCapabilities()['SystemBatteriesPresent']: + self.assertIsNotNone(psutil.sensors_battery()) + else: + self.assertIsNone(psutil.sensors_battery()) + def test_emulate_no_battery(self): with mock.patch("psutil._pswindows.cext.sensors_battery", return_value=(0, 128, 0, 0)) as m: From cccc9789568d91cd07ad31a145c0b5113068c454 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 13:01:53 +0100 Subject: [PATCH 194/922] add Process.cpu_times() windows specific test --- psutil/tests/test_windows.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 4514c61fd..5b3e981b3 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -20,6 +20,7 @@ try: import win32api # requires "pip install pypiwin32" / "make setup-dev-env" import win32con + import win32process import wmi # requires "pip install wmi" / "make setup-dev-env" except ImportError: if os.name == 'nt': @@ -378,6 +379,17 @@ def test_cmdline(self): psutil_value = ' '.join(psutil.Process().cmdline()) self.assertEqual(sys_value, psutil_value) + def test_cpu_times(self): + handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, + win32con.FALSE, os.getpid()) + self.addCleanup(win32api.CloseHandle, handle) + sys_times = win32process.GetProcessTimes(handle) + psutil_times = psutil.Process().cpu_times() + self.assertAlmostEqual( + psutil_times.user, sys_times['UserTime'] / 10000000.0, delta=0.2) + self.assertAlmostEqual( + psutil_times.user, sys_times['KernelTime'] / 10000000.0, delta=0.2) + @unittest.skipUnless(WINDOWS, "WINDOWS only") class TestProcessWMI(unittest.TestCase): From 471df54d1aa471a695c982e55e3af95b656494b3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 13:06:16 +0100 Subject: [PATCH 195/922] add Process.nice() windows specific test --- psutil/tests/test_windows.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 5b3e981b3..3cff652e4 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -383,12 +383,20 @@ def test_cpu_times(self): handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, win32con.FALSE, os.getpid()) self.addCleanup(win32api.CloseHandle, handle) - sys_times = win32process.GetProcessTimes(handle) - psutil_times = psutil.Process().cpu_times() + sys_value = win32process.GetProcessTimes(handle) + psutil_value = psutil.Process().cpu_times() self.assertAlmostEqual( - psutil_times.user, sys_times['UserTime'] / 10000000.0, delta=0.2) + psutil_value.user, sys_value['UserTime'] / 10000000.0, delta=0.2) self.assertAlmostEqual( - psutil_times.user, sys_times['KernelTime'] / 10000000.0, delta=0.2) + psutil_value.user, sys_value['KernelTime'] / 10000000.0, delta=0.2) + + def test_nice(self): + handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, + win32con.FALSE, os.getpid()) + self.addCleanup(win32api.CloseHandle, handle) + sys_value = win32process.GetPriorityClass(handle) + psutil_value = psutil.Process().nice() + self.assertEqual(psutil_value, sys_value) @unittest.skipUnless(WINDOWS, "WINDOWS only") From c476be7021be9fb03d4f0bc8f6d4eb9b7ac48170 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 13:26:12 +0100 Subject: [PATCH 196/922] add Process.memory_info() windows specific test --- psutil/tests/test_windows.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 3cff652e4..7be2b25d5 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -398,6 +398,33 @@ def test_nice(self): psutil_value = psutil.Process().nice() self.assertEqual(psutil_value, sys_value) + def test_memory_info(self): + handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, + win32con.FALSE, self.pid) + self.addCleanup(win32api.CloseHandle, handle) + sys_value = win32process.GetProcessMemoryInfo(handle) + psutil_value = psutil.Process(self.pid).memory_info() + self.assertEqual( + sys_value['PeakWorkingSetSize'], psutil_value.peak_wset) + self.assertEqual( + sys_value['WorkingSetSize'], psutil_value.wset) + self.assertEqual( + sys_value['QuotaPeakPagedPoolUsage'], psutil_value.peak_paged_pool) + self.assertEqual( + sys_value['QuotaPagedPoolUsage'], psutil_value.paged_pool) + self.assertEqual( + sys_value['QuotaPeakNonPagedPoolUsage'], + psutil_value.peak_nonpaged_pool) + self.assertEqual( + sys_value['QuotaNonPagedPoolUsage'], psutil_value.nonpaged_pool) + self.assertEqual( + sys_value['PagefileUsage'], psutil_value.pagefile) + self.assertEqual( + sys_value['PeakPagefileUsage'], psutil_value.peak_pagefile) + + self.assertEqual(psutil_value.rss, psutil_value.wset) + self.assertEqual(psutil_value.vms, psutil_value.pagefile) + @unittest.skipUnless(WINDOWS, "WINDOWS only") class TestProcessWMI(unittest.TestCase): From 95a25e0391994cff643a25c570eb5fd47c6e29b2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 13:33:37 +0100 Subject: [PATCH 197/922] add Process.wait() windows specific test --- psutil/tests/test_windows.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 7be2b25d5..37e5021d4 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -425,6 +425,16 @@ def test_memory_info(self): self.assertEqual(psutil_value.rss, psutil_value.wset) self.assertEqual(psutil_value.vms, psutil_value.pagefile) + def test_wait(self): + handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, + win32con.FALSE, self.pid) + self.addCleanup(win32api.CloseHandle, handle) + p = psutil.Process(self.pid) + p.terminate() + psutil_value = p.wait() + sys_value = win32process.GetExitCodeProcess(handle) + self.assertEqual(psutil_value, sys_value) + @unittest.skipUnless(WINDOWS, "WINDOWS only") class TestProcessWMI(unittest.TestCase): From 277e66e05c0fc515335d9e41ca4343282fb0009f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 13:37:39 +0100 Subject: [PATCH 198/922] add Process.cpu_affinity() windows specific test --- psutil/tests/test_windows.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 37e5021d4..35d069597 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -435,6 +435,18 @@ def test_wait(self): sys_value = win32process.GetExitCodeProcess(handle) self.assertEqual(psutil_value, sys_value) + def test_cpu_affinity(self): + def from_bitmask(x): + return [i for i in range(64) if (1 << i) & x] + + handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, + win32con.FALSE, self.pid) + self.addCleanup(win32api.CloseHandle, handle) + sys_value = from_bitmask( + win32process.GetProcessAffinityMask(handle)[0]) + psutil_value = psutil.Process(self.pid).cpu_affinity() + self.assertEqual(psutil_value, sys_value) + @unittest.skipUnless(WINDOWS, "WINDOWS only") class TestProcessWMI(unittest.TestCase): From caeaa6864a8211e8e6ae93c8ebda5e17a8b9f738 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 13:44:19 +0100 Subject: [PATCH 199/922] add Process.io_counters() windows specific test --- psutil/tests/test_windows.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 35d069597..0c3c8a43f 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -447,6 +447,21 @@ def from_bitmask(x): psutil_value = psutil.Process(self.pid).cpu_affinity() self.assertEqual(psutil_value, sys_value) + def test_io_counters(self): + handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, + win32con.FALSE, os.getpid()) + self.addCleanup(win32api.CloseHandle, handle) + sys_value = win32process.GetProcessIoCounters(handle) + psutil_value = psutil.Process().io_counters() + self.assertEqual( + psutil_value.read_count, sys_value['ReadOperationCount']) + self.assertEqual( + psutil_value.write_count, sys_value['WriteOperationCount']) + self.assertEqual( + psutil_value.read_bytes, sys_value['ReadTransferCount']) + self.assertEqual( + psutil_value.write_bytes, sys_value['WriteTransferCount']) + @unittest.skipUnless(WINDOWS, "WINDOWS only") class TestProcessWMI(unittest.TestCase): From 9333bed78e9c8ece2173aee188b17293a7c94cc8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 15:21:48 +0100 Subject: [PATCH 200/922] add Process.num_handles() windows specific test --- psutil/tests/test_windows.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 0c3c8a43f..069ed6445 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -291,7 +291,7 @@ def test_exe(self): except psutil.Error: pass - def test_num_handles(self): + def test_num_handles_increment(self): p = psutil.Process(os.getpid()) before = p.num_handles() handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, @@ -462,6 +462,21 @@ def test_io_counters(self): self.assertEqual( psutil_value.write_bytes, sys_value['WriteTransferCount']) + def test_num_handles(self): + import ctypes + import ctypes.wintypes + PROCESS_QUERY_INFORMATION = 0x400 + handle = ctypes.windll.kernel32.OpenProcess( + PROCESS_QUERY_INFORMATION, 0, os.getpid()) + self.addCleanup(ctypes.windll.kernel32.CloseHandle, handle) + hndcnt = ctypes.wintypes.DWORD() + ctypes.windll.kernel32.GetProcessHandleCount( + handle, ctypes.byref(hndcnt)) + sys_value = hndcnt.value + psutil_value = psutil.Process().num_handles() + ctypes.windll.kernel32.CloseHandle(handle) + self.assertEqual(psutil_value, sys_value + 1) + @unittest.skipUnless(WINDOWS, "WINDOWS only") class TestProcessWMI(unittest.TestCase): @@ -591,8 +606,6 @@ def test_cpu_times(self): def test_io_counters(self): io_counters_1 = psutil.Process(self.pid).io_counters() - print("") - print(io_counters_1) with mock.patch("psutil._psplatform.cext.proc_io_counters", side_effect=OSError(errno.EPERM, "msg")) as fun: io_counters_2 = psutil.Process(self.pid).io_counters() From 57745cfb4914742f047195717bb31c26bac9c23a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 16:23:19 +0100 Subject: [PATCH 201/922] fix #976 / windows / io_counters(): add 2 new fields --- HISTORY.rst | 2 ++ docs/index.rst | 22 ++++++++++++++++------ psutil/_psutil_windows.c | 13 +++++++++---- psutil/_pswindows.py | 30 +++++++++++++++++++----------- psutil/tests/test_process.py | 4 ++++ psutil/tests/test_windows.py | 6 ++++-- 6 files changed, 54 insertions(+), 23 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 603bc1c02..c641998a8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,8 @@ **Enhancements** - 971_: [Linux] Add psutil.sensors_fans() function. (patch by Nicolas Hennion) +- 976_: [Windows] Process.io_counters() has 2 new fields: *other_count* and + *other_bytes*. 5.1.3 ===== diff --git a/docs/index.rst b/docs/index.rst index af5abd4bb..9a847a661 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1213,13 +1213,20 @@ Process class .. method:: io_counters() - Return process I/O statistics as a named tuple including the number of read - and write operations performed by the process and the amount of bytes read - and written (cumulative). For Linux you can refer to + Return process I/O statistics as a named tuple. + For Linux you can refer to `/proc filesysem documentation `__. - On BSD there's apparently no way to retrieve bytes counters, hence ``-1`` - is returned for **read_bytes** and **write_bytes** fields. OSX is not - supported. + + - **read_count**: the number of read operations performed (cumulative). + - **write_count**: the number of write operations performed (cumulative). + - **read_bytes**: the number of bytes read (cumulative). + Always ``-1`` on BSD. + - **write_bytes**: the number of bytes written (cumulative). + Always ``-1`` on BSD. + - **other_count** *(Windows)*: the number of I/O operations performed, + other than read and write operations. + - **other_bytes** *(Windows)*: the number of bytes transferred during + operations other than read and write operations. >>> import psutil >>> p = psutil.Process() @@ -1228,6 +1235,9 @@ Process class Availability: all platforms except OSX and Solaris + .. versionchanged:: 5.2.0 added *other_count* and *other_bytes* Windows + metrics. + .. method:: num_ctx_switches() The number voluntary and involuntary context switches performed by diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index e1e537236..430f518e5 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2101,11 +2101,13 @@ psutil_proc_io_counters(PyObject *self, PyObject *args) { return PyErr_SetFromWindowsErr(0); } CloseHandle(hProcess); - return Py_BuildValue("(KKKK)", + return Py_BuildValue("(KKKKKK)", IoCounters.ReadOperationCount, IoCounters.WriteOperationCount, IoCounters.ReadTransferCount, - IoCounters.WriteTransferCount); + IoCounters.WriteTransferCount, + IoCounters.OtherOperationCount, + IoCounters.OtherTransferCount); } @@ -2793,9 +2795,9 @@ psutil_proc_info(PyObject *self, PyObject *args) { py_retlist = Py_BuildValue( #if defined(_WIN64) - "kkdddiKKKK" "kKKKKKKKKK", + "kkdddiKKKKKK" "kKKKKKKKKK", #else - "kkdddiKKKK" "kIIIIIIIII", + "kkdddiKKKKKK" "kIIIIIIIII", #endif process->HandleCount, // num handles ctx_switches, // num ctx switches @@ -2803,10 +2805,13 @@ psutil_proc_info(PyObject *self, PyObject *args) { kernel_time, // cpu kernel time (double)create_time, // create time (int)process->NumberOfThreads, // num threads + // IO counters process->ReadOperationCount.QuadPart, // io rcount process->WriteOperationCount.QuadPart, // io wcount process->ReadTransferCount.QuadPart, // io rbytes process->WriteTransferCount.QuadPart, // io wbytes + process->OtherOperationCount.QuadPart, // io others count + process->OtherTransferCount.QuadPart, // io others bytes // memory process->PageFaultCount, // num page faults process->PeakWorkingSetSize, // peak wset diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 271b4c7d7..2b60c55ba 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -120,16 +120,18 @@ class Priority(enum.IntEnum): io_wcount=7, io_rbytes=8, io_wbytes=9, - num_page_faults=10, - peak_wset=11, - wset=12, - peak_paged_pool=13, - paged_pool=14, - peak_non_paged_pool=15, - non_paged_pool=16, - pagefile=17, - peak_pagefile=18, - mem_private=19, + io_count_others=10, + io_bytes_others=11, + num_page_faults=12, + peak_wset=13, + wset=14, + peak_paged_pool=15, + paged_pool=16, + peak_non_paged_pool=17, + non_paged_pool=18, + pagefile=19, + peak_pagefile=20, + mem_private=21, ) @@ -154,6 +156,10 @@ class Priority(enum.IntEnum): 'ntpinfo', ['num_handles', 'ctx_switches', 'user_time', 'kernel_time', 'create_time', 'num_threads', 'io_rcount', 'io_wcount', 'io_rbytes', 'io_wbytes']) +# psutil.Process.io_counters() +pio = namedtuple('pio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'other_count', 'other_bytes']) # ===================================================================== @@ -900,10 +906,12 @@ def io_counters(self): info[pinfo_map['io_wcount']], info[pinfo_map['io_rbytes']], info[pinfo_map['io_wbytes']], + info[pinfo_map['io_count_others']], + info[pinfo_map['io_bytes_others']], ) else: raise - return _common.pio(*ret) + return pio(*ret) @wrap_exceptions def status(self): diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 4d1a88a3c..44a60e7b4 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -348,6 +348,10 @@ def test_io_counters(self): assert io2.write_bytes >= io1.write_bytes, (io1, io2) assert io2.read_count >= io1.read_count, (io1, io2) assert io2.read_bytes >= io1.read_bytes, (io1, io2) + # sanity check + for i in range(len(io2)): + self.assertGreaterEqual(io2[i], 0) + self.assertGreaterEqual(io2[i], 0) @unittest.skipUnless(LINUX or (WINDOWS and get_winver() >= WIN_VISTA), 'platform not supported') diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 069ed6445..2fb93ad11 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -461,6 +461,10 @@ def test_io_counters(self): psutil_value.read_bytes, sys_value['ReadTransferCount']) self.assertEqual( psutil_value.write_bytes, sys_value['WriteTransferCount']) + self.assertEqual( + psutil_value.other_count, sys_value['OtherOperationCount']) + self.assertEqual( + psutil_value.other_bytes, sys_value['OtherTransferCount']) def test_num_handles(self): import ctypes @@ -610,8 +614,6 @@ def test_io_counters(self): side_effect=OSError(errno.EPERM, "msg")) as fun: io_counters_2 = psutil.Process(self.pid).io_counters() for i in range(len(io_counters_1)): - self.assertGreaterEqual(io_counters_1[i], 0) - self.assertGreaterEqual(io_counters_2[i], 0) self.assertAlmostEqual( io_counters_1[i], io_counters_2[i], delta=5) assert fun.called From 3630f631a319aa0101b239c2711a92fb6ffcf91f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 17:10:36 +0100 Subject: [PATCH 202/922] update doc --- docs/index.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 9a847a661..811f9fb21 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1215,23 +1215,27 @@ Process class Return process I/O statistics as a named tuple. For Linux you can refer to - `/proc filesysem documentation `__. + `/proc filesysem documentation `__, ``/proc//io`` section. - **read_count**: the number of read operations performed (cumulative). + This is supposed to count the number of read-related syscalls such as + ``read()`` and ``pread()`` on UNIX. - **write_count**: the number of write operations performed (cumulative). + This is supposed to count the number of write-related syscalls such as + ``write()`` and ``pwrite()`` on UNIX. - **read_bytes**: the number of bytes read (cumulative). Always ``-1`` on BSD. - **write_bytes**: the number of bytes written (cumulative). Always ``-1`` on BSD. - - **other_count** *(Windows)*: the number of I/O operations performed, + - **other_count** *(Windows)*: the number of I/O operations performed other than read and write operations. - **other_bytes** *(Windows)*: the number of bytes transferred during operations other than read and write operations. - >>> import psutil - >>> p = psutil.Process() - >>> p.io_counters() - pio(read_count=454556, write_count=3456, read_bytes=110592, write_bytes=0) + >>> import psutil + >>> p = psutil.Process() + >>> p.io_counters() + pio(read_count=454556, write_count=3456, read_bytes=110592, write_bytes=0) Availability: all platforms except OSX and Solaris From deb5ec3e7042fd7639fe27f50336b041c9b40c69 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 20:13:13 +0100 Subject: [PATCH 203/922] #976 / Linux / Process.io_counters(): return also read_chars and write_chars fields --- HISTORY.rst | 2 ++ docs/index.rst | 14 +++++++++++--- psutil/_pslinux.py | 36 ++++++++++++++++++++++-------------- psutil/tests/test_process.py | 13 ++++++++++--- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c641998a8..d760cdf08 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,8 @@ - 971_: [Linux] Add psutil.sensors_fans() function. (patch by Nicolas Hennion) - 976_: [Windows] Process.io_counters() has 2 new fields: *other_count* and *other_bytes*. +- 976_: [Linux] Process.io_counters() has 2 new fields: *read_chars* and + *write_chars*. 5.1.3 ===== diff --git a/docs/index.rst b/docs/index.rst index 811f9fb21..343571396 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1215,7 +1215,7 @@ Process class Return process I/O statistics as a named tuple. For Linux you can refer to - `/proc filesysem documentation `__, ``/proc//io`` section. + `/proc filesysem documentation `__. - **read_count**: the number of read operations performed (cumulative). This is supposed to count the number of read-related syscalls such as @@ -1227,6 +1227,14 @@ Process class Always ``-1`` on BSD. - **write_bytes**: the number of bytes written (cumulative). Always ``-1`` on BSD. + - **read_chars** *(Linux)*: the amount of bytes which this process passed + to ``read()`` and ``pread()`` syscalls (cumulative). + Differently from *read_bytes* it doesn't care whether or not actual + physical disk IO occurred. + - **write_chars** *(Linux)*: the amount of bytes which this process passed + to ``write()`` and ``pwrite()`` syscalls (cumulative). + Differently from *write_bytes* it doesn't care whether or not actual + physical disk IO occurred. - **other_count** *(Windows)*: the number of I/O operations performed other than read and write operations. - **other_bytes** *(Windows)*: the number of bytes transferred during @@ -1239,8 +1247,8 @@ Process class Availability: all platforms except OSX and Solaris - .. versionchanged:: 5.2.0 added *other_count* and *other_bytes* Windows - metrics. + .. versionchanged:: 5.2.0 added *read_chars* and *write_chars* on Linux; + added *other_count* and *other_bytes* Windows. .. method:: num_ctx_switches() diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 52c67302c..2c5e97e28 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -157,25 +157,36 @@ class IOPriority(enum.IntEnum): # ===================================================================== +# psutil.virtual_memory() svmem = namedtuple( 'svmem', ['total', 'available', 'percent', 'used', 'free', 'active', 'inactive', 'buffers', 'cached', 'shared']) +# psutil.disk_io_counters() sdiskio = namedtuple( 'sdiskio', ['read_count', 'write_count', 'read_bytes', 'write_bytes', 'read_time', 'write_time', 'read_merged_count', 'write_merged_count', 'busy_time']) +# psutil.Process().open_files() popenfile = namedtuple( 'popenfile', ['path', 'fd', 'position', 'mode', 'flags']) +# psutil.Process().memory_info() pmem = namedtuple('pmem', 'rss vms shared text lib data dirty') +# psutil.Process().memory_full_info() pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', 'pss', 'swap')) +# psutil.Process().memory_maps(grouped=True) pmmap_grouped = namedtuple( 'pmmap_grouped', ['path', 'rss', 'size', 'pss', 'shared_clean', 'shared_dirty', 'private_clean', 'private_dirty', 'referenced', 'anonymous', 'swap']) +# psutil.Process().memory_maps(grouped=False) pmmap_ext = namedtuple( 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) +# psutil.Process.io_counters() +pio = namedtuple('pio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'read_chars', 'write_chars']) # ===================================================================== @@ -1436,22 +1447,19 @@ def terminal(self): @wrap_exceptions def io_counters(self): fname = "%s/%s/io" % (self._procfs_path, self.pid) + fields = {} with open_binary(fname) as f: - rcount = wcount = rbytes = wbytes = None for line in f: - if rcount is None and line.startswith(b"syscr"): - rcount = int(line.split()[1]) - elif wcount is None and line.startswith(b"syscw"): - wcount = int(line.split()[1]) - elif rbytes is None and line.startswith(b"read_bytes"): - rbytes = int(line.split()[1]) - elif wbytes is None and line.startswith(b"write_bytes"): - wbytes = int(line.split()[1]) - for x in (rcount, wcount, rbytes, wbytes): - if x is None: - raise NotImplementedError( - "couldn't read all necessary info from %r" % fname) - return _common.pio(rcount, wcount, rbytes, wbytes) + name, value = line.split(b': ') + fields[name] = int(value) + return pio( + fields[b'syscr'], # read syscalls + fields[b'syscw'], # write syscalls + fields[b'read_bytes'], # read bytes + fields[b'write_bytes'], # write bytes + fields[b'rchar'], # read chars + fields[b'wchar'], # write chars + ) else: def io_counters(self): raise NotImplementedError("couldn't find /proc/%s/io (kernel " diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 44a60e7b4..255d9b1c5 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -326,16 +326,22 @@ def test_terminal(self): @skip_on_not_implemented(only_if=LINUX) def test_io_counters(self): p = psutil.Process() + # test reads io1 = p.io_counters() with open(PYTHON, 'rb') as f: f.read() io2 = p.io_counters() if not BSD: - assert io2.read_count > io1.read_count, (io1, io2) + self.assertGreater(io2.read_count, io1.read_count) self.assertEqual(io2.write_count, io1.write_count) - assert io2.read_bytes >= io1.read_bytes, (io1, io2) - assert io2.write_bytes >= io1.write_bytes, (io1, io2) + if LINUX: + self.assertGreater(io2.read_chars, io1.read_chars) + self.assertEqual(io2.write_chars, io1.write_chars) + else: + self.assertGreaterEqual(io2.read_bytes, io1.read_bytes) + self.assertGreaterEqual(io2.write_bytes, io1.write_bytes) + # test writes io1 = p.io_counters() with tempfile.TemporaryFile(prefix=TESTFILE_PREFIX) as f: @@ -348,6 +354,7 @@ def test_io_counters(self): assert io2.write_bytes >= io1.write_bytes, (io1, io2) assert io2.read_count >= io1.read_count, (io1, io2) assert io2.read_bytes >= io1.read_bytes, (io1, io2) + # sanity check for i in range(len(io2)): self.assertGreaterEqual(io2[i], 0) From 7ff84c208af10e618e23aacb3691c668eb25a12d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 20:33:58 +0100 Subject: [PATCH 204/922] cosmetic changes --- psutil/_psbsd.py | 29 +++++++++++++++-------------- psutil/_pslinux.py | 14 +++----------- psutil/_psosx.py | 30 +++++++++++++++++++----------- psutil/_pssunos.py | 26 +++++++++++++------------- psutil/_pswindows.py | 28 ++++++++++++---------------- psutil/tests/test_linux.py | 7 ------- 6 files changed, 62 insertions(+), 72 deletions(-) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 72ef71e8b..fc5e1dc8b 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -30,7 +30,7 @@ # ===================================================================== -# --- constants +# --- globals # ===================================================================== @@ -126,26 +126,39 @@ name=24, ) +# these get overwritten on "import psutil" from the __init__.py file +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + # ===================================================================== # --- named tuples # ===================================================================== -# extend base mem ntuple with BSD-specific memory metrics +# psutil.virtual_memory() svmem = namedtuple( 'svmem', ['total', 'available', 'percent', 'used', 'free', 'active', 'inactive', 'buffers', 'cached', 'shared', 'wired']) +# psutil.cpu_times() scputimes = namedtuple( 'scputimes', ['user', 'nice', 'system', 'idle', 'irq']) +# psutil.Process.memory_info() pmem = namedtuple('pmem', ['rss', 'vms', 'text', 'data', 'stack']) +# psutil.Process.memory_full_info() pfullmem = pmem +# psutil.Process.cpu_times() pcputimes = namedtuple('pcputimes', ['user', 'system', 'children_user', 'children_system']) +# psutil.Process.memory_maps(grouped=True) pmmap_grouped = namedtuple( 'pmmap_grouped', 'path rss, private, ref_count, shadow_count') +# psutil.Process.memory_maps(grouped=False) pmmap_ext = namedtuple( 'pmmap_ext', 'addr, perms path rss, private, ref_count, shadow_count') +# psutil.disk_io_counters() if FREEBSD: sdiskio = namedtuple('sdiskio', ['read_count', 'write_count', 'read_bytes', 'write_bytes', @@ -156,18 +169,6 @@ 'read_bytes', 'write_bytes']) -# ===================================================================== -# --- exceptions -# ===================================================================== - - -# these get overwritten on "import psutil" from the __init__.py file -NoSuchProcess = None -ZombieProcess = None -AccessDenied = None -TimeoutExpired = None - - # ===================================================================== # --- memory # ===================================================================== diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 2c5e97e28..2884e4d14 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -59,11 +59,11 @@ # ===================================================================== -# --- constants +# --- globals # ===================================================================== -POWER_SUPPLY_PATH = "/sys/class/power_supply" +POWER_SUPPLY_PATH = "/sys/class/power_supply" HAS_SMAPS = os.path.exists('/proc/%s/smaps' % os.getpid()) HAS_PRLIMIT = hasattr(cext, "linux_prlimit") _DEFAULT = object() @@ -137,14 +137,6 @@ class IOPriority(enum.IntEnum): "0B": _common.CONN_CLOSING } -_DEFAULT = object() - - -# ===================================================================== -# -- exceptions -# ===================================================================== - - # these get overwritten on "import psutil" from the __init__.py file NoSuchProcess = None ZombieProcess = None @@ -529,7 +521,7 @@ def swap_memory(): # ===================================================================== -# --- CPUs +# --- CPU # ===================================================================== diff --git a/psutil/_psosx.py b/psutil/_psosx.py index f22ef2fbd..f780d4594 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -26,7 +26,7 @@ # ===================================================================== -# --- constants +# --- globals # ===================================================================== @@ -82,28 +82,36 @@ volctxsw=7, ) -scputimes = namedtuple('scputimes', ['user', 'nice', 'system', 'idle']) +# these get overwritten on "import psutil" from the __init__.py file +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + + +# ===================================================================== +# --- named tuples +# ===================================================================== + +# psutil.cpu_times() +scputimes = namedtuple('scputimes', ['user', 'nice', 'system', 'idle']) +# psutil.virtual_memory() svmem = namedtuple( 'svmem', ['total', 'available', 'percent', 'used', 'free', 'active', 'inactive', 'wired']) - +# psutil.Process.memory_info() pmem = namedtuple('pmem', ['rss', 'vms', 'pfaults', 'pageins']) +# psutil.Process.memory_full_info() pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', )) - +# psutil.Process.memory_maps(grouped=True) pmmap_grouped = namedtuple( 'pmmap_grouped', 'path rss private swapped dirtied ref_count shadow_depth') - +# psutil.Process.memory_maps(grouped=False) pmmap_ext = namedtuple( 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) -# these get overwritten on "import psutil" from the __init__.py file -NoSuchProcess = None -ZombieProcess = None -AccessDenied = None -TimeoutExpired = None - # ===================================================================== # --- memory diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 41547a8f5..ad72de259 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -28,7 +28,7 @@ # ===================================================================== -# --- constants +# --- globals # ===================================================================== @@ -66,36 +66,36 @@ cext.TCPS_BOUND: CONN_BOUND, # sunos specific } +# these get overwritten on "import psutil" from the __init__.py file +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + # ===================================================================== # --- named tuples # ===================================================================== +# psutil.cpu_times() scputimes = namedtuple('scputimes', ['user', 'system', 'idle', 'iowait']) +# psutil.cpu_times(percpu=True) pcputimes = namedtuple('pcputimes', ['user', 'system', 'children_user', 'children_system']) +# psutil.virtual_memory() svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) +# psutil.Process.memory_info() pmem = namedtuple('pmem', ['rss', 'vms']) pfullmem = pmem +# psutil.Process.memory_maps(grouped=True) pmmap_grouped = namedtuple('pmmap_grouped', ['path', 'rss', 'anonymous', 'locked']) +# psutil.Process.memory_maps(grouped=False) pmmap_ext = namedtuple( 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) -# ===================================================================== -# --- exceptions -# ===================================================================== - - -# these get overwritten on "import psutil" from the __init__.py file -NoSuchProcess = None -ZombieProcess = None -AccessDenied = None -TimeoutExpired = None - - # ===================================================================== # --- utils # ===================================================================== diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 2b60c55ba..0105d6c8b 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -67,7 +67,7 @@ # ===================================================================== -# --- constants +# --- globals # ===================================================================== @@ -134,45 +134,41 @@ class Priority(enum.IntEnum): mem_private=21, ) +# these get overwritten on "import psutil" from the __init__.py file +NoSuchProcess = None +AccessDenied = None +TimeoutExpired = None + # ===================================================================== # --- named tuples # ===================================================================== +# psutil.cpu_times() scputimes = namedtuple('scputimes', ['user', 'system', 'idle', 'interrupt', 'dpc']) +# psutil.virtual_memory() svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) +# psutil.Process.memory_info() pmem = namedtuple( 'pmem', ['rss', 'vms', 'num_page_faults', 'peak_wset', 'wset', 'peak_paged_pool', 'paged_pool', 'peak_nonpaged_pool', 'nonpaged_pool', 'pagefile', 'peak_pagefile', 'private']) +# psutil.Process.memory_full_info() pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', )) +# psutil.Process.memory_maps(grouped=True) pmmap_grouped = namedtuple('pmmap_grouped', ['path', 'rss']) +# psutil.Process.memory_maps(grouped=False) pmmap_ext = namedtuple( 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) -ntpinfo = namedtuple( - 'ntpinfo', ['num_handles', 'ctx_switches', 'user_time', 'kernel_time', - 'create_time', 'num_threads', 'io_rcount', 'io_wcount', - 'io_rbytes', 'io_wbytes']) # psutil.Process.io_counters() pio = namedtuple('pio', ['read_count', 'write_count', 'read_bytes', 'write_bytes', 'other_count', 'other_bytes']) -# ===================================================================== -# --- exceptions -# ===================================================================== - - -# these get overwritten on "import psutil" from the __init__.py file -NoSuchProcess = None -AccessDenied = None -TimeoutExpired = None - - # ===================================================================== # --- utils # ===================================================================== diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 35310b639..8c64afc49 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1293,13 +1293,6 @@ def test_cmdline_mocked(self): p.cmdline() == ['foo', 'bar', ''] assert m.called - def test_io_counters_mocked(self): - with mock.patch('psutil._pslinux.open', create=True) as m: - self.assertRaises( - NotImplementedError, - psutil._pslinux.Process(os.getpid()).io_counters) - assert m.called - def test_readlink_path_deleted_mocked(self): with mock.patch('psutil._pslinux.os.readlink', return_value='/home/foo (deleted)'): From 31f17dfddbac84e7aa131dd74f1633ca93b3dc2b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 20:43:26 +0100 Subject: [PATCH 205/922] update doc --- docs/index.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 343571396..096ec9045 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1227,14 +1227,20 @@ Process class Always ``-1`` on BSD. - **write_bytes**: the number of bytes written (cumulative). Always ``-1`` on BSD. + + Linux specific: + - **read_chars** *(Linux)*: the amount of bytes which this process passed to ``read()`` and ``pread()`` syscalls (cumulative). Differently from *read_bytes* it doesn't care whether or not actual - physical disk IO occurred. + physical disk I/O occurred. - **write_chars** *(Linux)*: the amount of bytes which this process passed to ``write()`` and ``pwrite()`` syscalls (cumulative). Differently from *write_bytes* it doesn't care whether or not actual - physical disk IO occurred. + physical disk I/O occurred. + + Windows specific: + - **other_count** *(Windows)*: the number of I/O operations performed other than read and write operations. - **other_bytes** *(Windows)*: the number of bytes transferred during @@ -1248,7 +1254,7 @@ Process class Availability: all platforms except OSX and Solaris .. versionchanged:: 5.2.0 added *read_chars* and *write_chars* on Linux; - added *other_count* and *other_bytes* Windows. + added *other_count* and *other_bytes* on Windows. .. method:: num_ctx_switches() From 3354a2bac38d4eae40f17a7b31cfceabffbc1ad2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 20:48:40 +0100 Subject: [PATCH 206/922] update doc --- docs/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 096ec9045..da307700b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1003,6 +1003,7 @@ Process class call. Not on POSIX because `ppid may change `__ if process becomes a zombie. + See also :meth:`parent` method. .. method:: name() @@ -1085,6 +1086,7 @@ Process class Utility method which returns the parent process as a :class:`Process` object preemptively checking whether PID has been reused. If no parent PID is known return ``None``. + See also :meth:`ppid` method. .. method:: status() From 42f5f9e86aa67e32cf7de146e6960cdc9914f14d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 21:03:10 +0100 Subject: [PATCH 207/922] update doc --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index da307700b..ad2a78d3c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1385,7 +1385,7 @@ Process class The returned number should be ``<=`` :func:`psutil.cpu_count()`. It may be used in conjunction with ``psutil.cpu_percent(percpu=True)`` to observe the system workload distributed across multiple CPUs as shown by - `cpu_workload.py `__ example script. + `cpu_distribution.py `__ example script. Availability: Linux, FreeBSD, SunOS From 915bb2199cac2ec8ee3c01b36f3545d49a87486d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Feb 2017 21:13:21 +0100 Subject: [PATCH 208/922] add linux test --- psutil/tests/test_process.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 255d9b1c5..6580fe9b8 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -350,10 +350,13 @@ def test_io_counters(self): else: f.write("x" * 1000000) io2 = p.io_counters() - assert io2.write_count >= io1.write_count, (io1, io2) - assert io2.write_bytes >= io1.write_bytes, (io1, io2) - assert io2.read_count >= io1.read_count, (io1, io2) - assert io2.read_bytes >= io1.read_bytes, (io1, io2) + self.assertGreaterEqual(io2.write_count, io1.write_count) + self.assertGreaterEqual(io2.write_bytes, io1.write_bytes) + self.assertGreaterEqual(io2.read_count, io1.read_count) + self.assertGreaterEqual(io2.read_bytes, io1.read_bytes) + if LINUX: + self.assertGreater(io2.write_chars, io1.write_chars) + self.assertGreaterEqual(io2.read_chars, io1.read_chars) # sanity check for i in range(len(io2)): From d964eeb6e85a98b3e27e16ecbf89cf06c37c06f1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 15 Feb 2017 13:21:25 +0100 Subject: [PATCH 209/922] Update INSTALL.rst update INSTALL instructions --- INSTALL.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/INSTALL.rst b/INSTALL.rst index d731bd3de..d86c022ce 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -141,6 +141,8 @@ Install: Install from sources ==================== +:: + git clone https://github.com/giampaolo/psutil.git cd psutil python setup.py install From c414ecd9b9151b05542ead65da1e7bf20ee21861 Mon Sep 17 00:00:00 2001 From: Baruch Siach Date: Thu, 16 Feb 2017 19:07:19 +0200 Subject: [PATCH 210/922] Fix build with musl libc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suppress inclusion of linux/sysinfo.h to fix redefinition of struct sysinfo that musl libc defines in sys/sysinfo.h, which least to the following build failure (paths abbreviated): In file included from .../usr/include/linux/kernel.h:4:0, from .../usr/include/linux/ethtool.h:16, from psutil/_psutil_linux.c:35: .../usr/include/linux/sysinfo.h:7:8: error: redefinition of ‘struct sysinfo’ struct sysinfo { ^ In file included from psutil/_psutil_linux.c:21:0: .../usr/include/sys/sysinfo.h:10:8: note: originally defined here struct sysinfo { ^ Fixes #872 --- psutil/_psutil_linux.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 4923ead6a..0296dd544 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -32,6 +32,8 @@ typedef __u16 u16; typedef __u8 u8; #endif +/* Avoid redefinition of struct sysinfo with musl libc */ +#define _LINUX_SYSINFO_H #include /* The minimum number of CPUs allocated in a cpu_set_t */ From 76d1fb61c14d286aa645f154f4a2b7a7bae8a828 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 16 Feb 2017 20:00:43 +0100 Subject: [PATCH 211/922] update HISTORY and CREDITS --- CREDITS | 4 ++++ HISTORY.rst | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/CREDITS b/CREDITS index 75d86db5b..518caf0be 100644 --- a/CREDITS +++ b/CREDITS @@ -433,3 +433,7 @@ I: 959 N: Nicolas Hennion W: https://github.com/nicolargo I: 974 + +N: Baruch Siach +W: https://github.com/baruchsiach +I: 872 diff --git a/HISTORY.rst b/HISTORY.rst index d760cdf08..024629a8e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,10 @@ - 976_: [Linux] Process.io_counters() has 2 new fields: *read_chars* and *write_chars*. +**Bug fixes** + +- 872_: [Linux] can now compile on Linux by using MUSL C library. + 5.1.3 ===== From f03b786250f0dee4f203450d212f12e78a931363 Mon Sep 17 00:00:00 2001 From: Max Belanger Date: Wed, 1 Mar 2017 19:14:11 -0300 Subject: [PATCH 212/922] first pass --- psutil/arch/windows/process_handles.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/psutil/arch/windows/process_handles.c b/psutil/arch/windows/process_handles.c index b260450e5..d1d04627e 100644 --- a/psutil/arch/windows/process_handles.c +++ b/psutil/arch/windows/process_handles.c @@ -312,8 +312,8 @@ psutil_NtQueryObject() { } -void -psutil_NtQueryObjectThread() { +DWORD WINAPI +psutil_NtQueryObjectThread(LPVOID lpvParam) { // Prevent the thread stack from leaking when this // thread gets terminated due to NTQueryObject hanging g_fiber = ConvertThreadToFiber(NULL); @@ -329,6 +329,8 @@ psutil_NtQueryObjectThread() { &g_dwLength); SetEvent(g_hEvtFinish); } + + return 0; } From e69033dfea975bbb8be34e464fd1b734c177a6f5 Mon Sep 17 00:00:00 2001 From: Max Belanger Date: Thu, 2 Mar 2017 12:05:22 -0800 Subject: [PATCH 213/922] correct signature --- psutil/arch/windows/process_handles.c | 2 +- psutil/arch/windows/process_handles.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/arch/windows/process_handles.c b/psutil/arch/windows/process_handles.c index d1d04627e..e4d552881 100644 --- a/psutil/arch/windows/process_handles.c +++ b/psutil/arch/windows/process_handles.c @@ -280,7 +280,7 @@ psutil_NtQueryObject() { g_hThread = CreateThread( NULL, 0, - (LPTHREAD_START_ROUTINE)psutil_NtQueryObjectThread, + psutil_NtQueryObjectThread, NULL, 0, NULL); diff --git a/psutil/arch/windows/process_handles.h b/psutil/arch/windows/process_handles.h index ea5fbdbeb..4a022c1c1 100644 --- a/psutil/arch/windows/process_handles.h +++ b/psutil/arch/windows/process_handles.h @@ -106,6 +106,6 @@ PyObject* psutil_get_open_files(long pid, HANDLE processHandle); PyObject* psutil_get_open_files_ntqueryobject(long dwPid, HANDLE hProcess); PyObject* psutil_get_open_files_getmappedfilename(long dwPid, HANDLE hProcess); DWORD psutil_NtQueryObject(void); -void psutil_NtQueryObjectThread(void); +DWORD WINAPI psutil_NtQueryObjectThread(LPVOID lpvParam); #endif // __PROCESS_HANDLES_H__ From d1d87fdf8c832d01b7165ad6af213fe2b1e8cfa6 Mon Sep 17 00:00:00 2001 From: Max Belanger Date: Thu, 2 Mar 2017 12:49:26 -0800 Subject: [PATCH 214/922] remove fiber code, not needed as we don't support xp anymore --- psutil/arch/windows/process_handles.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/psutil/arch/windows/process_handles.c b/psutil/arch/windows/process_handles.c index e4d552881..670d74f0b 100644 --- a/psutil/arch/windows/process_handles.c +++ b/psutil/arch/windows/process_handles.c @@ -19,7 +19,6 @@ HANDLE g_hThread = NULL; PUNICODE_STRING g_pNameBuffer = NULL; ULONG g_dwSize = 0; ULONG g_dwLength = 0; -PVOID g_fiber = NULL; PVOID @@ -300,11 +299,6 @@ psutil_NtQueryObject() { WaitForSingleObject(g_hThread, INFINITE); CloseHandle(g_hThread); - // Cleanup Fiber - if (g_fiber != NULL) - DeleteFiber(g_fiber); - g_fiber = NULL; - g_hThread = NULL; } @@ -314,10 +308,6 @@ psutil_NtQueryObject() { DWORD WINAPI psutil_NtQueryObjectThread(LPVOID lpvParam) { - // Prevent the thread stack from leaking when this - // thread gets terminated due to NTQueryObject hanging - g_fiber = ConvertThreadToFiber(NULL); - // Loop infinitely waiting for work while (TRUE) { WaitForSingleObject(g_hEvtStart, INFINITE); @@ -329,8 +319,6 @@ psutil_NtQueryObjectThread(LPVOID lpvParam) { &g_dwLength); SetEvent(g_hEvtFinish); } - - return 0; } From aa372c034e8deaf7e1acf981bf4f665e2d501a67 Mon Sep 17 00:00:00 2001 From: Max Belanger Date: Fri, 3 Mar 2017 11:25:52 -0800 Subject: [PATCH 215/922] add history entry --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 024629a8e..b96cf034e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,7 @@ **Bug fixes** - 872_: [Linux] can now compile on Linux by using MUSL C library. +- 985_: [Windows] Fix a crash in `Process.open_files` when the worker thread for `NtQueryObject` times out. 5.1.3 ===== From 1922a7674011852b64f3405e47113ba540a86c77 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 4 Mar 2017 16:13:50 +0700 Subject: [PATCH 216/922] fix #986: [Linux] Process.cwd() may raise NoSuchProcess instead of ZombieProcess. --- HISTORY.rst | 1 + psutil/_pslinux.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index b96cf034e..ad931e770 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ - 872_: [Linux] can now compile on Linux by using MUSL C library. - 985_: [Windows] Fix a crash in `Process.open_files` when the worker thread for `NtQueryObject` times out. +- 986_: [Linux] Process.cwd() may raise NoSuchProcess instead of ZombieProcess. 5.1.3 ===== diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 2884e4d14..533b5485d 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1617,7 +1617,16 @@ def memory_maps(self): @wrap_exceptions def cwd(self): - return readlink("%s/%s/cwd" % (self._procfs_path, self.pid)) + try: + return readlink("%s/%s/cwd" % (self._procfs_path, self.pid)) + except OSError as err: + # https://github.com/giampaolo/psutil/issues/986 + if err.errno in (errno.ENOENT, errno.ESRCH): + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + raise @wrap_exceptions def num_ctx_switches(self, _ctxsw_re=re.compile(b'ctxt_switches:\t(\d+)')): From ea61ed5a21b02878d14b1552166aafd0b12e36c7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 4 Mar 2017 16:17:18 +0700 Subject: [PATCH 217/922] fix failing test --- psutil/tests/test_linux.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 8c64afc49..f9731bea5 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1041,6 +1041,8 @@ def test_percent(self): @unittest.skipUnless(which("acpi"), "acpi utility not available") def test_power_plugged(self): out = sh("acpi -b") + if 'unknown' in out.lower(): + return unittest.skip("acpi output not reliable") plugged = "Charging" in out.split('\n')[0] self.assertEqual(psutil.sensors_battery().power_plugged, plugged) From bc7e8d82fd412c1157e8259157a7c61ca0c7a7c9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 4 Mar 2017 16:23:48 +0700 Subject: [PATCH 218/922] windows: disable test causing occasional failures --- docs/index.rst | 2 +- psutil/tests/test_windows.py | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ad2a78d3c..2f69480f0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1251,7 +1251,7 @@ Process class >>> import psutil >>> p = psutil.Process() >>> p.io_counters() - pio(read_count=454556, write_count=3456, read_bytes=110592, write_bytes=0) + pio(read_count=454556, write_count=3456, read_bytes=110592, write_bytes=0, read_chars=769931, write_chars=203) Availability: all platforms except OSX and Solaris diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 2fb93ad11..3fcc20ede 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -375,20 +375,24 @@ def test_username(self): self.assertEqual(sys_value, psutil_value.split('\\')[1]) def test_cmdline(self): - sys_value = re.sub(' +', ' ', win32api.GetCommandLine()) + sys_value = re.sub(' +', ' ', win32api.GetCommandLine()).strip() psutil_value = ' '.join(psutil.Process().cmdline()) self.assertEqual(sys_value, psutil_value) - def test_cpu_times(self): - handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, - win32con.FALSE, os.getpid()) - self.addCleanup(win32api.CloseHandle, handle) - sys_value = win32process.GetProcessTimes(handle) - psutil_value = psutil.Process().cpu_times() - self.assertAlmostEqual( - psutil_value.user, sys_value['UserTime'] / 10000000.0, delta=0.2) - self.assertAlmostEqual( - psutil_value.user, sys_value['KernelTime'] / 10000000.0, delta=0.2) + # XXX - occasional failures + + # def test_cpu_times(self): + # handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, + # win32con.FALSE, os.getpid()) + # self.addCleanup(win32api.CloseHandle, handle) + # sys_value = win32process.GetProcessTimes(handle) + # psutil_value = psutil.Process().cpu_times() + # self.assertAlmostEqual( + # psutil_value.user, sys_value['UserTime'] / 10000000.0, + # delta=0.2) + # self.assertAlmostEqual( + # psutil_value.user, sys_value['KernelTime'] / 10000000.0, + # delta=0.2) def test_nice(self): handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, From 48d633f0be7cf439351c445cab08fdc215fa4106 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 4 Mar 2017 16:29:44 +0700 Subject: [PATCH 219/922] fix #983: remove CPU% column from test() output --- psutil/__init__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index c72b7a7d2..0d4680e8c 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2320,13 +2320,13 @@ def test(): # pragma: no cover import datetime today_day = datetime.date.today() - templ = "%-10s %5s %4s %4s %7s %7s %-13s %5s %7s %s" - attrs = ['pid', 'cpu_percent', 'memory_percent', 'name', 'cpu_times', + templ = "%-10s %5s %4s %7s %7s %-13s %5s %7s %s" + attrs = ['pid', 'memory_percent', 'name', 'cpu_times', 'create_time', 'memory_info'] if POSIX: attrs.append('uids') attrs.append('terminal') - print(templ % ("USER", "PID", "%CPU", "%MEM", "VSZ", "RSS", "TTY", + print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "TTY", "START", "TIME", "COMMAND")) for p in process_iter(): try: @@ -2359,7 +2359,6 @@ def test(): # pragma: no cover print(templ % ( user[:10], pinfo['pid'], - pinfo['cpu_percent'], memp, vms, rss, From 6bfab5d4c7961a29fe9f195ccf89dccdc1637a41 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 5 Mar 2017 11:47:26 +0700 Subject: [PATCH 220/922] pre-release --- HISTORY.rst | 2 +- docs/index.rst | 2 +- psutil/__init__.py | 8 ++++---- scripts/sensors.py | 12 ++++++------ 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index ad931e770..cf09ae000 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,6 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -*2017-02-07* +*2017-03-05* 5.2.0 ===== diff --git a/docs/index.rst b/docs/index.rst index 2f69480f0..6ac908189 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -659,7 +659,7 @@ Sensors Return hardware fans speed. Each entry is a named tuple representing a certain hardware sensor fan. - Fan speed is expressed in RPM (round per minute). + Fan speed is expressed in RPM (rounds per minute). If sensors are not supported by the OS an empty dict is returned. Example:: diff --git a/psutil/__init__.py b/psutil/__init__.py index 0d4680e8c..6b887761f 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2321,13 +2321,13 @@ def test(): # pragma: no cover today_day = datetime.date.today() templ = "%-10s %5s %4s %7s %7s %-13s %5s %7s %s" - attrs = ['pid', 'memory_percent', 'name', 'cpu_times', - 'create_time', 'memory_info'] + attrs = ['pid', 'memory_percent', 'name', 'cpu_times', 'create_time', + 'memory_info'] if POSIX: attrs.append('uids') attrs.append('terminal') - print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "TTY", - "START", "TIME", "COMMAND")) + print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "TTY", "START", "TIME", + "COMMAND")) for p in process_iter(): try: pinfo = p.as_dict(attrs, ad_value='') diff --git a/scripts/sensors.py b/scripts/sensors.py index 4c055efac..277ec215e 100755 --- a/scripts/sensors.py +++ b/scripts/sensors.py @@ -11,17 +11,17 @@ $ python scripts/sensors.py asus Temperatures: - asus 57.0 °C (high=None °C, critical=None °C) + asus 57.0°C (high=None°C, critical=None°C) Fans: cpu_fan 3500 RPM acpitz Temperatures: - acpitz 57.0 °C (high=108.0 °C, critical=108.0 °C) + acpitz 57.0°C (high=108.0°C, critical=108.0°C) coretemp Temperatures: - Physical id 0 61.0 °C (high=87.0 °C, critical=105.0 °C) - Core 0 61.0 °C (high=87.0 °C, critical=105.0 °C) - Core 1 59.0 °C (high=87.0 °C, critical=105.0 °C) + Physical id 0 61.0°C (high=87.0°C, critical=105.0°C) + Core 0 61.0°C (high=87.0°C, critical=105.0°C) + Core 1 59.0°C (high=87.0°C, critical=105.0°C) Battery: charge: 84.95% status: charging @@ -64,7 +64,7 @@ def main(): if name in temps: print(" Temperatures:") for entry in temps[name]: - print(" %-20s %s °C (high=%s °C, critical=%s °C)" % ( + print(" %-20s %s°C (high=%s°C, critical=%s°C)" % ( entry.label or name, entry.current, entry.high, entry.critical)) # Fans. From 33928a6acd21626db7a90b40e05b3d5f53fb68d2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 5 Mar 2017 16:06:45 +0700 Subject: [PATCH 221/922] update doc --- README.rst | 2 +- docs/index.rst | 1 + scripts/sensors.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 32fde350e..b40a0a0e0 100644 --- a/README.rst +++ b/README.rst @@ -73,7 +73,7 @@ Projects using psutil ===================== At the time of writing there are over -`4200 open source projects `__ +`4600 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: diff --git a/docs/index.rst b/docs/index.rst index 6ac908189..4421ad325 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2189,6 +2189,7 @@ take a look at the Timeline ======== +- 2017-03-05: `5.2.0 `__ - `what's new `__ - 2017-02-07: `5.1.3 `__ - `what's new `__ - 2017-02-03: `5.1.2 `__ - `what's new `__ - 2017-02-03: `5.1.1 `__ - `what's new `__ diff --git a/scripts/sensors.py b/scripts/sensors.py index 277ec215e..f2927a104 100755 --- a/scripts/sensors.py +++ b/scripts/sensors.py @@ -6,7 +6,8 @@ # found in the LICENSE file. """ -A clone of 'sensors' utility on Linux printing hardware temperatures. +A clone of 'sensors' utility on Linux printing hardware temperatures, +fans speed and battery info. $ python scripts/sensors.py asus From 5bb89fd6794313febaa293a82e248973eff353f0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 23 Mar 2017 07:16:39 +0100 Subject: [PATCH 222/922] #997 / virtual_memory() / FreeBSD: sysctl vm.stats.vm.v_cache_count fails on FreeBSD 12; set it to 0 --- HISTORY.rst | 10 ++++++++++ IDEAS | 4 ++-- psutil/arch/bsd/freebsd.c | 3 ++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index cf09ae000..831f2dd7f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +*XXXX-XX-XX* + +5.2.1 +===== + +**Bug fixes** + +- 997_: [FreeBSD] virtual_memory() may fail due to missing sysctl parameter on + FreeBSD 12. + *2017-03-05* 5.2.0 diff --git a/IDEAS b/IDEAS index f565b991f..dab54431d 100644 --- a/IDEAS +++ b/IDEAS @@ -9,8 +9,8 @@ https://github.com/giampaolo/psutil/issues PLATFORMS ========= -- #355 (patch): Android -- #605 (branch): AIX +- #355: Android (with patch) +- #605: AIX (with branch) - #276: GNU/Hurd - DragonFlyBSD - HP-UX diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index 0bec81d87..4aac5a616 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -468,8 +468,9 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { goto error; if (sysctlbyname("vm.stats.vm.v_wire_count", &wired, &size, NULL, 0)) goto error; + // https://github.com/giampaolo/psutil/issues/997 if (sysctlbyname("vm.stats.vm.v_cache_count", &cached, &size, NULL, 0)) - goto error; + cached = 0; if (sysctlbyname("vm.stats.vm.v_free_count", &free, &size, NULL, 0)) goto error; if (sysctlbyname("vfs.bufspace", &buffers, &buffers_size, NULL, 0)) From 26a430aac311aa8a6a6ca07730af930cd6017852 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 23 Mar 2017 07:57:05 +0000 Subject: [PATCH 223/922] disable failing test on BSD --- psutil/tests/test_process.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 6580fe9b8..af5cceef1 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -360,6 +360,9 @@ def test_io_counters(self): # sanity check for i in range(len(io2)): + if BSD and i >= 2: + # On BSD read_bytes and write_bytes are always set to -1. + continue self.assertGreaterEqual(io2[i], 0) self.assertGreaterEqual(io2[i], 0) From 1fd183c4d4e6fda474388c59c6b6fd24d98f2f7a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 23 Mar 2017 11:07:58 +0100 Subject: [PATCH 224/922] #996: [Linux] sensors_temperatures() may not show all temperatures. --- HISTORY.rst | 1 + psutil/_pslinux.py | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 831f2dd7f..5ca9dbd4d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,7 @@ **Bug fixes** +- 996_: [Linux] sensors_temperatures() may not show all temperatures. - 997_: [FreeBSD] virtual_memory() may fail due to missing sysctl parameter on FreeBSD 12. diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 533b5485d..33b23bcef 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1097,12 +1097,12 @@ def sensors_temperatures(): """ ret = collections.defaultdict(list) basenames = glob.glob('/sys/class/hwmon/hwmon*/temp*_*') - if not basenames: - # CentOS has an intermediate /device directory: - # https://github.com/giampaolo/psutil/issues/971 - basenames = glob.glob('/sys/class/hwmon/hwmon*/device/temp*_*') - + # CentOS has an intermediate /device directory: + # https://github.com/giampaolo/psutil/issues/971 + # https://github.com/nicolargo/glances/issues/1060 + basenames.extend(glob.glob('/sys/class/hwmon/hwmon*/device/temp*_*')) basenames = sorted(set([x.split('_')[0] for x in basenames])) + for base in basenames: unit_name = cat(os.path.join(os.path.dirname(base), 'name'), binary=False) From c16db211624f6c42a62623b924022f46a651927e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Mar 2017 12:48:15 +0100 Subject: [PATCH 225/922] fix #981: [Linux] cpu_freq() may return an empty list. --- HISTORY.rst | 1 + psutil/__init__.py | 2 +- psutil/_pslinux.py | 20 +++++++++++++------- psutil/tests/test_linux.py | 22 ++++++++++++++++++++++ 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 5ca9dbd4d..0870c296e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ - 996_: [Linux] sensors_temperatures() may not show all temperatures. - 997_: [FreeBSD] virtual_memory() may fail due to missing sysctl parameter on FreeBSD 12. +- 981_: [Linux] cpu_freq() may return an empty list. *2017-03-05* diff --git a/psutil/__init__.py b/psutil/__init__.py index 6b887761f..e64db5fb5 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1898,7 +1898,7 @@ def cpu_freq(percpu=False): else: num_cpus = float(len(ret)) if num_cpus == 0: - return [] + return None elif num_cpus == 1: return ret[0] else: diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 33b23bcef..b2f993908 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -636,8 +636,8 @@ def cpu_stats(): ctx_switches, interrupts, soft_interrupts, syscalls) -if os.path.exists("/sys/devices/system/cpu/cpufreq"): - +if os.path.exists("/sys/devices/system/cpu/cpufreq") or \ + os.path.exists("/sys/devices/system/cpu/cpu0"): def cpu_freq(): """Return frequency metrics for all CPUs. Contrarily to other OSes, Linux updates these values in @@ -647,11 +647,17 @@ def cpu_freq(): # http://unix.stackexchange.com/a/87537/168884 ret = [] ls = glob.glob("/sys/devices/system/cpu/cpufreq/policy*") - # Sort the list so that '10' comes after '2'. This should - # ensure the CPU order is consistent with other CPU functions - # having a 'percpu' argument and returning results for multiple - # CPUs (cpu_times(), cpu_percent(), cpu_times_percent()). - ls.sort(key=lambda x: int(os.path.basename(x)[6:])) + if ls: + # Sort the list so that '10' comes after '2'. This should + # ensure the CPU order is consistent with other CPU functions + # having a 'percpu' argument and returning results for multiple + # CPUs (cpu_times(), cpu_percent(), cpu_times_percent()). + ls.sort(key=lambda x: int(os.path.basename(x)[6:])) + else: + # https://github.com/giampaolo/psutil/issues/981 + ls = glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq") + ls.sort(key=lambda x: int(re.search('[0-9]+', x).group(0))) + for path in ls: curr = int(cat(os.path.join(path, "scaling_cur_freq"))) / 1000 max_ = int(cat(os.path.join(path, "scaling_max_freq"))) / 1000 diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index f9731bea5..cf539e3df 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -10,6 +10,7 @@ import collections import contextlib import errno +import glob import io import os import pprint @@ -536,6 +537,27 @@ def test_cpu_count_physical_mocked(self): self.assertIsNone(psutil._pslinux.cpu_count_physical()) assert m.called + def test_cpu_freq_no_result(self): + with mock.patch("psutil._pslinux.glob.glob", return_value=[]): + self.assertIsNone(psutil.cpu_freq()) + + def test_cpu_freq_use_second_file(self): + # https://github.com/giampaolo/psutil/issues/981 + def glob_mock(pattern): + if pattern.startswith("/sys/devices/system/cpu/cpufreq/policy"): + flags.append(None) + return [] + else: + flags.append(None) + return orig_glob(pattern) + + flags = [] + orig_glob = glob.glob + with mock.patch("psutil._pslinux.glob.glob", side_effect=glob_mock, + create=True): + assert psutil.cpu_freq() + self.assertEqual(len(flags), 2) + # ===================================================================== # --- system CPU stats From a3f7f7e7f40c0b812d0b0cc4a31a49e7d6391343 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Mar 2017 13:01:59 +0100 Subject: [PATCH 226/922] fix failure on TRAVIS --- psutil/tests/test_linux.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index cf539e3df..a28644d4c 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -555,7 +555,8 @@ def glob_mock(pattern): orig_glob = glob.glob with mock.patch("psutil._pslinux.glob.glob", side_effect=glob_mock, create=True): - assert psutil.cpu_freq() + if not TRAVIS: + assert psutil.cpu_freq() self.assertEqual(len(flags), 2) From ba475eb52299d4cc057f66b4d341353a99c72dd4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Mar 2017 13:08:36 +0100 Subject: [PATCH 227/922] update docstring --- .git-pre-commit | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.git-pre-commit b/.git-pre-commit index 071957d14..372d80912 100755 --- a/.git-pre-commit +++ b/.git-pre-commit @@ -6,8 +6,15 @@ """ This gets executed on 'git commit' and rejects the commit in case the -submitted code does not pass validation. -Install it with "make install-git-hooks". +submitted code does not pass validation. Validation is run only against +the *.py files which were modified in the commit. Checks: + +- assert no space at EOLs +- assert not pdb.set_trace in code +- assert no bare except clause ("except:") in code +- assert "flake8" returns no warnings + +Install this with "make install-git-hooks". """ from __future__ import print_function From 92b67f7151025388b87f069d83a4fb4c2908df07 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Mar 2017 14:35:49 +0100 Subject: [PATCH 228/922] #999 update disk_usage doc --- docs/index.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4421ad325..a2d53ff3c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -334,9 +334,9 @@ Disks .. function:: disk_usage(path) - Return disk usage statistics about the given *path* as a named tuple including - **total**, **used** and **free** space expressed in bytes, plus the - **percentage** usage. + Return disk usage statistics about the partition which contains the given + *path* as a named tuple including **total**, **used** and **free** space + expressed in bytes, plus the **percentage** usage. `OSError `__ is raised if *path* does not exist. Starting from `Python 3.3 `__ this is From 6e02e2bfcdabbd2830d998d9f9101d04c0e12792 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Mar 2017 16:04:46 +0100 Subject: [PATCH 229/922] #993 / windows / memory_maps: fix UnicodeDecodeError occurring on python 3 --- psutil/_psutil_windows.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 430f518e5..80b54e2bd 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2873,6 +2873,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { LPVOID maxAddr; PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; + PyObject *py_str = NULL; if (py_retlist == NULL) return NULL; @@ -2896,17 +2897,28 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { if (GetMappedFileNameA(hProcess, baseAddress, mappedFileName, sizeof(mappedFileName))) { + +#if PY_MAJOR_VERSION >= 3 + py_str = PyUnicode_Decode( + mappedFileName, _tcslen(mappedFileName), + Py_FileSystemDefaultEncoding, "surrogateescape"); +#else + py_str = Py_BuildValue("s", mappedFileName); +#endif + if (py_str == NULL) + goto error; + #ifdef _WIN64 py_tuple = Py_BuildValue( - "(KssI)", + "(KsOI)", (unsigned long long)baseAddress, #else py_tuple = Py_BuildValue( - "(kssI)", + "(ksOI)", (unsigned long)baseAddress, #endif get_region_protection_string(basicInfo.Protect), - mappedFileName, + py_str, basicInfo.RegionSize); if (!py_tuple) @@ -2924,6 +2936,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { error: Py_XDECREF(py_tuple); + Py_XDECREF(py_str); Py_DECREF(py_retlist); if (hProcess != NULL) CloseHandle(hProcess); From ad91a2e60838443c9d5efbe93eba3baf8b609cb2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Mar 2017 16:17:21 +0100 Subject: [PATCH 230/922] pre release --- HISTORY.rst | 4 +++- docs/index.rst | 1 + psutil/__init__.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0870c296e..c457e033d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,6 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -*XXXX-XX-XX* +*2017-03-24* 5.2.1 ===== @@ -11,6 +11,8 @@ - 997_: [FreeBSD] virtual_memory() may fail due to missing sysctl parameter on FreeBSD 12. - 981_: [Linux] cpu_freq() may return an empty list. +- 993_: [Windows] Process.memory_maps() on Python 3 may raise + UnicodeDecodeError. *2017-03-05* diff --git a/docs/index.rst b/docs/index.rst index a2d53ff3c..72a123299 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2189,6 +2189,7 @@ take a look at the Timeline ======== +- 2017-03-24: `5.2.1 `__ - `what's new `__ - 2017-03-05: `5.2.0 `__ - `what's new `__ - 2017-02-07: `5.1.3 `__ - `what's new `__ - 2017-02-03: `5.1.2 `__ - `what's new `__ diff --git a/psutil/__init__.py b/psutil/__init__.py index e64db5fb5..95b1110d3 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -192,7 +192,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.2.0" +__version__ = "5.2.1" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From 2ab6417338cfa4b674a2e39c7592b265f3cadc80 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Mar 2017 16:19:51 +0100 Subject: [PATCH 231/922] fix travis failure --- psutil/tests/test_linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index a28644d4c..183f5cdab 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -541,6 +541,7 @@ def test_cpu_freq_no_result(self): with mock.patch("psutil._pslinux.glob.glob", return_value=[]): self.assertIsNone(psutil.cpu_freq()) + @unittest.skipIf(TRAVIS, "fails on Travis") def test_cpu_freq_use_second_file(self): # https://github.com/giampaolo/psutil/issues/981 def glob_mock(pattern): @@ -555,8 +556,7 @@ def glob_mock(pattern): orig_glob = glob.glob with mock.patch("psutil._pslinux.glob.glob", side_effect=glob_mock, create=True): - if not TRAVIS: - assert psutil.cpu_freq() + assert psutil.cpu_freq() self.assertEqual(len(flags), 2) From bd4ed4d391343a64ee716583dec8106f5833dae3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 25 Mar 2017 02:56:32 +0100 Subject: [PATCH 232/922] fix #1000: setup.py: suppress module name: module references __file__warning --- HISTORY.rst | 9 +++++++++ Makefile | 1 - psutil/__init__.py | 2 +- setup.py | 1 + 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c457e033d..099bca9a7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,14 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +*XXXX-XX-XX* + +5.2.2 +===== + +**Bug fixes** + +- 1000_: fixed some setup.py warnings. + *2017-03-24* 5.2.1 diff --git a/Makefile b/Makefile index cad65e0c8..40a5495c3 100644 --- a/Makefile +++ b/Makefile @@ -217,7 +217,6 @@ pre-release: assert ver in history, '%r not in HISTORY.rst' % ver; \ assert 'XXXX' not in history; \ " - ${MAKE} setup-dev-env # mainly to update sphinx and install twine ${MAKE} win-download-exes $(PYTHON) setup.py sdist diff --git a/psutil/__init__.py b/psutil/__init__.py index 95b1110d3..0944d5d93 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -192,7 +192,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.2.1" +__version__ = "5.2.2" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED diff --git a/setup.py b/setup.py index 47772da9e..58a46c83f 100755 --- a/setup.py +++ b/setup.py @@ -272,6 +272,7 @@ def main(): ext_modules=extensions, test_suite="psutil.tests.runner.get_suite", tests_require=['ipaddress', 'mock', 'unittest2'], + zip_safe=False, # http://stackoverflow.com/questions/19548957 # see: python setup.py register --list-classifiers classifiers=[ 'Development Status :: 5 - Production/Stable', From f3f4b72f536fa13a4fa0a8b2f879cece8427a878 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 26 Mar 2017 07:15:59 +0200 Subject: [PATCH 233/922] update README --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index b40a0a0e0..ec01df1dc 100644 --- a/README.rst +++ b/README.rst @@ -50,7 +50,7 @@ iotop, uptime, pidof, tty, taskset, pmap. It currently supports **Linux**, **Windows**, **OSX**, **Sun Solaris**, **FreeBSD**, **OpenBSD** and **NetBSD**, both **32-bit** and **64-bit** architectures, with Python versions from **2.6 -to 3.5** (users of Python 2.4 and 2.5 may use +to 3.6** (users of Python 2.4 and 2.5 may use `2.1.3 `__ version). `PyPy `__ is also known to work. @@ -73,7 +73,7 @@ Projects using psutil ===================== At the time of writing there are over -`4600 open source projects `__ +`4800 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: @@ -306,7 +306,7 @@ Process management ...] >>> >>> p.io_counters() - pio(read_count=478001, write_count=59371, read_bytes=700416, write_bytes=69632) + pio(read_count=478001, write_count=59371, read_bytes=700416, write_bytes=69632, read_chars=456232, write_chars=517543) >>> >>> p.open_files() [popenfile(path='/home/giampaolo/svn/psutil/setup.py', fd=3, position=0, mode='r', flags=32768), From 2437c11538659ae28ff44d8ca1b05ec9e6925b92 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 27 Mar 2017 15:14:46 +0200 Subject: [PATCH 234/922] update README --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index ec01df1dc..470d52ca8 100644 --- a/README.rst +++ b/README.rst @@ -93,6 +93,7 @@ Portings - Node: https://github.com/christkv/node-psutil - Rust: https://github.com/borntyping/rust-psutil - Ruby: https://github.com/spacewander/posixpsutil +- Nim: https://github.com/johnscillieri/psutil-nim ============== Example usages From 6a2c351d0c450def95ee0505c84d1bc82ebe9f7b Mon Sep 17 00:00:00 2001 From: Danek Duvall Date: Tue, 28 Mar 2017 11:17:43 -0700 Subject: [PATCH 235/922] remove use of MA_RESERVED1 from SunOS module MA_RESERVED1 never meant anything, and the macro is going away in the next release of Solaris. MA_NORESERVE wasn't really mapped to anything useful, either. Removing the use of both macros shouldn't make any material difference, and will be compatible across more versions of Solaris. Fixes #1002. --- psutil/_psutil_sunos.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 48767add9..68b0a89ea 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -713,9 +713,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { sprintf(perms, "%c%c%c%c%c%c", p->pr_mflags & MA_READ ? 'r' : '-', p->pr_mflags & MA_WRITE ? 'w' : '-', p->pr_mflags & MA_EXEC ? 'x' : '-', - p->pr_mflags & MA_SHARED ? 's' : '-', - p->pr_mflags & MA_NORESERVE ? 'R' : '-', - p->pr_mflags & MA_RESERVED1 ? '*' : ' '); + p->pr_mflags & MA_SHARED ? 's' : '-'); // name if (strlen(p->pr_mapname) > 0) { From 9009349b22c2be000178a762070efb4064d8968a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 28 Mar 2017 21:06:26 +0200 Subject: [PATCH 236/922] #1002 give CREDITs for #1002 --- CREDITS | 4 ++++ HISTORY.rst | 2 ++ 2 files changed, 6 insertions(+) diff --git a/CREDITS b/CREDITS index 518caf0be..8c6f4cde6 100644 --- a/CREDITS +++ b/CREDITS @@ -437,3 +437,7 @@ I: 974 N: Baruch Siach W: https://github.com/baruchsiach I: 872 + +N: Danek Duvall +W: https://github.com/dhduvall +I: 1002 diff --git a/HISTORY.rst b/HISTORY.rst index 099bca9a7..0e9d92d14 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,6 +8,8 @@ **Bug fixes** - 1000_: fixed some setup.py warnings. +- 1002_: remove C macro which will not be available on new Solaris versions. + (patch by Danek Duvall) *2017-03-24* From feeb71389fd5f0e414e6ea8554ab273d296b1b68 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 5 Apr 2017 12:15:19 +0200 Subject: [PATCH 237/922] fix #1004 / linux / Process.io_counters(): fix possible ValueError --- HISTORY.rst | 11 ++++++----- README.rst | 2 +- psutil/_pslinux.py | 9 +++++++-- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 0e9d92d14..17429169a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -8,8 +8,9 @@ **Bug fixes** - 1000_: fixed some setup.py warnings. -- 1002_: remove C macro which will not be available on new Solaris versions. - (patch by Danek Duvall) +- 1002_: [SunOS] remove C macro which will not be available on new Solaris + versions. (patch by Danek Duvall) +- 1004_: [Linux] Process.io_counters() may raise ValueError. *2017-03-24* @@ -18,12 +19,12 @@ **Bug fixes** -- 996_: [Linux] sensors_temperatures() may not show all temperatures. -- 997_: [FreeBSD] virtual_memory() may fail due to missing sysctl parameter on - FreeBSD 12. - 981_: [Linux] cpu_freq() may return an empty list. - 993_: [Windows] Process.memory_maps() on Python 3 may raise UnicodeDecodeError. +- 996_: [Linux] sensors_temperatures() may not show all temperatures. +- 997_: [FreeBSD] virtual_memory() may fail due to missing sysctl parameter on + FreeBSD 12. *2017-03-05* diff --git a/README.rst b/README.rst index 470d52ca8..c670b9b1b 100644 --- a/README.rst +++ b/README.rst @@ -41,7 +41,7 @@ Summary psutil (process and system utilities) is a cross-platform library for retrieving information on **running processes** and **system utilization** -(CPU, memory, disks, networkm sensors) in Python. +(CPU, memory, disks, network, sensors) in Python. It is useful mainly for **system monitoring**, **profiling and limiting process resources** and **management of running processes**. It implements many functionalities offered by command line tools such as: diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index b2f993908..0af9aa7a3 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1448,8 +1448,13 @@ def io_counters(self): fields = {} with open_binary(fname) as f: for line in f: - name, value = line.split(b': ') - fields[name] = int(value) + # https://github.com/giampaolo/psutil/issues/1004 + line = line.strip() + if line: + name, value = line.split(b': ') + fields[name] = int(value) + if not fields: + raise RuntimeError("%s file was empty" % fname) return pio( fields[b'syscr'], # read syscalls fields[b'syscw'], # write syscalls From 17a866b72f5cbd06094529c99a15407b0a9d886a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 6 Apr 2017 18:02:38 +0200 Subject: [PATCH 238/922] fix #1006 / linux / cpu_freq(): do not define the function if not available instead of returning None --- HISTORY.rst | 302 +++++++++++++++++++++++++++++++++++++++++++++ IDEAS | 1 + README.rst | 2 +- psutil/_pslinux.py | 2 +- 4 files changed, 305 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 17429169a..2c01ee8db 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,8 @@ - 1002_: [SunOS] remove C macro which will not be available on new Solaris versions. (patch by Danek Duvall) - 1004_: [Linux] Process.io_counters() may raise ValueError. +- 1006_: [Linux] cpu_freq() may return None on some Linux versions does not + support the function; now the function is not declared instead. *2017-03-24* @@ -2706,3 +2708,303 @@ DeprecationWarning. .. _1198: https://github.com/giampaolo/psutil/issues/1198 .. _1199: https://github.com/giampaolo/psutil/issues/1199 .. _1200: https://github.com/giampaolo/psutil/issues/1200 +.. _1201: https://github.com/giampaolo/psutil/issues/1201 +.. _1202: https://github.com/giampaolo/psutil/issues/1202 +.. _1203: https://github.com/giampaolo/psutil/issues/1203 +.. _1204: https://github.com/giampaolo/psutil/issues/1204 +.. _1205: https://github.com/giampaolo/psutil/issues/1205 +.. _1206: https://github.com/giampaolo/psutil/issues/1206 +.. _1207: https://github.com/giampaolo/psutil/issues/1207 +.. _1208: https://github.com/giampaolo/psutil/issues/1208 +.. _1209: https://github.com/giampaolo/psutil/issues/1209 +.. _1210: https://github.com/giampaolo/psutil/issues/1210 +.. _1211: https://github.com/giampaolo/psutil/issues/1211 +.. _1212: https://github.com/giampaolo/psutil/issues/1212 +.. _1213: https://github.com/giampaolo/psutil/issues/1213 +.. _1214: https://github.com/giampaolo/psutil/issues/1214 +.. _1215: https://github.com/giampaolo/psutil/issues/1215 +.. _1216: https://github.com/giampaolo/psutil/issues/1216 +.. _1217: https://github.com/giampaolo/psutil/issues/1217 +.. _1218: https://github.com/giampaolo/psutil/issues/1218 +.. _1219: https://github.com/giampaolo/psutil/issues/1219 +.. _1220: https://github.com/giampaolo/psutil/issues/1220 +.. _1221: https://github.com/giampaolo/psutil/issues/1221 +.. _1222: https://github.com/giampaolo/psutil/issues/1222 +.. _1223: https://github.com/giampaolo/psutil/issues/1223 +.. _1224: https://github.com/giampaolo/psutil/issues/1224 +.. _1225: https://github.com/giampaolo/psutil/issues/1225 +.. _1226: https://github.com/giampaolo/psutil/issues/1226 +.. _1227: https://github.com/giampaolo/psutil/issues/1227 +.. _1228: https://github.com/giampaolo/psutil/issues/1228 +.. _1229: https://github.com/giampaolo/psutil/issues/1229 +.. _1230: https://github.com/giampaolo/psutil/issues/1230 +.. _1231: https://github.com/giampaolo/psutil/issues/1231 +.. _1232: https://github.com/giampaolo/psutil/issues/1232 +.. _1233: https://github.com/giampaolo/psutil/issues/1233 +.. _1234: https://github.com/giampaolo/psutil/issues/1234 +.. _1235: https://github.com/giampaolo/psutil/issues/1235 +.. _1236: https://github.com/giampaolo/psutil/issues/1236 +.. _1237: https://github.com/giampaolo/psutil/issues/1237 +.. _1238: https://github.com/giampaolo/psutil/issues/1238 +.. _1239: https://github.com/giampaolo/psutil/issues/1239 +.. _1240: https://github.com/giampaolo/psutil/issues/1240 +.. _1241: https://github.com/giampaolo/psutil/issues/1241 +.. _1242: https://github.com/giampaolo/psutil/issues/1242 +.. _1243: https://github.com/giampaolo/psutil/issues/1243 +.. _1244: https://github.com/giampaolo/psutil/issues/1244 +.. _1245: https://github.com/giampaolo/psutil/issues/1245 +.. _1246: https://github.com/giampaolo/psutil/issues/1246 +.. _1247: https://github.com/giampaolo/psutil/issues/1247 +.. _1248: https://github.com/giampaolo/psutil/issues/1248 +.. _1249: https://github.com/giampaolo/psutil/issues/1249 +.. _1250: https://github.com/giampaolo/psutil/issues/1250 +.. _1251: https://github.com/giampaolo/psutil/issues/1251 +.. _1252: https://github.com/giampaolo/psutil/issues/1252 +.. _1253: https://github.com/giampaolo/psutil/issues/1253 +.. _1254: https://github.com/giampaolo/psutil/issues/1254 +.. _1255: https://github.com/giampaolo/psutil/issues/1255 +.. _1256: https://github.com/giampaolo/psutil/issues/1256 +.. _1257: https://github.com/giampaolo/psutil/issues/1257 +.. _1258: https://github.com/giampaolo/psutil/issues/1258 +.. _1259: https://github.com/giampaolo/psutil/issues/1259 +.. _1260: https://github.com/giampaolo/psutil/issues/1260 +.. _1261: https://github.com/giampaolo/psutil/issues/1261 +.. _1262: https://github.com/giampaolo/psutil/issues/1262 +.. _1263: https://github.com/giampaolo/psutil/issues/1263 +.. _1264: https://github.com/giampaolo/psutil/issues/1264 +.. _1265: https://github.com/giampaolo/psutil/issues/1265 +.. _1266: https://github.com/giampaolo/psutil/issues/1266 +.. _1267: https://github.com/giampaolo/psutil/issues/1267 +.. _1268: https://github.com/giampaolo/psutil/issues/1268 +.. _1269: https://github.com/giampaolo/psutil/issues/1269 +.. _1270: https://github.com/giampaolo/psutil/issues/1270 +.. _1271: https://github.com/giampaolo/psutil/issues/1271 +.. _1272: https://github.com/giampaolo/psutil/issues/1272 +.. _1273: https://github.com/giampaolo/psutil/issues/1273 +.. _1274: https://github.com/giampaolo/psutil/issues/1274 +.. _1275: https://github.com/giampaolo/psutil/issues/1275 +.. _1276: https://github.com/giampaolo/psutil/issues/1276 +.. _1277: https://github.com/giampaolo/psutil/issues/1277 +.. _1278: https://github.com/giampaolo/psutil/issues/1278 +.. _1279: https://github.com/giampaolo/psutil/issues/1279 +.. _1280: https://github.com/giampaolo/psutil/issues/1280 +.. _1281: https://github.com/giampaolo/psutil/issues/1281 +.. _1282: https://github.com/giampaolo/psutil/issues/1282 +.. _1283: https://github.com/giampaolo/psutil/issues/1283 +.. _1284: https://github.com/giampaolo/psutil/issues/1284 +.. _1285: https://github.com/giampaolo/psutil/issues/1285 +.. _1286: https://github.com/giampaolo/psutil/issues/1286 +.. _1287: https://github.com/giampaolo/psutil/issues/1287 +.. _1288: https://github.com/giampaolo/psutil/issues/1288 +.. _1289: https://github.com/giampaolo/psutil/issues/1289 +.. _1290: https://github.com/giampaolo/psutil/issues/1290 +.. _1291: https://github.com/giampaolo/psutil/issues/1291 +.. _1292: https://github.com/giampaolo/psutil/issues/1292 +.. _1293: https://github.com/giampaolo/psutil/issues/1293 +.. _1294: https://github.com/giampaolo/psutil/issues/1294 +.. _1295: https://github.com/giampaolo/psutil/issues/1295 +.. _1296: https://github.com/giampaolo/psutil/issues/1296 +.. _1297: https://github.com/giampaolo/psutil/issues/1297 +.. _1298: https://github.com/giampaolo/psutil/issues/1298 +.. _1299: https://github.com/giampaolo/psutil/issues/1299 +.. _1300: https://github.com/giampaolo/psutil/issues/1300 +.. _1301: https://github.com/giampaolo/psutil/issues/1301 +.. _1302: https://github.com/giampaolo/psutil/issues/1302 +.. _1303: https://github.com/giampaolo/psutil/issues/1303 +.. _1304: https://github.com/giampaolo/psutil/issues/1304 +.. _1305: https://github.com/giampaolo/psutil/issues/1305 +.. _1306: https://github.com/giampaolo/psutil/issues/1306 +.. _1307: https://github.com/giampaolo/psutil/issues/1307 +.. _1308: https://github.com/giampaolo/psutil/issues/1308 +.. _1309: https://github.com/giampaolo/psutil/issues/1309 +.. _1310: https://github.com/giampaolo/psutil/issues/1310 +.. _1311: https://github.com/giampaolo/psutil/issues/1311 +.. _1312: https://github.com/giampaolo/psutil/issues/1312 +.. _1313: https://github.com/giampaolo/psutil/issues/1313 +.. _1314: https://github.com/giampaolo/psutil/issues/1314 +.. _1315: https://github.com/giampaolo/psutil/issues/1315 +.. _1316: https://github.com/giampaolo/psutil/issues/1316 +.. _1317: https://github.com/giampaolo/psutil/issues/1317 +.. _1318: https://github.com/giampaolo/psutil/issues/1318 +.. _1319: https://github.com/giampaolo/psutil/issues/1319 +.. _1320: https://github.com/giampaolo/psutil/issues/1320 +.. _1321: https://github.com/giampaolo/psutil/issues/1321 +.. _1322: https://github.com/giampaolo/psutil/issues/1322 +.. _1323: https://github.com/giampaolo/psutil/issues/1323 +.. _1324: https://github.com/giampaolo/psutil/issues/1324 +.. _1325: https://github.com/giampaolo/psutil/issues/1325 +.. _1326: https://github.com/giampaolo/psutil/issues/1326 +.. _1327: https://github.com/giampaolo/psutil/issues/1327 +.. _1328: https://github.com/giampaolo/psutil/issues/1328 +.. _1329: https://github.com/giampaolo/psutil/issues/1329 +.. _1330: https://github.com/giampaolo/psutil/issues/1330 +.. _1331: https://github.com/giampaolo/psutil/issues/1331 +.. _1332: https://github.com/giampaolo/psutil/issues/1332 +.. _1333: https://github.com/giampaolo/psutil/issues/1333 +.. _1334: https://github.com/giampaolo/psutil/issues/1334 +.. _1335: https://github.com/giampaolo/psutil/issues/1335 +.. _1336: https://github.com/giampaolo/psutil/issues/1336 +.. _1337: https://github.com/giampaolo/psutil/issues/1337 +.. _1338: https://github.com/giampaolo/psutil/issues/1338 +.. _1339: https://github.com/giampaolo/psutil/issues/1339 +.. _1340: https://github.com/giampaolo/psutil/issues/1340 +.. _1341: https://github.com/giampaolo/psutil/issues/1341 +.. _1342: https://github.com/giampaolo/psutil/issues/1342 +.. _1343: https://github.com/giampaolo/psutil/issues/1343 +.. _1344: https://github.com/giampaolo/psutil/issues/1344 +.. _1345: https://github.com/giampaolo/psutil/issues/1345 +.. _1346: https://github.com/giampaolo/psutil/issues/1346 +.. _1347: https://github.com/giampaolo/psutil/issues/1347 +.. _1348: https://github.com/giampaolo/psutil/issues/1348 +.. _1349: https://github.com/giampaolo/psutil/issues/1349 +.. _1350: https://github.com/giampaolo/psutil/issues/1350 +.. _1351: https://github.com/giampaolo/psutil/issues/1351 +.. _1352: https://github.com/giampaolo/psutil/issues/1352 +.. _1353: https://github.com/giampaolo/psutil/issues/1353 +.. _1354: https://github.com/giampaolo/psutil/issues/1354 +.. _1355: https://github.com/giampaolo/psutil/issues/1355 +.. _1356: https://github.com/giampaolo/psutil/issues/1356 +.. _1357: https://github.com/giampaolo/psutil/issues/1357 +.. _1358: https://github.com/giampaolo/psutil/issues/1358 +.. _1359: https://github.com/giampaolo/psutil/issues/1359 +.. _1360: https://github.com/giampaolo/psutil/issues/1360 +.. _1361: https://github.com/giampaolo/psutil/issues/1361 +.. _1362: https://github.com/giampaolo/psutil/issues/1362 +.. _1363: https://github.com/giampaolo/psutil/issues/1363 +.. _1364: https://github.com/giampaolo/psutil/issues/1364 +.. _1365: https://github.com/giampaolo/psutil/issues/1365 +.. _1366: https://github.com/giampaolo/psutil/issues/1366 +.. _1367: https://github.com/giampaolo/psutil/issues/1367 +.. _1368: https://github.com/giampaolo/psutil/issues/1368 +.. _1369: https://github.com/giampaolo/psutil/issues/1369 +.. _1370: https://github.com/giampaolo/psutil/issues/1370 +.. _1371: https://github.com/giampaolo/psutil/issues/1371 +.. _1372: https://github.com/giampaolo/psutil/issues/1372 +.. _1373: https://github.com/giampaolo/psutil/issues/1373 +.. _1374: https://github.com/giampaolo/psutil/issues/1374 +.. _1375: https://github.com/giampaolo/psutil/issues/1375 +.. _1376: https://github.com/giampaolo/psutil/issues/1376 +.. _1377: https://github.com/giampaolo/psutil/issues/1377 +.. _1378: https://github.com/giampaolo/psutil/issues/1378 +.. _1379: https://github.com/giampaolo/psutil/issues/1379 +.. _1380: https://github.com/giampaolo/psutil/issues/1380 +.. _1381: https://github.com/giampaolo/psutil/issues/1381 +.. _1382: https://github.com/giampaolo/psutil/issues/1382 +.. _1383: https://github.com/giampaolo/psutil/issues/1383 +.. _1384: https://github.com/giampaolo/psutil/issues/1384 +.. _1385: https://github.com/giampaolo/psutil/issues/1385 +.. _1386: https://github.com/giampaolo/psutil/issues/1386 +.. _1387: https://github.com/giampaolo/psutil/issues/1387 +.. _1388: https://github.com/giampaolo/psutil/issues/1388 +.. _1389: https://github.com/giampaolo/psutil/issues/1389 +.. _1390: https://github.com/giampaolo/psutil/issues/1390 +.. _1391: https://github.com/giampaolo/psutil/issues/1391 +.. _1392: https://github.com/giampaolo/psutil/issues/1392 +.. _1393: https://github.com/giampaolo/psutil/issues/1393 +.. _1394: https://github.com/giampaolo/psutil/issues/1394 +.. _1395: https://github.com/giampaolo/psutil/issues/1395 +.. _1396: https://github.com/giampaolo/psutil/issues/1396 +.. _1397: https://github.com/giampaolo/psutil/issues/1397 +.. _1398: https://github.com/giampaolo/psutil/issues/1398 +.. _1399: https://github.com/giampaolo/psutil/issues/1399 +.. _1400: https://github.com/giampaolo/psutil/issues/1400 +.. _1401: https://github.com/giampaolo/psutil/issues/1401 +.. _1402: https://github.com/giampaolo/psutil/issues/1402 +.. _1403: https://github.com/giampaolo/psutil/issues/1403 +.. _1404: https://github.com/giampaolo/psutil/issues/1404 +.. _1405: https://github.com/giampaolo/psutil/issues/1405 +.. _1406: https://github.com/giampaolo/psutil/issues/1406 +.. _1407: https://github.com/giampaolo/psutil/issues/1407 +.. _1408: https://github.com/giampaolo/psutil/issues/1408 +.. _1409: https://github.com/giampaolo/psutil/issues/1409 +.. _1410: https://github.com/giampaolo/psutil/issues/1410 +.. _1411: https://github.com/giampaolo/psutil/issues/1411 +.. _1412: https://github.com/giampaolo/psutil/issues/1412 +.. _1413: https://github.com/giampaolo/psutil/issues/1413 +.. _1414: https://github.com/giampaolo/psutil/issues/1414 +.. _1415: https://github.com/giampaolo/psutil/issues/1415 +.. _1416: https://github.com/giampaolo/psutil/issues/1416 +.. _1417: https://github.com/giampaolo/psutil/issues/1417 +.. _1418: https://github.com/giampaolo/psutil/issues/1418 +.. _1419: https://github.com/giampaolo/psutil/issues/1419 +.. _1420: https://github.com/giampaolo/psutil/issues/1420 +.. _1421: https://github.com/giampaolo/psutil/issues/1421 +.. _1422: https://github.com/giampaolo/psutil/issues/1422 +.. _1423: https://github.com/giampaolo/psutil/issues/1423 +.. _1424: https://github.com/giampaolo/psutil/issues/1424 +.. _1425: https://github.com/giampaolo/psutil/issues/1425 +.. _1426: https://github.com/giampaolo/psutil/issues/1426 +.. _1427: https://github.com/giampaolo/psutil/issues/1427 +.. _1428: https://github.com/giampaolo/psutil/issues/1428 +.. _1429: https://github.com/giampaolo/psutil/issues/1429 +.. _1430: https://github.com/giampaolo/psutil/issues/1430 +.. _1431: https://github.com/giampaolo/psutil/issues/1431 +.. _1432: https://github.com/giampaolo/psutil/issues/1432 +.. _1433: https://github.com/giampaolo/psutil/issues/1433 +.. _1434: https://github.com/giampaolo/psutil/issues/1434 +.. _1435: https://github.com/giampaolo/psutil/issues/1435 +.. _1436: https://github.com/giampaolo/psutil/issues/1436 +.. _1437: https://github.com/giampaolo/psutil/issues/1437 +.. _1438: https://github.com/giampaolo/psutil/issues/1438 +.. _1439: https://github.com/giampaolo/psutil/issues/1439 +.. _1440: https://github.com/giampaolo/psutil/issues/1440 +.. _1441: https://github.com/giampaolo/psutil/issues/1441 +.. _1442: https://github.com/giampaolo/psutil/issues/1442 +.. _1443: https://github.com/giampaolo/psutil/issues/1443 +.. _1444: https://github.com/giampaolo/psutil/issues/1444 +.. _1445: https://github.com/giampaolo/psutil/issues/1445 +.. _1446: https://github.com/giampaolo/psutil/issues/1446 +.. _1447: https://github.com/giampaolo/psutil/issues/1447 +.. _1448: https://github.com/giampaolo/psutil/issues/1448 +.. _1449: https://github.com/giampaolo/psutil/issues/1449 +.. _1450: https://github.com/giampaolo/psutil/issues/1450 +.. _1451: https://github.com/giampaolo/psutil/issues/1451 +.. _1452: https://github.com/giampaolo/psutil/issues/1452 +.. _1453: https://github.com/giampaolo/psutil/issues/1453 +.. _1454: https://github.com/giampaolo/psutil/issues/1454 +.. _1455: https://github.com/giampaolo/psutil/issues/1455 +.. _1456: https://github.com/giampaolo/psutil/issues/1456 +.. _1457: https://github.com/giampaolo/psutil/issues/1457 +.. _1458: https://github.com/giampaolo/psutil/issues/1458 +.. _1459: https://github.com/giampaolo/psutil/issues/1459 +.. _1460: https://github.com/giampaolo/psutil/issues/1460 +.. _1461: https://github.com/giampaolo/psutil/issues/1461 +.. _1462: https://github.com/giampaolo/psutil/issues/1462 +.. _1463: https://github.com/giampaolo/psutil/issues/1463 +.. _1464: https://github.com/giampaolo/psutil/issues/1464 +.. _1465: https://github.com/giampaolo/psutil/issues/1465 +.. _1466: https://github.com/giampaolo/psutil/issues/1466 +.. _1467: https://github.com/giampaolo/psutil/issues/1467 +.. _1468: https://github.com/giampaolo/psutil/issues/1468 +.. _1469: https://github.com/giampaolo/psutil/issues/1469 +.. _1470: https://github.com/giampaolo/psutil/issues/1470 +.. _1471: https://github.com/giampaolo/psutil/issues/1471 +.. _1472: https://github.com/giampaolo/psutil/issues/1472 +.. _1473: https://github.com/giampaolo/psutil/issues/1473 +.. _1474: https://github.com/giampaolo/psutil/issues/1474 +.. _1475: https://github.com/giampaolo/psutil/issues/1475 +.. _1476: https://github.com/giampaolo/psutil/issues/1476 +.. _1477: https://github.com/giampaolo/psutil/issues/1477 +.. _1478: https://github.com/giampaolo/psutil/issues/1478 +.. _1479: https://github.com/giampaolo/psutil/issues/1479 +.. _1480: https://github.com/giampaolo/psutil/issues/1480 +.. _1481: https://github.com/giampaolo/psutil/issues/1481 +.. _1482: https://github.com/giampaolo/psutil/issues/1482 +.. _1483: https://github.com/giampaolo/psutil/issues/1483 +.. _1484: https://github.com/giampaolo/psutil/issues/1484 +.. _1485: https://github.com/giampaolo/psutil/issues/1485 +.. _1486: https://github.com/giampaolo/psutil/issues/1486 +.. _1487: https://github.com/giampaolo/psutil/issues/1487 +.. _1488: https://github.com/giampaolo/psutil/issues/1488 +.. _1489: https://github.com/giampaolo/psutil/issues/1489 +.. _1490: https://github.com/giampaolo/psutil/issues/1490 +.. _1491: https://github.com/giampaolo/psutil/issues/1491 +.. _1492: https://github.com/giampaolo/psutil/issues/1492 +.. _1493: https://github.com/giampaolo/psutil/issues/1493 +.. _1494: https://github.com/giampaolo/psutil/issues/1494 +.. _1495: https://github.com/giampaolo/psutil/issues/1495 +.. _1496: https://github.com/giampaolo/psutil/issues/1496 +.. _1497: https://github.com/giampaolo/psutil/issues/1497 +.. _1498: https://github.com/giampaolo/psutil/issues/1498 +.. _1499: https://github.com/giampaolo/psutil/issues/1499 +.. _1500: https://github.com/giampaolo/psutil/issues/1500 diff --git a/IDEAS b/IDEAS index dab54431d..e37ba5236 100644 --- a/IDEAS +++ b/IDEAS @@ -11,6 +11,7 @@ PLATFORMS - #355: Android (with patch) - #605: AIX (with branch) +- #82: Cygwin (PR at #998) - #276: GNU/Hurd - DragonFlyBSD - HP-UX diff --git a/README.rst b/README.rst index c670b9b1b..e7dff4ccd 100644 --- a/README.rst +++ b/README.rst @@ -73,7 +73,7 @@ Projects using psutil ===================== At the time of writing there are over -`4800 open source projects `__ +`4900 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 0af9aa7a3..ddc3ce50c 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -637,7 +637,7 @@ def cpu_stats(): if os.path.exists("/sys/devices/system/cpu/cpufreq") or \ - os.path.exists("/sys/devices/system/cpu/cpu0"): + os.path.exists("/sys/devices/system/cpu/cpu0/cpufreq"): def cpu_freq(): """Return frequency metrics for all CPUs. Contrarily to other OSes, Linux updates these values in From 5994e49648ef98d668fb0517dcca79e5a1a750fd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 8 Apr 2017 12:41:05 +0200 Subject: [PATCH 239/922] fix sensors.py script on python 3 + fix linux test --- psutil/tests/test_linux.py | 5 ++++- scripts/sensors.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 183f5cdab..2e3183ca3 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1066,7 +1066,10 @@ def test_power_plugged(self): out = sh("acpi -b") if 'unknown' in out.lower(): return unittest.skip("acpi output not reliable") - plugged = "Charging" in out.split('\n')[0] + if 'discharging at zero rate' in out: + plugged = True + else: + plugged = "Charging" in out.split('\n')[0] self.assertEqual(psutil.sensors_battery().power_plugged, plugged) def test_emulate_power_plugged(self): diff --git a/scripts/sensors.py b/scripts/sensors.py index f2927a104..e3301ebfe 100755 --- a/scripts/sensors.py +++ b/scripts/sensors.py @@ -58,7 +58,7 @@ def main(): if not any((temps, fans, battery)): return sys.exit("can't read any temperature, fans or battery info") - names = set(temps.keys() + fans.keys()) + names = set(list(temps.keys()) + list(fans.keys())) for name in names: print(name) # Temperatures. From 3f67acceb089029563c423517b5508bdc9cb6356 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 9 Apr 2017 17:29:43 +0200 Subject: [PATCH 240/922] fix #1009: sensors_temperatures() may raise OSError. --- HISTORY.rst | 1 + psutil/_pslinux.py | 12 ++++++++++-- psutil/tests/test_linux.py | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2c01ee8db..a57152872 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,6 +13,7 @@ - 1004_: [Linux] Process.io_counters() may raise ValueError. - 1006_: [Linux] cpu_freq() may return None on some Linux versions does not support the function; now the function is not declared instead. +- 1009_: [Linux] sensors_temperatures() may raise OSError. *2017-03-24* diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index ddc3ce50c..71eae5bd8 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1112,10 +1112,18 @@ def sensors_temperatures(): for base in basenames: unit_name = cat(os.path.join(os.path.dirname(base), 'name'), binary=False) - label = cat(base + '_label', fallback='', binary=False) - current = float(cat(base + '_input')) / 1000.0 high = cat(base + '_max', fallback=None) critical = cat(base + '_crit', fallback=None) + label = cat(base + '_label', fallback='', binary=False) + try: + current = float(cat(base + '_input')) / 1000.0 + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1009 + if err.errno == errno.EIO: + warnings.warn("ignoring %r" % err, RuntimeWarning) + continue + else: + raise if high is not None: high = float(high) / 1000.0 diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 2e3183ca3..730ffeced 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1200,6 +1200,25 @@ def open_mock(name, *args, **kwargs): assert m.called +@unittest.skipUnless(LINUX, "LINUX only") +class TestSensorsTemperatures(unittest.TestCase): + + def test_emulate_eio_error(self): + def open_mock(name, *args, **kwargs): + if name.endswith("_input"): + raise OSError(errno.EIO, "") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: + with warnings.catch_warnings(record=True) as ws: + self.assertEqual(psutil.sensors_temperatures(), {}) + assert m.called + self.assertIn("ignoring", str(ws[0].message)) + + # ===================================================================== # --- test process # ===================================================================== From 57c776140ccb913c3ecc3be7142e7d575358290a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 9 Apr 2017 17:41:19 +0200 Subject: [PATCH 241/922] disable unreliable tests on travis --- psutil/tests/test_linux.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 730ffeced..2d37b4646 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -172,6 +172,7 @@ def test_used(self): free_value, psutil_value, delta=MEMORY_TOLERANCE, msg='%s %s \n%s' % (free_value, psutil_value, free.output)) + @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") @retry_before_failing() def test_free(self): # _, _, free_value, _ = free_physmem() @@ -1203,6 +1204,7 @@ def open_mock(name, *args, **kwargs): @unittest.skipUnless(LINUX, "LINUX only") class TestSensorsTemperatures(unittest.TestCase): + @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_emulate_eio_error(self): def open_mock(name, *args, **kwargs): if name.endswith("_input"): From 5e1d5d772ce1c796269d8b2195af073208a7ed99 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 10 Apr 2017 19:16:24 +0200 Subject: [PATCH 242/922] fix #1010 / linux: virtual_memory() may raise ValueError --- HISTORY.rst | 1 + psutil/_pslinux.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a57152872..848cc434a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,7 @@ - 1006_: [Linux] cpu_freq() may return None on some Linux versions does not support the function; now the function is not declared instead. - 1009_: [Linux] sensors_temperatures() may raise OSError. +- 1010_: [Linux] virtual_memory() may raise ValueError on Ubuntu 14.04. *2017-03-24* diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 71eae5bd8..a6181a86c 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -396,8 +396,12 @@ def virtual_memory(): # returned by sysinfo(2); as such we assume they are always there. total = mems[b'MemTotal:'] free = mems[b'MemFree:'] - buffers = mems[b'Buffers:'] - + try: + buffers = mems[b'Buffers:'] + except KeyError: + # https://github.com/giampaolo/psutil/issues/1010 + buffers = 0 + missing_fields.append('buffers') try: cached = mems[b"Cached:"] except KeyError: From c54a0f4376b6c66c031ff094dcf040f8f0e3e3b2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 10 Apr 2017 19:17:44 +0200 Subject: [PATCH 243/922] pre-release --- HISTORY.rst | 2 +- docs/index.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 848cc434a..7b6c19ec1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,6 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -*XXXX-XX-XX* +*2017-04-10* 5.2.2 ===== diff --git a/docs/index.rst b/docs/index.rst index 72a123299..5c3096871 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2189,6 +2189,7 @@ take a look at the Timeline ======== +- 2017-04-17: `5.2.2 `__ - `what's new `__ - 2017-03-24: `5.2.1 `__ - `what's new `__ - 2017-03-05: `5.2.0 `__ - `what's new `__ - 2017-02-07: `5.1.3 `__ - `what's new `__ From dd5ce9cfd7c6fc7450237628034017699862c645 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 13 Apr 2017 15:53:05 +0200 Subject: [PATCH 244/922] small refactoring --- IDEAS | 2 +- README.rst | 2 +- psutil/_pslinux.py | 10 ++++++---- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/IDEAS b/IDEAS index e37ba5236..fec7e1960 100644 --- a/IDEAS +++ b/IDEAS @@ -13,10 +13,10 @@ PLATFORMS - #605: AIX (with branch) - #82: Cygwin (PR at #998) - #276: GNU/Hurd +- #693: Windows Nano - DragonFlyBSD - HP-UX - FEATURES ======== diff --git a/README.rst b/README.rst index e7dff4ccd..89342e840 100644 --- a/README.rst +++ b/README.rst @@ -73,7 +73,7 @@ Projects using psutil ===================== At the time of writing there are over -`4900 open source projects `__ +`5000 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index a6181a86c..1884484e6 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -83,6 +83,7 @@ # speedup, see: https://github.com/giampaolo/psutil/issues/708 BIGGER_FILE_BUFFERING = -1 if PY3 else 8192 LITTLE_ENDIAN = sys.byteorder == 'little' +SECTOR_SIZE_FALLBACK = 512 if PY3: FS_ENCODING = sys.getfilesystemencoding() ENCODING_ERRORS_HANDLER = 'surrogateescape' @@ -250,7 +251,7 @@ def file_flags_to_mode(flags): return mode -def get_sector_size(partition): +def get_sector_size(partition, fallback=SECTOR_SIZE_FALLBACK): """Return the sector size of a partition. Used by disk_io_counters(). """ @@ -260,7 +261,7 @@ def get_sector_size(partition): except (IOError, ValueError): # man iostat states that sectors are equivalent with blocks and # have a size of 512 bytes since 2.4 kernels. - return 512 + return fallback @memoize @@ -298,9 +299,10 @@ def cat(fname, fallback=_DEFAULT, binary=True): with open_binary(fname) if binary else open_text(fname) as f: return f.read().strip() except IOError: - if fallback != _DEFAULT: + if fallback is not _DEFAULT: return fallback - raise + else: + raise try: From b303690afb7b6dad50cfb12056c133da5795af81 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 13 Apr 2017 21:05:15 +0200 Subject: [PATCH 245/922] update README --- IDEAS | 4 ---- README.rst | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/IDEAS b/IDEAS index fec7e1960..d122be871 100644 --- a/IDEAS +++ b/IDEAS @@ -5,7 +5,6 @@ A collection of ideas and notes about stuff to implement in future versions. "#NNN" occurrences refer to bug tracker issues at: https://github.com/giampaolo/psutil/issues - PLATFORMS ========= @@ -148,19 +147,16 @@ FEATURES - #550: number of threads per core. - BUGFIXES ======== - #600: windows / open_files(): support network file handles. - REJECTED ======== - #550: threads per core - RESOURCES ========= diff --git a/README.rst b/README.rst index 89342e840..c65ec1afa 100644 --- a/README.rst +++ b/README.rst @@ -376,6 +376,7 @@ Further process APIs .. code-block:: python + >>> import psutil >>> for p in psutil.process_iter(): ... print(p) ... @@ -384,6 +385,9 @@ Further process APIs psutil.Process(pid=3, name='ksoftirqd/0') ... >>> + >>> psutil.pid_exists(3) + True + >>> >>> def on_terminate(proc): ... print("process {} terminated".format(proc)) ... From 22651a6ca3142aeb405b0c23b4f668ed6d3f1de3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 21 Apr 2017 17:17:36 +0200 Subject: [PATCH 246/922] fix #1014: Linux can mask legitimate ENOENT exceptions as NoSuchProcess. --- HISTORY.rst | 9 +++++++++ psutil/__init__.py | 2 +- psutil/_pslinux.py | 19 +++++++++++++------ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7b6c19ec1..db5bb70ce 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,14 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +*XXXX-XX-XX* + +5.2.3 +===== + +**Bug fixes** + +- 1014_: Linux can mask legitimate ENOENT exceptions as NoSuchProcess. + *2017-04-10* 5.2.2 diff --git a/psutil/__init__.py b/psutil/__init__.py index 0944d5d93..5ce4f42b7 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -192,7 +192,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.2.2" +__version__ = "5.2.3" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 1884484e6..fa4299edf 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1296,7 +1296,9 @@ def pids(): def pid_exists(pid): - """Check for the existence of a unix PID.""" + """Check for the existence of a unix PID. Linux TIDs are not + supported (always return False). + """ if not _psposix.pid_exists(pid): return False else: @@ -1333,13 +1335,18 @@ def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) except EnvironmentError as err: - # ENOENT (no such file or directory) gets raised on open(). - # ESRCH (no such process) can get raised on read() if - # process is gone in meantime. - if err.errno in (errno.ENOENT, errno.ESRCH): - raise NoSuchProcess(self.pid, self._name) if err.errno in (errno.EPERM, errno.EACCES): raise AccessDenied(self.pid, self._name) + # ESRCH (no such process) can be raised on read() if + # process is gone in the meantime. + if err.errno == errno.ESRCH: + raise NoSuchProcess(self.pid, self._name) + # ENOENT (no such file or directory) can be raised on open(). + if err.errno == errno.ENOENT and not os.path.exists("%s/%s" % ( + self._procfs_path, self.pid)): + raise NoSuchProcess(self.pid, self._name) + # Note: zombies will keep existing under /proc until they're + # gone so there's no way to distinguish them in here. raise return wrapper From a115de4a07627cd7676a4c2da141a1c856d7af96 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 21 Apr 2017 17:25:23 +0200 Subject: [PATCH 247/922] test case for #1014 --- psutil/tests/test_linux.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 2d37b4646..eec33546a 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1397,6 +1397,24 @@ def test_exe_mocked(self): return_value=False): self.assertRaises(psutil.ZombieProcess, psutil.Process().exe) + def test_issue_1014(self): + # Emulates a case where smaps file does not exist. In this case + # wrap_exception decorator should not raise NoSuchProcess. + def open_mock(name, *args, **kwargs): + if name.startswith('/proc/%s/smaps' % os.getpid()): + raise IOError(errno.ENOENT, "") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: + p = psutil.Process() + with self.assertRaises(IOError) as err: + p.memory_maps() + self.assertEqual(err.exception.errno, errno.ENOENT) + assert m.called + @unittest.skipUnless(LINUX, "LINUX only") class TestProcessAgainstStatus(unittest.TestCase): From 9d24ec29c3f865bfb4849beb57cb95255dfb4c2b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 22 Apr 2017 23:25:56 +0200 Subject: [PATCH 248/922] fix #1015 / linux: have swap_memory() rely on /proc fs instead of sysinfo() syscall in order to be nice with Linux containers such as Docker and Heroku --- .ci/travis/install.sh | 2 +- .ci/travis/run.sh | 6 +++--- .travis.yml | 14 +------------- HISTORY.rst | 6 ++++++ docs/index.rst | 18 ++++++++++++++++-- psutil/_pslinux.py | 20 +++++++++++++++++--- psutil/tests/test_linux.py | 22 ++++++++++++++++++++-- 7 files changed, 64 insertions(+), 24 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index 677dc4653..6563251c2 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -51,4 +51,4 @@ elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]] || [[ $PYVER == 'py33' ]]; then pip install -U ipaddress fi -pip install -U coverage coveralls flake8 pep8 setuptools +pip install --upgrade coverage coveralls flake8 pep8 setuptools diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index b3a6a4a09..aaef347a7 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -24,10 +24,10 @@ else python psutil/tests/runner.py fi -if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.5" ]; then - # run mem leaks test +# Run memory leak tests and linter only on Linux and latest major Python +# versions. +if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then python psutil/tests/test_memory_leaks.py - # run linter (on Linux only) if [[ "$(uname -s)" != 'Darwin' ]]; then python -m flake8 fi diff --git a/.travis.yml b/.travis.yml index 48c84a7b3..2d05de174 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,30 +10,18 @@ matrix: - python: 3.5 - python: 3.6 - "pypy" - # XXX - commented because OSX builds are deadly slow - # - language: generic - # os: osx - # env: PYVER=py26 - language: generic os: osx env: PYVER=py27 - # XXX - commented because OSX builds are deadly slow - # - language: generic - # os: osx - # env: PYVER=py33 - language: generic os: osx env: PYVER=py34 - # XXX - not supported yet - # - language: generic - # os: osx - # env: PYVER=py35 install: - ./.ci/travis/install.sh script: - ./.ci/travis/run.sh after_success: - # upload reports to coveralls.io + # upload test reports to coveralls.io - | if [ "$(uname -s)" != 'Darwin' ]; then coveralls diff --git a/HISTORY.rst b/HISTORY.rst index db5bb70ce..c08104584 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,12 @@ 5.2.3 ===== +**Enhancements** + +- 1015_: swap_memory() now relies on /proc/meminfo instead of sysinfo() syscall + so that it can be used in conjunction with PROCFS_PATH in order to retrieve + memory info about Linux containers such as Docker and Heroku. + **Bug fixes** - 1014_: Linux can mask legitimate ENOENT exceptions as NoSuchProcess. diff --git a/docs/index.rst b/docs/index.rst index 5c3096871..fabfb22b9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -302,6 +302,11 @@ Memory >>> psutil.swap_memory() sswap(total=2097147904L, used=886620160L, free=1210527744L, percent=42.3, sin=1050411008, sout=1906720768) + .. versionchanged:: 5.2.3 on Linux this function relies on /proc fs instead + of sysinfo() syscall so that it can be used in conjunction with + :const:`psutil.PROCFS_PATH` in order to retrieve memory info about + Linux containers such as Docker and Heroku. + Disks ----- @@ -1979,9 +1984,18 @@ Constants .. _const-procfs_path: .. data:: PROCFS_PATH - The path of the /proc filesystem on Linux and Solaris (defaults to "/proc"). + The path of the /proc filesystem on Linux and Solaris (defaults to + ``"/proc"``). You may want to re-set this constant right after importing psutil in case - your /proc filesystem is mounted elsewhere. + your /proc filesystem is mounted elsewhere or if you want to retrieve + information about Linux containers such as + `Docker `__, + `Heroku `__ or + `LXC `__ (see + `here `__ + for more info). + It must be noted that this trick works only for APIs which rely on /proc + filesystem (e.g. `memory`_ APIs and most :class:`Process` class methods). Availability: Linux, Solaris diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index fa4299edf..6b3a951c2 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -489,9 +489,23 @@ def virtual_memory(): def swap_memory(): """Return swap memory metrics.""" - _, _, _, _, total, free, unit_multiplier = cext.linux_sysinfo() - total *= unit_multiplier - free *= unit_multiplier + mems = {} + with open_binary('%s/meminfo' % get_procfs_path()) as f: + for line in f: + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + # We prefer /proc/meminfo over sysinfo() syscall so that + # psutil.PROCFS_PATH can be used in order to allow retrieval + # for linux containers, see: + # https://github.com/giampaolo/psutil/issues/1015 + try: + total = mems['SwapTotal:'] + free = mems['SwapFree:'] + except KeyError: + _, _, _, _, total, free, unit_multiplier = cext.linux_sysinfo() + total *= unit_multiplier + free *= unit_multiplier + used = total - free percent = usage_percent(used, total, _round=1) # get pgin/pgouts diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index eec33546a..3c4da84c0 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -421,8 +421,15 @@ def test_warnings_mocked(self): def test_no_vmstat_mocked(self): # see https://github.com/giampaolo/psutil/issues/722 - with mock.patch('psutil._pslinux.open', create=True, - side_effect=IOError) as m: + def open_mock(name, *args, **kwargs): + if name == "/proc/vmstat": + raise IOError(errno.ENOENT, 'no such file or directory') + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: with warnings.catch_warnings(record=True) as ws: warnings.simplefilter("always") ret = psutil.swap_memory() @@ -437,6 +444,17 @@ def test_no_vmstat_mocked(self): self.assertEqual(ret.sin, 0) self.assertEqual(ret.sout, 0) + def test_against_sysinfo(self): + with mock.patch('psutil._pslinux.cext.linux_sysinfo') as m: + swap = psutil.swap_memory() + assert not m.called + import psutil._psutil_linux as cext + _, _, _, _, total, free, unit_multiplier = cext.linux_sysinfo() + total *= unit_multiplier + free *= unit_multiplier + self.assertEqual(swap.total, total) + self.assertEqual(swap.free, free) + # ===================================================================== # --- system CPU From 5608fff80bd077cef984935432e43de7dbfd3184 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 22 Apr 2017 23:36:47 +0200 Subject: [PATCH 249/922] #1015: fix python3 failure https://travis-ci.org/giampaolo/psutil/jobs/224770921 --- psutil/_pslinux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 6b3a951c2..3ae4b29aa 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -499,8 +499,8 @@ def swap_memory(): # for linux containers, see: # https://github.com/giampaolo/psutil/issues/1015 try: - total = mems['SwapTotal:'] - free = mems['SwapFree:'] + total = mems[b'SwapTotal:'] + free = mems[b'SwapFree:'] except KeyError: _, _, _, _, total, free, unit_multiplier = cext.linux_sysinfo() total *= unit_multiplier From df64140292be0c740fb43eda1a47c6c4b214dd7d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 22 Apr 2017 23:57:46 +0200 Subject: [PATCH 250/922] #1015: add test case which makes sure sysinfo() syscall is used as a fallback in case /proc/meminfo provides no swap metrics --- psutil/tests/test_linux.py | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 3c4da84c0..4e8627072 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -381,6 +381,13 @@ def open_mock(name, *args, **kwargs): # ===================================================================== +def meminfo_has_swap_info(): + """Return True if /proc/meminfo provides swap metrics.""" + with open("/proc/meminfo") as f: + data = f.read() + return 'SwapTotal:' in data and 'SwapFree:' in data + + @unittest.skipUnless(LINUX, "LINUX only") class TestSystemSwapMemory(unittest.TestCase): @@ -444,7 +451,12 @@ def open_mock(name, *args, **kwargs): self.assertEqual(ret.sin, 0) self.assertEqual(ret.sout, 0) - def test_against_sysinfo(self): + @unittest.skipUnless(meminfo_has_swap_info(), + "/proc/meminfo has no swap metrics") + def test_meminfo_against_sysinfo(self): + # Make sure the content of /proc/meminfo about swap memory + # matches sysinfo() syscall, see: + # https://github.com/giampaolo/psutil/issues/1015 with mock.patch('psutil._pslinux.cext.linux_sysinfo') as m: swap = psutil.swap_memory() assert not m.called @@ -455,6 +467,22 @@ def test_against_sysinfo(self): self.assertEqual(swap.total, total) self.assertEqual(swap.free, free) + def test_emulate_meminfo_has_no_metrics(self): + # Emulate a case where /proc/meminfo provides no swap metrics + # in which case sysinfo() syscall is supposed to be used + # as a fallback. + def open_mock(name, *args, **kwargs): + if name == "/proc/meminfo": + return io.BytesIO(b"") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, create=True, side_effect=open_mock) as m: + psutil.swap_memory() + assert m.called + # ===================================================================== # --- system CPU From 30db86ce39a1aae677a5fb5735806f3380de2520 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 00:00:29 +0200 Subject: [PATCH 251/922] refactor test --- psutil/tests/test_linux.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 4e8627072..e4d9d2889 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -381,16 +381,16 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -def meminfo_has_swap_info(): - """Return True if /proc/meminfo provides swap metrics.""" - with open("/proc/meminfo") as f: - data = f.read() - return 'SwapTotal:' in data and 'SwapFree:' in data - - @unittest.skipUnless(LINUX, "LINUX only") class TestSystemSwapMemory(unittest.TestCase): + @staticmethod + def meminfo_has_swap_info(): + """Return True if /proc/meminfo provides swap metrics.""" + with open("/proc/meminfo") as f: + data = f.read() + return 'SwapTotal:' in data and 'SwapFree:' in data + def test_total(self): free_value = free_swap().total psutil_value = psutil.swap_memory().total @@ -451,12 +451,12 @@ def open_mock(name, *args, **kwargs): self.assertEqual(ret.sin, 0) self.assertEqual(ret.sout, 0) - @unittest.skipUnless(meminfo_has_swap_info(), - "/proc/meminfo has no swap metrics") def test_meminfo_against_sysinfo(self): # Make sure the content of /proc/meminfo about swap memory # matches sysinfo() syscall, see: # https://github.com/giampaolo/psutil/issues/1015 + if not self.meminfo_has_swap_info(): + return unittest.skip("/proc/meminfo has no swap metrics") with mock.patch('psutil._pslinux.cext.linux_sysinfo') as m: swap = psutil.swap_memory() assert not m.called From 0eee5698d7c80a05f67c39dc87ea456df40c034c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 00:02:32 +0200 Subject: [PATCH 252/922] skip test on OSX --- psutil/tests/test_misc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 84215d30c..7abb28e83 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -492,6 +492,7 @@ def test_battery(self): self.assert_syntax('battery.py') @unittest.skipIf(APPVEYOR or TRAVIS, "unreliable on CI") + @unittest.skipIf(OSX, "platform not supported") def test_sensors(self): self.assert_stdout('sensors.py') From 1392d4763724c8efdfd42d17bf77fe3c0bd327fa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 00:46:52 +0200 Subject: [PATCH 253/922] try to make coveralls show the right results --- .ci/travis/run.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index aaef347a7..6b0baf1d7 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -5,20 +5,20 @@ set -x PYVER=`python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))'` -# setup OSX -if [[ "$(uname -s)" == 'Darwin' ]]; then +# setup OSX venv +if [ "$(uname -s)" == 'Darwin' ]; then if which pyenv > /dev/null; then eval "$(pyenv init -)" fi pyenv activate psutil fi -# install psutil +# install python setup.py build python setup.py develop -# run tests (with coverage) -if [[ "$(uname -s)" != 'Darwin' ]]; then +# run tests +if [ "$PYVER" == "2.7" ] && [ "$(uname -s)" != 'Darwin' ]; then coverage run psutil/tests/runner.py --include="psutil/*" --omit="test/*,*setup*" else python psutil/tests/runner.py @@ -28,7 +28,8 @@ fi # versions. if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then python psutil/tests/test_memory_leaks.py - if [[ "$(uname -s)" != 'Darwin' ]]; then + + if [ "$(uname -s)" != 'Darwin' ]; then python -m flake8 fi fi From 098dcf9bc991826dd3dc0f251123c00076c7b8c9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 00:59:56 +0200 Subject: [PATCH 254/922] try to make coveralls show the right results --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2d05de174..961403869 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,5 +24,6 @@ after_success: # upload test reports to coveralls.io - | if [ "$(uname -s)" != 'Darwin' ]; then + echo "sending results to coveralls.io" coveralls fi From e3583d87dec31be15df9a0b1627cc19d664b4c21 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 01:05:54 +0200 Subject: [PATCH 255/922] try to make coveralls show the right results --- .ci/travis/install.sh | 13 ++++++++++--- .travis.yml | 5 ++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index 6563251c2..f1ae928dc 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -41,14 +41,21 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then pyenv activate psutil fi +# old python versions if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]] || [[ $PYVER == 'py26' ]]; then pip install -U ipaddress unittest2 argparse mock==1.0.1 elif [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] || [[ $PYVER == 'py27' ]]; then pip install -U ipaddress mock -elif [[ $TRAVIS_PYTHON_VERSION == '3.2' ]] || [[ $PYVER == 'py32' ]]; then - pip install -U ipaddress mock elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]] || [[ $PYVER == 'py33' ]]; then pip install -U ipaddress fi -pip install --upgrade coverage coveralls flake8 pep8 setuptools +pip install --upgrade setuptools + +if [ "$PYVER" == "2.7" ] && [ "$(uname -s)" != 'Darwin' ]; then + pip install --upgrade coverage coveralls flake8 +fi + +if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then + pip install --upgrade flake8 +fi diff --git a/.travis.yml b/.travis.yml index 961403869..c7d095bf2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,9 +21,8 @@ install: script: - ./.ci/travis/run.sh after_success: - # upload test reports to coveralls.io - | - if [ "$(uname -s)" != 'Darwin' ]; then - echo "sending results to coveralls.io" + if [ "$PYVER" == "2.7" ] && [ "$(uname -s)" != 'Darwin' ]; then + echo "sending test coverage results to coveralls.io" coveralls fi From 1b23fda7b84debb27cf4f3ac6fb77e62869ace1f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 01:10:00 +0200 Subject: [PATCH 256/922] try to make coveralls show the right results --- .ci/travis/install.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index f1ae928dc..70c2696a3 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -50,12 +50,4 @@ elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]] || [[ $PYVER == 'py33' ]]; then pip install -U ipaddress fi -pip install --upgrade setuptools - -if [ "$PYVER" == "2.7" ] && [ "$(uname -s)" != 'Darwin' ]; then - pip install --upgrade coverage coveralls flake8 -fi - -if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then - pip install --upgrade flake8 -fi +pip install --upgrade coverage coveralls setuptools flake8 From 8e291c35a11a8bad883cfeb2473d056e2c9d6e16 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 01:17:02 +0200 Subject: [PATCH 257/922] try to make coveralls show the right results --- .ci/travis/install.sh | 6 +++--- .ci/travis/run.sh | 6 +++--- .travis.yml | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index 70c2696a3..24996c3d1 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -42,11 +42,11 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then fi # old python versions -if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]] || [[ $PYVER == 'py26' ]]; then +if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install -U ipaddress unittest2 argparse mock==1.0.1 -elif [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] || [[ $PYVER == 'py27' ]]; then +elif [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install -U ipaddress mock -elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]] || [[ $PYVER == 'py33' ]]; then +elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]]; then pip install -U ipaddress fi diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 6b0baf1d7..ae5b09e24 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -18,7 +18,7 @@ python setup.py build python setup.py develop # run tests -if [ "$PYVER" == "2.7" ] && [ "$(uname -s)" != 'Darwin' ]; then +if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then coverage run psutil/tests/runner.py --include="psutil/*" --omit="test/*,*setup*" else python psutil/tests/runner.py @@ -26,10 +26,10 @@ fi # Run memory leak tests and linter only on Linux and latest major Python # versions. -if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then +if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] || [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then python psutil/tests/test_memory_leaks.py - if [ "$(uname -s)" != 'Darwin' ]; then + if [[ "$(uname -s)" != 'Darwin' ]]; then python -m flake8 fi fi diff --git a/.travis.yml b/.travis.yml index c7d095bf2..3d75b15c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ script: - ./.ci/travis/run.sh after_success: - | - if [ "$PYVER" == "2.7" ] && [ "$(uname -s)" != 'Darwin' ]; then + if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then echo "sending test coverage results to coveralls.io" coveralls fi From b78f2f20128fb541b7808d8e96be68aa0e33def5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 01:55:27 +0200 Subject: [PATCH 258/922] increase test coverage for sensors_temperatures() + refactor test constants + bump travis tollerance during tests --- .ci/travis/install.sh | 21 ++++----------- .ci/travis/run.sh | 4 +-- psutil/__init__.py | 14 +++++----- psutil/tests/__init__.py | 51 ++++++++++++++++++++++--------------- psutil/tests/test_system.py | 9 +++++-- 5 files changed, 51 insertions(+), 48 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index 24996c3d1..0d69e56da 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -3,8 +3,9 @@ set -e set -x +PYVER=`python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))'` + uname -a -python -c "import sys; print(sys.version)" if [[ "$(uname -s)" == 'Darwin' ]]; then brew update || brew update @@ -16,22 +17,10 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then fi case "${PYVER}" in - # py26) - # pyenv install 2.6.9 - # pyenv virtualenv 2.6.9 psutil - # ;; py27) pyenv install 2.7.10 pyenv virtualenv 2.7.10 psutil ;; - # py32) - # pyenv install 3.2.6 - # pyenv virtualenv 3.2.6 psutil - # ;; - # py33) - # pyenv install 3.3.6 - # pyenv virtualenv 3.3.6 psutil - # ;; py34) pyenv install 3.4.3 pyenv virtualenv 3.4.3 psutil @@ -42,11 +31,11 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then fi # old python versions -if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then +if [[ $PYVER == '2.6' ]]; then pip install -U ipaddress unittest2 argparse mock==1.0.1 -elif [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then +elif [[ $PYVER == '2.7' ]]; then pip install -U ipaddress mock -elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]]; then +elif [[ $PYVER == '3.3' ]]; then pip install -U ipaddress fi diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index ae5b09e24..eec282ce0 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -18,7 +18,7 @@ python setup.py build python setup.py develop # run tests -if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then +if [[ $PYVER == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then coverage run psutil/tests/runner.py --include="psutil/*" --omit="test/*,*setup*" else python psutil/tests/runner.py @@ -26,7 +26,7 @@ fi # Run memory leak tests and linter only on Linux and latest major Python # versions. -if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] || [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then +if [[ $PYVER == '2.7' ]] || [[ $PYVER == '3.6' ]]; then python psutil/tests/test_memory_leaks.py if [[ "$(uname -s)" != 'Darwin' ]]; then diff --git a/psutil/__init__.py b/psutil/__init__.py index 5ce4f42b7..46a819576 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2205,8 +2205,9 @@ def sensors_temperatures(fahrenheit=False): All temperatures are expressed in celsius unless *fahrenheit* is set to True. """ - def to_fahrenheit(n): - return (float(n) * 9 / 5) + 32 + def convert(n): + if n is not None: + return (float(n) * 9 / 5) + 32 if fahrenheit else n ret = collections.defaultdict(list) rawdict = _psplatform.sensors_temperatures() @@ -2214,12 +2215,9 @@ def to_fahrenheit(n): for name, values in rawdict.items(): while values: label, current, high, critical = values.pop(0) - if fahrenheit: - current = to_fahrenheit(current) - if high is not None: - high = to_fahrenheit(high) - if critical is not None: - critical = to_fahrenheit(critical) + current = convert(current) + high = convert(high) + critical = convert(critical) if high and not critical: critical = high diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 13c4cfca2..d18bc738a 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -90,18 +90,35 @@ # --- constants # =================================================================== +# --- platforms -# conf for retry_before_failing() decorator +TOX = os.getenv('TOX') or '' in ('1', 'true') +PYPY = '__pypy__' in sys.builtin_module_names +WIN_VISTA = (6, 0, 0) if WINDOWS else None +# whether we're running this test suite on Travis (https://travis-ci.org/) +TRAVIS = bool(os.environ.get('TRAVIS')) +# whether we're running this test suite on Appveyor for Windows +# (http://www.appveyor.com/) +APPVEYOR = bool(os.environ.get('APPVEYOR')) + +# --- configurable defaults + +# how many times retry_before_failing() decorator will retry NO_RETRIES = 10 -# bytes tolerance for OS memory related tests +# bytes tolerance for system-wide memory related tests MEMORY_TOLERANCE = 500 * 1024 # 500KB # the timeout used in functions which have to wait GLOBAL_TIMEOUT = 3 +# test output verbosity +VERBOSITY = 1 if os.getenv('SILENT') or TOX else 2 +# be more tolerant if we're on travis / appveyor in order to avoid +# false positives +if TRAVIS or APPVEYOR: + NO_RETRIES *= 3 + MEMORY_TOLERANCE *= 3 + GLOBAL_TIMEOUT *= 3 -AF_INET6 = getattr(socket, "AF_INET6") -AF_UNIX = getattr(socket, "AF_UNIX", None) -PYTHON = os.path.realpath(sys.executable) -DEVNULL = open(os.devnull, 'r+') +# --- files TESTFILE_PREFIX = '$testfn' TESTFN = os.path.join(os.path.realpath(os.getcwd()), TESTFILE_PREFIX) @@ -113,25 +130,19 @@ except UnicodeDecodeError: TESTFN_UNICODE = TESTFN + "-???" -TOX = os.getenv('TOX') or '' in ('1', 'true') -PYPY = '__pypy__' in sys.builtin_module_names +# --- paths -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), - '..', '..')) +ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') -WIN_VISTA = (6, 0, 0) if WINDOWS else None +# --- misc + +AF_INET6 = getattr(socket, "AF_INET6") +AF_UNIX = getattr(socket, "AF_UNIX", None) +PYTHON = os.path.realpath(sys.executable) +DEVNULL = open(os.devnull, 'r+') VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) if x.startswith('STATUS_')] -# whether we're running this test suite on Travis (https://travis-ci.org/) -TRAVIS = bool(os.environ.get('TRAVIS')) -# whether we're running this test suite on Appveyor for Windows -# (http://www.appveyor.com/) -APPVEYOR = bool(os.environ.get('APPVEYOR')) - -if TRAVIS or APPVEYOR: - GLOBAL_TIMEOUT = GLOBAL_TIMEOUT * 4 -VERBOSITY = 1 if os.getenv('SILENT') or TOX else 2 # assertRaisesRegexp renamed to assertRaisesRegex in 3.3; add support # for the new name diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 013ae8e39..fa9e87916 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -768,8 +768,8 @@ def test_os_constants(self): @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), "platform not supported") - def test_sensors_temperatures(self): - temps = psutil.sensors_temperatures() + def test_sensors_temperatures(self, fahrenheit=False): + temps = psutil.sensors_temperatures(fahrenheit=fahrenheit) for name, entries in temps.items(): self.assertIsInstance(name, (str, unicode)) for entry in entries: @@ -781,6 +781,11 @@ def test_sensors_temperatures(self): if entry.critical is not None: self.assertGreaterEqual(entry.critical, 0) + @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), + "platform not supported") + def test_sensors_temperatures_fahreneit(self): + self.test_sensors_temperatures(fahrenheit=True) + @unittest.skipUnless(hasattr(psutil, "sensors_battery"), "platform not supported") def test_sensors_battery(self): From 684f04fc23f2397d46eec493321391d05a006d6a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 02:09:45 +0200 Subject: [PATCH 259/922] refactoring --- .ci/travis/install.sh | 1 + psutil/tests/__init__.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index 0d69e56da..f05853e86 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -6,6 +6,7 @@ set -x PYVER=`python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))'` uname -a +echo $PYVER if [[ "$(uname -s)" == 'Darwin' ]]; then brew update || brew update diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index d18bc738a..ad9ac140b 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -115,7 +115,6 @@ # false positives if TRAVIS or APPVEYOR: NO_RETRIES *= 3 - MEMORY_TOLERANCE *= 3 GLOBAL_TIMEOUT *= 3 # --- files @@ -144,11 +143,6 @@ VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) if x.startswith('STATUS_')] -# assertRaisesRegexp renamed to assertRaisesRegex in 3.3; add support -# for the new name -if not hasattr(unittest.TestCase, 'assertRaisesRegex'): - unittest.TestCase.assertRaisesRegex = unittest.TestCase.assertRaisesRegexp - # =================================================================== # --- classes @@ -554,14 +548,20 @@ def create_exe(outpath, c_code=None): class TestCase(unittest.TestCase): + # Print a full path representation of the single unit tests + # being run. def __str__(self): return "%s.%s.%s" % ( self.__class__.__module__, self.__class__.__name__, self._testMethodName) + # assertRaisesRegexp renamed to assertRaisesRegex in 3.3; + # add support for the new name. + if not hasattr(unittest.TestCase, 'assertRaisesRegex'): + assertRaisesRegex = unittest.TestCase.assertRaisesRegexp + -# Hack that overrides default unittest.TestCase in order to print -# a full path representation of the single unit tests being run. +# override default unittest.TestCase unittest.TestCase = TestCase From 3b55e544fcc25e539b1e80907abe1eb1a4e9c823 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 02:17:48 +0200 Subject: [PATCH 260/922] attempt to fix travis + osx --- .ci/travis/install.sh | 44 +++++++++++++++++++++++-------------------- .travis.yml | 6 +++--- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index f05853e86..ca90f4006 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -9,26 +9,30 @@ uname -a echo $PYVER if [[ "$(uname -s)" == 'Darwin' ]]; then - brew update || brew update - brew outdated pyenv || brew upgrade pyenv - brew install pyenv-virtualenv - - if which pyenv > /dev/null; then - eval "$(pyenv init -)" - fi - - case "${PYVER}" in - py27) - pyenv install 2.7.10 - pyenv virtualenv 2.7.10 psutil - ;; - py34) - pyenv install 3.4.3 - pyenv virtualenv 3.4.3 psutil - ;; - esac - pyenv rehash - pyenv activate psutil + brew update + brew install python + virtualenv env -p python + source env/bin/activate + # brew update || brew update + # brew outdated pyenv || brew upgrade pyenv + # brew install pyenv-virtualenv + + # if which pyenv > /dev/null; then + # eval "$(pyenv init -)" + # fi + + # case "${PYVER}" in + # py27) + # pyenv install 2.7.10 + # pyenv virtualenv 2.7.10 psutil + # ;; + # py34) + # pyenv install 3.4.3 + # pyenv virtualenv 3.4.3 psutil + # ;; + # esac + # pyenv rehash + # pyenv activate psutil fi # old python versions diff --git a/.travis.yml b/.travis.yml index 3d75b15c1..746e7af37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,9 +13,9 @@ matrix: - language: generic os: osx env: PYVER=py27 - - language: generic - os: osx - env: PYVER=py34 + # - language: generic + # os: osx + # env: PYVER=py34 install: - ./.ci/travis/install.sh script: From 74ca8a025f8b3c772512c9ce59831e7d73ddd90f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 02:25:08 +0200 Subject: [PATCH 261/922] attempt to fix travis + osx --- .ci/travis/install.sh | 48 ++++++++++++++++++++----------------------- .travis.yml | 23 +++++++++++++-------- 2 files changed, 37 insertions(+), 34 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index ca90f4006..5f445522d 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -8,32 +8,28 @@ PYVER=`python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))'` uname -a echo $PYVER -if [[ "$(uname -s)" == 'Darwin' ]]; then - brew update - brew install python - virtualenv env -p python - source env/bin/activate - # brew update || brew update - # brew outdated pyenv || brew upgrade pyenv - # brew install pyenv-virtualenv - - # if which pyenv > /dev/null; then - # eval "$(pyenv init -)" - # fi - - # case "${PYVER}" in - # py27) - # pyenv install 2.7.10 - # pyenv virtualenv 2.7.10 psutil - # ;; - # py34) - # pyenv install 3.4.3 - # pyenv virtualenv 3.4.3 psutil - # ;; - # esac - # pyenv rehash - # pyenv activate psutil -fi +# if [[ "$(uname -s)" == 'Darwin' ]]; then +# brew update || brew update +# brew outdated pyenv || brew upgrade pyenv +# brew install pyenv-virtualenv + +# if which pyenv > /dev/null; then +# eval "$(pyenv init -)" +# fi + +# case "${PYVER}" in +# py27) +# pyenv install 2.7.10 +# pyenv virtualenv 2.7.10 psutil +# ;; +# py34) +# pyenv install 3.4.3 +# pyenv virtualenv 3.4.3 psutil +# ;; +# esac +# pyenv rehash +# pyenv activate psutil +# fi # old python versions if [[ $PYVER == '2.6' ]]; then diff --git a/.travis.yml b/.travis.yml index 746e7af37..63fcb316a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,16 +3,23 @@ language: python cache: pip matrix: include: - - python: 2.6 - - python: 2.7 - - python: 3.3 - - python: 3.4 - - python: 3.5 - - python: 3.6 - - "pypy" + # - python: 2.6 + # - python: 2.7 + # - python: 3.3 + # - python: 3.4 + # - python: 3.5 + # - python: 3.6 + # - "pypy" - language: generic os: osx - env: PYVER=py27 + before_install: + - brew update + - brew install python + - virtualenv env -p python + - source env/bin/activate + # - language: generic + # os: osx + # env: PYVER=py27 # - language: generic # os: osx # env: PYVER=py34 From 6e4089bf8930fac22cf44fc02a29d67bef9d3a18 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 02:28:43 +0200 Subject: [PATCH 262/922] attempt to fix travis + osx --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 63fcb316a..3608c5e37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ matrix: os: osx before_install: - brew update + - brew unlink python - brew install python - virtualenv env -p python - source env/bin/activate From 1cb346ceb70c4da7b1c9a717a922ed9ce5ee4175 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 02:34:17 +0200 Subject: [PATCH 263/922] attempt to fix travis + osx --- .ci/travis/run.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index eec282ce0..87485cdbc 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -6,12 +6,12 @@ set -x PYVER=`python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))'` # setup OSX venv -if [ "$(uname -s)" == 'Darwin' ]; then - if which pyenv > /dev/null; then - eval "$(pyenv init -)" - fi - pyenv activate psutil -fi +# if [ "$(uname -s)" == 'Darwin' ]; then +# if which pyenv > /dev/null; then +# eval "$(pyenv init -)" +# fi +# pyenv activate psutil +# fi # install python setup.py build From d82e725de0fa2a785db3225b4fd2a7c090681276 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 02:42:32 +0200 Subject: [PATCH 264/922] attempt to fix travis + osx --- .ci/travis/run.sh | 8 -------- .travis.yml | 27 ++++++++++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 87485cdbc..92c1db39a 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -5,14 +5,6 @@ set -x PYVER=`python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))'` -# setup OSX venv -# if [ "$(uname -s)" == 'Darwin' ]; then -# if which pyenv > /dev/null; then -# eval "$(pyenv init -)" -# fi -# pyenv activate psutil -# fi - # install python setup.py build python setup.py develop diff --git a/.travis.yml b/.travis.yml index 3608c5e37..60e4cc56e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ matrix: # - python: 3.5 # - python: 3.6 # - "pypy" - - language: generic + - language: generic # osx + python2 os: osx before_install: - brew update @@ -18,14 +18,23 @@ matrix: - brew install python - virtualenv env -p python - source env/bin/activate - # - language: generic - # os: osx - # env: PYVER=py27 - # - language: generic - # os: osx - # env: PYVER=py34 -install: - - ./.ci/travis/install.sh + - language: generic # osx + python3 + os: osx + before_install: + - brew update + - brew unlink python3 + - brew install python3 + - virtualenv env -p python3 + - source env/bin/activate +install: | + if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then + pip install -U ipaddress unittest2 argparse mock==1.0.1 + elif [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then + pip install -U ipaddress mock + elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]]; then + pip install -U ipaddress + fi + pip install --upgrade coverage coveralls setuptools flake8 script: - ./.ci/travis/run.sh after_success: From 6a59ad8825a2d18cc444b1149691f42980478cd4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 02:51:54 +0200 Subject: [PATCH 265/922] restore travis config to original status --- .ci/travis/install.sh | 73 +++++++++++++++++++++++++------------------ .ci/travis/run.sh | 19 +++++++---- .travis.yml | 55 +++++++++++++++----------------- 3 files changed, 80 insertions(+), 67 deletions(-) diff --git a/.ci/travis/install.sh b/.ci/travis/install.sh index 5f445522d..677dc4653 100755 --- a/.ci/travis/install.sh +++ b/.ci/travis/install.sh @@ -3,41 +3,52 @@ set -e set -x -PYVER=`python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))'` - uname -a -echo $PYVER - -# if [[ "$(uname -s)" == 'Darwin' ]]; then -# brew update || brew update -# brew outdated pyenv || brew upgrade pyenv -# brew install pyenv-virtualenv - -# if which pyenv > /dev/null; then -# eval "$(pyenv init -)" -# fi - -# case "${PYVER}" in -# py27) -# pyenv install 2.7.10 -# pyenv virtualenv 2.7.10 psutil -# ;; -# py34) -# pyenv install 3.4.3 -# pyenv virtualenv 3.4.3 psutil -# ;; -# esac -# pyenv rehash -# pyenv activate psutil -# fi +python -c "import sys; print(sys.version)" + +if [[ "$(uname -s)" == 'Darwin' ]]; then + brew update || brew update + brew outdated pyenv || brew upgrade pyenv + brew install pyenv-virtualenv + + if which pyenv > /dev/null; then + eval "$(pyenv init -)" + fi + + case "${PYVER}" in + # py26) + # pyenv install 2.6.9 + # pyenv virtualenv 2.6.9 psutil + # ;; + py27) + pyenv install 2.7.10 + pyenv virtualenv 2.7.10 psutil + ;; + # py32) + # pyenv install 3.2.6 + # pyenv virtualenv 3.2.6 psutil + # ;; + # py33) + # pyenv install 3.3.6 + # pyenv virtualenv 3.3.6 psutil + # ;; + py34) + pyenv install 3.4.3 + pyenv virtualenv 3.4.3 psutil + ;; + esac + pyenv rehash + pyenv activate psutil +fi -# old python versions -if [[ $PYVER == '2.6' ]]; then +if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]] || [[ $PYVER == 'py26' ]]; then pip install -U ipaddress unittest2 argparse mock==1.0.1 -elif [[ $PYVER == '2.7' ]]; then +elif [[ $TRAVIS_PYTHON_VERSION == '2.7' ]] || [[ $PYVER == 'py27' ]]; then + pip install -U ipaddress mock +elif [[ $TRAVIS_PYTHON_VERSION == '3.2' ]] || [[ $PYVER == 'py32' ]]; then pip install -U ipaddress mock -elif [[ $PYVER == '3.3' ]]; then +elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]] || [[ $PYVER == 'py33' ]]; then pip install -U ipaddress fi -pip install --upgrade coverage coveralls setuptools flake8 +pip install -U coverage coveralls flake8 pep8 setuptools diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 92c1db39a..0f453dd72 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -5,22 +5,29 @@ set -x PYVER=`python -c 'import sys; print(".".join(map(str, sys.version_info[:2])))'` -# install +# setup OSX +if [[ "$(uname -s)" == 'Darwin' ]]; then + if which pyenv > /dev/null; then + eval "$(pyenv init -)" + fi + pyenv activate psutil +fi + +# install psutil python setup.py build python setup.py develop -# run tests +# run tests (with coverage) if [[ $PYVER == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then coverage run psutil/tests/runner.py --include="psutil/*" --omit="test/*,*setup*" else python psutil/tests/runner.py fi -# Run memory leak tests and linter only on Linux and latest major Python -# versions. -if [[ $PYVER == '2.7' ]] || [[ $PYVER == '3.6' ]]; then +if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then + # run mem leaks test python psutil/tests/test_memory_leaks.py - + # run linter (on Linux only) if [[ "$(uname -s)" != 'Darwin' ]]; then python -m flake8 fi diff --git a/.travis.yml b/.travis.yml index 60e4cc56e..0f1919385 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,38 +3,33 @@ language: python cache: pip matrix: include: - # - python: 2.6 - # - python: 2.7 - # - python: 3.3 - # - python: 3.4 - # - python: 3.5 - # - python: 3.6 - # - "pypy" - - language: generic # osx + python2 + - python: 2.6 + - python: 2.7 + - python: 3.3 + - python: 3.4 + - python: 3.5 + - python: 3.6 + - "pypy" + # XXX - commented because OSX builds are deadly slow + # - language: generic + # os: osx + # env: PYVER=py26 + - language: generic os: osx - before_install: - - brew update - - brew unlink python - - brew install python - - virtualenv env -p python - - source env/bin/activate - - language: generic # osx + python3 + env: PYVER=py27 + # XXX - commented because OSX builds are deadly slow + # - language: generic + # os: osx + # env: PYVER=py33 + - language: generic os: osx - before_install: - - brew update - - brew unlink python3 - - brew install python3 - - virtualenv env -p python3 - - source env/bin/activate -install: | - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then - pip install -U ipaddress unittest2 argparse mock==1.0.1 - elif [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then - pip install -U ipaddress mock - elif [[ $TRAVIS_PYTHON_VERSION == '3.3' ]]; then - pip install -U ipaddress - fi - pip install --upgrade coverage coveralls setuptools flake8 + env: PYVER=py34 + # XXX - not supported yet + # - language: generic + # os: osx + # env: PYVER=py35 +install: + - ./.ci/travis/install.sh script: - ./.ci/travis/run.sh after_success: From d172d7ced683241d0be7e0111b6435757491ea17 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 03:22:12 +0200 Subject: [PATCH 266/922] add test case for Process.parent() --- psutil/tests/test_process.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index af5cceef1..17313ae62 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1183,7 +1183,6 @@ def test_ppid(self): sproc = get_test_subprocess() p = psutil.Process(sproc.pid) self.assertEqual(p.ppid(), this_parent) - self.assertEqual(p.parent().pid, this_parent) # no other process is supposed to have us as parent reap_children(recursive=True) if APPVEYOR: @@ -1197,6 +1196,20 @@ def test_ppid(self): # XXX: sometimes this fails on Windows; not sure why. self.assertNotEqual(p.ppid(), this_parent, msg=p) + def test_parent(self): + this_parent = os.getpid() + sproc = get_test_subprocess() + p = psutil.Process(sproc.pid) + self.assertEqual(p.parent().pid, this_parent) + + def test_parent_disappeared(self): + # Emulate a case where the parent process disappeared. + sproc = get_test_subprocess() + p = psutil.Process(sproc.pid) + with mock.patch("psutil.Process", + side_effect=psutil.NoSuchProcess(0, 'foo')): + self.assertIsNone(p.parent()) + def test_children(self): p = psutil.Process() self.assertEqual(p.children(), []) From 9f8f663941262f50ea84b9876ae2148567dc2043 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 03:33:42 +0200 Subject: [PATCH 267/922] increase test coverage --- psutil/tests/test_process.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 17313ae62..a74833e16 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1514,6 +1514,26 @@ def succeed_or_zombie_p_exc(fun, *args, **kwargs): finally: reap_children(recursive=True) + @unittest.skipUnless(POSIX, 'POSIX only') + def test_zombie_process_is_running_w_exc(self): + # Emulate a case where internally is_running() raises + # ZombieProcess. + p = psutil.Process() + with mock.patch("psutil.Process", + side_effect=psutil.ZombieProcess(0)) as m: + assert p.is_running() + assert m.called + + @unittest.skipUnless(POSIX, 'POSIX only') + def test_zombie_process_status_w_exc(self): + # Emulate a case where internally status() raises + # ZombieProcess. + p = psutil.Process() + with mock.patch("psutil._psplatform.Process.status", + side_effect=psutil.ZombieProcess(0)) as m: + self.assertEqual(p.status(), psutil.STATUS_ZOMBIE) + assert m.called + def test_pid_0(self): # Process(0) is supposed to work on all platforms except Linux if 0 not in psutil.pids(): From b499cc2ddce57d73d1af842a0abbd171cfe9a71c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 03:42:15 +0200 Subject: [PATCH 268/922] fix #1016: do not raise RuntimeError in case no disks are installed --- HISTORY.rst | 1 + psutil/__init__.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c08104584..e76fc48a8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,7 @@ **Bug fixes** - 1014_: Linux can mask legitimate ENOENT exceptions as NoSuchProcess. +- 1016_: disk_io_counters() raises RuntimeError on a system with no disks. *2017-04-10* diff --git a/psutil/__init__.py b/psutil/__init__.py index 46a819576..20794e70b 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2049,8 +2049,6 @@ def disk_io_counters(perdisk=False): executed first otherwise this function won't find any disk. """ rawdict = _psplatform.disk_io_counters() - if not rawdict: - raise RuntimeError("couldn't find any physical disk") nt = getattr(_psplatform, "sdiskio", _common.sdiskio) if perdisk: for disk, fields in rawdict.items(): From 91068b52e21997b273e5ed678409bdefcd52a3f1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 03:44:13 +0200 Subject: [PATCH 269/922] fix #1017 / net_io_counters(): do not raise RuntimeError in case no NICs are installed --- HISTORY.rst | 2 ++ psutil/__init__.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index e76fc48a8..6dfd44208 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,8 @@ - 1014_: Linux can mask legitimate ENOENT exceptions as NoSuchProcess. - 1016_: disk_io_counters() raises RuntimeError on a system with no disks. +- 1017_: net_io_counters() raises RuntimeError on a system with no network + cards installed. *2017-04-10* diff --git a/psutil/__init__.py b/psutil/__init__.py index 20794e70b..abe596256 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2083,8 +2083,6 @@ def net_io_counters(pernic=False): described above as the values. """ rawdict = _psplatform.net_io_counters() - if not rawdict: - raise RuntimeError("couldn't find any network interface") if pernic: for nic, fields in rawdict.items(): rawdict[nic] = _common.snetio(*fields) From f3147652571589393b5a7fb4dfbbfa65aa684375 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 04:14:41 +0200 Subject: [PATCH 270/922] increase test coverage --- psutil/_pslinux.py | 6 +++--- psutil/tests/test_linux.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 3ae4b29aa..8bfd3db6a 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -251,7 +251,7 @@ def file_flags_to_mode(flags): return mode -def get_sector_size(partition, fallback=SECTOR_SIZE_FALLBACK): +def get_sector_size(partition): """Return the sector size of a partition. Used by disk_io_counters(). """ @@ -261,7 +261,7 @@ def get_sector_size(partition, fallback=SECTOR_SIZE_FALLBACK): except (IOError, ValueError): # man iostat states that sectors are equivalent with blocks and # have a size of 512 bytes since 2.4 kernels. - return fallback + return SECTOR_SIZE_FALLBACK @memoize @@ -1654,7 +1654,7 @@ def get_blocks(lines, current_block): )) return ls - else: + else: # pragma: no cover def memory_maps(self): raise NotImplementedError( "/proc/%s/smaps does not exist on kernels < 2.6.14 or " diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index e4d9d2889..e1d8918b3 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -24,6 +24,7 @@ import warnings import psutil +from psutil import _pslinux from psutil import LINUX from psutil._compat import PY3 from psutil._compat import u @@ -1541,5 +1542,35 @@ def test_cpu_affinity_eligible_cpus(self): assert m.called +# ===================================================================== +# --- test utils +# ===================================================================== + + +@unittest.skipUnless(LINUX, "LINUX only") +class TestUtils(unittest.TestCase): + + def test_open_text(self): + with _pslinux.open_text(__file__) as f: + self.assertEqual(f.mode, 'rt') + + def test_open_binary(self): + with _pslinux.open_binary(__file__) as f: + self.assertEqual(f.mode, 'rb') + + def test_readlink(self): + with mock.patch("os.readlink", return_value="foo (deleted)") as m: + self.assertEqual(_pslinux.readlink("bar"), "foo") + assert m.called + + def test_cat(self): + fname = os.path.abspath(TESTFN) + with open(fname, "wt") as f: + f.write("foo ") + self.assertEqual(_pslinux.cat(TESTFN, binary=False), "foo") + self.assertEqual(_pslinux.cat(TESTFN, binary=True), b"foo") + self.assertEqual(_pslinux.cat(TESTFN + '??', fallback="bar"), "bar") + + if __name__ == '__main__': run_test_module_by_name(__file__) From c9571a1ba020cfa9030fba05f886143f9a49f754 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 04:26:42 +0200 Subject: [PATCH 271/922] increase test coverage on travis --- psutil/tests/test_process.py | 41 +++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index a74833e16..053f30ec3 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -399,20 +399,6 @@ def test_ionice(self): ioclass, value = p.ionice() self.assertEqual(ioclass, 2) self.assertEqual(value, 7) - # - self.assertRaises(ValueError, p.ionice, 2, 10) - self.assertRaises(ValueError, p.ionice, 2, -1) - self.assertRaises(ValueError, p.ionice, 4) - self.assertRaises(TypeError, p.ionice, 2, "foo") - self.assertRaisesRegex( - ValueError, "can't specify value with IOPRIO_CLASS_NONE", - p.ionice, psutil.IOPRIO_CLASS_NONE, 1) - self.assertRaisesRegex( - ValueError, "can't specify value with IOPRIO_CLASS_IDLE", - p.ionice, psutil.IOPRIO_CLASS_IDLE, 1) - self.assertRaisesRegex( - ValueError, "'ioclass' argument must be specified", - p.ionice, value=1) finally: p.ionice(IOPRIO_CLASS_NONE) else: @@ -427,7 +413,27 @@ def test_ionice(self): self.assertEqual(p.ionice(), value) finally: p.ionice(original) - # + + @unittest.skipUnless(LINUX or (WINDOWS and get_winver() >= WIN_VISTA), + 'platform not supported') + def test_ionice_errs(self): + sproc = get_test_subprocess() + p = psutil.Process(sproc.pid) + if LINUX: + self.assertRaises(ValueError, p.ionice, 2, 10) + self.assertRaises(ValueError, p.ionice, 2, -1) + self.assertRaises(ValueError, p.ionice, 4) + self.assertRaises(TypeError, p.ionice, 2, "foo") + self.assertRaisesRegex( + ValueError, "can't specify value with IOPRIO_CLASS_NONE", + p.ionice, psutil.IOPRIO_CLASS_NONE, 1) + self.assertRaisesRegex( + ValueError, "can't specify value with IOPRIO_CLASS_IDLE", + p.ionice, psutil.IOPRIO_CLASS_IDLE, 1) + self.assertRaisesRegex( + ValueError, "'ioclass' argument must be specified", + p.ionice, value=1) + else: self.assertRaises(ValueError, p.ionice, 3) self.assertRaises(TypeError, p.ionice, 2, 1) @@ -915,6 +921,11 @@ def test_cpu_affinity(self): # it should work with all iterables, not only lists p.cpu_affinity(set(all_cpus)) p.cpu_affinity(tuple(all_cpus)) + + @unittest.skipUnless(WINDOWS or LINUX or FREEBSD, 'platform not supported') + def test_cpu_affinity_errs(self): + sproc = get_test_subprocess() + p = psutil.Process(sproc.pid) invalid_cpu = [len(psutil.cpu_times(percpu=True)) + 10] self.assertRaises(ValueError, p.cpu_affinity, invalid_cpu) self.assertRaises(ValueError, p.cpu_affinity, range(10000, 11000)) From 0e14bcbb90200c3336e4a52c496a56cd4cf45323 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 04:30:07 +0200 Subject: [PATCH 272/922] fix failure on travis --- psutil/tests/test_linux.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index e1d8918b3..01ac93573 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -24,7 +24,6 @@ import warnings import psutil -from psutil import _pslinux from psutil import LINUX from psutil._compat import PY3 from psutil._compat import u @@ -1551,25 +1550,26 @@ def test_cpu_affinity_eligible_cpus(self): class TestUtils(unittest.TestCase): def test_open_text(self): - with _pslinux.open_text(__file__) as f: + with psutil._psplatform.open_text(__file__) as f: self.assertEqual(f.mode, 'rt') def test_open_binary(self): - with _pslinux.open_binary(__file__) as f: + with psutil._psplatform.open_binary(__file__) as f: self.assertEqual(f.mode, 'rb') def test_readlink(self): with mock.patch("os.readlink", return_value="foo (deleted)") as m: - self.assertEqual(_pslinux.readlink("bar"), "foo") + self.assertEqual(psutil._psplatform.readlink("bar"), "foo") assert m.called def test_cat(self): fname = os.path.abspath(TESTFN) with open(fname, "wt") as f: f.write("foo ") - self.assertEqual(_pslinux.cat(TESTFN, binary=False), "foo") - self.assertEqual(_pslinux.cat(TESTFN, binary=True), b"foo") - self.assertEqual(_pslinux.cat(TESTFN + '??', fallback="bar"), "bar") + self.assertEqual(psutil._psplatform.cat(TESTFN, binary=False), "foo") + self.assertEqual(psutil._psplatform.cat(TESTFN, binary=True), b"foo") + self.assertEqual( + psutil._psplatform.cat(TESTFN + '??', fallback="bar"), "bar") if __name__ == '__main__': From 6926720e65693c2ba6e14b01e9485db4d8397c1c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 04:45:21 +0200 Subject: [PATCH 273/922] provide an actual test for sensors_temperatures(fahrenheit=True) --- psutil/tests/test_system.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index fa9e87916..b3fb710dd 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -768,8 +768,8 @@ def test_os_constants(self): @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), "platform not supported") - def test_sensors_temperatures(self, fahrenheit=False): - temps = psutil.sensors_temperatures(fahrenheit=fahrenheit) + def test_sensors_temperatures(self): + temps = psutil.sensors_temperatures() for name, entries in temps.items(): self.assertIsInstance(name, (str, unicode)) for entry in entries: @@ -784,7 +784,15 @@ def test_sensors_temperatures(self, fahrenheit=False): @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), "platform not supported") def test_sensors_temperatures_fahreneit(self): - self.test_sensors_temperatures(fahrenheit=True) + d = {'coretemp': [('label', 50.0, 60.0, 70.0)]} + with mock.patch("psutil._psplatform.sensors_temperatures", + return_value=d) as m: + temps = psutil.sensors_temperatures( + fahrenheit=True)['coretemp'][0] + assert m.called + self.assertEqual(temps.current, 122.0) + self.assertEqual(temps.high, 140.0) + self.assertEqual(temps.critical, 158.0) @unittest.skipUnless(hasattr(psutil, "sensors_battery"), "platform not supported") From 47060dbdff8645a2c35103230e7440ad4db50f61 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 04:58:24 +0200 Subject: [PATCH 274/922] some refactoring --- psutil/tests/test_system.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index b3fb710dd..2ddc6e3b7 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -403,8 +403,6 @@ def test_per_cpu_times_percent_negative(self): for percent in cpu: self._test_cpu_percent(percent, None, None) - @unittest.skipIf(POSIX and not hasattr(os, 'statvfs'), - "os.statvfs() not available") def test_disk_usage(self): usage = psutil.disk_usage(os.getcwd()) assert usage.total > 0, usage @@ -434,17 +432,14 @@ def test_disk_usage(self): else: self.fail("OSError not raised") - @unittest.skipIf(POSIX and not hasattr(os, 'statvfs'), - "os.statvfs() not available") def test_disk_usage_unicode(self): - # see: https://github.com/giampaolo/psutil/issues/416 + # Related to https://github.com/giampaolo/psutil/issues/416 + # but doesn't really excercise it. safe_rmpath(TESTFN_UNICODE) self.addCleanup(safe_rmpath, TESTFN_UNICODE) os.mkdir(TESTFN_UNICODE) psutil.disk_usage(TESTFN_UNICODE) - @unittest.skipIf(POSIX and not hasattr(os, 'statvfs'), - "os.statvfs() not available") @unittest.skipIf(LINUX and TRAVIS, "unknown failure on travis") def test_disk_partitions(self): # all = False From 027ec99c1291e0edeccbf5953e8c332cc0b320d6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 05:10:45 +0200 Subject: [PATCH 275/922] really provide a unicode test for disk_usage() (we were doing it wrong) --- psutil/tests/__init__.py | 7 +------ psutil/tests/test_system.py | 3 +-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index ad9ac140b..d08aef019 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -122,12 +122,7 @@ TESTFILE_PREFIX = '$testfn' TESTFN = os.path.join(os.path.realpath(os.getcwd()), TESTFILE_PREFIX) _TESTFN = TESTFN + '-internal' -TESTFN_UNICODE = TESTFN + "-ƒőő" -if not PY3: - try: - TESTFN_UNICODE = unicode(TESTFN, sys.getfilesystemencoding()) - except UnicodeDecodeError: - TESTFN_UNICODE = TESTFN + "-???" +TESTFN_UNICODE = TESTFN + u"-ƒőő" # --- paths diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 2ddc6e3b7..b3018da74 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -433,8 +433,7 @@ def test_disk_usage(self): self.fail("OSError not raised") def test_disk_usage_unicode(self): - # Related to https://github.com/giampaolo/psutil/issues/416 - # but doesn't really excercise it. + # See: https://github.com/giampaolo/psutil/issues/416 safe_rmpath(TESTFN_UNICODE) self.addCleanup(safe_rmpath, TESTFN_UNICODE) os.mkdir(TESTFN_UNICODE) From fc66da427cb176355d9d7a2d34796ecdff5dc870 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 05:20:12 +0200 Subject: [PATCH 276/922] assume AF_INET6 is always available --- psutil/tests/__init__.py | 2 +- psutil/tests/test_misc.py | 7 +++---- psutil/tests/test_process.py | 2 +- psutil/tests/test_system.py | 5 ++--- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index d08aef019..05d2de2ea 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -27,6 +27,7 @@ import time import warnings from socket import AF_INET +from socket import AF_INET6 from socket import SOCK_DGRAM from socket import SOCK_STREAM @@ -131,7 +132,6 @@ # --- misc -AF_INET6 = getattr(socket, "AF_INET6") AF_UNIX = getattr(socket, "AF_UNIX", None) PYTHON = os.path.realpath(sys.executable) DEVNULL = open(os.devnull, 'r+') diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 7abb28e83..5b34a343f 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -282,10 +282,9 @@ def test_supports_ipv6(self): assert not supports_ipv6() assert s.called else: - if hasattr(socket, 'AF_INET6'): - with self.assertRaises(Exception): - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - sock.bind(("::1", 0)) + with self.assertRaises(Exception): + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock.bind(("::1", 0)) def test_isfile_strict(self): from psutil._common import isfile_strict diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 053f30ec3..8b8006da1 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -23,6 +23,7 @@ import traceback import types from socket import AF_INET +from socket import AF_INET6 from socket import SOCK_DGRAM from socket import SOCK_STREAM @@ -42,7 +43,6 @@ from psutil._compat import long from psutil._compat import PY3 from psutil._compat import unicode -from psutil.tests import AF_INET6 from psutil.tests import AF_UNIX from psutil.tests import APPVEYOR from psutil.tests import call_until diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index b3018da74..79e217d4a 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -30,7 +30,6 @@ from psutil import WINDOWS from psutil._compat import long from psutil._compat import unicode -from psutil.tests import AF_INET6 from psutil.tests import APPVEYOR from psutil.tests import check_net_address from psutil.tests import DEVNULL @@ -559,7 +558,7 @@ def test_net_if_addrs(self): # self.assertEqual(sorted(nics.keys()), # sorted(psutil.net_io_counters(pernic=True).keys())) - families = set([socket.AF_INET, AF_INET6, psutil.AF_LINK]) + families = set([socket.AF_INET, socket.AF_INET6, psutil.AF_LINK]) for nic, addrs in nics.items(): self.assertIsInstance(nic, (str, unicode)) self.assertEqual(len(set(addrs)), len(addrs)) @@ -592,7 +591,7 @@ def test_net_if_addrs(self): # TODO: skip AF_INET6 for now because I get: # AddressValueError: Only hex digits permitted in # u'c6f3%lxcbr0' in u'fe80::c8e0:fff:fe54:c6f3%lxcbr0' - if addr.family != AF_INET6: + if addr.family != socket.AF_INET6: check_net_address(ip, addr.family) # broadcast and ptp addresses are mutually exclusive if addr.broadcast: From 2577bd234219422d845bb1bf7f1bc43966596eaa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 20:49:42 +0200 Subject: [PATCH 277/922] try to re-enable some travis testst --- psutil/tests/test_linux.py | 4 ++-- psutil/tests/test_process.py | 6 +++--- psutil/tests/test_system.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 01ac93573..3ee2ee8b8 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1425,9 +1425,9 @@ def open_mock(name, *args, **kwargs): # not sure why (doesn't fail locally) # https://travis-ci.org/giampaolo/psutil/jobs/108629915 - @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") + # @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_exe_mocked(self): - with mock.patch('psutil._pslinux.os.readlink', + with mock.patch('psutil._pslinux.readlink', side_effect=OSError(errno.ENOENT, "")) as m: # No such file error; might be raised also if /proc/pid/exe # path actually exists for system processes with low pids diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 8b8006da1..2b4a67952 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -368,7 +368,7 @@ def test_io_counters(self): @unittest.skipUnless(LINUX or (WINDOWS and get_winver() >= WIN_VISTA), 'platform not supported') - @unittest.skipIf(LINUX and TRAVIS, "unknown failure on travis") + # @unittest.skipIf(LINUX and TRAVIS, "unknown failure on travis") def test_ionice(self): if LINUX: from psutil import (IOPRIO_CLASS_NONE, IOPRIO_CLASS_RT, @@ -574,7 +574,7 @@ def test_threads(self): @retry_before_failing() # see: https://travis-ci.org/giampaolo/psutil/jobs/111842553 - @unittest.skipIf(OSX and TRAVIS, "fails on TRAVIS + OSX") + # @unittest.skipIf(OSX and TRAVIS, "fails on TRAVIS + OSX") @skip_on_access_denied(only_if=OSX) def test_threads_2(self): sproc = get_test_subprocess() @@ -879,7 +879,7 @@ def test_cwd_2(self): call_until(p.cwd, "ret == os.path.dirname(os.getcwd())") @unittest.skipUnless(WINDOWS or LINUX or FREEBSD, 'platform not supported') - @unittest.skipIf(LINUX and TRAVIS, "unreliable on TRAVIS") + # @unittest.skipIf(LINUX and TRAVIS, "unreliable on TRAVIS") def test_cpu_affinity(self): p = psutil.Process() initial = p.cpu_affinity() diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 79e217d4a..1ff798159 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -438,7 +438,7 @@ def test_disk_usage_unicode(self): os.mkdir(TESTFN_UNICODE) psutil.disk_usage(TESTFN_UNICODE) - @unittest.skipIf(LINUX and TRAVIS, "unknown failure on travis") + # @unittest.skipIf(LINUX and TRAVIS, "unknown failure on travis") def test_disk_partitions(self): # all = False ls = psutil.disk_partitions(all=False) From 3ecc248a05cef8f62af572431dfe465c8ac0f3b4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 21:05:22 +0200 Subject: [PATCH 278/922] fix travis failure --- psutil/tests/test_linux.py | 2 +- psutil/tests/test_process.py | 4 ---- psutil/tests/test_system.py | 3 +-- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 3ee2ee8b8..74d4dc6f3 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1425,7 +1425,7 @@ def open_mock(name, *args, **kwargs): # not sure why (doesn't fail locally) # https://travis-ci.org/giampaolo/psutil/jobs/108629915 - # @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") + @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_exe_mocked(self): with mock.patch('psutil._pslinux.readlink', side_effect=OSError(errno.ENOENT, "")) as m: diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 2b4a67952..f6442a538 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -368,7 +368,6 @@ def test_io_counters(self): @unittest.skipUnless(LINUX or (WINDOWS and get_winver() >= WIN_VISTA), 'platform not supported') - # @unittest.skipIf(LINUX and TRAVIS, "unknown failure on travis") def test_ionice(self): if LINUX: from psutil import (IOPRIO_CLASS_NONE, IOPRIO_CLASS_RT, @@ -573,8 +572,6 @@ def test_threads(self): thread.stop() @retry_before_failing() - # see: https://travis-ci.org/giampaolo/psutil/jobs/111842553 - # @unittest.skipIf(OSX and TRAVIS, "fails on TRAVIS + OSX") @skip_on_access_denied(only_if=OSX) def test_threads_2(self): sproc = get_test_subprocess() @@ -879,7 +876,6 @@ def test_cwd_2(self): call_until(p.cwd, "ret == os.path.dirname(os.getcwd())") @unittest.skipUnless(WINDOWS or LINUX or FREEBSD, 'platform not supported') - # @unittest.skipIf(LINUX and TRAVIS, "unreliable on TRAVIS") def test_cpu_affinity(self): p = psutil.Process() initial = p.cpu_affinity() diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 1ff798159..ff82f67e6 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -438,7 +438,6 @@ def test_disk_usage_unicode(self): os.mkdir(TESTFN_UNICODE) psutil.disk_usage(TESTFN_UNICODE) - # @unittest.skipIf(LINUX and TRAVIS, "unknown failure on travis") def test_disk_partitions(self): # all = False ls = psutil.disk_partitions(all=False) @@ -459,7 +458,7 @@ def test_disk_partitions(self): # we cannot make any assumption about this, see: # http://goo.gl/p9c43 disk.device - if SUNOS: + if SUNOS or TRAVIS: # on solaris apparently mount points can also be files assert os.path.exists(disk.mountpoint), disk else: From 76d36b417ab1f0c7fa9070a2887cbcdae1bea66c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 21:14:57 +0200 Subject: [PATCH 279/922] fix travis failure --- psutil/tests/test_system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index ff82f67e6..96f5480b1 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -480,7 +480,7 @@ def test_disk_partitions(self): if err.errno not in (errno.EPERM, errno.EACCES): raise else: - if SUNOS: + if SUNOS or TRAVIS: # on solaris apparently mount points can also be files assert os.path.exists(disk.mountpoint), disk else: From f112ef9a0c7cc65e6604e4572de154e9cf27c95f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 23 Apr 2017 23:12:23 +0200 Subject: [PATCH 280/922] add UNIX test case for name()s longer than 15 chars --- psutil/tests/test_posix.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 16d1eb7e6..8ecca4773 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -123,6 +123,42 @@ def test_name(self): name_psutil = psutil.Process(self.pid).name().lower() self.assertEqual(name_ps, name_psutil) + def test_name_long(self): + # On UNIX the kernel truncates the name to the first 15 + # characters. In sich a case psutil tries to determine the + # full name from the cmdline. + name = "long-program-name" + cmdline = ["long-program-name-extended", "foo", "bar"] + with mock.patch("psutil._psplatform.Process.name", + return_value=name): + with mock.patch("psutil._psplatform.Process.cmdline", + return_value=cmdline): + p = psutil.Process() + self.assertEqual(p.name(), "long-program-name-extended") + + def test_name_long_cmdline_ad_exc(self): + # Same as above but emulates a case where cmdline() raises + # AccessDenied in which case psutil is supposed to return + # the truncated name instead of crashing. + name = "long-program-name" + with mock.patch("psutil._psplatform.Process.name", + return_value=name): + with mock.patch("psutil._psplatform.Process.cmdline", + side_effect=psutil.AccessDenied(0, "")): + p = psutil.Process() + self.assertEqual(p.name(), "long-program-name") + + def test_name_long_cmdline_nsp_exc(self): + # Same as above but emulates a case where cmdline() raises NSP + # which is supposed to propagate. + name = "long-program-name" + with mock.patch("psutil._psplatform.Process.name", + return_value=name): + with mock.patch("psutil._psplatform.Process.cmdline", + side_effect=psutil.NoSuchProcess(0, "")): + p = psutil.Process() + self.assertRaises(psutil.NoSuchProcess, p.name) + @unittest.skipIf(OSX or BSD, 'ps -o start not available') def test_create_time(self): time_ps = ps("ps --no-headers -o start -p %s" % self.pid).split(' ')[0] From b42ab4dddb295abf000a351f127004410eb31ed3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 24 Apr 2017 04:38:13 +0200 Subject: [PATCH 281/922] improve test coverage --- .coveragerc | 35 +++++++++++++++++------------------ psutil/__init__.py | 2 +- psutil/tests/test_linux.py | 26 ++++++++++++++++++++++++++ psutil/tests/test_posix.py | 2 +- 4 files changed, 45 insertions(+), 20 deletions(-) diff --git a/.coveragerc b/.coveragerc index 6b6309b9f..7d3f185f5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,32 +2,31 @@ include = *psutil* - omit = + psutil/_compat.py psutil/tests/* setup.py - psutil/_compat.py - exclude_lines = - pragma: no cover - if PY3: + enum.IntEnum + except ImportError: + globals().update if __name__ == .__main__.: - if sys.platform.startswith if _WINDOWS: - import enum - if enum is not None: + if BSD if enum is None: + if enum is not None: + if FREEBSD if has_enums: + if LINUX if LITTLE_ENDIAN: - enum.IntEnum - except ImportError: - raise NotImplementedError - if WINDOWS - if OSX - if BSD - if FREEBSD - if OPENBSD if NETBSD - if SUNOS - if LINUX + if OPENBSD + if OSX if ppid_map is None: + if PY3: + if SUNOS + if sys.platform.startswith + if WINDOWS + import enum + pragma: no cover + raise NotImplementedError diff --git a/psutil/__init__.py b/psutil/__init__.py index abe596256..68a110da4 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -627,7 +627,7 @@ def ppid(self): # Process.parent()? if POSIX: return self._proc.ppid() - else: + else: # pragma: no cover self._ppid = self._ppid or self._proc.ppid() return self._ppid diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 74d4dc6f3..cea36c2bd 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1266,6 +1266,32 @@ def open_mock(name, *args, **kwargs): assert m.called self.assertIn("ignoring", str(ws[0].message)) + def test_emulate_data(self): + def open_mock(name, *args, **kwargs): + if name.endswith('name'): + return io.BytesIO(b"name") + elif name.endswith('label'): + return io.BytesIO(b"label") + elif name.endswith('temp1_input'): + return io.BytesIO(b"30000") + elif name.endswith('temp1_max'): + return io.BytesIO(b"40000") + elif name.endswith('temp1_crit'): + return io.BytesIO(b"50000") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock): + with mock.patch('glob.glob', + return_value=['/sys/class/hwmon/hwmon0/temp1']): + temp = psutil.sensors_temperatures()['name'][0] + self.assertEqual(temp.label, 'label') + self.assertEqual(temp.current, 30.0) + self.assertEqual(temp.high, 40.0) + self.assertEqual(temp.critical, 50.0) + # ===================================================================== # --- test process diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 8ecca4773..c9b176a1f 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -125,7 +125,7 @@ def test_name(self): def test_name_long(self): # On UNIX the kernel truncates the name to the first 15 - # characters. In sich a case psutil tries to determine the + # characters. In such a case psutil tries to determine the # full name from the cmdline. name = "long-program-name" cmdline = ["long-program-name-extended", "foo", "bar"] From 8690a308216838be95e94eec0d298b4d97e66d9a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 24 Apr 2017 04:54:37 +0200 Subject: [PATCH 282/922] improve test coverage --- psutil/tests/test_linux.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index cea36c2bd..2e55cf016 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -606,6 +606,28 @@ def glob_mock(pattern): assert psutil.cpu_freq() self.assertEqual(len(flags), 2) + def test_cpu_freq_emulate_data(self): + def open_mock(name, *args, **kwargs): + if name.endswith('/scaling_cur_freq'): + return io.BytesIO(b"500000") + elif name.endswith('/scaling_min_freq'): + return io.BytesIO(b"600000") + elif name.endswith('/scaling_max_freq'): + return io.BytesIO(b"700000") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock): + with mock.patch( + 'glob.glob', + return_value=['/sys/devices/system/cpu/cpufreq/policy0']): + freq = psutil.cpu_freq() + self.assertEqual(freq.current, 500.0) + self.assertEqual(freq.min, 600.0) + self.assertEqual(freq.max, 700.0) + # ===================================================================== # --- system CPU stats @@ -1269,9 +1291,9 @@ def open_mock(name, *args, **kwargs): def test_emulate_data(self): def open_mock(name, *args, **kwargs): if name.endswith('name'): - return io.BytesIO(b"name") + return io.StringIO("name") elif name.endswith('label'): - return io.BytesIO(b"label") + return io.StringIO("label") elif name.endswith('temp1_input'): return io.BytesIO(b"30000") elif name.endswith('temp1_max'): From c8c07eae02042715bc7a38f2f08a9e070e401131 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 24 Apr 2017 05:15:20 +0200 Subject: [PATCH 283/922] improve test coverage --- psutil/tests/test_linux.py | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 2e55cf016..7f9483d0f 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1290,15 +1290,15 @@ def open_mock(name, *args, **kwargs): def test_emulate_data(self): def open_mock(name, *args, **kwargs): - if name.endswith('name'): - return io.StringIO("name") - elif name.endswith('label'): - return io.StringIO("label") - elif name.endswith('temp1_input'): + if name.endswith('/name'): + return io.StringIO(u("name")) + elif name.endswith('/temp1_label'): + return io.StringIO(u("label")) + elif name.endswith('/temp1_input'): return io.BytesIO(b"30000") - elif name.endswith('temp1_max'): + elif name.endswith('/temp1_max'): return io.BytesIO(b"40000") - elif name.endswith('temp1_crit'): + elif name.endswith('/temp1_crit'): return io.BytesIO(b"50000") else: return orig_open(name, *args, **kwargs) @@ -1315,6 +1315,30 @@ def open_mock(name, *args, **kwargs): self.assertEqual(temp.critical, 50.0) +@unittest.skipUnless(LINUX, "LINUX only") +class TestSensorsFans(unittest.TestCase): + + def test_emulate_data(self): + def open_mock(name, *args, **kwargs): + if name.endswith('/name'): + return io.StringIO(u("name")) + elif name.endswith('/fan1_label'): + return io.StringIO(u("label")) + elif name.endswith('/fan1_input'): + return io.StringIO(u("2000")) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock): + with mock.patch('glob.glob', + return_value=['/sys/class/hwmon/hwmon2/fan1']): + fan = psutil.sensors_fans()['name'][0] + self.assertEqual(fan.label, 'label') + self.assertEqual(fan.current, 2000) + + # ===================================================================== # --- test process # ===================================================================== From a2e36c9b7b6eecfefc204f1368033b7c22c9e219 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 24 Apr 2017 05:30:12 +0200 Subject: [PATCH 284/922] improve test coverage --- psutil/__init__.py | 15 +++------------ psutil/tests/test_linux.py | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 68a110da4..71ba34127 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1907,18 +1907,9 @@ def cpu_freq(percpu=False): currs += cpu.current mins += cpu.min maxs += cpu.max - try: - current = currs / num_cpus - except ZeroDivisionError: - current = 0.0 - try: - min_ = mins / num_cpus - except ZeroDivisionError: - min_ = 0.0 - try: - max_ = maxs / num_cpus - except ZeroDivisionError: - max_ = 0.0 + current = currs / num_cpus + min_ = mins / num_cpus + max_ = maxs / num_cpus return _common.scpufreq(current, min_, max_) __all__.append("cpu_freq") diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 7f9483d0f..747f1456e 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -628,6 +628,29 @@ def open_mock(name, *args, **kwargs): self.assertEqual(freq.min, 600.0) self.assertEqual(freq.max, 700.0) + def test_cpu_freq_emulate_multi_cpu(self): + def open_mock(name, *args, **kwargs): + if name.endswith('/scaling_cur_freq'): + return io.BytesIO(b"100000") + elif name.endswith('/scaling_min_freq'): + return io.BytesIO(b"200000") + elif name.endswith('/scaling_max_freq'): + return io.BytesIO(b"300000") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + policies = ['/sys/devices/system/cpu/cpufreq/policy0', + '/sys/devices/system/cpu/cpufreq/policy1', + '/sys/devices/system/cpu/cpufreq/policy2'] + with mock.patch(patch_point, side_effect=open_mock): + with mock.patch('glob.glob', return_value=policies): + freq = psutil.cpu_freq() + self.assertEqual(freq.current, 100.0) + self.assertEqual(freq.min, 200.0) + self.assertEqual(freq.max, 300.0) + # ===================================================================== # --- system CPU stats From ee722d4a14224e2331a8b7e44af0e5f6755c0c03 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 24 Apr 2017 13:46:00 +0200 Subject: [PATCH 285/922] update README --- README.rst | 15 +++++++++++++++ psutil/__init__.py | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c65ec1afa..4f23b2a2d 100644 --- a/README.rst +++ b/README.rst @@ -395,6 +395,21 @@ Further process APIs >>> gone, alive = psutil.wait_procs(procs_list, timeout=3, callback=on_terminate) >>> +Popen wrapper: + + >>> import psutil + >>> from subprocess import PIPE + >>> p = psutil.Popen(["/usr/bin/python", "-c", "print('hello')"], stdout=PIPE) + >>> p.name() + 'python' + >>> p.username() + 'giampaolo' + >>> p.communicate() + ('hello\n', None) + >>> p.wait(timeout=2) + 0 + >>> + Windows services ================ diff --git a/psutil/__init__.py b/psutil/__init__.py index 71ba34127..4e68a46dc 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1067,7 +1067,7 @@ def timer(): # interval was too low return 0.0 else: - # Note 1. + # Note 1: # in order to emulate "top" we multiply the value for the num # of CPU cores. This way the busy process will be reported as # having 100% (or more) usage. @@ -1076,7 +1076,7 @@ def timer(): # taskmgr.exe on Windows differs in that it will show 50% # instead. # - # Note #3: + # Note 3: # a percentage > 100 is legitimate as it can result from a # process with multiple threads running on different CPU # cores (top does the same), see: From 1d6df8d52530c6143dcd0260bbcfd384d1315c8f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 24 Apr 2017 13:46:29 +0200 Subject: [PATCH 286/922] update README --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 4f23b2a2d..a8d62bd2f 100644 --- a/README.rst +++ b/README.rst @@ -397,6 +397,8 @@ Further process APIs Popen wrapper: +.. code-block:: python + >>> import psutil >>> from subprocess import PIPE >>> p = psutil.Popen(["/usr/bin/python", "-c", "print('hello')"], stdout=PIPE) From 1da757c4408911dae34a707b7a63df89cc530727 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 24 Apr 2017 19:38:21 +0200 Subject: [PATCH 287/922] #1018: enable 'python -m psutil.tests' to run tests --- .ci/travis/run.sh | 4 ++-- Makefile | 8 ++++---- appveyor.yml | 4 ++-- docs/index.rst | 12 ++++++++++++ psutil/tests/README.rst | 25 +++++++++++++++---------- psutil/tests/__init__.py | 30 +++++++++++++++++++++++++----- psutil/tests/__main__.py | 14 ++++++++++++++ psutil/tests/runner.py | 37 ------------------------------------- setup.py | 15 ++++++--------- tox.ini | 2 +- 10 files changed, 81 insertions(+), 70 deletions(-) create mode 100755 psutil/tests/__main__.py delete mode 100755 psutil/tests/runner.py diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 0f453dd72..2bc0dfe91 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -19,9 +19,9 @@ python setup.py develop # run tests (with coverage) if [[ $PYVER == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then - coverage run psutil/tests/runner.py --include="psutil/*" --omit="test/*,*setup*" + coverage run psutil/tests/__main__.py --include="psutil/*" --omit="test/*,*setup*" else - python psutil/tests/runner.py + python psutil/tests/__main__.py fi if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then diff --git a/Makefile b/Makefile index 40a5495c3..65762df6a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # You can set the variables below from the command line. PYTHON = python -TSCRIPT = psutil/tests/runner.py +TSCRIPT = psutil/tests/__main__.py ARGS = # List of nice-to-have dev libs. @@ -121,11 +121,11 @@ test: install # Test psutil process-related APIs. test-process: install - $(PYTHON) -m unittest -v psutil.tests.test_process + $(PYTHON) -m unittest -v psutil.test.test_process # Test psutil system-related APIs. test-system: install - $(PYTHON) -m unittest -v psutil.tests.test_system + $(PYTHON) -m unittest -v psutil.test.test_system # Test misc. test-misc: install @@ -144,7 +144,7 @@ test-platform: install $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py # Run a specific test by name, e.g. -# make test-by-name psutil.tests.test_system.TestSystemAPIs.test_cpu_times +# make test-by-name psutil.test.test_system.TestSystemAPIs.test_cpu_times test-by-name: install @$(PYTHON) -m unittest -v $(ARGS) diff --git a/appveyor.yml b/appveyor.yml index 896f550f3..af7a63192 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -90,7 +90,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "%WITH_COMPILER% %PYTHON%/python psutil/tests/runner.py" + - "%WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" @@ -118,7 +118,7 @@ only_commits: psutil/_pswindows.py psutil/arch/windows/* psutil/tests/__init__.py - psutil/tests/runner.py + psutil/tests/__main__.py psutil/tests/test_memory_leaks.py psutil/tests/test_misc.py psutil/tests/test_process.py diff --git a/docs/index.rst b/docs/index.rst index fabfb22b9..72c2881cd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2193,6 +2193,18 @@ Q&A in python as `os.getloadavg `__ +Running tests +============= + +There are two ways of running tests. If psutil is already installed use:: + + $ python -m psutil.tests + +You can use this method as a quick way to make sure psutil fully works on your +platform. If you have a copy of the source code you can also use:: + + $ make test + Development guide ================= diff --git a/psutil/tests/README.rst b/psutil/tests/README.rst index 2ad91c143..637fb7ddc 100644 --- a/psutil/tests/README.rst +++ b/psutil/tests/README.rst @@ -1,19 +1,24 @@ Instructions for running tests ============================== -- The recommended way to run tests (also on Windows) is to cd into psutil root - directory and run ``make test``. +* There are two ways of running tests. If psutil is already installed: -- Depending on the Python version, dependencies for running tests include + python -m psutil.tests + + If you have a copy of the source code: + + make test + +* Depending on the Python version, dependencies for running tests include ``ipaddress``, ``mock`` and ``unittest2`` modules. - On Windows also ``pywin32`` and ``wmi`` modules are recommended - (although optional). - Run ``make setup-dev-env`` to install all deps (also on Windows). + On Windows you'll also need ``pywin32`` and ``wmi`` modules. + If you have a copy of the source code you can run ``make setup-dev-env`` to + install all deps (also on Windows). -- To run tests on all supported Python versions install tox +* To run tests on all supported Python versions install tox (``pip install tox``) then run ``tox`` from psutil root directory. -- Every time a commit is pushed tests are automatically run on Travis +* Every time a commit is pushed tests are automatically run on Travis (Linux, OSX) and appveyor (Windows): - - https://travis-ci.org/giampaolo/psutil/ - - https://ci.appveyor.com/project/giampaolo/psutil + * https://travis-ci.org/giampaolo/psutil/ + * https://ci.appveyor.com/project/giampaolo/psutil diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 05d2de2ea..84eafe9e8 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -127,6 +127,7 @@ # --- paths +HERE = os.path.abspath(os.path.dirname(__file__)) ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') @@ -560,11 +561,23 @@ def __str__(self): unittest.TestCase = TestCase -def retry_before_failing(retries=NO_RETRIES): - """Decorator which runs a test function and retries N times before - actually failing. - """ - return retry(exception=AssertionError, timeout=None, retries=retries) +def get_suite(): + testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) + if x.endswith('.py') and x.startswith('test_') and not + x.startswith('test_memory_leaks')] + suite = unittest.TestSuite() + for tm in testmodules: + # ...so that the full test paths are printed on screen + tm = "psutil.tests.%s" % tm + suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) + return suite + + +def run_suite(): + """Run unit tests.""" + result = unittest.TextTestRunner(verbosity=VERBOSITY).run(get_suite()) + success = result.wasSuccessful() + sys.exit(0 if success else 1) def run_test_module_by_name(name): @@ -578,6 +591,13 @@ def run_test_module_by_name(name): sys.exit(0 if success else 1) +def retry_before_failing(retries=NO_RETRIES): + """Decorator which runs a test function and retries N times before + actually failing. + """ + return retry(exception=AssertionError, timeout=None, retries=retries) + + def skip_on_access_denied(only_if=None): """Decorator to Ignore AccessDenied exceptions.""" def decorator(fun): diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py new file mode 100755 index 000000000..5f7bb5287 --- /dev/null +++ b/psutil/tests/__main__.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Run unit tests. This is invoked by: + +$ python -m psutil.tests +""" + +from psutil.tests import run_suite +run_suite() diff --git a/psutil/tests/runner.py b/psutil/tests/runner.py deleted file mode 100755 index 88bcd6208..000000000 --- a/psutil/tests/runner.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2007-2016 Giampaolo Rodola' . -# Use of this source code is governed by MIT license that can be -# found in the LICENSE file. - -"""Script for running all test files (except memory leaks tests).""" - -import os -import sys - -from psutil.tests import unittest -from psutil.tests import VERBOSITY - - -def get_suite(): - HERE = os.path.abspath(os.path.dirname(__file__)) - testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) - if x.endswith('.py') and x.startswith('test_') and not - x.startswith('test_memory_leaks')] - suite = unittest.TestSuite() - for tm in testmodules: - # ...so that "make test" will print the full test paths - tm = "psutil.tests.%s" % tm - suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) - return suite - - -def main(): - # run tests - result = unittest.TextTestRunner(verbosity=VERBOSITY).run(get_suite()) - success = result.wasSuccessful() - sys.exit(0 if success else 1) - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index 58a46c83f..3f2fd0dfb 100755 --- a/setup.py +++ b/setup.py @@ -9,7 +9,6 @@ in Python. """ -import atexit import contextlib import io import os @@ -195,13 +194,6 @@ def get_ethtool_macro(): suffix='.c', delete=False, mode="wt") as f: f.write("#include ") - @atexit.register - def on_exit(): - try: - os.remove(f.name) - except OSError: - pass - compiler = UnixCCompiler() try: with silenced_output('stderr'): @@ -211,6 +203,11 @@ def on_exit(): return ("PSUTIL_ETHTOOL_MISSING_TYPES", 1) else: return None + finally: + try: + os.remove(f.name) + except OSError: + pass ETHTOOL_MACRO = get_ethtool_macro() @@ -270,7 +267,7 @@ def main(): license='BSD', packages=['psutil', 'psutil.tests'], ext_modules=extensions, - test_suite="psutil.tests.runner.get_suite", + test_suite="psutil.tests.get_suite", tests_require=['ipaddress', 'mock', 'unittest2'], zip_safe=False, # http://stackoverflow.com/questions/19548957 # see: python setup.py register --list-classifiers diff --git a/tox.ini b/tox.ini index be87cf082..20b9f229d 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ setenv = TOX = 1 commands = - python psutil/tests/runner.py + python psutil/tests/__main__.py git ls-files | grep \\.py$ | xargs flake8 # suppress "WARNING: 'git' command found but not installed in testenv From d6d90b1483c3f1233d043955111af2ecd7fd424e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 01:37:36 +0200 Subject: [PATCH 288/922] #1018: add --install-deps opt --- Makefile | 4 +-- psutil/tests/__init__.py | 60 +++++++++++++++++++++++++++++++++++++++- psutil/tests/__main__.py | 35 ++++++++++++++++++++++- 3 files changed, 95 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 65762df6a..e70aefe4c 100644 --- a/Makefile +++ b/Makefile @@ -121,11 +121,11 @@ test: install # Test psutil process-related APIs. test-process: install - $(PYTHON) -m unittest -v psutil.test.test_process + $(PYTHON) -m unittest -v psutil.tests.test_process # Test psutil system-related APIs. test-system: install - $(PYTHON) -m unittest -v psutil.test.test_system + $(PYTHON) -m unittest -v psutil.tests.test_system # Test misc. test-misc: install diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 84eafe9e8..59684ba25 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -13,11 +13,11 @@ import contextlib import errno import functools -import ipaddress # python >= 3.3 / requires "pip install ipaddress" import os import re import shutil import socket +import ssl import stat import subprocess import sys @@ -31,6 +31,11 @@ from socket import SOCK_DGRAM from socket import SOCK_STREAM +try: + from urllib.request import urlopen # py3 +except ImportError: + from urllib2 import urlopen + try: from unittest import mock # py3 except ImportError: @@ -74,6 +79,8 @@ 'check_connection_ntuple', 'check_net_address', 'unittest', 'cleanup', 'skip_on_access_denied', 'skip_on_not_implemented', 'retry_before_failing', 'run_test_module_by_name', + # install utils + 'install_pip', 'install_test_deps', # fs utils 'chdir', 'safe_rmpath', 'create_exe', # subprocesses @@ -138,6 +145,14 @@ DEVNULL = open(os.devnull, 'r+') VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) if x.startswith('STATUS_')] +GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" +TEST_DEPS = [] +if sys.version_info[:2] == (2, 6): + TEST_DEPS.extend(["ipaddress", "unittest2", "argparse", "mock==1.0.1"]) +elif sys.version_info[:2] == (2, 7) or sys.version_info[:2] <= (3, 2): + TEST_DEPS.extend(["ipaddress", "mock"]) +elif sys.version_info[:2] == (3, 3): + TEST_DEPS.extend(["ipaddress"]) # =================================================================== @@ -638,6 +653,7 @@ def check_net_address(addr, family): """Check a net address validity. Supported families are IPv4, IPv6 and MAC addresses. """ + import ipaddress # python >= 3.3 / requires "pip install ipaddress" if enum and PY3: assert isinstance(family, enum.IntEnum), family if family == AF_INET: @@ -734,6 +750,48 @@ def cleanup(): atexit.register(lambda: DEVNULL.close()) +# =================================================================== +# --- install +# =================================================================== + + +def install_pip(): + """Install pip. Returns the exit code of the subprocess.""" + try: + import pip # NOQA + except ImportError: + f = tempfile.NamedTemporaryFile(suffix='.py') + with contextlib.closing(f): + print("downloading %s to %s" % (GET_PIP_URL, f.name)) + if hasattr(ssl, '_create_unverified_context'): + ctx = ssl._create_unverified_context() + else: + ctx = None + kwargs = dict(context=ctx) if ctx else {} + req = urlopen(GET_PIP_URL, **kwargs) + data = req.read() + f.write(data) + f.flush() + + print("installing pip") + code = os.system('%s %s --user' % (sys.executable, f.name)) + return code + + +def install_test_deps(deps=None): + """Install test dependencies via pip.""" + if deps is None: + deps = TEST_DEPS + deps = set(deps) + if deps: + is_venv = hasattr(sys, 'real_prefix') + opts = "--user" if not is_venv else "" + install_pip() + code = os.system('%s -m pip install %s --upgrade %s' % ( + sys.executable, opts, " ".join(deps))) + return code + + # =================================================================== # --- others # =================================================================== diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py index 5f7bb5287..b2f9c0bff 100755 --- a/psutil/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -10,5 +10,38 @@ $ python -m psutil.tests """ +import optparse +import os +import sys + +from psutil.tests import install_pip +from psutil.tests import install_test_deps from psutil.tests import run_suite -run_suite() +from psutil.tests import TEST_DEPS + + +PYTHON = os.path.basename(sys.executable) + + +def main(): + usage = "%s -m psutil.tests [opts]" % PYTHON + parser = optparse.OptionParser(usage=usage, description="run unit tests") + parser.add_option("-i", "--install-deps", + action="store_true", default=False, + help="don't print status messages to stdout") + + opts, args = parser.parse_args() + if opts.install_deps: + install_pip() + install_test_deps() + else: + for dep in TEST_DEPS: + try: + __import__(dep) + except ImportError: + sys.exit("%r lib is not installed; run:\n" + "%s -m psutil.tests --install-deps" % (dep, PYTHON)) + run_suite() + + +main() From 8e8aa1c62b44a725e0e7f8e1417ea4819591901d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 02:16:13 +0200 Subject: [PATCH 289/922] fix travis --- .ci/travis/run.sh | 2 +- psutil/tests/__main__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 2bc0dfe91..a0cdd1b67 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -19,7 +19,7 @@ python setup.py develop # run tests (with coverage) if [[ $PYVER == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then - coverage run psutil/tests/__main__.py --include="psutil/*" --omit="test/*,*setup*" + coverage run psutil/tests/__main__.py else python psutil/tests/__main__.py fi diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py index b2f9c0bff..b57914d48 100755 --- a/psutil/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -37,7 +37,7 @@ def main(): else: for dep in TEST_DEPS: try: - __import__(dep) + __import__(dep.split("==")[0]) except ImportError: sys.exit("%r lib is not installed; run:\n" "%s -m psutil.tests --install-deps" % (dep, PYTHON)) From d491d8fb762bb20ad5b88254749940fbc3434d82 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 02:29:34 +0200 Subject: [PATCH 290/922] minor change --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index e70aefe4c..4b362a326 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ TSCRIPT = psutil/tests/__main__.py ARGS = # List of nice-to-have dev libs. -DEPS = argparse \ +DEPS = \ + argparse \ check-manifest \ coverage \ flake8 \ @@ -144,7 +145,7 @@ test-platform: install $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py # Run a specific test by name, e.g. -# make test-by-name psutil.test.test_system.TestSystemAPIs.test_cpu_times +# make test-by-name psutil.testss.test_system.TestSystemAPIs.test_cpu_times test-by-name: install @$(PYTHON) -m unittest -v $(ARGS) From 4f170f378cffe2dac03f591dcb31e74a48c64b5c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 03:19:57 +0200 Subject: [PATCH 291/922] #1020 / windows / open_files: update doc and inform that open_files() only lists files living in the C:\ drive --- docs/index.rst | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 72c2881cd..ce66e966a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1633,6 +1633,9 @@ Process class - **path**: the absolute file name. - **fd**: the file descriptor number; on Windows this is always ``-1``. + + Linux only: + - **position** (*Linux*): the file (offset) position. - **mode** (*Linux*): a string indicating how the file was opened, similarly `open `__'s @@ -1653,18 +1656,20 @@ Process class [popenfile(path='/home/giampaolo/svn/psutil/file.ext', fd=3, position=0, mode='w', flags=32769)] .. warning:: - on Windows this is not fully reliable as due to some limitations of the - Windows API the underlying implementation may hang when retrieving - certain file handles. - In order to work around that psutil on Windows Vista (and higher) spawns - a thread and kills it if it's not responding after 100ms. - That implies that on Windows this method is not guaranteed to enumerate - all regular file handles (see full - `discussion `_). + on Windows this method is not reliable due to some limitations of the + underlying Windows API which may hang when retrieving certain file + handles. + In order to work around that psutil spawns a thread for each handle and + kills it if it's not responding after 100ms. + That implies that this method on Windows is not guaranteed to enumerate + all regular file handles (see + `issue 597 `_). + Also, it will only list files living in the C:\\ drive (see + `issue 1020 `_). .. warning:: - on BSD this method can return files with a 'null' path due to a kernel - bug hence it's not reliable + on BSD this method can return files with a null path ("") due to a + kernel bug, hence it's not reliable (see `issue 595 `_). .. versionchanged:: From 1ab8a0bff56163956a2140142481b83ddbfc8f3c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 11:15:39 +0200 Subject: [PATCH 292/922] update doc --- DEVGUIDE.rst | 22 ++++++++--------- Makefile | 52 ++++++++++++++++++++++++----------------- psutil/tests/README.rst | 20 +++++++--------- 3 files changed, 50 insertions(+), 44 deletions(-) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index 93dfa6903..2494816f8 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -8,23 +8,21 @@ If you plan on hacking on psutil this is what you're supposed to do first: $ git clone git@github.com:giampaolo/psutil.git -- install system deps (see `install instructions `__). - -- install development deps; these are useful for running tests (e.g. mock, - unittest2), building doc (e.g. sphinx), running linters (flake8), etc. :: +- install test deps and GIT hooks:: $ make setup-dev-env -- bear in mind that ``make`` (see `Makefile `_) +- run tests:: + + $ make test + +- bear in mind that ``make`` + (see `Makefile `_) is the designated tool to run tests, build, install etc. and that it is also available on Windows (see `make.bat `_). -- bear in mind that both psutil (``make install``) and any other lib - (``make setup-dev-env``) is installed as a limited user - (``pip install --user ...``), so develop as such (don't use root). -- (UNIX only) run ``make install-git-hooks``: this will reject your commits - if python code is not PEP8 compliant. -- run ``make test`` to run tests. +- do not use ``sudo``; ``make install`` installs psutil as a limited user in + "edit" mode; also ``make setup-dev-env`` installs deps as a limited user. ============ Coding style @@ -43,7 +41,7 @@ Some useful make commands:: $ make install # install $ make setup-dev-env # install useful dev libs (pyflakes, unittest2, etc.) - $ make test # run all tests + $ make test # run unit tests $ make test-memleaks # run memory leak tests $ make coverage # run test coverage $ make flake8 # run PEP8 linter diff --git a/Makefile b/Makefile index 4b362a326..e2d27c949 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,6 @@ DEPS = \ # In not in a virtualenv, add --user options for install commands. INSTALL_OPTS = `$(PYTHON) -c "import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"` - all: test # =================================================================== @@ -73,7 +72,8 @@ build: _ # Install this package + GIT hooks. Install is done: # - as the current user, in order to avoid permission issues # - in development / edit mode, so that source can be modified on the fly -install: build +install: + ${MAKE} build # make sure setuptools is installed (needed for 'develop' / edit mode) $(PYTHON) -c "import setuptools" $(PYTHON) setup.py develop $(INSTALL_OPTS) @@ -103,12 +103,10 @@ install-pip: f.close(); \ sys.exit(code);" -# Install: -# - GIT hooks -# - pip (if necessary) -# - useful deps which are nice to have while developing / testing; -# deps these are also upgraded -setup-dev-env: install-git-hooks install-pip +# Install GIT hooks, pip, test deps (also upgrades them). +setup-dev-env: + ${MAKE} install-git-hooks + ${MAKE} install-pip $(PYTHON) -m pip install $(INSTALL_OPTS) --upgrade pip $(PYTHON) -m pip install $(INSTALL_OPTS) --upgrade $(DEPS) @@ -117,39 +115,48 @@ setup-dev-env: install-git-hooks install-pip # =================================================================== # Run all tests. -test: install +test: + ${MAKE} install $(PYTHON) $(TSCRIPT) # Test psutil process-related APIs. -test-process: install +test-process: + ${MAKE} install $(PYTHON) -m unittest -v psutil.tests.test_process # Test psutil system-related APIs. -test-system: install +test-system: + ${MAKE} install $(PYTHON) -m unittest -v psutil.tests.test_system # Test misc. -test-misc: install +test-misc: + ${MAKE} install $(PYTHON) psutil/tests/test_misc.py # Test POSIX. -test-posix: install +test-posix: + ${MAKE} install $(PYTHON) psutil/tests/test_posix.py # Test memory leaks. -test-memleaks: install +test-memleaks: + ${MAKE} install $(PYTHON) psutil/tests/test_memory_leaks.py # Run specific platform tests only. -test-platform: install +test-platform: + ${MAKE} install $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py # Run a specific test by name, e.g. -# make test-by-name psutil.testss.test_system.TestSystemAPIs.test_cpu_times -test-by-name: install +# make test-by-name psutil.tests.test_system.TestSystemAPIs.test_cpu_times +test-by-name: + ${MAKE} install @$(PYTHON) -m unittest -v $(ARGS) -coverage: install +coverage: + ${MAKE} install # Note: coverage options are controlled by .coveragerc file rm -rf .coverage htmlcov $(PYTHON) -m coverage run $(TSCRIPT) @@ -194,7 +201,8 @@ install-git-hooks: # =================================================================== # Upload source tarball on https://pypi.python.org/pypi/psutil. -upload-src: clean +upload-src: + ${MAKE} clean $(PYTHON) setup.py sdist upload # Download exes/wheels hosted on appveyor. @@ -240,11 +248,13 @@ grep-todos: git grep -EIn "TODO|FIXME|XXX" # run script which benchmarks oneshot() ctx manager (see #799) -bench-oneshot: install +bench-oneshot: + ${MAKE} install $(PYTHON) scripts/internal/bench_oneshot.py # same as above but using perf module (supposed to be more precise) -bench-oneshot-2: install +bench-oneshot-2: + ${MAKE} install $(PYTHON) scripts/internal/bench_oneshot_2.py # generate a doc.zip file and manually upload it to PYPI. diff --git a/psutil/tests/README.rst b/psutil/tests/README.rst index 637fb7ddc..ab78aa8e6 100644 --- a/psutil/tests/README.rst +++ b/psutil/tests/README.rst @@ -1,24 +1,22 @@ Instructions for running tests ============================== -* There are two ways of running tests. If psutil is already installed: +* There are two ways of running tests. As a "user", if psutil is already + installed and you just want to test it works:: + python -m psutil.tests --install-deps # install test deps python -m psutil.tests - If you have a copy of the source code: + As a "developer", if you have a copy of the source code and you whish to hack + on psutil:: + make setup-dev-env # install test deps (+ other things) make test -* Depending on the Python version, dependencies for running tests include - ``ipaddress``, ``mock`` and ``unittest2`` modules. - On Windows you'll also need ``pywin32`` and ``wmi`` modules. - If you have a copy of the source code you can run ``make setup-dev-env`` to - install all deps (also on Windows). - * To run tests on all supported Python versions install tox - (``pip install tox``) then run ``tox`` from psutil root directory. + (``pip install tox``) then run ``tox`` from within psutil root directory. * Every time a commit is pushed tests are automatically run on Travis (Linux, OSX) and appveyor (Windows): - * https://travis-ci.org/giampaolo/psutil/ - * https://ci.appveyor.com/project/giampaolo/psutil + * Travis builds: https://travis-ci.org/giampaolo/psutil + * AppVeyor builds: https://ci.appveyor.com/project/giampaolo/psutil From 38a1632b312e66d213b3ab665ed5736c9f379321 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 11:20:36 +0200 Subject: [PATCH 293/922] update doc --- DEVGUIDE.rst | 9 ++++----- psutil/tests/README.rst | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index 2494816f8..df2113917 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -99,18 +99,18 @@ Test files controlling these are and `appveyor.yml `_. Both services run psutil test suite against all supported python version -(2.6 - 3.5). +(2.6 - 3.6). Two icons in the home page (README) always show the build status: -.. image:: https://api.travis-ci.org/giampaolo/psutil.png?branch=master +.. image:: https://img.shields.io/travis/giampaolo/psutil/master.svg?maxAge=3600&label=Linux%20/%20OSX :target: https://travis-ci.org/giampaolo/psutil :alt: Linux tests (Travis) -.. image:: https://ci.appveyor.com/api/projects/status/qdwvw7v1t915ywr5/branch/master?svg=true +.. image:: https://img.shields.io/appveyor/ci/giampaolo/psutil/master.svg?maxAge=3600&label=Windows :target: https://ci.appveyor.com/project/giampaolo/psutil :alt: Windows tests (Appveyor) -OSX, FreeBSD and Solaris are currently tested manually (sigh!). +OSX, BSD and Solaris are currently tested manually (sigh!). Test coverage ------------- @@ -133,7 +133,6 @@ Documentation and it's built with `sphinx `_. - doc can be built with ``make setup-dev-env; cd docs; make html``. - public doc is hosted on http://pythonhosted.org/psutil/. -- it is uploaded on every new release with ``make upload-doc``. ======================= Releasing a new version diff --git a/psutil/tests/README.rst b/psutil/tests/README.rst index ab78aa8e6..515abf772 100644 --- a/psutil/tests/README.rst +++ b/psutil/tests/README.rst @@ -18,5 +18,6 @@ Instructions for running tests * Every time a commit is pushed tests are automatically run on Travis (Linux, OSX) and appveyor (Windows): + * Travis builds: https://travis-ci.org/giampaolo/psutil * AppVeyor builds: https://ci.appveyor.com/project/giampaolo/psutil From 1353533343ecfff505daec238bb309a5be05ae83 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 13:45:48 +0200 Subject: [PATCH 294/922] increase test coverage --- psutil/tests/test_linux.py | 7 +++++-- psutil/tests/test_process.py | 3 +++ psutil/tests/test_system.py | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 747f1456e..c514db30a 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -237,10 +237,9 @@ def open_mock(name, *args, **kwargs): return io.BytesIO(textwrap.dedent("""\ Active(anon): 6145416 kB Active(file): 2950064 kB - Buffers: 287952 kB Inactive(anon): 574764 kB Inactive(file): 1567648 kB - MemAvailable: 6574984 kB + MemAvailable: -1 kB MemFree: 2057400 kB MemTotal: 16325648 kB SReclaimable: 346648 kB @@ -264,10 +263,14 @@ def open_mock(name, *args, **kwargs): self.assertIn("shared", str(w.message)) self.assertIn("active", str(w.message)) self.assertIn("inactive", str(w.message)) + self.assertIn("buffers", str(w.message)) + self.assertIn("available", str(w.message)) self.assertEqual(ret.cached, 0) self.assertEqual(ret.active, 0) self.assertEqual(ret.inactive, 0) self.assertEqual(ret.shared, 0) + self.assertEqual(ret.buffers, 0) + self.assertEqual(ret.available, 0) def test_avail_old_percent(self): # Make sure that our calculation of avail mem for old kernels diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index f6442a538..ba682b774 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1096,6 +1096,9 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): psutil.CONN_NONE, ("all", "inet", "inet6", "udp", "udp6")) + # err + self.assertRaises(ValueError, p.connections, kind='???') + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'AF_UNIX not supported') @skip_on_access_denied(only_if=OSX) def test_connections_unix(self): diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 96f5480b1..4076beecc 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -518,6 +518,8 @@ def check(cons, families, types_): self.assertEqual(len(cons), len(set(cons))) check(cons, families, types_) + self.assertRaises(ValueError, psutil.net_connections, kind='???') + def test_net_io_counters(self): def check_ntuple(nt): self.assertEqual(nt[0], nt.bytes_sent) From fe58626e3dd8ee352e2b4615c38c25167c80c253 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 13:58:12 +0200 Subject: [PATCH 295/922] increase test coverage --- psutil/tests/test_linux.py | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index c514db30a..bc4c0f79f 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1183,12 +1183,44 @@ def open_mock(name, *args, **kwargs): psutil.sensors_battery().secsleft, psutil.POWER_TIME_UNLIMITED) assert m.called + def test_emulate_power_plugged_2(self): + # Same as above but pretend /AC0/online does not exist in which + # case code relies on /status file. + def open_mock(name, *args, **kwargs): + if name.endswith("AC0/online") or name.endswith("AC/online"): + raise IOError(errno.ENOENT, "") + elif name.endswith("/status"): + return io.BytesIO(b"charging") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: + self.assertEqual(psutil.sensors_battery().power_plugged, True) + assert m.called + def test_emulate_power_not_plugged(self): # Pretend the AC power cable is not connected. def open_mock(name, *args, **kwargs): - if name.startswith("/sys/class/power_supply/AC0/online"): + if name.endswith("AC0/online") or name.endswith("AC/online"): return io.BytesIO(b"0") - elif name.startswith("/sys/class/power_supply/BAT0/status"): + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock) as m: + self.assertEqual(psutil.sensors_battery().power_plugged, False) + assert m.called + + def test_emulate_power_not_plugged_2(self): + # Same as above but pretend /AC0/online does not exist in which + # case code relies on /status file. + def open_mock(name, *args, **kwargs): + if name.endswith("AC0/online") or name.endswith("AC/online"): + raise IOError(errno.ENOENT, "") + elif name.endswith("/status"): return io.BytesIO(b"discharging") else: return orig_open(name, *args, **kwargs) From 5abb9a34c10e707a11f01f4025a643cc7fdf889c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 14:06:11 +0200 Subject: [PATCH 296/922] increase test coverage --- psutil/tests/test_linux.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index bc4c0f79f..8a265ba59 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1137,6 +1137,21 @@ def test_issue_687(self): finally: t.stop() + def test_pid_exists_no_proc_status(self): + # Internally pid_exists relies on /proc/{pid}/status. + # Emulate a case where this file is empty in which case + # psutil is supposed to fall back on using pids(). + def open_mock(name, *args, **kwargs): + if name == "/proc/%s/status" % os.getpid(): + return io.BytesIO("") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock): + assert psutil.pid_exists(os.getpid()) + # ===================================================================== # --- sensors @@ -1170,7 +1185,7 @@ def test_power_plugged(self): def test_emulate_power_plugged(self): # Pretend the AC power cable is connected. def open_mock(name, *args, **kwargs): - if name.startswith("/sys/class/power_supply/AC0/online"): + if name.endswith("AC0/online") or name.endswith("AC/online"): return io.BytesIO(b"1") else: return orig_open(name, *args, **kwargs) From e0a25e902a40faf13f6c20ad78e804bf20285106 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 14:23:47 +0200 Subject: [PATCH 297/922] increase test coverage --- psutil/tests/test_linux.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 8a265ba59..2be2e6584 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -231,7 +231,10 @@ def test_available(self): free_value, psutil_value, delta=MEMORY_TOLERANCE, msg='%s %s \n%s' % (free_value, psutil_value, out)) - def test_warnings_mocked(self): + def test_warnings_on_misses(self): + # Emulate a case where /proc/meminfo provides few info. + # psutil is supposed to set the missing fields to 0 and + # raise a warning. def open_mock(name, *args, **kwargs): if name == '/proc/meminfo': return io.BytesIO(textwrap.dedent("""\ @@ -414,7 +417,7 @@ def test_free(self): return self.assertAlmostEqual( free_value, psutil_value, delta=MEMORY_TOLERANCE) - def test_warnings_mocked(self): + def test_missing_sin_sout(self): with mock.patch('psutil._pslinux.open', create=True) as m: with warnings.catch_warnings(record=True) as ws: warnings.simplefilter("always") @@ -1606,6 +1609,20 @@ def open_mock(name, *args, **kwargs): self.assertEqual(err.exception.errno, errno.ENOENT) assert m.called + def test_rlimit_zombie(self): + # Emulate a case where rlimit() raises ENOSYS, which may + # happen in case of zombie process: + # https://travis-ci.org/giampaolo/psutil/jobs/51368273 + with mock.patch("psutil._pslinux.cext.linux_prlimit", + side_effect=OSError(errno.ENOSYS, "")) as m: + p = psutil.Process() + p.name() + with self.assertRaises(psutil.ZombieProcess) as exc: + p.rlimit(psutil.RLIMIT_NOFILE) + assert m.called + self.assertEqual(exc.exception.pid, p.pid) + self.assertEqual(exc.exception.name, p.name()) + @unittest.skipUnless(LINUX, "LINUX only") class TestProcessAgainstStatus(unittest.TestCase): From 477f47720fc87330e4041525ca4aa34140641444 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 14:26:08 +0200 Subject: [PATCH 298/922] increase test coverage --- psutil/tests/test_linux.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 2be2e6584..98d30bba6 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1623,6 +1623,17 @@ def test_rlimit_zombie(self): self.assertEqual(exc.exception.pid, p.pid) self.assertEqual(exc.exception.name, p.name()) + def test_cwd_zombie(self): + with mock.patch("psutil._pslinux.os.readlink", + side_effect=OSError(errno.ENOENT, "")) as m: + p = psutil.Process() + p.name() + with self.assertRaises(psutil.ZombieProcess) as exc: + p.cwd() + assert m.called + self.assertEqual(exc.exception.pid, p.pid) + self.assertEqual(exc.exception.name, p.name()) + @unittest.skipUnless(LINUX, "LINUX only") class TestProcessAgainstStatus(unittest.TestCase): From 9b29a184bfcf5a8434b544a139440efe2003da1b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 14:39:09 +0200 Subject: [PATCH 299/922] fix py3 failures --- psutil/tests/test_linux.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 98d30bba6..162d7a29d 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1146,7 +1146,7 @@ def test_pid_exists_no_proc_status(self): # psutil is supposed to fall back on using pids(). def open_mock(name, *args, **kwargs): if name == "/proc/%s/status" % os.getpid(): - return io.BytesIO("") + return io.StringIO(u("")) else: return orig_open(name, *args, **kwargs) @@ -1208,7 +1208,7 @@ def open_mock(name, *args, **kwargs): if name.endswith("AC0/online") or name.endswith("AC/online"): raise IOError(errno.ENOENT, "") elif name.endswith("/status"): - return io.BytesIO(b"charging") + return io.StringIO(u("charging")) else: return orig_open(name, *args, **kwargs) @@ -1239,7 +1239,7 @@ def open_mock(name, *args, **kwargs): if name.endswith("AC0/online") or name.endswith("AC/online"): raise IOError(errno.ENOENT, "") elif name.endswith("/status"): - return io.BytesIO(b"discharging") + return io.StringIO(u("discharging")) else: return orig_open(name, *args, **kwargs) From 8ce574092408d7a697cb71eb27d57caa6abdcfdf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 15:08:19 +0200 Subject: [PATCH 300/922] fix some code smells --- psutil/__init__.py | 2 +- psutil/tests/__init__.py | 13 ++++++------- psutil/tests/test_bsd.py | 3 ++- psutil/tests/test_linux.py | 7 +++---- psutil/tests/test_misc.py | 6 ++++-- psutil/tests/test_process.py | 2 +- psutil/tests/test_system.py | 2 -- psutil/tests/test_windows.py | 3 ++- setup.py | 3 +-- 9 files changed, 20 insertions(+), 21 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 4e68a46dc..dc2c063c3 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -272,7 +272,7 @@ class ZombieProcess(NoSuchProcess): """ def __init__(self, pid, name=None, ppid=None, msg=None): - Error.__init__(self, msg) + NoSuchProcess.__init__(self, msg) self.pid = pid self.ppid = ppid self.name = name diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 59684ba25..b3dda439c 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -166,20 +166,19 @@ class ThreadTask(threading.Thread): def __init__(self): threading.Thread.__init__(self) self._running = False - self._interval = None + self._interval = 0.001 self._flag = threading.Event() def __repr__(self): name = self.__class__.__name__ return '<%s running=%s at %#x>' % (name, self._running, id(self)) - def start(self, interval=0.001): + def start(self): """Start thread and keep it running until an explicit stop() request. Polls for shutdown every 'timeout' seconds. """ if self._running: raise ValueError("already started") - self._interval = interval threading.Thread.start(self) self._flag.wait() @@ -435,11 +434,11 @@ def wrapper(*args, **kwargs): if self.logfun is not None: self.logfun(exc) self.sleep() + continue + if PY3: + raise exc else: - if PY3: - raise exc - else: - raise + raise # This way the user of the decorated function can change config # parameters. diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 9c1753d5e..77bb7bffe 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -415,7 +415,8 @@ def test_boot_time(self): @unittest.skipUnless(NETBSD, "NETBSD only") class NetBSDSpecificTestCase(unittest.TestCase): - def parse_meminfo(self, look_for): + @staticmethod + def parse_meminfo(look_for): with open('/proc/meminfo', 'rb') as f: for line in f: if line.startswith(look_for): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 162d7a29d..0d8c73465 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1528,12 +1528,12 @@ def test_cmdline_mocked(self): fake_file = io.StringIO(u('foo\x00bar\x00')) with mock.patch('psutil._pslinux.open', return_value=fake_file, create=True) as m: - p.cmdline() == ['foo', 'bar'] + self.assertEqual(p.cmdline(), ['foo', 'bar']) assert m.called fake_file = io.StringIO(u('foo\x00bar\x00\x00')) with mock.patch('psutil._pslinux.open', return_value=fake_file, create=True) as m: - p.cmdline() == ['foo', 'bar', ''] + self.assertEqual(p.cmdline(), ['foo', 'bar', '']) assert m.called def test_readlink_path_deleted_mocked(self): @@ -1659,8 +1659,7 @@ def read_status_file(self, linestart): return int(value) except ValueError: return value - else: - raise ValueError("can't find %r" % linestart) + raise ValueError("can't find %r" % linestart) def test_name(self): value = self.read_status_file("Name:") diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 5b34a343f..3ca0ebfdc 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -367,7 +367,8 @@ def test_sanity_version_check(self): class TestScripts(unittest.TestCase): """Tests for scripts in the "scripts" directory.""" - def assert_stdout(self, exe, args=None): + @staticmethod + def assert_stdout(exe, args=None): exe = '"%s"' % os.path.join(SCRIPTS_DIR, exe) if args: exe = exe + ' ' + args @@ -381,7 +382,8 @@ def assert_stdout(self, exe, args=None): assert out, out return out - def assert_syntax(self, exe, args=None): + @staticmethod + def assert_syntax(exe, args=None): exe = os.path.join(SCRIPTS_DIR, exe) with open(exe, 'r') as f: src = f.read() diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index ba682b774..45f88fe5d 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -852,7 +852,7 @@ def test_username(self): self.assertEqual(p.username(), pwd.getpwuid(os.getuid()).pw_name) with mock.patch("psutil.pwd.getpwuid", side_effect=KeyError) as fun: - p.username() == str(p.uids().real) + self.assertEqual(p.username(), str(p.uids().real)) assert fun.called elif WINDOWS and 'USERNAME' in os.environ: diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 4076beecc..1b838fa87 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -196,8 +196,6 @@ def test_pid_exists(self): self.assertFalse(psutil.pid_exists(sproc.pid)) self.assertFalse(psutil.pid_exists(-1)) self.assertEqual(psutil.pid_exists(0), 0 in psutil.pids()) - # pid 0 - psutil.pid_exists(0) == 0 in psutil.pids() def test_pid_exists_2(self): reap_children() diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 3fcc20ede..1ca796d05 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -626,7 +626,8 @@ def test_num_handles(self): num_handles = psutil.Process(self.pid).num_handles() with mock.patch("psutil._psplatform.cext.proc_num_handles", side_effect=OSError(errno.EPERM, "msg")) as fun: - psutil.Process(self.pid).num_handles() == num_handles + self.assertEqual(psutil.Process(self.pid).num_handles(), + num_handles) assert fun.called diff --git a/setup.py b/setup.py index 3f2fd0dfb..2fad970e3 100755 --- a/setup.py +++ b/setup.py @@ -54,8 +54,7 @@ def get_version(): for num in ret.split('.'): assert num.isdigit(), ret return ret - else: - raise ValueError("couldn't find version string") + raise ValueError("couldn't find version string") def get_description(): From 9ad2d0f09d14af999ca0e31d3a4ed77fe41c5b81 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 17:23:53 +0200 Subject: [PATCH 301/922] small refactoring --- psutil/tests/__init__.py | 6 +++--- psutil/tests/test_misc.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index b3dda439c..3739b0025 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -220,7 +220,7 @@ def get_test_subprocess(cmd=None, **kwds): pyline += "sleep(60)" cmd = [PYTHON, "-c", pyline] sproc = subprocess.Popen(cmd, **kwds) - wait_for_file(_TESTFN, delete_file=True, empty=True) + wait_for_file(_TESTFN, delete=True, empty=True) else: sproc = subprocess.Popen(cmd, **kwds) wait_for_pid(sproc.pid) @@ -460,13 +460,13 @@ def wait_for_pid(pid): @retry(exception=(EnvironmentError, AssertionError), logfun=None, timeout=GLOBAL_TIMEOUT, interval=0.001) -def wait_for_file(fname, delete_file=True, empty=False): +def wait_for_file(fname, delete=True, empty=False): """Wait for a file to be written on disk with some content.""" with open(fname, "rb") as f: data = f.read() if not empty: assert data - if delete_file: + if delete: os.remove(fname) return data diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 3ca0ebfdc..354736929 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -600,7 +600,7 @@ def test_wait_for_file_no_file(self): def test_wait_for_file_no_delete(self): with open(TESTFN, 'w') as f: f.write('foo') - wait_for_file(TESTFN, delete_file=False) + wait_for_file(TESTFN, delete=False) assert os.path.exists(TESTFN) From 267de56fe0e424f86d3c615b2d770efefac2f35f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 18:32:00 +0200 Subject: [PATCH 302/922] add utility to create a child, grandchild process pair --- psutil/tests/__init__.py | 40 +++++++++++++++++++++++++++++++++++---- psutil/tests/test_misc.py | 20 ++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 3739b0025..41b5c213a 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -202,6 +202,7 @@ def stop(self): _subprocesses_started = set() +_pids_started = set() def get_test_subprocess(cmd=None, **kwds): @@ -228,6 +229,27 @@ def get_test_subprocess(cmd=None, **kwds): return sproc +def create_proc_children_pair(): + """Create a subprocess which creates another one as in: + A (us) -> B (child) -> C (grandchild). + Return a (child, grandchild) tuple. + """ + s = "import subprocess, os, sys, time;" + s += "PYTHON = os.path.realpath(sys.executable);" + s += "cmd = [PYTHON, '-c', 'import time; time.sleep(60);'];" + s += "sproc = subprocess.Popen(cmd);" + s += "f = open('%s', 'w');" % TESTFN + s += "f.write(str(sproc.pid));" + s += "f.close();" + s += "time.sleep(60);" + child1 = psutil.Process(get_test_subprocess(cmd=[PYTHON, "-c", s]).pid) + data = wait_for_file(TESTFN, delete=False, empty=False) + child2_pid = int(data) + _pids_started.add(child2_pid) + child2 = psutil.Process(child2_pid) + return (child1, child2) + + _testfiles = [] @@ -279,9 +301,9 @@ def reap_children(recursive=False): # processes as we don't want to lose the intermediate reference # in case of grandchildren. if recursive: - children = psutil.Process().children(recursive=True) + children = set(psutil.Process().children(recursive=True)) else: - children = [] + children = set() # Terminate subprocess.Popen instances "cleanly" by closing their # fds and wiat()ing for them in order to avoid zombies. @@ -309,7 +331,17 @@ def reap_children(recursive=False): if err.errno != errno.ECHILD: raise - # Terminates grandchildren. + # Terminate started pids. + for pid in _pids_started: + try: + p = psutil.Process(pid) + except psutil.NoSuchProcess: + pass + else: + children.add(p) + _pids_started.clear() + + # Terminate grandchildren. if children: for p in children: try: @@ -323,7 +355,7 @@ def reap_children(recursive=False): p.kill() except psutil.NoSuchProcess: pass - _, alive = psutil.wait_procs(alive, timeout=GLOBAL_TIMEOUT) + gone, alive = psutil.wait_procs(alive, timeout=GLOBAL_TIMEOUT) if alive: for p in alive: warn("process %r survived kill()" % p) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 354736929..13649755a 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -30,6 +30,7 @@ from psutil._common import supports_ipv6 from psutil.tests import APPVEYOR from psutil.tests import chdir +from psutil.tests import create_proc_children_pair from psutil.tests import get_test_subprocess from psutil.tests import importlib from psutil.tests import mock @@ -46,6 +47,7 @@ from psutil.tests import unittest from psutil.tests import wait_for_file from psutil.tests import wait_for_pid +import psutil.tests class TestMisc(unittest.TestCase): @@ -645,6 +647,24 @@ def test_reap_children(self): assert p.is_running() reap_children() assert not p.is_running() + assert not psutil.tests._pids_started + assert not psutil.tests._subprocesses_started + + def test_create_proc_children_pair(self): + p1, p2 = create_proc_children_pair() + self.assertNotEqual(p1.pid, p2.pid) + assert p1.is_running() + assert p2.is_running() + children = psutil.Process().children(recursive=True) + self.assertEqual(len(children), 2) + self.assertIn(p1, children) + self.assertIn(p2, children) + # make sure both or them are cleanup up + reap_children() + assert not p1.is_running() + assert not p2.is_running() + assert not psutil.tests._pids_started + assert not psutil.tests._subprocesses_started if __name__ == '__main__': From 7666871497e930ff1a318d90bbcec93da851afbf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 18:47:21 +0200 Subject: [PATCH 303/922] reuse create_proc_children_pair in tests --- psutil/tests/__init__.py | 20 +++++++++++--------- psutil/tests/test_misc.py | 5 ++++- psutil/tests/test_process.py | 29 +++++++++++------------------ 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 41b5c213a..ccd2d3750 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -234,15 +234,17 @@ def create_proc_children_pair(): A (us) -> B (child) -> C (grandchild). Return a (child, grandchild) tuple. """ - s = "import subprocess, os, sys, time;" - s += "PYTHON = os.path.realpath(sys.executable);" - s += "cmd = [PYTHON, '-c', 'import time; time.sleep(60);'];" - s += "sproc = subprocess.Popen(cmd);" - s += "f = open('%s', 'w');" % TESTFN - s += "f.write(str(sproc.pid));" - s += "f.close();" - s += "time.sleep(60);" - child1 = psutil.Process(get_test_subprocess(cmd=[PYTHON, "-c", s]).pid) + s = textwrap.dedent("""\ + import subprocess, os, sys, time + PYTHON = os.path.realpath(sys.executable) + cmd = [PYTHON, '-c', 'import time; time.sleep(60);'] + sproc = subprocess.Popen(cmd) + f = open('%s', 'w') + f.write(str(sproc.pid)) + f.close() + time.sleep(60) + """ % TESTFN) + child1 = psutil.Process(pyrun(s).pid) data = wait_for_file(TESTFN, delete=False, empty=False) child2_pid = int(data) _pids_started.add(child2_pid) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 13649755a..ea488b23b 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -659,7 +659,10 @@ def test_create_proc_children_pair(self): self.assertEqual(len(children), 2) self.assertIn(p1, children) self.assertIn(p2, children) - # make sure both or them are cleanup up + self.assertEqual(p1.ppid(), os.getpid()) + self.assertEqual(p2.ppid(), p1.pid) + + # make sure both of them are cleaned up reap_children() assert not p1.is_running() assert not p2.is_running() diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 45f88fe5d..50fcd95bf 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -49,6 +49,7 @@ from psutil.tests import chdir from psutil.tests import check_connection_ntuple from psutil.tests import create_exe +from psutil.tests import create_proc_children_pair from psutil.tests import enum from psutil.tests import get_test_subprocess from psutil.tests import get_winver @@ -1233,25 +1234,17 @@ def test_children(self): self.assertEqual(children[0].ppid(), os.getpid()) def test_children_recursive(self): - # here we create a subprocess which creates another one as in: - # A (parent) -> B (child) -> C (grandchild) - s = "import subprocess, os, sys, time;" - s += "PYTHON = os.path.realpath(sys.executable);" - s += "cmd = [PYTHON, '-c', 'import time; time.sleep(60);'];" - s += "subprocess.Popen(cmd);" - s += "time.sleep(60);" - get_test_subprocess(cmd=[PYTHON, "-c", s]) + # Test children() against two sub processes, A and B, where + # A (our child) spawned B (our grandchild). + p1, p2 = create_proc_children_pair() p = psutil.Process() - self.assertEqual(len(p.children(recursive=False)), 1) - # give the grandchild some time to start - stop_at = time.time() + GLOBAL_TIMEOUT - while time.time() < stop_at: - children = p.children(recursive=True) - if len(children) > 1: - break - self.assertEqual(len(children), 2) - self.assertEqual(children[0].ppid(), os.getpid()) - self.assertEqual(children[1].ppid(), children[0].pid) + self.assertEqual(p.children(), [p1]) + self.assertEqual(p.children(recursive=True), [p1, p2]) + # If the intermediate process is gone there's no wait for + # children() to recursively find it. + p1.terminate() + p1.wait() + self.assertEqual(p.children(recursive=True), []) def test_children_duplicates(self): # find the process which has the highest number of children From fb9ae861cf3cf175c3da4a3cd4e558c6cbd6af91 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 18:54:28 +0200 Subject: [PATCH 304/922] update doc --- docs/index.rst | 2 ++ psutil/tests/test_process.py | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ce66e966a..be8d1f2b5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1625,6 +1625,8 @@ Process class Note that in the example above if process X disappears process Y won't be returned either as the reference to process A is lost. + :meth:`children()` behaviour is well summaried by this + `unit test `__. .. method:: open_files() diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 50fcd95bf..f6824c480 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1234,13 +1234,13 @@ def test_children(self): self.assertEqual(children[0].ppid(), os.getpid()) def test_children_recursive(self): - # Test children() against two sub processes, A and B, where - # A (our child) spawned B (our grandchild). + # Test children() against two sub processes, p1 and p2, where + # p1 (our child) spawned p2 (our grandchild). p1, p2 = create_proc_children_pair() p = psutil.Process() self.assertEqual(p.children(), [p1]) self.assertEqual(p.children(recursive=True), [p1, p2]) - # If the intermediate process is gone there's no wait for + # If the intermediate process is gone there's no way for # children() to recursively find it. p1.terminate() p1.wait() From 9c192592076de7bf402ce6522b4ef2bd0727bb7e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 19:30:07 +0200 Subject: [PATCH 305/922] create_proc_children_pair: have the grandchild create the intermediate test file so that it has more time to initialize --- docs/index.rst | 4 +- psutil/tests/__init__.py | 83 +++++++++++++++++++++------------------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index be8d1f2b5..aa68012fe 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1625,8 +1625,8 @@ Process class Note that in the example above if process X disappears process Y won't be returned either as the reference to process A is lost. - :meth:`children()` behaviour is well summaried by this - `unit test `__. + This concept is well summaried by this + `unit test `__. .. method:: open_files() diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index ccd2d3750..9b9572c39 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -46,6 +46,7 @@ from psutil import POSIX from psutil import WINDOWS from psutil._compat import PY3 +from psutil._compat import u from psutil._compat import unicode from psutil._compat import which @@ -82,15 +83,16 @@ # install utils 'install_pip', 'install_test_deps', # fs utils - 'chdir', 'safe_rmpath', 'create_exe', + 'chdir', 'safe_rmpath', 'create_exe', 'decode_path', 'encode_path', # subprocesses 'pyrun', 'reap_children', 'get_test_subprocess', + 'create_proc_children_pair', # os 'get_winver', 'get_kernel_version', # sync primitives 'call_until', 'wait_for_pid', 'wait_for_file', # others - 'warn', 'decode_path', 'encode_path', + 'warn', ] @@ -130,7 +132,7 @@ TESTFILE_PREFIX = '$testfn' TESTFN = os.path.join(os.path.realpath(os.getcwd()), TESTFILE_PREFIX) _TESTFN = TESTFN + '-internal' -TESTFN_UNICODE = TESTFN + u"-ƒőő" +TESTFN_UNICODE = TESTFN + u("-ƒőő") # --- paths @@ -154,6 +156,10 @@ elif sys.version_info[:2] == (3, 3): TEST_DEPS.extend(["ipaddress"]) +_subprocesses_started = set() +_pids_started = set() +_testfiles = set() + # =================================================================== # --- classes @@ -201,14 +207,11 @@ def stop(self): # =================================================================== -_subprocesses_started = set() -_pids_started = set() - - def get_test_subprocess(cmd=None, **kwds): - """Return a subprocess.Popen object to use in tests. - By default stdout and stderr are redirected to /dev/null and the - python interpreter is used as test process. + """Creates a python subprocess which does nothing for 60 secs and + return it as subprocess.Popen instance. + If "cmd" is specified that is used instead of python. + By default stdout and stderr are redirected to /dev/null. It also attemps to make sure the process is in a reasonably initialized state. """ @@ -233,37 +236,37 @@ def create_proc_children_pair(): """Create a subprocess which creates another one as in: A (us) -> B (child) -> C (grandchild). Return a (child, grandchild) tuple. + The 2 processes are fully initialized and will live for 60 secs. """ s = textwrap.dedent("""\ import subprocess, os, sys, time PYTHON = os.path.realpath(sys.executable) - cmd = [PYTHON, '-c', 'import time; time.sleep(60);'] - sproc = subprocess.Popen(cmd) - f = open('%s', 'w') - f.write(str(sproc.pid)) - f.close() + s = "import os, time;" + s += "f = open('%s', 'w');" + s += "f.write(str(os.getpid()));" + s += "f.close();" + s += "time.sleep(60);" + subprocess.Popen([PYTHON, '-c', s]) time.sleep(60) - """ % TESTFN) + """ % _TESTFN) child1 = psutil.Process(pyrun(s).pid) - data = wait_for_file(TESTFN, delete=False, empty=False) + data = wait_for_file(_TESTFN, delete=False, empty=False) + os.remove(_TESTFN) child2_pid = int(data) _pids_started.add(child2_pid) child2 = psutil.Process(child2_pid) return (child1, child2) -_testfiles = [] - - def pyrun(src): """Run python 'src' code in a separate interpreter. - Return interpreter subprocess. + Returns a subprocess.Popen instance. """ if PY3: src = bytes(src, 'ascii') with tempfile.NamedTemporaryFile( prefix=TESTFILE_PREFIX, delete=False) as f: - _testfiles.append(f.name) + _testfiles.add(f.name) f.write(src) f.flush() subp = get_test_subprocess([PYTHON, f.name], stdout=None, @@ -585,6 +588,24 @@ def create_exe(outpath, c_code=None): os.chmod(outpath, st.st_mode | stat.S_IEXEC) +# In Python 3 paths are unicode objects by default. Surrogate escapes +# are used to handle non-character data. +def encode_path(path): + if PY3: + return path.encode(sys.getfilesystemencoding(), + errors="surrogateescape") + else: + return path + + +def decode_path(path): + if PY3: + return path.decode(sys.getfilesystemencoding(), + errors="surrogateescape") + else: + return path + + # =================================================================== # --- testing # =================================================================== @@ -833,21 +854,3 @@ def install_test_deps(deps=None): def warn(msg): """Raise a warning msg.""" warnings.warn(msg, UserWarning) - - -# In Python 3 paths are unicode objects by default. Surrogate escapes -# are used to handle non-character data. -def encode_path(path): - if PY3: - return path.encode(sys.getfilesystemencoding(), - errors="surrogateescape") - else: - return path - - -def decode_path(path): - if PY3: - return path.decode(sys.getfilesystemencoding(), - errors="surrogateescape") - else: - return path From ef2e1b1868364f307645724cb615e1f80ad559bc Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 20:11:48 +0200 Subject: [PATCH 306/922] fix #1021 / linux / open_files: open_files() may erroneously raise NoSuchProcess instead of skipping a file which gets deleted while open files are retrieved --- HISTORY.rst | 5 ++++- psutil/_pslinux.py | 23 +++++++++++++++++------ psutil/tests/test_linux.py | 16 ++++++++++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 6dfd44208..453ea009d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -13,10 +13,13 @@ **Bug fixes** -- 1014_: Linux can mask legitimate ENOENT exceptions as NoSuchProcess. +- 1014_: [Linux] Process class can mask legitimate ENOENT exceptions as + NoSuchProcess. - 1016_: disk_io_counters() raises RuntimeError on a system with no disks. - 1017_: net_io_counters() raises RuntimeError on a system with no network cards installed. +- 1021_: [Linux] open_files() may erroneously raise NoSuchProcess instead of + skipping a file which gets deleted while open files are retrieved. *2017-04-10* diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 8bfd3db6a..a84752d82 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1879,12 +1879,23 @@ def open_files(self): # Get file position and flags. file = "%s/%s/fdinfo/%s" % ( self._procfs_path, self.pid, fd) - with open_binary(file) as f: - pos = int(f.readline().split()[1]) - flags = int(f.readline().split()[1], 8) - mode = file_flags_to_mode(flags) - ntuple = popenfile(path, int(fd), int(pos), mode, flags) - retlist.append(ntuple) + try: + with open_binary(file) as f: + pos = int(f.readline().split()[1]) + flags = int(f.readline().split()[1], 8) + except IOError as err: + if err.errno == errno.ENOENT: + # fd gone in the meantime; does not + # necessarily mean the process disappeared + # on us. + hit_enoent = True + else: + raise + else: + mode = file_flags_to_mode(flags) + ntuple = popenfile( + path, int(fd), int(pos), mode, flags) + retlist.append(ntuple) if hit_enoent: # raise NSP if the process disappeared on us os.stat('%s/%s' % (self._procfs_path, self.pid)) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 0d8c73465..a164d69a7 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1506,6 +1506,22 @@ def test_open_files_file_gone(self): self.assertEqual(p.open_files(), []) assert m.called + def test_open_files_fd_gone(self): + # Simulate a case where /proc/{pid}/fdinfo/{fd} disappears + # while iterating through fds. + # https://travis-ci.org/giampaolo/psutil/jobs/225694530 + p = psutil.Process() + files = p.open_files() + with tempfile.NamedTemporaryFile(): + # give the kernel some time to see the new file + call_until(p.open_files, "len(ret) != %i" % len(files)) + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, + side_effect=IOError(errno.ENOENT, "")) as m: + files = p.open_files() + assert not files + assert m.called + # --- mocked tests def test_terminal_mocked(self): From c218867050f5ec22d43171d73846330b77fc3399 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 22:03:43 +0200 Subject: [PATCH 307/922] reuse create_proc_children_pair --- psutil/tests/test_process.py | 62 ++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index f6824c480..db6a9e59c 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -199,26 +199,26 @@ def test_wait(self): # timeout < 0 not allowed self.assertRaises(ValueError, p.wait, -1) - # XXX why is this skipped on Windows? - @unittest.skipUnless(POSIX, 'skipped on Windows') def test_wait_non_children(self): - # test wait() against processes which are not our children - code = "import sys;" - code += "from subprocess import Popen, PIPE;" - code += "cmd = ['%s', '-c', 'import time; time.sleep(60)'];" % PYTHON - code += "sp = Popen(cmd, stdout=PIPE);" - code += "sys.stdout.write(str(sp.pid));" - sproc = get_test_subprocess([PYTHON, "-c", code], - stdout=subprocess.PIPE) - grandson_pid = int(sproc.stdout.read()) - grandson_proc = psutil.Process(grandson_pid) - try: - self.assertRaises(psutil.TimeoutExpired, grandson_proc.wait, 0.01) - grandson_proc.kill() - ret = grandson_proc.wait() - self.assertEqual(ret, None) - finally: - reap_children(recursive=True) + # Test wait() against a process which is not our direct + # child. + p1, p2 = create_proc_children_pair() + self.assertRaises(psutil.TimeoutExpired, p1.wait, 0.01) + self.assertRaises(psutil.TimeoutExpired, p2.wait, 0.01) + # We also terminate the direct child otherwise the + # grandchild will hang until the parent is gone. + p1.terminate() + p2.terminate() + ret1 = p1.wait() + ret2 = p2.wait() + if POSIX: + self.assertEqual(ret1, -signal.SIGTERM) + # For processes which are not our children we're supposed + # to get None. + self.assertEqual(ret2, None) + else: + self.assertEqual(ret1, 0) + self.assertEqual(ret1, 0) def test_wait_timeout_0(self): sproc = get_test_subprocess() @@ -1633,18 +1633,18 @@ def test_environ(self): def test_weird_environ(self): # environment variables can contain values without an equals sign code = textwrap.dedent(""" - #include - #include - char * const argv[] = {"cat", 0}; - char * const envp[] = {"A=1", "X", "C=3", 0}; - int main(void) { - /* Close stderr on exec so parent can wait for the execve to - * finish. */ - if (fcntl(2, F_SETFD, FD_CLOEXEC) != 0) - return 0; - return execve("/bin/cat", argv, envp); - } - """) + #include + #include + char * const argv[] = {"cat", 0}; + char * const envp[] = {"A=1", "X", "C=3", 0}; + int main(void) { + /* Close stderr on exec so parent can wait for the execve to + * finish. */ + if (fcntl(2, F_SETFD, FD_CLOEXEC) != 0) + return 0; + return execve("/bin/cat", argv, envp); + } + """) path = TESTFN create_exe(path, c_code=code) self.addCleanup(safe_rmpath, path) From 77268aef6ae11408e41cd51bea2be19aa551f625 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 22:21:38 +0200 Subject: [PATCH 308/922] minor refactoring --- psutil/_pslinux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index a84752d82..97ee7304c 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1279,7 +1279,7 @@ def users(): # to use them in the future. if not user_process: continue - if hostname == ':0.0' or hostname == ':0': + if hostname in (':0.0', ':0'): hostname = 'localhost' nt = _common.suser(user, tty or None, hostname, tstamp) retlist.append(nt) From a9e3effa73a31c6ca77356964f0563f8531e4467 Mon Sep 17 00:00:00 2001 From: Alexander Hasselhuhn Date: Tue, 25 Apr 2017 22:41:45 +0200 Subject: [PATCH 309/922] make users() include pid into suser tuple --- psutil/_common.py | 2 +- psutil/_psbsd.py | 4 ++-- psutil/_pslinux.py | 4 ++-- psutil/_psosx.py | 4 ++-- psutil/_pssunos.py | 4 ++-- psutil/_psutil_bsd.c | 5 +++-- psutil/_psutil_linux.c | 3 ++- psutil/_psutil_osx.c | 5 +++-- psutil/_psutil_sunos.c | 3 ++- psutil/_pswindows.py | 2 +- 10 files changed, 20 insertions(+), 16 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 2497226af..54cb1ff5d 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -156,7 +156,7 @@ class BatteryTime(enum.IntEnum): 'errin', 'errout', 'dropin', 'dropout']) # psutil.users() -suser = namedtuple('suser', ['name', 'terminal', 'host', 'started']) +suser = namedtuple('suser', ['name', 'terminal', 'host', 'started', 'pid']) # psutil.net_connections() sconn = namedtuple('sconn', ['fd', 'family', 'type', 'laddr', 'raddr', 'status', 'pid']) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index fc5e1dc8b..86c0bdd57 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -434,10 +434,10 @@ def users(): retlist = [] rawlist = cext.users() for item in rawlist: - user, tty, hostname, tstamp = item + user, tty, hostname, tstamp, pid = item if tty == '~': continue # reboot or shutdown - nt = _common.suser(user, tty or None, hostname, tstamp) + nt = _common.suser(user, tty or None, hostname, tstamp, pid) retlist.append(nt) return retlist diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 97ee7304c..445a20a65 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1273,7 +1273,7 @@ def users(): retlist = [] rawlist = cext.users() for item in rawlist: - user, tty, hostname, tstamp, user_process = item + user, tty, hostname, tstamp, pid, user_process = item # note: the underlying C function includes entries about # system boot, run level and others. We might want # to use them in the future. @@ -1281,7 +1281,7 @@ def users(): continue if hostname in (':0.0', ':0'): hostname = 'localhost' - nt = _common.suser(user, tty or None, hostname, tstamp) + nt = _common.suser(user, tty or None, hostname, tstamp, pid) retlist.append(nt) return retlist diff --git a/psutil/_psosx.py b/psutil/_psosx.py index f780d4594..f5851b81a 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -264,12 +264,12 @@ def users(): retlist = [] rawlist = cext.users() for item in rawlist: - user, tty, hostname, tstamp = item + user, tty, hostname, tstamp, pid = item if tty == '~': continue # reboot or shutdown if not tstamp: continue - nt = _common.suser(user, tty or None, hostname or None, tstamp) + nt = _common.suser(user, tty or None, hostname or None, tstamp, pid) retlist.append(nt) return retlist diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index ad72de259..b3c6bb397 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -291,7 +291,7 @@ def users(): rawlist = cext.users() localhost = (':0.0', ':0') for item in rawlist: - user, tty, hostname, tstamp, user_process = item + user, tty, hostname, tstamp, pid, user_process = item # note: the underlying C function includes entries about # system boot, run level and others. We might want # to use them in the future. @@ -299,7 +299,7 @@ def users(): continue if hostname in localhost: hostname = 'localhost' - nt = _common.suser(user, tty, hostname, tstamp) + nt = _common.suser(user, tty, hostname, tstamp, pid) retlist.append(nt) return retlist diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index de748dccb..d811078ec 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -799,11 +799,12 @@ psutil_users(PyObject *self, PyObject *args) { if (*ut.ut_name == '\0') continue; py_tuple = Py_BuildValue( - "(sssf)", + "(sssfi)", ut.ut_name, // username ut.ut_line, // tty ut.ut_host, // hostname - (float)ut.ut_time); // start time + (float)ut.ut_time, // start time + ut.ut_pid); // process id if (!py_tuple) { fclose(fp); goto error; diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 0296dd544..a4fe940bd 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -450,11 +450,12 @@ psutil_users(PyObject *self, PyObject *args) { else py_user_proc = Py_False; py_tuple = Py_BuildValue( - "(sssfO)", + "(sssfiO)", ut->ut_user, // username ut->ut_line, // tty ut->ut_host, // hostname (float)ut->ut_tv.tv_sec, // tstamp + ut->ut_pid, // process id py_user_proc // (bool) user process ); if (! py_tuple) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index fb26dc9b4..1e7a5ac58 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1697,11 +1697,12 @@ psutil_users(PyObject *self, PyObject *args) { if (utx->ut_type != USER_PROCESS) continue; py_tuple = Py_BuildValue( - "(sssf)", + "(sssfi)", utx->ut_user, // username utx->ut_line, // tty utx->ut_host, // hostname - (float)utx->ut_tv.tv_sec // start time + (float)utx->ut_tv.tv_sec, // start time + utx->ut_pid // process id ); if (!py_tuple) { endutxent(); diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 68b0a89ea..d41ebe973 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -461,11 +461,12 @@ psutil_users(PyObject *self, PyObject *args) { else py_user_proc = Py_False; py_tuple = Py_BuildValue( - "(sssfO)", + "(sssfiO)", ut->ut_user, // username ut->ut_line, // tty ut->ut_host, // hostname (float)ut->ut_tv.tv_sec, // tstamp + ut->ut_pid, // process id py_user_proc); // (bool) user process if (py_tuple == NULL) goto error; diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 0105d6c8b..a5525df28 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -411,7 +411,7 @@ def users(): for item in rawlist: user, hostname, tstamp = item user = py2_strencode(user) - nt = _common.suser(user, None, hostname, tstamp) + nt = _common.suser(user, None, hostname, tstamp, None) retlist.append(nt) return retlist From b353b5fa58cfd996258f45691044e8fb95cd0444 Mon Sep 17 00:00:00 2001 From: Alexander Hasselhuhn Date: Tue, 25 Apr 2017 23:05:21 +0200 Subject: [PATCH 310/922] in psutil_users() move pid to the back --- psutil/_pslinux.py | 2 +- psutil/_pssunos.py | 2 +- psutil/_psutil_bsd.c | 3 ++- psutil/_psutil_linux.c | 4 ++-- psutil/_psutil_sunos.c | 5 +++-- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 445a20a65..c0ccb4669 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1273,7 +1273,7 @@ def users(): retlist = [] rawlist = cext.users() for item in rawlist: - user, tty, hostname, tstamp, pid, user_process = item + user, tty, hostname, tstamp, user_process, pid = item # note: the underlying C function includes entries about # system boot, run level and others. We might want # to use them in the future. diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index b3c6bb397..6782f7f3d 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -291,7 +291,7 @@ def users(): rawlist = cext.users() localhost = (':0.0', ':0') for item in rawlist: - user, tty, hostname, tstamp, pid, user_process = item + user, tty, hostname, tstamp, user_process, pid = item # note: the underlying C function includes entries about # system boot, run level and others. We might want # to use them in the future. diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index d811078ec..9c8c6a292 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -804,7 +804,8 @@ psutil_users(PyObject *self, PyObject *args) { ut.ut_line, // tty ut.ut_host, // hostname (float)ut.ut_time, // start time - ut.ut_pid); // process id + ut.ut_pid // process id + ); if (!py_tuple) { fclose(fp); goto error; diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index a4fe940bd..5ab69d2b2 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -455,8 +455,8 @@ psutil_users(PyObject *self, PyObject *args) { ut->ut_line, // tty ut->ut_host, // hostname (float)ut->ut_tv.tv_sec, // tstamp - ut->ut_pid, // process id - py_user_proc // (bool) user process + py_user_proc, // (bool) user process + ut->ut_pid // process id ); if (! py_tuple) goto error; diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index d41ebe973..6de0bc921 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -466,8 +466,9 @@ psutil_users(PyObject *self, PyObject *args) { ut->ut_line, // tty ut->ut_host, // hostname (float)ut->ut_tv.tv_sec, // tstamp - ut->ut_pid, // process id - py_user_proc); // (bool) user process + py_user_proc, // (bool) user process + ut->ut_pid // process id + ); if (py_tuple == NULL) goto error; if (PyList_Append(py_retlist, py_tuple)) From 427457a3763c2a631890703e40e2be0192c66c1e Mon Sep 17 00:00:00 2001 From: Alexander Hasselhuhn Date: Tue, 25 Apr 2017 23:17:24 +0200 Subject: [PATCH 311/922] update CREDITS and HISTORY --- CREDITS | 5 +++++ HISTORY.rst | 2 ++ 2 files changed, 7 insertions(+) diff --git a/CREDITS b/CREDITS index 8c6f4cde6..7f9da69d6 100644 --- a/CREDITS +++ b/CREDITS @@ -441,3 +441,8 @@ I: 872 N: Danek Duvall W: https://github.com/dhduvall I: 1002 + +N: Alexander Hasselhuhn +C: Germany +W: https://github.com/alexanha + diff --git a/HISTORY.rst b/HISTORY.rst index 453ea009d..60339a7e1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,8 @@ **Enhancements** +- 1022_: [Linux, BSD, OSX, SunOS] Provide the process id in the output of + users(), which is provided by utmp. - 1015_: swap_memory() now relies on /proc/meminfo instead of sysinfo() syscall so that it can be used in conjunction with PROCFS_PATH in order to retrieve memory info about Linux containers such as Docker and Heroku. From 3272d71d96f8275d1ecfd24eab9745ec73817788 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 25 Apr 2017 23:50:42 +0200 Subject: [PATCH 312/922] #1022: fix users() on Linux; update doc; bump up version --- HISTORY.rst | 5 ++--- README.rst | 4 ++-- docs/index.rst | 9 +++++++-- psutil/__init__.py | 2 +- psutil/_psutil_linux.c | 2 +- psutil/_psutil_sunos.c | 2 +- psutil/tests/test_posix.py | 6 +++--- psutil/tests/test_system.py | 4 ++++ scripts/who.py | 15 +++++++++------ 9 files changed, 30 insertions(+), 19 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 60339a7e1..412e75451 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,16 +2,15 @@ *XXXX-XX-XX* -5.2.3 +5.3.0 ===== **Enhancements** -- 1022_: [Linux, BSD, OSX, SunOS] Provide the process id in the output of - users(), which is provided by utmp. - 1015_: swap_memory() now relies on /proc/meminfo instead of sysinfo() syscall so that it can be used in conjunction with PROCFS_PATH in order to retrieve memory info about Linux containers such as Docker and Heroku. +- 1022_: psutil.users() provides a new "pid" field. **Bug fixes** diff --git a/README.rst b/README.rst index a8d62bd2f..2a9ed6ca9 100644 --- a/README.rst +++ b/README.rst @@ -229,8 +229,8 @@ Other system info >>> import psutil >>> psutil.users() - [user(name='giampaolo', terminal='pts/2', host='localhost', started=1340737536.0), - user(name='giampaolo', terminal='pts/3', host='localhost', started=1340737792.0)] + [suser(name='giampaolo', terminal='pts/2', host='localhost', started=1340737536.0, pid=1352), + suser(name='giampaolo', terminal='pts/3', host='localhost', started=1340737792.0, pid=1788)] >>> >>> psutil.boot_time() 1365519115.0 diff --git a/docs/index.rst b/docs/index.rst index aa68012fe..faf1edd06 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -753,13 +753,18 @@ Other system info - **host**: the host name associated with the entry, if any. - **started**: the creation time as a floating point number expressed in seconds since the epoch. + - **pid**: the PID of the login process (like sshd, tmux, gdm-session-worker, + ...). On Windows this is always set to ``None``. Example:: >>> import psutil >>> psutil.users() - [suser(name='giampaolo', terminal='pts/2', host='localhost', started=1340737536.0), - suser(name='giampaolo', terminal='pts/3', host='localhost', started=1340737792.0)] + [suser(name='giampaolo', terminal='pts/2', host='localhost', started=1340737536.0, pid=1352), + suser(name='giampaolo', terminal='pts/3', host='localhost', started=1340737792.0, pid=1788)] + + .. versionchanged:: + 5.3.0 added "pid" field Processes ========= diff --git a/psutil/__init__.py b/psutil/__init__.py index dc2c063c3..513bfa445 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -192,7 +192,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.2.3" +__version__ = "5.3.0" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 5ab69d2b2..a7eb789bc 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -450,7 +450,7 @@ psutil_users(PyObject *self, PyObject *args) { else py_user_proc = Py_False; py_tuple = Py_BuildValue( - "(sssfiO)", + "(sssfOi)", ut->ut_user, // username ut->ut_line, // tty ut->ut_host, // hostname diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 6de0bc921..e26c92d0f 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -461,7 +461,7 @@ psutil_users(PyObject *self, PyObject *args) { else py_user_proc = Py_False; py_tuple = Py_BuildValue( - "(sssfiO)", + "(sssfOi)", ut->ut_user, // username ut->ut_line, // tty ut->ut_host, // hostname diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index c9b176a1f..f72fb20a9 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -313,11 +313,11 @@ def test_users(self): out = sh("who") lines = out.split('\n') users = [x.split()[0] for x in lines] - self.assertEqual(len(users), len(psutil.users())) terminals = [x.split()[1] for x in lines] + self.assertEqual(len(users), len(psutil.users())) for u in psutil.users(): - self.assertTrue(u.name in users, u.name) - self.assertTrue(u.terminal in terminals, u.terminal) + self.assertIn(u.name, users) + self.assertIn(u.terminal, terminals) def test_pid_exists_let_raise(self): # According to "man 2 kill" possible error values for kill diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 1b838fa87..81d47ab96 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -690,6 +690,10 @@ def test_users(self): user.host assert user.started > 0.0, user datetime.datetime.fromtimestamp(user.started) + if POSIX: + psutil.Process(user.pid) + else: + self.assertIsNone(user.pid) def test_cpu_stats(self): # Tested more extensively in per-platform test modules. diff --git a/scripts/who.py b/scripts/who.py index 046ec23f0..dbaa97274 100755 --- a/scripts/who.py +++ b/scripts/who.py @@ -9,10 +9,10 @@ currently logged in. $ python scripts/who.py -giampaolo tty7 2014-02-23 17:25 (:0) -giampaolo pts/7 2014-02-24 18:25 (:192.168.1.56) -giampaolo pts/8 2014-02-24 18:25 (:0) -giampaolo pts/9 2014-02-27 01:32 (:0) +giampaolo tty7 2014-02-23 17:25 (:0) upstart +giampaolo pts/7 2014-02-24 18:25 (:192.168.1.56) sshd +giampaolo pts/8 2014-02-24 18:25 (:0) upstart +giampaolo pts/9 2014-02-27 01:32 (:0) upstart """ from datetime import datetime @@ -23,11 +23,14 @@ def main(): users = psutil.users() for user in users: - print("%-15s %-15s %s (%s)" % ( + proc_name = psutil.Process(user.pid).name() if user.pid else "" + print("%-12s %-10s %s (%s) %10s" % ( user.name, user.terminal or '-', datetime.fromtimestamp(user.started).strftime("%Y-%m-%d %H:%M"), - user.host)) + user.host, + proc_name + )) if __name__ == '__main__': From a2f947fda49c42420967395246b1d334bcbe5b96 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 26 Apr 2017 00:00:21 +0200 Subject: [PATCH 313/922] adjust who.py script formatting --- scripts/who.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/who.py b/scripts/who.py index dbaa97274..748d936c9 100755 --- a/scripts/who.py +++ b/scripts/who.py @@ -9,10 +9,8 @@ currently logged in. $ python scripts/who.py -giampaolo tty7 2014-02-23 17:25 (:0) upstart -giampaolo pts/7 2014-02-24 18:25 (:192.168.1.56) sshd -giampaolo pts/8 2014-02-24 18:25 (:0) upstart -giampaolo pts/9 2014-02-27 01:32 (:0) upstart +giampaolo console 2017-03-25 22:24 loginwindow +giampaolo ttys000 2017-03-25 23:28 (10.0.2.2) sshd """ from datetime import datetime @@ -24,11 +22,11 @@ def main(): users = psutil.users() for user in users: proc_name = psutil.Process(user.pid).name() if user.pid else "" - print("%-12s %-10s %s (%s) %10s" % ( + print("%-12s %-10s %-10s %-14s %s" % ( user.name, user.terminal or '-', datetime.fromtimestamp(user.started).strftime("%Y-%m-%d %H:%M"), - user.host, + "(%s)" % user.host if user.host else "", proc_name )) From 28d2add3a4e490ff0014a9fcf4e6aa134f043d43 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 26 Apr 2017 00:09:28 +0200 Subject: [PATCH 314/922] fix linux test --- psutil/tests/test_linux.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index a164d69a7..afa7a173d 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1069,18 +1069,18 @@ def test_users_mocked(self): # to 'localhost'. with mock.patch('psutil._pslinux.cext.users', return_value=[('giampaolo', 'pts/2', ':0', - 1436573184.0, True)]) as m: + 1436573184.0, True, 2)]) as m: self.assertEqual(psutil.users()[0].host, 'localhost') assert m.called with mock.patch('psutil._pslinux.cext.users', return_value=[('giampaolo', 'pts/2', ':0.0', - 1436573184.0, True)]) as m: + 1436573184.0, True, 2)]) as m: self.assertEqual(psutil.users()[0].host, 'localhost') assert m.called # ...otherwise it should be returned as-is with mock.patch('psutil._pslinux.cext.users', return_value=[('giampaolo', 'pts/2', 'foo', - 1436573184.0, True)]) as m: + 1436573184.0, True, 2)]) as m: self.assertEqual(psutil.users()[0].host, 'foo') assert m.called From 936a9194783f79e796519a80f5b088c01a767df4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 26 Apr 2017 19:23:23 +0200 Subject: [PATCH 315/922] update INSTALL instructions --- INSTALL.rst | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index d86c022ce..9999eccf7 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -1,19 +1,21 @@ -PIP -=== +Install pip +=========== pip is the easiest way to install psutil. -It is shipped by default with Python 2.7.9+ and 3.4+. If you're using an -older Python version `install pip `__ -first. -If you GIT cloned psutil source code you can also install pip and/or upgrade -it to latest version with:: - - make install-pip - -Unless you're on Windows, in order to install psutil with pip you'll also need -a C compiler installed (e.g. gcc). -pip will retrieve psutil source code or binaries from -`PYPI `__ repository. +It is shipped by default with Python 2.7.9+ and 3.4+. For other Python versions +you can install it manually. +On Linux or via wget:: + + wget https://bootstrap.pypa.io/get-pip.py -O - | python + +On OSX or via curl:: + + python < <(curl -s https://bootstrap.pypa.io/get-pip.py) + +On Windows, `download pip `, open +cmd.exe and install it:: + + C:\Python27\python.exe get-pip.py Permission issues (UNIX) ======================== From 3850cb84ce5cf7515487e6287895d2cc47f16ad6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 26 Apr 2017 19:24:23 +0200 Subject: [PATCH 316/922] update INSTALL instructions --- INSTALL.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.rst b/INSTALL.rst index 9999eccf7..8737c94a1 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -12,7 +12,7 @@ On OSX or via curl:: python < <(curl -s https://bootstrap.pypa.io/get-pip.py) -On Windows, `download pip `, open +On Windows, `download pip `__, open cmd.exe and install it:: C:\Python27\python.exe get-pip.py From 1c4019f1298eca752c1e6b85ca4c8c461ff488ea Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 26 Apr 2017 20:49:58 +0200 Subject: [PATCH 317/922] #1025: implement process_iter(attrs, ad_value) --- HISTORY.rst | 4 ++++ README.rst | 10 ++++----- docs/index.rst | 55 +++++++++++++++++++++++++++++++++++++--------- psutil/__init__.py | 14 +++++++++++- 4 files changed, 67 insertions(+), 16 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 412e75451..309397ef1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,10 @@ so that it can be used in conjunction with PROCFS_PATH in order to retrieve memory info about Linux containers such as Docker and Heroku. - 1022_: psutil.users() provides a new "pid" field. +- 1025_: process_iter() accepts two new parameters in order to invoke + Process.as_dict(): "attrs" and "ad_value". With this you can iterate over all + processes in one shot without needing to catch NoSuchProcess and do list/dict + comprehensions. **Bug fixes** diff --git a/README.rst b/README.rst index 2a9ed6ca9..41fe28f8e 100644 --- a/README.rst +++ b/README.rst @@ -377,12 +377,12 @@ Further process APIs .. code-block:: python >>> import psutil - >>> for p in psutil.process_iter(): - ... print(p) + >>> for proc in psutil.process_iter(attrs=['pid', 'name']): + ... print(proc.info) ... - psutil.Process(pid=1, name='init') - psutil.Process(pid=2, name='kthreadd') - psutil.Process(pid=3, name='ksoftirqd/0') + {'pid': 1, 'name': 'systemd'} + {'pid': 2, 'name': 'kthreadd'} + {'pid': 3, 'name': 'ksoftirqd/0'} ... >>> >>> psutil.pid_exists(3) diff --git a/docs/index.rst b/docs/index.rst index faf1edd06..359bf3a58 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -786,7 +786,7 @@ Functions Check whether the given PID exists in the current process list. This is faster than doing ``pid in psutil.pids()`` and should be preferred. -.. function:: process_iter() +.. function:: process_iter(attrs=None, ad_value=None) Return an iterator yielding a :class:`Process` class instance for all running processes on the local machine. @@ -798,17 +798,52 @@ Functions This is should be preferred over :func:`psutil.pids()` for iterating over processes. Sorting order in which processes are returned is - based on their PID. Example usage:: + based on their PID. + *attrs* and *ad_value* have the same meaning as in :meth:`Process.as_dict()`. + If *attrs* is specified :meth:`Process.as_dict()` is called and the resulting + dict is stored as a ``info`` attribute which is attached to the returned + :class:`Process` instance. + If *attrs* is an empty list it will retrieve all process info (slow). + Example usage:: - import psutil + >>> import psutil + >>> for proc in psutil.process_iter(): + ... try: + ... pinfo = proc.as_dict(attrs=['pid', 'name', 'username']) + ... except psutil.NoSuchProcess: + ... pass + ... else: + ... print(pinfo) + ... + {'name': 'systemd', 'pid': 1, 'username': 'root'} + {'name': 'kthreadd', 'pid': 2, 'username': 'root'} + {'name': 'ksoftirqd/0', 'pid': 3, 'username': 'root'} + ... + + More compact version using *attrs* parameter:: + + >>> import psutil + >>> for proc in psutil.process_iter(attrs=['pid', 'name', 'username']): + ... print(proc.info) + ... + {'username': 'root', 'pid': 1, 'name': 'systemd'} + {'username': 'root', 'pid': 2, 'name': 'kthreadd'} + {'username': 'root', 'pid': 3, 'name': 'ksoftirqd/0'} + ...} + + Example of a dict comprehensions to create a ``{pid: info, ...}`` data + structure: - for proc in psutil.process_iter(): - try: - pinfo = proc.as_dict(attrs=['pid', 'name']) - except psutil.NoSuchProcess: - pass - else: - print(pinfo) + >>> import psutil + >>> info = dict([(p.pid, p.info) for p in psutil.process_iter(attrs=['pid', 'name', 'username'])]) + >>> info + {1: {'name': 'systemd', 'pid': 1, 'username': 'root'}, + 2: {'name': 'kthreadd', 'pid': 2, 'username': 'root'}, + 3: {'name': 'ksoftirqd/0', 'pid': 3, 'username': 'root'}, + ...} + + .. versionchanged:: + 5.3.0 added "attrs" and "ad_value" parameters. .. function:: wait_procs(procs, timeout=None, callback=None) diff --git a/psutil/__init__.py b/psutil/__init__.py index 513bfa445..6cad1d1d0 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1451,7 +1451,7 @@ def pid_exists(pid): _pmap = {} -def process_iter(): +def process_iter(attrs=None, ad_value=None): """Return a generator yielding a Process instance for all running processes. @@ -1464,9 +1464,18 @@ def process_iter(): The sorting order in which processes are yielded is based on their PIDs. + + "attrs" and "ad_value" have the same meaning as in + Process.as_dict(). If "attrs" is specified as_dict() is called + and the resulting dict is stored as a 'info' attribute attached + to returned Process instance. + If "attrs" is an empty list it will retrieve all process info + (slow). """ def add(pid): proc = Process(pid) + if attrs is not None: + proc.info = proc.as_dict(attrs=attrs, ad_value=ad_value) _pmap[proc.pid] = proc return proc @@ -1489,6 +1498,9 @@ def remove(pid): # use is_running() to check whether PID has been reused by # another process in which case yield a new Process instance if proc.is_running(): + if attrs is not None: + proc.info = proc.as_dict( + attrs=attrs, ad_value=ad_value) yield proc else: yield add(pid) From bde13c410bd0e266d49ad1e24ae3b78dba571e29 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 26 Apr 2017 21:00:44 +0200 Subject: [PATCH 318/922] #1025: add process_iter(attrs, ad_value) test case --- psutil/tests/test_system.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 81d47ab96..3360877b1 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -78,6 +78,25 @@ def test_process_iter(self): with self.assertRaises(psutil.AccessDenied): list(psutil.process_iter()) + def test_prcess_iter_w_params(self): + for p in psutil.process_iter(attrs=['pid']): + self.assertEqual(p.info.keys(), ['pid']) + with self.assertRaises(ValueError): + list(psutil.process_iter(attrs=['foo'])) + with mock.patch("psutil._psplatform.Process.name", + side_effect=psutil.AccessDenied(0, "")) as m: + for p in psutil.process_iter(attrs=["pid", "name"]): + self.assertIsNone(p.info['name']) + self.assertGreaterEqual(p.info['pid'], 0) + assert m.called + with mock.patch("psutil._psplatform.Process.name", + side_effect=psutil.AccessDenied(0, "")) as m: + flag = object() + for p in psutil.process_iter(attrs=["pid", "name"], ad_value=flag): + self.assertIs(p.info['name'], flag) + self.assertGreaterEqual(p.info['pid'], 0) + assert m.called + def test_wait_procs(self): def callback(p): pids.append(p.pid) From 888eebfd4fecfaf936838a2055496485e2af8b65 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 26 Apr 2017 21:30:59 +0200 Subject: [PATCH 319/922] #1025: take advantage of process_iter(attrs) in scripts --- psutil/__init__.py | 69 +++++++++++++++++-------------------- scripts/cpu_distribution.py | 10 ++---- scripts/netstat.py | 7 ++-- scripts/pidof.py | 25 +++----------- 4 files changed, 41 insertions(+), 70 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 6cad1d1d0..d39f54f1d 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2325,44 +2325,39 @@ def test(): # pragma: no cover attrs.append('terminal') print(templ % ("USER", "PID", "%MEM", "VSZ", "RSS", "TTY", "START", "TIME", "COMMAND")) - for p in process_iter(): - try: - pinfo = p.as_dict(attrs, ad_value='') - except NoSuchProcess: - pass - else: - if pinfo['create_time']: - ctime = datetime.datetime.fromtimestamp(pinfo['create_time']) - if ctime.date() == today_day: - ctime = ctime.strftime("%H:%M") - else: - ctime = ctime.strftime("%b%d") + for p in process_iter(attrs=attrs, ad_value=''): + if p.info['create_time']: + ctime = datetime.datetime.fromtimestamp(p.info['create_time']) + if ctime.date() == today_day: + ctime = ctime.strftime("%H:%M") else: - ctime = '' - cputime = time.strftime("%M:%S", - time.localtime(sum(pinfo['cpu_times']))) - try: - user = p.username() - except Error: - user = '' - if WINDOWS and '\\' in user: - user = user.split('\\')[1] - vms = pinfo['memory_info'] and \ - int(pinfo['memory_info'].vms / 1024) or '?' - rss = pinfo['memory_info'] and \ - int(pinfo['memory_info'].rss / 1024) or '?' - memp = pinfo['memory_percent'] and \ - round(pinfo['memory_percent'], 1) or '?' - print(templ % ( - user[:10], - pinfo['pid'], - memp, - vms, - rss, - pinfo.get('terminal', '') or '?', - ctime, - cputime, - pinfo['name'].strip() or '?')) + ctime = ctime.strftime("%b%d") + else: + ctime = '' + cputime = time.strftime("%M:%S", + time.localtime(sum(p.info['cpu_times']))) + try: + user = p.username() + except Error: + user = '' + if WINDOWS and '\\' in user: + user = user.split('\\')[1] + vms = p.info['memory_info'] and \ + int(p.info['memory_info'].vms / 1024) or '?' + rss = p.info['memory_info'] and \ + int(p.info['memory_info'].rss / 1024) or '?' + memp = p.info['memory_percent'] and \ + round(p.info['memory_percent'], 1) or '?' + print(templ % ( + user[:10], + p.info['pid'], + memp, + vms, + rss, + p.info.get('terminal', '') or '?', + ctime, + cputime, + p.info['name'].strip() or '?')) del memoize, memoize_when_activated, division, deprecated_method diff --git a/scripts/cpu_distribution.py b/scripts/cpu_distribution.py index 31cdbb863..a9f76b4e3 100755 --- a/scripts/cpu_distribution.py +++ b/scripts/cpu_distribution.py @@ -74,14 +74,8 @@ def main(): # processes procs = collections.defaultdict(list) - for p in psutil.process_iter(): - try: - name = p.name()[:5] - cpunum = p.cpu_num() - except psutil.Error: - continue - else: - procs[cpunum].append(name) + for p in psutil.process_iter(attrs=['name', 'cpu_num']): + procs[p.info['cpu_num']].append(p.info['name'][:5]) end_marker = [[] for x in range(total)] while True: diff --git a/scripts/netstat.py b/scripts/netstat.py index 1426cd76d..490b429f2 100755 --- a/scripts/netstat.py +++ b/scripts/netstat.py @@ -41,11 +41,8 @@ def main(): "Proto", "Local address", "Remote address", "Status", "PID", "Program name")) proc_names = {} - for p in psutil.process_iter(): - try: - proc_names[p.pid] = p.name() - except psutil.Error: - pass + for p in psutil.process_iter(attrs=['pid', 'name']): + proc_names[p.info['pid']] = p.info['name'] for c in psutil.net_connections(kind='inet'): laddr = "%s:%s" % (c.laddr) raddr = "" diff --git a/scripts/pidof.py b/scripts/pidof.py index 1c23900f4..bcb8a2e6d 100755 --- a/scripts/pidof.py +++ b/scripts/pidof.py @@ -18,26 +18,11 @@ def pidof(pgname): pids = [] - for proc in psutil.process_iter(): - with proc.oneshot(): - # search for matches in the process name and cmdline - try: - name = proc.name() - except psutil.Error: - pass - else: - if name == pgname: - pids.append(str(proc.pid)) - continue - - try: - cmdline = proc.cmdline() - except psutil.Error: - pass - else: - if cmdline and cmdline[0] == pgname: - pids.append(str(proc.pid)) - + for proc in psutil.process_iter(attrs=['name', 'cmdline']): + # search for matches in the process name and cmdline + if proc.info['name'] == pgname or \ + proc.info['cmdline'] and proc.info['cmdline'][0] == pgname: + pids.append(str(proc.pid)) return pids From 45fcea42c4236055d5eefc346b14b08c09871062 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 26 Apr 2017 21:49:13 +0200 Subject: [PATCH 320/922] fix test; update doc --- docs/index.rst | 12 ++++++------ psutil/tests/test_system.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 359bf3a58..c434a9004 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -832,14 +832,14 @@ Functions ...} Example of a dict comprehensions to create a ``{pid: info, ...}`` data - structure: + structure as a one-liner: >>> import psutil - >>> info = dict([(p.pid, p.info) for p in psutil.process_iter(attrs=['pid', 'name', 'username'])]) - >>> info - {1: {'name': 'systemd', 'pid': 1, 'username': 'root'}, - 2: {'name': 'kthreadd', 'pid': 2, 'username': 'root'}, - 3: {'name': 'ksoftirqd/0', 'pid': 3, 'username': 'root'}, + >>> procs = dict([(p.pid, p.info) for p in psutil.process_iter(attrs=['name', 'username'])]) + >>> procs + {1: {'name': 'systemd', 'username': 'root'}, + 2: {'name': 'kthreadd', 'username': 'root'}, + 3: {'name': 'ksoftirqd/0', 'username': 'root'}, ...} .. versionchanged:: diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 3360877b1..92b48693c 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -80,7 +80,7 @@ def test_process_iter(self): def test_prcess_iter_w_params(self): for p in psutil.process_iter(attrs=['pid']): - self.assertEqual(p.info.keys(), ['pid']) + self.assertEqual(list(p.info.keys()), ['pid']) with self.assertRaises(ValueError): list(psutil.process_iter(attrs=['foo'])) with mock.patch("psutil._psplatform.Process.name", From 9e34798ece5f231cafdd0da215c8b568b4f82d09 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 01:01:31 +0200 Subject: [PATCH 321/922] update doc --- docs/index.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index c434a9004..12ea4b7b0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -829,18 +829,18 @@ Functions {'username': 'root', 'pid': 1, 'name': 'systemd'} {'username': 'root', 'pid': 2, 'name': 'kthreadd'} {'username': 'root', 'pid': 3, 'name': 'ksoftirqd/0'} - ...} + ... Example of a dict comprehensions to create a ``{pid: info, ...}`` data structure as a one-liner: - >>> import psutil - >>> procs = dict([(p.pid, p.info) for p in psutil.process_iter(attrs=['name', 'username'])]) - >>> procs - {1: {'name': 'systemd', 'username': 'root'}, - 2: {'name': 'kthreadd', 'username': 'root'}, - 3: {'name': 'ksoftirqd/0', 'username': 'root'}, - ...} + >>> import psutil + >>> procs = dict([(p.pid, p.info) for p in psutil.process_iter(attrs=['name', 'username'])]) + >>> procs + {1: {'name': 'systemd', 'username': 'root'}, + 2: {'name': 'kthreadd', 'username': 'root'}, + 3: {'name': 'ksoftirqd/0', 'username': 'root'}, + ...} .. versionchanged:: 5.3.0 added "attrs" and "ad_value" parameters. From d214ea04e72d7132c9598e89a2a1dd84a3cd3ee3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 01:19:35 +0200 Subject: [PATCH 322/922] update doc --- docs/index.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 12ea4b7b0..db65d3d52 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -826,9 +826,9 @@ Functions >>> for proc in psutil.process_iter(attrs=['pid', 'name', 'username']): ... print(proc.info) ... - {'username': 'root', 'pid': 1, 'name': 'systemd'} - {'username': 'root', 'pid': 2, 'name': 'kthreadd'} - {'username': 'root', 'pid': 3, 'name': 'ksoftirqd/0'} + {'name': 'systemd', 'pid': 1, 'username': 'root'} + {'name': 'kthreadd', 'pid': 2, 'username': 'root'} + {'name': 'ksoftirqd/0', 'pid': 3, 'username': 'root'} ... Example of a dict comprehensions to create a ``{pid: info, ...}`` data @@ -842,6 +842,12 @@ Functions 3: {'name': 'ksoftirqd/0', 'username': 'root'}, ...} + Example showing how to filter processes by name:: + + >>> [p.info for p in psutil.process_iter(attrs=['pid', 'name']) if 'python' in p.info['name']] + [{'name': 'python3', 'pid': 21947}, + {'name': 'python', 'pid': 23835}] + .. versionchanged:: 5.3.0 added "attrs" and "ad_value" parameters. From 87a112858ee35e77ea4fe91f4b1dd945b0503754 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 03:12:18 +0200 Subject: [PATCH 323/922] #1026 / doc: add recipes section --- README.rst | 12 ++++-- docs/index.rst | 112 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 109 insertions(+), 15 deletions(-) diff --git a/README.rst b/README.rst index 41fe28f8e..9f849d6ec 100644 --- a/README.rst +++ b/README.rst @@ -66,7 +66,8 @@ Example applications | :target: https://github.com/giampaolo/psutil/blob/master/docs/_static/procsmem.png | :target: https://github.com/giampaolo/psutil/blob/master/docs/_static/pmap.png | +------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------+ -Also see https://github.com/giampaolo/psutil/tree/master/scripts. +Also see https://github.com/giampaolo/psutil/tree/master/scripts and +`doc recipes `__. ===================== Projects using psutil @@ -434,16 +435,21 @@ Windows services 'status': 'stopped', 'username': 'NT AUTHORITY\\LocalService'} +Other samples +============= + +See `doc recipes `__. + ====== Author ====== psutil was created and is maintained by -`Giampaolo Rodola' `_. +`Giampaolo Rodola' `__. A lot of time and effort went into making psutil as it is right now. If you feel psutil is useful to you or your business and want to support its future development please consider donating me -(`Giampaolo `_) some money. +(`Giampaolo `__) some money. .. image:: http://www.paypal.com/en_US/i/btn/x-click-but04.gif :target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=A9ZS7PKKRM3S8 diff --git a/docs/index.rst b/docs/index.rst index db65d3d52..e005dfe33 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -781,11 +781,6 @@ Functions >>> psutil.pids() [1, 2, 3, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, ..., 32498] -.. function:: pid_exists(pid) - - Check whether the given PID exists in the current process list. This is - faster than doing ``pid in psutil.pids()`` and should be preferred. - .. function:: process_iter(attrs=None, ad_value=None) Return an iterator yielding a :class:`Process` class instance for all running @@ -832,7 +827,7 @@ Functions ... Example of a dict comprehensions to create a ``{pid: info, ...}`` data - structure as a one-liner: + structure:: >>> import psutil >>> procs = dict([(p.pid, p.info) for p in psutil.process_iter(attrs=['name', 'username'])]) @@ -844,6 +839,7 @@ Functions Example showing how to filter processes by name:: + >>> import psutil >>> [p.info for p in psutil.process_iter(attrs=['pid', 'name']) if 'python' in p.info['name']] [{'name': 'python3', 'pid': 21947}, {'name': 'python', 'pid': 23835}] @@ -851,16 +847,24 @@ Functions .. versionchanged:: 5.3.0 added "attrs" and "ad_value" parameters. +.. function:: pid_exists(pid) + + Check whether the given PID exists in the current process list. This is + faster than doing ``pid in psutil.pids()`` and should be preferred. + .. function:: wait_procs(procs, timeout=None, callback=None) Convenience function which waits for a list of :class:`Process` instances to terminate. Return a ``(gone, alive)`` tuple indicating which processes are gone and which ones are still alive. The *gone* ones will have a new - *returncode* attribute indicating process exit status (it may be ``None``). - ``callback`` is a function which gets called every time a process terminates - (a :class:`Process` instance is passed as callback argument). Function will - return as soon as all processes terminate or when timeout occurs. Typical use - case is: + *returncode* attribute indicating process exit status (will be ``None`` for + processes which are not our children). + ``callback`` is a function which gets called when one of the processes being + waited on is terminated and a :class:`Process` instance is passed as callback + argument). + This tunction will return as soon as all processes terminate or when + *timeout* occurs, if specified. + A typical use case may be: - send SIGTERM to a list of processes - give them some time to terminate @@ -876,7 +880,7 @@ Functions procs = psutil.Process().children() for p in procs: p.terminate() - gone, still_alive = psutil.wait_procs(procs, timeout=3, callback=on_terminate) + gone, alive = psutil.wait_procs(procs, timeout=3, callback=on_terminate) for p in still_alive: p.kill() @@ -2208,6 +2212,90 @@ Constants >>> if psutil.version_info >= (4, 5): ... pass +Recipes +======= + +Follows a collection of utilities and examples which are common but not generic +enough to be part of the public API. + +Find process by name +-------------------- + +Check string against process :meth:`Process.name()`: + +:: + + import psutil + + def find_procs_by_name(name): + "Return a list of processes matching 'name'." + ls = [] + for p in psutil.process_iter(attrs=['name']): + if p.info['name'] == name: + ls.append(p) + return ls + +A bit more advanced, check string against process :meth:`Process.name()`, +:meth:`Process.exe()` and :meth:`Process.cmdline()`: + +:: + + import os + import psutil + + def find_procs_by_name(name): + "Return a list of processes matching 'name'." + ls = [] + for p in psutil.process_iter(): + name_, exe, cmdline = "", "", [] + try: + name_ = p.name() + cmdline = p.cmdline() + exe = p.exe() + except psutil.NoSuchProcess: + continue + except psutil.AccessDenied: + pass + if name == name_ or cmdline[0] == name or os.path.basename(exe) == name: + ls.append(name) + return ls + +Terminate my children +--------------------- + +This may be useful in unit tests whenever sub-processes are started. +This will help ensure that no extra children (zombies) stick around to hog +resources. + +:: + + import psutil + + def reap_children(timeout=3): + "Tries hard to terminate and ultimately kill all the children of this process." + def on_terminate(proc): + print("process {} terminated with exit code {}".format(proc, proc.returncode)) + + procs = psutil.Process().children() + # send SIGTERM + for p in procs: + p.terminate() + gone, alive = psutil.wait_procs(procs, timeout=timeout, callback=on_terminate) + if not alive: + return + # send SIGKILL to the survivors + for p in alive: + print("process {} survived SIGTERM; trying SIGKILL" % p) + p.kill() + gone, alive = psutil.wait_procs(alive, timeout=timeout, callback=on_terminate) + if not alive: + return + # give up + for p in alive: + print("process {} survived SIGKILL; giving up" % p) + + reap_children() + Q&A === From 5325aaf8c994744c6a2f2d3a88553199fa4a0293 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 03:39:14 +0200 Subject: [PATCH 324/922] #1026 / doc: add kill_proc_tree() recipe --- docs/index.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index e005dfe33..1be4b3f1a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2245,6 +2245,7 @@ A bit more advanced, check string against process :meth:`Process.name()`, def find_procs_by_name(name): "Return a list of processes matching 'name'." + assert name, name ls = [] for p in psutil.process_iter(): name_, exe, cmdline = "", "", [] @@ -2296,6 +2297,33 @@ resources. reap_children() +Kill process tree +----------------- + +:: + + import psutil + import signal + import os + + def kill_proc_tree(pid, sig=signal.SIGTERM, recursive=True, include_parent=True, + timeout=None, on_terminate=None): + """Kill a process tree with signal "sig" and return a + (gone, still_alive) tuple. + If recursive is True also attempts to kill grandchildren. + """ + if pid == os.getpid(): + raise RuntimeError("I refuse to kill myself") + parent = psutil.Process(pid) + children = parent.children(recursive=recursive) + if include_parent: + children.append(parent) + for p in children: + p.send_signal(sig) + gone, alive = psutil.wait_procs(children, timeout=timeout, + callback=on_terminate) + return (gone, alive) + Q&A === From aba3b3953a79d7c1b5ee4eef56ea17477ac3219d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 04:40:01 +0200 Subject: [PATCH 325/922] #1026 / doc / recipes: add one-liners --- docs/index.rst | 185 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 153 insertions(+), 32 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 1be4b3f1a..4d9da9a98 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -844,6 +844,9 @@ Functions [{'name': 'python3', 'pid': 21947}, {'name': 'python', 'pid': 23835}] + See also `process filtering <#filtering-and-sorting-processes>`__ section for + more examples. + .. versionchanged:: 5.3.0 added "attrs" and "ad_value" parameters. @@ -1065,6 +1068,7 @@ Process class The process name. On Windows the return value is cached after first call. Not on POSIX because the process name `may change `__. + See also how to `find a process by name <#find-process-by-name>`__. .. method:: exe() @@ -1677,6 +1681,8 @@ Process class returned either as the reference to process A is lost. This concept is well summaried by this `unit test `__. + See also how to `kill a process tree <#kill-process-tree>`__ and + `terminate my children <#terminate-my-children>`__. .. method:: open_files() @@ -1820,6 +1826,8 @@ Process class On UNIX this is the same as ``os.kill(pid, sig)``. On Windows only *SIGTERM*, *CTRL_C_EVENT* and *CTRL_BREAK_EVENT* signals are supported and *SIGTERM* is treated as an alias for :meth:`kill()`. + See also how to `kill a process tree <#kill-process-tree>`__ and + `terminate my children <#terminate-my-children>`__. .. versionchanged:: 3.2.0 support for CTRL_C_EVENT and CTRL_BREAK_EVENT signals on Windows @@ -1845,14 +1853,18 @@ Process class whether PID has been reused. On UNIX this is the same as ``os.kill(pid, signal.SIGTERM)``. On Windows this is an alias for :meth:`kill`. + See also how to `kill a process tree <#kill-process-tree>`__ and + `terminate my children <#terminate-my-children>`__. .. method:: kill() - Kill the current process by using *SIGKILL* signal preemptively - checking whether PID has been reused. - On UNIX this is the same as ``os.kill(pid, signal.SIGKILL)``. - On Windows this is done by using - `TerminateProcess `__. + Kill the current process by using *SIGKILL* signal preemptively + checking whether PID has been reused. + On UNIX this is the same as ``os.kill(pid, signal.SIGKILL)``. + On Windows this is done by using + `TerminateProcess `__. + See also how to `kill a process tree <#kill-process-tree>`__ and + `terminate my children <#terminate-my-children>`__. .. method:: wait(timeout=None) @@ -2221,7 +2233,7 @@ enough to be part of the public API. Find process by name -------------------- -Check string against process :meth:`Process.name()`: +Check string against :meth:`Process.name()`: :: @@ -2235,7 +2247,7 @@ Check string against process :meth:`Process.name()`: ls.append(p) return ls -A bit more advanced, check string against process :meth:`Process.name()`, +A bit more advanced, check string against :meth:`Process.name()`, :meth:`Process.exe()` and :meth:`Process.cmdline()`: :: @@ -2253,14 +2265,41 @@ A bit more advanced, check string against process :meth:`Process.name()`, name_ = p.name() cmdline = p.cmdline() exe = p.exe() + except (psutil.AccessDenied, psutil.ZombieProcess): + pass except psutil.NoSuchProcess: continue - except psutil.AccessDenied: - pass if name == name_ or cmdline[0] == name or os.path.basename(exe) == name: ls.append(name) return ls +Kill process tree +----------------- + +:: + + import psutil + import signal + import os + + def kill_proc_tree(pid, sig=signal.SIGTERM, recursive=True, include_parent=True, + timeout=None, on_terminate=None): + """Kill a process tree with signal "sig" and return a + (gone, still_alive) tuple. + If recursive is True also attempts to kill grandchildren. + """ + if pid == os.getpid(): + raise RuntimeError("I refuse to kill myself") + parent = psutil.Process(pid) + children = parent.children(recursive=recursive) + if include_parent: + children.append(parent) + for p in children: + p.send_signal(sig) + gone, alive = psutil.wait_procs(children, timeout=timeout, + callback=on_terminate) + return (gone, alive) + Terminate my children --------------------- @@ -2297,32 +2336,114 @@ resources. reap_children() -Kill process tree ------------------ +Filtering and sorting processes +------------------------------- -:: +This is a collection of one-liners showing how to use :func:`process_iter()` in +order to filter for processes and sort them. - import psutil - import signal - import os +Setup:: - def kill_proc_tree(pid, sig=signal.SIGTERM, recursive=True, include_parent=True, - timeout=None, on_terminate=None): - """Kill a process tree with signal "sig" and return a - (gone, still_alive) tuple. - If recursive is True also attempts to kill grandchildren. - """ - if pid == os.getpid(): - raise RuntimeError("I refuse to kill myself") - parent = psutil.Process(pid) - children = parent.children(recursive=recursive) - if include_parent: - children.append(parent) - for p in children: - p.send_signal(sig) - gone, alive = psutil.wait_procs(children, timeout=timeout, - callback=on_terminate) - return (gone, alive) + >>> import psutil + >>> from pprint import pprint as pp + +Processes having "python" in their name:: + + >>> pp([p.info for p in psutil.process_iter(attrs=['pid', 'name']) if 'python' in p.info['name']]) + [{'name': 'python3', 'pid': 21947}, + {'name': 'python', 'pid': 23835}] + +Processes owned by user:: + + >>> import getpass + >>> pp([(p.pid, p.info['name']) for p in psutil.process_iter(attrs=['name', 'username']) if p.info['username'] == getpass.getuser()]) + (16832, 'bash'), + (18841, 'chrome'), + (19772, 'ssh'), + (20028, 'chrome'), + (20492, 'python'), + (31658, 'VBoxSVC')] + +Processes having open network connections:: + + >>> pp([(p.pid, p.info) for p in psutil.process_iter(attrs=['name', 'connections']) if p.info['connections']]) + [(2650, + {'name': 'chrome', + 'connections': [pconn(fd=131, family=2, type=2, laddr=('192.168.1.7', 48653), raddr=('151.5.1.14', 443), status='NONE'), + pconn(fd=223, family=2, type=1, laddr=('192.168.1.7', 42642), raddr=('192.168.1.2', 8008), status='ESTABLISHED')]}), + (19772, + 'name': 'ssh' + {'connections': [pconn(fd=3, family=2, type=1, laddr=('127.0.0.1', 39082), raddr=('127.0.0.1', 2222), status='ESTABLISHED')],}), + (21932, + 'name': 'plugin_host' + {'connections': [pconn(fd=26, family=2, type=1, laddr=('192.168.1.7', 59698), raddr=('209.20.75.76', 80), status='CLOSE_WAIT'), + pconn(fd=27, family=2, type=2, laddr=('127.0.0.1', 55580), raddr=('127.0.1.1', 53), status='NONE')]})] + +Processes actively running:: + + >>> pp([(p.pid, p.info) for p in psutil.process_iter(attrs=['name', 'status']) if p.info['status'] == psutil.STATUS_RUNNING]) + [(1150, {'name': 'Xorg', 'status': 'running'}), + (1776, {'name': 'unity-panel-service', 'status': 'running'}), + (20492, {'name': 'python', 'status': 'running'})] + +Processes using log files:: + + >>> import os + >>> import psutil + >>> for p in psutil.process_iter(attrs=['name', 'open_files']): + ... for file in p.info['open_files'] or []: + ... if os.path.splitext(file.path)[1] == '.log': + ... print("%-5s %-10s %s" % (p.pid, p.info['name'][:10], file.path)) + ... + 1510 upstart /home/giampaolo/.cache/upstart/unity-settings-daemon.log + 2174 nautilus /home/giampaolo/.local/share/gvfs-metadata/home-ce08efac.log + 2274 gvfsd-meta /home/giampaolo/.local/share/gvfs-metadata/root-1d9eaa2d.log + 2650 chrome /home/giampaolo/.config/google-chrome/Default/data_reduction_proxy_leveldb/000003.log + +Processes consuming more than 500M of memory:: + + >>> pp([(p.pid, p.info['name'], p.info['memory_info'].rss) for p in psutil.process_iter(attrs=['name', 'memory_info']) if p.info['memory_info'].rss > 500 * 1024 * 1024]) + [(2650, 'chrome', 532324352), + (3038, 'chrome', 1120088064), + (3249, 'chrome', 1503940608), + (18841, 'chrome', 582803456), + (21915, 'sublime_text', 615407616)] + +Top 5 most memory consuming processes:: + + >>> pp([(p.pid, p.info) for p in sorted(psutil.process_iter(attrs=['name', 'memory_percent']), key=lambda p: p.info['memory_percent'])][-5:]) + [(2650, {'memory_percent': 3.1836352240031873, 'name': 'chrome'}), + (18841, {'memory_percent': 3.482724332758809, 'name': 'chrome'}), + (21915, {'memory_percent': 3.6815453247662737, 'name': 'sublime_text'}), + (3038, {'memory_percent': 6.732935429979187, 'name': 'chrome'}), + (3249, {'memory_percent': 8.994554843376399, 'name': 'chrome'})] + +Top 5 processes which consumed the most CPU time:: + + >>> pp([(p.pid, p.info['name'], sum(p.info['cpu_times'])) for p in sorted(psutil.process_iter(attrs=['name', 'cpu_times']), key=lambda p: sum(p.info['cpu_times'][:2]))][-5:]) + [(3249, 'chrome', 6392.240000000001), + (1888, 'compiz', 8833.04), + (2721, 'chrome', 10219.73), + (1150, 'Xorg', 11116.989999999998), + (2650, 'chrome', 18451.97)] + +Top 5 processes which caused the most I/O:: + + >>> pp([(p.pid, p.info['name']) for p in sorted(psutil.process_iter(attrs=['name', 'io_counters']), key=lambda p: p.info['io_counters'] and p.info['io_counters'][:2])][-5:]) + [(21915, 'sublime_text'), + (2175, 'indicator-multiload'), + (1871, 'pulseaudio'), + (1510, 'upstart'), + (2650, 'chrome')] + +Top 5 processes opening more file descriptors:: + + >>> pp([(p.pid, p.info) for p in sorted(psutil.process_iter(attrs=['name', 'num_fds']), key=lambda p: p.info['num_fds'])][-5:]) + [(3038, {'name': 'chrome', 'num_fds': 100}), + (21915, {'name': 'sublime_text', 'num_fds': 105}), + (18841, {'name': 'chrome', 'num_fds': 144}), + (2721, {'name': 'chrome', 'num_fds': 185}), + (2650, {'name': 'chrome', 'num_fds': 354})] Q&A === From f7194e669393b59e5146c608f33fd78f719c2a9e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 04:44:58 +0200 Subject: [PATCH 326/922] update doc --- docs/index.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4d9da9a98..0ee32ffa5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2278,20 +2278,21 @@ Kill process tree :: - import psutil - import signal import os + import signal + import psutil - def kill_proc_tree(pid, sig=signal.SIGTERM, recursive=True, include_parent=True, + def kill_proc_tree(pid, sig=signal.SIGTERM, include_parent=True, timeout=None, on_terminate=None): - """Kill a process tree with signal "sig" and return a - (gone, still_alive) tuple. - If recursive is True also attempts to kill grandchildren. + """Kill a process tree (including grandchildren) with signal + "sig" and return a (gone, still_alive) tuple. + "on_terminate", if specified, is a callabck as soon as a child + terminates. """ if pid == os.getpid(): raise RuntimeError("I refuse to kill myself") parent = psutil.Process(pid) - children = parent.children(recursive=recursive) + children = parent.children(recursive=True) if include_parent: children.append(parent) for p in children: @@ -2323,7 +2324,7 @@ resources. gone, alive = psutil.wait_procs(procs, timeout=timeout, callback=on_terminate) if not alive: return - # send SIGKILL to the survivors + # send SIGKILL for p in alive: print("process {} survived SIGTERM; trying SIGKILL" % p) p.kill() From ac61a5f9c80ae317167fdfb8787a7ea3a2c80e4c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 17:01:13 +0200 Subject: [PATCH 327/922] update doc --- docs/index.rst | 75 +++++++++++++++----------------------------------- 1 file changed, 22 insertions(+), 53 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 0ee32ffa5..ed5e52ce4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2323,17 +2323,15 @@ resources. p.terminate() gone, alive = psutil.wait_procs(procs, timeout=timeout, callback=on_terminate) if not alive: - return - # send SIGKILL - for p in alive: - print("process {} survived SIGTERM; trying SIGKILL" % p) - p.kill() - gone, alive = psutil.wait_procs(alive, timeout=timeout, callback=on_terminate) - if not alive: - return - # give up - for p in alive: - print("process {} survived SIGKILL; giving up" % p) + # send SIGKILL + for p in alive: + print("process {} survived SIGTERM; trying SIGKILL" % p) + p.kill() + gone, alive = psutil.wait_procs(alive, timeout=timeout, callback=on_terminate) + if not alive: + # give up + for p in alive: + print("process {} survived SIGKILL; giving up" % p) reap_children() @@ -2359,26 +2357,8 @@ Processes owned by user:: >>> import getpass >>> pp([(p.pid, p.info['name']) for p in psutil.process_iter(attrs=['name', 'username']) if p.info['username'] == getpass.getuser()]) (16832, 'bash'), - (18841, 'chrome'), (19772, 'ssh'), - (20028, 'chrome'), - (20492, 'python'), - (31658, 'VBoxSVC')] - -Processes having open network connections:: - - >>> pp([(p.pid, p.info) for p in psutil.process_iter(attrs=['name', 'connections']) if p.info['connections']]) - [(2650, - {'name': 'chrome', - 'connections': [pconn(fd=131, family=2, type=2, laddr=('192.168.1.7', 48653), raddr=('151.5.1.14', 443), status='NONE'), - pconn(fd=223, family=2, type=1, laddr=('192.168.1.7', 42642), raddr=('192.168.1.2', 8008), status='ESTABLISHED')]}), - (19772, - 'name': 'ssh' - {'connections': [pconn(fd=3, family=2, type=1, laddr=('127.0.0.1', 39082), raddr=('127.0.0.1', 2222), status='ESTABLISHED')],}), - (21932, - 'name': 'plugin_host' - {'connections': [pconn(fd=26, family=2, type=1, laddr=('192.168.1.7', 59698), raddr=('209.20.75.76', 80), status='CLOSE_WAIT'), - pconn(fd=27, family=2, type=2, laddr=('127.0.0.1', 55580), raddr=('127.0.1.1', 53), status='NONE')]})] + (20492, 'python')] Processes actively running:: @@ -2398,7 +2378,6 @@ Processes using log files:: ... 1510 upstart /home/giampaolo/.cache/upstart/unity-settings-daemon.log 2174 nautilus /home/giampaolo/.local/share/gvfs-metadata/home-ce08efac.log - 2274 gvfsd-meta /home/giampaolo/.local/share/gvfs-metadata/root-1d9eaa2d.log 2650 chrome /home/giampaolo/.config/google-chrome/Default/data_reduction_proxy_leveldb/000003.log Processes consuming more than 500M of memory:: @@ -2406,43 +2385,33 @@ Processes consuming more than 500M of memory:: >>> pp([(p.pid, p.info['name'], p.info['memory_info'].rss) for p in psutil.process_iter(attrs=['name', 'memory_info']) if p.info['memory_info'].rss > 500 * 1024 * 1024]) [(2650, 'chrome', 532324352), (3038, 'chrome', 1120088064), - (3249, 'chrome', 1503940608), - (18841, 'chrome', 582803456), (21915, 'sublime_text', 615407616)] -Top 5 most memory consuming processes:: +Top 3 most memory consuming processes:: - >>> pp([(p.pid, p.info) for p in sorted(psutil.process_iter(attrs=['name', 'memory_percent']), key=lambda p: p.info['memory_percent'])][-5:]) - [(2650, {'memory_percent': 3.1836352240031873, 'name': 'chrome'}), - (18841, {'memory_percent': 3.482724332758809, 'name': 'chrome'}), - (21915, {'memory_percent': 3.6815453247662737, 'name': 'sublime_text'}), + >>> pp([(p.pid, p.info) for p in sorted(psutil.process_iter(attrs=['name', 'memory_percent']), key=lambda p: p.info['memory_percent'])][-3:]) + [(21915, {'memory_percent': 3.6815453247662737, 'name': 'sublime_text'}), (3038, {'memory_percent': 6.732935429979187, 'name': 'chrome'}), (3249, {'memory_percent': 8.994554843376399, 'name': 'chrome'})] -Top 5 processes which consumed the most CPU time:: +Top 3 processes which consumed the most CPU time:: - >>> pp([(p.pid, p.info['name'], sum(p.info['cpu_times'])) for p in sorted(psutil.process_iter(attrs=['name', 'cpu_times']), key=lambda p: sum(p.info['cpu_times'][:2]))][-5:]) - [(3249, 'chrome', 6392.240000000001), - (1888, 'compiz', 8833.04), - (2721, 'chrome', 10219.73), + >>> pp([(p.pid, p.info['name'], sum(p.info['cpu_times'])) for p in sorted(psutil.process_iter(attrs=['name', 'cpu_times']), key=lambda p: sum(p.info['cpu_times'][:2]))][-3:]) + [(2721, 'chrome', 10219.73), (1150, 'Xorg', 11116.989999999998), (2650, 'chrome', 18451.97)] -Top 5 processes which caused the most I/O:: +Top 3 processes which caused the most I/O:: - >>> pp([(p.pid, p.info['name']) for p in sorted(psutil.process_iter(attrs=['name', 'io_counters']), key=lambda p: p.info['io_counters'] and p.info['io_counters'][:2])][-5:]) + >>> pp([(p.pid, p.info['name']) for p in sorted(psutil.process_iter(attrs=['name', 'io_counters']), key=lambda p: p.info['io_counters'] and p.info['io_counters'][:2])][-3:]) [(21915, 'sublime_text'), - (2175, 'indicator-multiload'), (1871, 'pulseaudio'), - (1510, 'upstart'), - (2650, 'chrome')] + (1510, 'upstart')] -Top 5 processes opening more file descriptors:: +Top 3 processes opening more file descriptors:: - >>> pp([(p.pid, p.info) for p in sorted(psutil.process_iter(attrs=['name', 'num_fds']), key=lambda p: p.info['num_fds'])][-5:]) - [(3038, {'name': 'chrome', 'num_fds': 100}), - (21915, {'name': 'sublime_text', 'num_fds': 105}), - (18841, {'name': 'chrome', 'num_fds': 144}), + >>> pp([(p.pid, p.info) for p in sorted(psutil.process_iter(attrs=['name', 'num_fds']), key=lambda p: p.info['num_fds'])][-3:]) + [(21915, {'name': 'sublime_text', 'num_fds': 105}), (2721, {'name': 'chrome', 'num_fds': 185}), (2650, {'name': 'chrome', 'num_fds': 354})] From c9babc9e87a8b0dd5561d1362550f72f32e9e2ce Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 17:03:50 +0200 Subject: [PATCH 328/922] always use sh() instead of subprocess.Popen --- psutil/tests/__init__.py | 9 ++------- psutil/tests/test_bsd.py | 9 +-------- psutil/tests/test_osx.py | 35 +++++++---------------------------- psutil/tests/test_posix.py | 12 ++---------- psutil/tests/test_process.py | 6 ++---- psutil/tests/test_windows.py | 7 ++----- 6 files changed, 16 insertions(+), 62 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 9b9572c39..ca5041b24 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -279,18 +279,13 @@ def sh(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE): """run cmd in a subprocess and return its output. raises RuntimeError on error. """ - p = subprocess.Popen(cmdline, shell=True, stdout=stdout, stderr=stderr) + p = subprocess.Popen(cmdline, shell=True, stdout=stdout, stderr=stderr, + universal_newlines=True) stdout, stderr = p.communicate() if p.returncode != 0: raise RuntimeError(stderr) if stderr: - if PY3: - stderr = str(stderr, sys.stderr.encoding or - sys.getfilesystemencoding()) warn(stderr) - if PY3: - stdout = str(stdout, sys.stdout.encoding or - sys.getfilesystemencoding()) return stdout.strip() diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 77bb7bffe..aeda555a9 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -13,8 +13,6 @@ import datetime import os import re -import subprocess -import sys import time import psutil @@ -22,7 +20,6 @@ from psutil import FREEBSD from psutil import NETBSD from psutil import OPENBSD -from psutil._compat import PY3 from psutil.tests import get_test_subprocess from psutil.tests import MEMORY_TOLERANCE from psutil.tests import reap_children @@ -87,11 +84,7 @@ def tearDownClass(cls): reap_children() def test_process_create_time(self): - cmdline = "ps -o lstart -p %s" % self.pid - p = subprocess.Popen(cmdline, shell=1, stdout=subprocess.PIPE) - output = p.communicate()[0] - if PY3: - output = str(output, sys.stdout.encoding) + output = sh("ps -o lstart -p %s" % self.pid) start_ps = output.replace('STARTED', '').strip() start_psutil = psutil.Process(self.pid).create_time() start_psutil = time.strftime("%a %b %e %H:%M:%S %Y", diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 69d6c8408..d30b65af7 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -8,13 +8,10 @@ import os import re -import subprocess -import sys import time import psutil from psutil import OSX -from psutil._compat import PY3 from psutil.tests import get_test_subprocess from psutil.tests import MEMORY_TOLERANCE from psutil.tests import reap_children @@ -27,20 +24,6 @@ PAGESIZE = os.sysconf("SC_PAGE_SIZE") if OSX else None -def sysctl(cmdline): - """Expects a sysctl command with an argument and parse the result - returning only the value of interest. - """ - p = subprocess.Popen(cmdline, shell=1, stdout=subprocess.PIPE) - result = p.communicate()[0].strip().split()[1] - if PY3: - result = str(result, sys.stdout.encoding) - try: - return int(result) - except ValueError: - return result - - def vm_stat(field): """Wrapper around 'vm_stat' cmdline utility.""" out = sh('vm_stat') @@ -91,11 +74,7 @@ def tearDownClass(cls): reap_children() def test_process_create_time(self): - cmdline = "ps -o lstart -p %s" % self.pid - p = subprocess.Popen(cmdline, shell=1, stdout=subprocess.PIPE) - output = p.communicate()[0] - if PY3: - output = str(output, sys.stdout.encoding) + output = sh("ps -o lstart -p %s" % self.pid) start_ps = output.replace('STARTED', '').strip() hhmmss = start_ps.split(' ')[-2] year = start_ps.split(' ')[-1] @@ -143,26 +122,26 @@ def df(path): # --- cpu def test_cpu_count_logical(self): - num = sysctl("sysctl hw.logicalcpu") + num = int(sh("sysctl hw.logicalcpu")) self.assertEqual(num, psutil.cpu_count(logical=True)) def test_cpu_count_physical(self): - num = sysctl("sysctl hw.physicalcpu") + num = int(sh("sysctl hw.physicalcpu")) self.assertEqual(num, psutil.cpu_count(logical=False)) def test_cpu_freq(self): freq = psutil.cpu_freq() self.assertEqual( - freq.current * 1000 * 1000, sysctl("sysctl hw.cpufrequency")) + freq.current * 1000 * 1000, int(sh("sysctl hw.cpufrequency"))) self.assertEqual( - freq.min * 1000 * 1000, sysctl("sysctl hw.cpufrequency_min")) + freq.min * 1000 * 1000, int(sh("sysctl hw.cpufrequency_min"))) self.assertEqual( - freq.max * 1000 * 1000, sysctl("sysctl hw.cpufrequency_max")) + freq.max * 1000 * 1000, int(sh("sysctl hw.cpufrequency_max"))) # --- virtual mem def test_vmem_total(self): - sysctl_hwphymem = sysctl('sysctl hw.memsize') + sysctl_hwphymem = int(sh('sysctl hw.memsize')) self.assertEqual(sysctl_hwphymem, psutil.virtual_memory().total) @retry_before_failing() diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index f72fb20a9..654b16276 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -46,10 +46,7 @@ def ps(cmd): if SUNOS: cmd = cmd.replace("-o command", "-o comm") cmd = cmd.replace("-o start", "-o stime") - p = subprocess.Popen(cmd, shell=1, stdout=subprocess.PIPE) - output = p.communicate()[0].strip() - if PY3: - output = str(output, sys.stdout.encoding) + output = sh(cmd) if not LINUX: output = output.split('\n')[1].strip() try: @@ -290,12 +287,7 @@ def test_pids(self): @unittest.skipIf(SUNOS, "unreliable on SUNOS") @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_nic_names(self): - p = subprocess.Popen("ifconfig -a", shell=1, stdout=subprocess.PIPE) - output = p.communicate()[0].strip() - if p.returncode != 0: - raise unittest.SkipTest('ifconfig returned no output') - if PY3: - output = str(output, sys.stdout.encoding) + output = sh("ifconfig -a") for nic in psutil.net_io_counters(pernic=True).keys(): for line in output.split(): if line.startswith(nic): diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index db6a9e59c..2d8d0e2c4 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -722,10 +722,8 @@ def test_exe(self): # Tipically OSX. Really not sure what to do here. pass - subp = subprocess.Popen([exe, '-c', 'import os; print("hey")'], - stdout=subprocess.PIPE) - out, _ = subp.communicate() - self.assertEqual(out.strip(), b'hey') + out = sh("""%s -c 'import os; print("hey")'""" % exe) + self.assertEqual(out, 'hey') def test_cmdline(self): cmdline = [PYTHON, "-c", "import time; time.sleep(60)"] diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 1ca796d05..c58c4f96d 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -30,13 +30,13 @@ from psutil import WINDOWS from psutil._compat import basestring from psutil._compat import callable -from psutil._compat import PY3 from psutil.tests import APPVEYOR from psutil.tests import get_test_subprocess from psutil.tests import mock from psutil.tests import reap_children from psutil.tests import retry_before_failing from psutil.tests import run_test_module_by_name +from psutil.tests import sh from psutil.tests import unittest @@ -69,10 +69,7 @@ def wrapper(self, *args, **kwargs): class TestSystemAPIs(unittest.TestCase): def test_nic_names(self): - p = subprocess.Popen(['ipconfig', '/all'], stdout=subprocess.PIPE) - out = p.communicate()[0] - if PY3: - out = str(out, sys.stdout.encoding or sys.getfilesystemencoding()) + out = sh('ipconfig', '/all') nics = psutil.net_io_counters(pernic=True).keys() for nic in nics: if "pseudo-interface" in nic.replace(' ', '-').lower(): From 20a2976811ebaac95423df3dd4187f55a04a1ab3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 17:10:32 +0200 Subject: [PATCH 329/922] change sh() signature; also use sh() in the GIT pre-commit script instead of subprocess.check_output (not avalaible on py 2.6) --- .git-pre-commit | 18 +++++++++++++++++- psutil/tests/__init__.py | 10 ++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.git-pre-commit b/.git-pre-commit index 372d80912..23ed51ab1 100755 --- a/.git-pre-commit +++ b/.git-pre-commit @@ -56,8 +56,24 @@ def exit(msg): sys.exit(1) +def sh(cmd): + """run cmd in a subprocess and return its output. + raises RuntimeError on error. + """ + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True) + stdout, stderr = p.communicate() + if p.returncode != 0: + raise RuntimeError(stderr) + if stderr: + print(stderr, file=sys.stderr) + if stdout.endswith('\n'): + stdout = stdout[:-1] + return stdout + + def main(): - out = subprocess.check_output("git diff --cached --name-only", shell=True) + out = sh("git diff --cached --name-only") py_files = [x for x in out.split(b'\n') if x.endswith(b'.py') and os.path.exists(x)] diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index ca5041b24..5ae499005 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -275,18 +275,20 @@ def pyrun(src): return subp -def sh(cmdline, stdout=subprocess.PIPE, stderr=subprocess.PIPE): +def sh(cmd): """run cmd in a subprocess and return its output. raises RuntimeError on error. """ - p = subprocess.Popen(cmdline, shell=True, stdout=stdout, stderr=stderr, - universal_newlines=True) + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, universal_newlines=True) stdout, stderr = p.communicate() if p.returncode != 0: raise RuntimeError(stderr) if stderr: warn(stderr) - return stdout.strip() + if stdout.endswith('\n'): + stdout = stdout[:-1] + return stdout def reap_children(recursive=False): From 1a13c2769a40b0094fdc0f66a3a2e6b4f6479f52 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 17:23:08 +0200 Subject: [PATCH 330/922] fix accidentally broken tests on osx --- psutil/tests/test_osx.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index d30b65af7..db40efbc8 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -24,6 +24,18 @@ PAGESIZE = os.sysconf("SC_PAGE_SIZE") if OSX else None +def sysctl(cmdline): + """Expects a sysctl command with an argument and parse the result + returning only the value of interest. + """ + out = sh(cmdline) + result = out.split()[1] + try: + return int(result) + except ValueError: + return result + + def vm_stat(field): """Wrapper around 'vm_stat' cmdline utility.""" out = sh('vm_stat') @@ -122,26 +134,26 @@ def df(path): # --- cpu def test_cpu_count_logical(self): - num = int(sh("sysctl hw.logicalcpu")) + num = sysctl("sysctl hw.logicalcpu") self.assertEqual(num, psutil.cpu_count(logical=True)) def test_cpu_count_physical(self): - num = int(sh("sysctl hw.physicalcpu")) + num = sysctl("sysctl hw.physicalcpu") self.assertEqual(num, psutil.cpu_count(logical=False)) def test_cpu_freq(self): freq = psutil.cpu_freq() self.assertEqual( - freq.current * 1000 * 1000, int(sh("sysctl hw.cpufrequency"))) + freq.current * 1000 * 1000, sysctl("sysctl hw.cpufrequency")) self.assertEqual( - freq.min * 1000 * 1000, int(sh("sysctl hw.cpufrequency_min"))) + freq.min * 1000 * 1000, sysctl("sysctl hw.cpufrequency_min")) self.assertEqual( - freq.max * 1000 * 1000, int(sh("sysctl hw.cpufrequency_max"))) + freq.max * 1000 * 1000, sysctl("sysctl hw.cpufrequency_max")) # --- virtual mem def test_vmem_total(self): - sysctl_hwphymem = int(sh('sysctl hw.memsize')) + sysctl_hwphymem = sysctl('sysctl hw.memsize') self.assertEqual(sysctl_hwphymem, psutil.virtual_memory().total) @retry_before_failing() From 41a4fd641ff669fa59f67d5532ea3483df5f05ad Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 17:32:04 +0200 Subject: [PATCH 331/922] skip rlimit() test if not availalble --- psutil/tests/test_linux.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index afa7a173d..6ceb5f848 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1625,6 +1625,9 @@ def open_mock(name, *args, **kwargs): self.assertEqual(err.exception.errno, errno.ENOENT) assert m.called + @unittest.skipUnless( + get_kernel_version() >= (2, 6, 36), + "prlimit() not available on this Linux kernel version") def test_rlimit_zombie(self): # Emulate a case where rlimit() raises ENOSYS, which may # happen in case of zombie process: From c7d059d860d3457f602f08ad44e788421ecd80d5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 17:58:58 +0200 Subject: [PATCH 332/922] docs/make.bat was wrong --- docs/make.bat | 274 ++++++++++++++++++++++++++++++++++++++++++++------ make.bat | 2 +- 2 files changed, 243 insertions(+), 33 deletions(-) diff --git a/docs/make.bat b/docs/make.bat index 185fc951b..f8473cff8 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,32 +1,242 @@ -@echo off - -rem ========================================================================== -rem Shortcuts for various tasks, emulating UNIX "make" on Windows. -rem It is primarly intended as a shortcut for compiling / installing -rem psutil ("make.bat build", "make.bat install") and running tests -rem ("make.bat test"). -rem -rem This script is modeled after my Windows installation which uses: -rem - Visual studio 2008 for Python 2.6, 2.7, 3.2 -rem - Visual studio 2010 for Python 3.3+ -rem ...therefore it might not work on your Windows installation. -rem -rem By default C:\Python27\python.exe is used. -rem To compile for a specific Python version run: -rem set PYTHON=C:\Python34\python.exe & make.bat build -rem -rem To use a different test script: -rem set PYTHON=C:\Python34\python.exe & set TSCRIPT=foo.py & make.bat test -rem ========================================================================== - -if "%PYTHON%" == "" ( - set PYTHON=C:\Python27\python.exe -) -if "%TSCRIPT%" == "" ( - set TSCRIPT=psutil\tests\runner.py -) - -rem Needed to locate the .pypirc file and upload exes on PYPI. -set HOME=%USERPROFILE% - -%PYTHON% scripts\internal\winmake.py %1 %2 +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. xml to make Docutils-native XML files + echo. pseudoxml to make pseudoxml-XML files for display purposes + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + + +%SPHINXBUILD% 2> nul +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pyftpdlib.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pyftpdlib.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdf" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "latexpdfja" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + cd %BUILDDIR%/latex + make all-pdf-ja + cd %BUILDDIR%/.. + echo. + echo.Build finished; the PDF files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +if "%1" == "xml" ( + %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The XML files are in %BUILDDIR%/xml. + goto end +) + +if "%1" == "pseudoxml" ( + %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. + goto end +) + +:end diff --git a/make.bat b/make.bat index 185fc951b..98456457d 100644 --- a/make.bat +++ b/make.bat @@ -23,7 +23,7 @@ if "%PYTHON%" == "" ( set PYTHON=C:\Python27\python.exe ) if "%TSCRIPT%" == "" ( - set TSCRIPT=psutil\tests\runner.py + set TSCRIPT=psutil\tests\__main__.py ) rem Needed to locate the .pypirc file and upload exes on PYPI. From 2783709a217db87fd44bceb5c6dcc037d04f559b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 18:12:37 +0200 Subject: [PATCH 333/922] make sh() utility function also accept a list and be smart so that it sets shell=False in that case --- psutil/tests/__init__.py | 3 ++- psutil/tests/test_process.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 5ae499005..60e409355 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -279,7 +279,8 @@ def sh(cmd): """run cmd in a subprocess and return its output. raises RuntimeError on error. """ - p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + shell = True if isinstance(cmd, (str, unicode)) else False + p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) stdout, stderr = p.communicate() if p.returncode != 0: diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 2d8d0e2c4..ad92ed3a5 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -722,7 +722,7 @@ def test_exe(self): # Tipically OSX. Really not sure what to do here. pass - out = sh("""%s -c 'import os; print("hey")'""" % exe) + out = sh([exe, "-c", "import os; print('hey')"]) self.assertEqual(out, 'hey') def test_cmdline(self): From 233a5674f3c924b3a88f9e40d6e625eccf5f92a0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 18:18:04 +0200 Subject: [PATCH 334/922] fix some windows tests --- psutil/tests/test_system.py | 4 ++-- psutil/tests/test_windows.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 92b48693c..abb8e6e5c 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -702,9 +702,9 @@ def test_users(self): for user in users: assert user.name, user self.assertIsInstance(user.name, (str, unicode)) - self.assertIsInstance(user.terminal, (str, unicode, None)) + self.assertIsInstance(user.terminal, (str, unicode, type(None))) if user.host is not None: - self.assertIsInstance(user.host, (str, unicode, None)) + self.assertIsInstance(user.host, (str, unicode, type(None))) user.terminal user.host assert user.started > 0.0, user diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index c58c4f96d..b4982f047 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -69,7 +69,7 @@ def wrapper(self, *args, **kwargs): class TestSystemAPIs(unittest.TestCase): def test_nic_names(self): - out = sh('ipconfig', '/all') + out = sh('ipconfig /all') nics = psutil.net_io_counters(pernic=True).keys() for nic in nics: if "pseudo-interface" in nic.replace(' ', '-').lower(): From f2286d4f92108f969f84d993b24badcc5823a17a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 18:33:29 +0200 Subject: [PATCH 335/922] fix windows test --- psutil/tests/test_system.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index abb8e6e5c..d72baa7bf 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -83,17 +83,18 @@ def test_prcess_iter_w_params(self): self.assertEqual(list(p.info.keys()), ['pid']) with self.assertRaises(ValueError): list(psutil.process_iter(attrs=['foo'])) - with mock.patch("psutil._psplatform.Process.name", + with mock.patch("psutil._psplatform.Process.cpu_times", side_effect=psutil.AccessDenied(0, "")) as m: - for p in psutil.process_iter(attrs=["pid", "name"]): - self.assertIsNone(p.info['name']) + for p in psutil.process_iter(attrs=["pid", "cpu_times"]): + self.assertIsNone(p.info['cpu_times']) self.assertGreaterEqual(p.info['pid'], 0) assert m.called - with mock.patch("psutil._psplatform.Process.name", + with mock.patch("psutil._psplatform.Process.cpu_times", side_effect=psutil.AccessDenied(0, "")) as m: flag = object() - for p in psutil.process_iter(attrs=["pid", "name"], ad_value=flag): - self.assertIs(p.info['name'], flag) + for p in psutil.process_iter( + attrs=["pid", "cpu_times"], ad_value=flag): + self.assertIs(p.info['cpu_times'], flag) self.assertGreaterEqual(p.info['pid'], 0) assert m.called From 07b8c2ea8f5c3070bf646e59407d50d04514c225 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 18:41:08 +0200 Subject: [PATCH 336/922] windows: remove unused C code --- psutil/arch/windows/process_info.c | 15 --------------- psutil/arch/windows/process_info.h | 2 -- 2 files changed, 17 deletions(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index e29f2161d..5b0b7726d 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -67,21 +67,6 @@ psutil_handle_from_pid(DWORD pid) { } -/* - * Given a Python int referencing a process handle close the process handle. - */ -PyObject * -psutil_win32_CloseHandle(PyObject *self, PyObject *args) { - unsigned long handle; - - if (! PyArg_ParseTuple(args, "k", &handle)) - return NULL; - // TODO: may want to check return value; - CloseHandle((HANDLE)handle); - Py_RETURN_NONE; -} - - DWORD * psutil_get_pids(DWORD *numberOfReturnedPIDs) { // Win32 SDK says the only way to know if our process array diff --git a/psutil/arch/windows/process_info.h b/psutil/arch/windows/process_info.h index 7c2c9c2be..6f7108f44 100644 --- a/psutil/arch/windows/process_info.h +++ b/psutil/arch/windows/process_info.h @@ -19,8 +19,6 @@ DWORD* psutil_get_pids(DWORD *numberOfReturnedPIDs); HANDLE psutil_handle_from_pid(DWORD pid); HANDLE psutil_handle_from_pid_waccess(DWORD pid, DWORD dwDesiredAccess); -PyObject* psutil_win32_OpenProcess(PyObject *self, PyObject *args); -PyObject* psutil_win32_CloseHandle(PyObject *self, PyObject *args); int psutil_handlep_is_running(HANDLE hProcess); int psutil_pid_in_proclist(DWORD pid); From 5c472ff7595cd58ad76189bdf7ff45d8f6fdb9f3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 18:43:33 +0200 Subject: [PATCH 337/922] windows: remove unused C code --- psutil/arch/windows/process_info.c | 21 --------------------- psutil/arch/windows/process_info.h | 1 - 2 files changed, 22 deletions(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 5b0b7726d..0719c966b 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -154,27 +154,6 @@ psutil_pid_is_running(DWORD pid) { } -int -psutil_pid_in_proclist(DWORD pid) { - DWORD *proclist = NULL; - DWORD numberOfReturnedPIDs; - DWORD i; - - proclist = psutil_get_pids(&numberOfReturnedPIDs); - if (proclist == NULL) - return -1; - for (i = 0; i < numberOfReturnedPIDs; i++) { - if (pid == proclist[i]) { - free(proclist); - return 1; - } - } - - free(proclist); - return 0; -} - - // Check exit code from a process handle. Return FALSE on an error also // XXX - not used anymore int diff --git a/psutil/arch/windows/process_info.h b/psutil/arch/windows/process_info.h index 6f7108f44..00272c2e3 100644 --- a/psutil/arch/windows/process_info.h +++ b/psutil/arch/windows/process_info.h @@ -21,7 +21,6 @@ HANDLE psutil_handle_from_pid(DWORD pid); HANDLE psutil_handle_from_pid_waccess(DWORD pid, DWORD dwDesiredAccess); int psutil_handlep_is_running(HANDLE hProcess); -int psutil_pid_in_proclist(DWORD pid); int psutil_pid_is_running(DWORD pid); PyObject* psutil_get_cmdline(long pid); PyObject* psutil_get_cwd(long pid); From 0521043d953c83cf10986a52d27eeb14a59f199b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 19:21:17 +0200 Subject: [PATCH 338/922] windows / c: refactor pid_is_running() code --- psutil/arch/windows/process_info.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 0719c966b..2f57489cb 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -105,10 +105,18 @@ psutil_get_pids(DWORD *numberOfReturnedPIDs) { } +/* +/* Check for PID existance by using OpenProcess() + GetExitCodeProcess. +/* Returns: + * 1: pid exists + * 0: it doesn't + * -1: error + */ int psutil_pid_is_running(DWORD pid) { HANDLE hProcess; DWORD exitCode; + DWORD WINAPI lasterr; // Special case for PID 0 System Idle Process if (pid == 0) @@ -119,21 +127,21 @@ psutil_pid_is_running(DWORD pid) { hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); if (NULL == hProcess) { - // invalid parameter is no such process - if (GetLastError() == ERROR_INVALID_PARAMETER) { - CloseHandle(hProcess); + lasterr = GetLastError(); + // Yeah, this is the actual error code in case of "no such process". + if (lasterr == ERROR_INVALID_PARAMETER) { return 0; } - - // access denied obviously means there's a process to deny access to... - if (GetLastError() == ERROR_ACCESS_DENIED) { - CloseHandle(hProcess); + // Access denied obviously means there's a process to deny access to. + else if (lasterr == ERROR_ACCESS_DENIED) { return 1; } - - CloseHandle(hProcess); - PyErr_SetFromWindowsErr(0); - return -1; + // Be strict and raise an exception; the caller is supposed + // to take -1 into account. + else { + PyErr_SetFromWindowsErr(0); + return -1; + } } if (GetExitCodeProcess(hProcess, &exitCode)) { From 7694555905ab8aae4e66248cda54788a1c32388e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 20:23:26 +0200 Subject: [PATCH 339/922] windows / c: refactor pid_is_running() code --- psutil/arch/windows/process_info.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 2f57489cb..65338c85d 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -116,24 +116,23 @@ int psutil_pid_is_running(DWORD pid) { HANDLE hProcess; DWORD exitCode; - DWORD WINAPI lasterr; + DWORD WINAPI err; // Special case for PID 0 System Idle Process if (pid == 0) return 1; if (pid < 0) return 0; - hProcess = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, FALSE, pid); if (NULL == hProcess) { - lasterr = GetLastError(); + err = GetLastError(); // Yeah, this is the actual error code in case of "no such process". - if (lasterr == ERROR_INVALID_PARAMETER) { + if (err == ERROR_INVALID_PARAMETER) { return 0; } // Access denied obviously means there's a process to deny access to. - else if (lasterr == ERROR_ACCESS_DENIED) { + else if (err == ERROR_ACCESS_DENIED) { return 1; } // Be strict and raise an exception; the caller is supposed @@ -146,19 +145,21 @@ psutil_pid_is_running(DWORD pid) { if (GetExitCodeProcess(hProcess, &exitCode)) { CloseHandle(hProcess); + // XXX - maybe STILL_ACTIVE is not fully reliable as per: + // http://stackoverflow.com/questions/1591342/#comment47830782_1591379 return (exitCode == STILL_ACTIVE); } - // access denied means there's a process there so we'll assume // it's running - if (GetLastError() == ERROR_ACCESS_DENIED) { - CloseHandle(hProcess); + err = GetLastError(); + CloseHandle(hProcess); + if (err == ERROR_ACCESS_DENIED) { return 1; } - - PyErr_SetFromWindowsErr(0); - CloseHandle(hProcess); - return -1; + else { + PyErr_SetFromWindowsErr(0); + return -1; + } } From 11461bd6e4623c6b3cc5734a96e23fd8dd0962be Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 20:25:49 +0200 Subject: [PATCH 340/922] windows / c: remove unused function --- psutil/arch/windows/process_info.c | 15 --------------- psutil/arch/windows/process_info.h | 7 +++---- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 65338c85d..47aa6bb86 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -163,21 +163,6 @@ psutil_pid_is_running(DWORD pid) { } -// Check exit code from a process handle. Return FALSE on an error also -// XXX - not used anymore -int -handlep_is_running(HANDLE hProcess) { - DWORD dwCode; - - if (NULL == hProcess) - return 0; - if (GetExitCodeProcess(hProcess, &dwCode)) { - if (dwCode == STILL_ACTIVE) - return 1; - } - return 0; -} - // Helper structures to access the memory correctly. Some of these might also // be defined in the winternl.h header file but unfortunately not in a usable // way. diff --git a/psutil/arch/windows/process_info.h b/psutil/arch/windows/process_info.h index 00272c2e3..a3b512e78 100644 --- a/psutil/arch/windows/process_info.h +++ b/psutil/arch/windows/process_info.h @@ -19,13 +19,12 @@ DWORD* psutil_get_pids(DWORD *numberOfReturnedPIDs); HANDLE psutil_handle_from_pid(DWORD pid); HANDLE psutil_handle_from_pid_waccess(DWORD pid, DWORD dwDesiredAccess); - -int psutil_handlep_is_running(HANDLE hProcess); int psutil_pid_is_running(DWORD pid); +int psutil_get_proc_info(DWORD pid, PSYSTEM_PROCESS_INFORMATION *retProcess, + PVOID *retBuffer); + PyObject* psutil_get_cmdline(long pid); PyObject* psutil_get_cwd(long pid); PyObject* psutil_get_environ(long pid); -int psutil_get_proc_info(DWORD pid, PSYSTEM_PROCESS_INFORMATION *retProcess, - PVOID *retBuffer); #endif From f95e6df60fd3e75d204e18b45ef7a14f576cfe49 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 20:31:14 +0200 Subject: [PATCH 341/922] windows / c: refactor pid_is_running() code --- psutil/arch/windows/process_info.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 47aa6bb86..b2e9eea6e 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -149,16 +149,18 @@ psutil_pid_is_running(DWORD pid) { // http://stackoverflow.com/questions/1591342/#comment47830782_1591379 return (exitCode == STILL_ACTIVE); } - // access denied means there's a process there so we'll assume - // it's running - err = GetLastError(); - CloseHandle(hProcess); - if (err == ERROR_ACCESS_DENIED) { - return 1; - } else { - PyErr_SetFromWindowsErr(0); - return -1; + err = GetLastError(); + CloseHandle(hProcess); + // Same as for OpenProcess, assume access denied means there's + // a process to deny access to. + if (err == ERROR_ACCESS_DENIED) { + return 1; + } + else { + PyErr_SetFromWindowsErr(0); + return -1; + } } } From a0a07759e822feff76fbc769ce4b0b6d9e5c75ae Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 20:42:28 +0200 Subject: [PATCH 342/922] update doc --- docs/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index ed5e52ce4..655791da9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -867,6 +867,8 @@ Functions argument). This tunction will return as soon as all processes terminate or when *timeout* occurs, if specified. + Differently from :meth:`Process.wait` it does not raise + :class:`TimeoutExpired` if timeout occurs. A typical use case may be: - send SIGTERM to a list of processes From 62fa5070735f77b14d707c63bc1b4ecc0e169ebf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 21:12:36 +0200 Subject: [PATCH 343/922] windows / c / pid_exists: return the right error code --- psutil/arch/windows/process_info.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index b2e9eea6e..bfcbe6246 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -158,7 +158,7 @@ psutil_pid_is_running(DWORD pid) { return 1; } else { - PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(err); return -1; } } From c2702117881eaff7bdd3dc3357db11107458e4de Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 21:19:45 +0200 Subject: [PATCH 344/922] windows / c: small refactoring --- psutil/_psutil_windows.c | 56 ++++++++++-------------------- psutil/arch/windows/process_info.c | 15 +++----- 2 files changed, 24 insertions(+), 47 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 80b54e2bd..b1a629774 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -379,10 +379,8 @@ psutil_proc_wait(PyObject *self, PyObject *args) { // return None instead. Py_RETURN_NONE; } - else { - PyErr_SetFromWindowsErr(0); - return NULL; - } + else + return PyErr_SetFromWindowsErr(0); } // wait until the process has terminated @@ -392,7 +390,7 @@ psutil_proc_wait(PyObject *self, PyObject *args) { if (retVal == WAIT_FAILED) { CloseHandle(hProcess); - return PyErr_SetFromWindowsErr(GetLastError()); + return PyErr_SetFromWindowsErr(0); } if (retVal == WAIT_TIMEOUT) { CloseHandle(hProcess); @@ -437,8 +435,7 @@ psutil_proc_cpu_times(PyObject *self, PyObject *args) { return NoSuchProcess(); } else { - PyErr_SetFromWindowsErr(0); - return NULL; + return PyErr_SetFromWindowsErr(0); } } @@ -494,8 +491,7 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { return NoSuchProcess(); } else { - PyErr_SetFromWindowsErr(0); - return NULL; + return PyErr_SetFromWindowsErr(0); } } @@ -513,10 +509,8 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { else { // Ignore access denied as it means the process is still alive. // For all other errors, we want an exception. - if (GetLastError() != ERROR_ACCESS_DENIED) { - PyErr_SetFromWindowsErr(0); - return NULL; - } + if (GetLastError() != ERROR_ACCESS_DENIED) + return PyErr_SetFromWindowsErr(0); } /* @@ -672,8 +666,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { return NULL; if (GetProcessImageFileNameW(hProcess, exe, MAX_PATH) == 0) { CloseHandle(hProcess); - PyErr_SetFromWindowsErr(0); - return NULL; + return PyErr_SetFromWindowsErr(0); } CloseHandle(hProcess); return Py_BuildValue("u", exe); @@ -696,16 +689,13 @@ psutil_proc_name(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; hSnapShot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, pid); - if (hSnapShot == INVALID_HANDLE_VALUE) { - PyErr_SetFromWindowsErr(0); - return NULL; - } + if (hSnapShot == INVALID_HANDLE_VALUE) + return PyErr_SetFromWindowsErr(0); pentry.dwSize = sizeof(PROCESSENTRY32W); ok = Process32FirstW(hSnapShot, &pentry); if (! ok) { CloseHandle(hSnapShot); - PyErr_SetFromWindowsErr(0); - return NULL; + return PyErr_SetFromWindowsErr(0); } while (ok) { if (pentry.th32ProcessID == pid) { @@ -1978,10 +1968,8 @@ psutil_proc_priority_get(PyObject *self, PyObject *args) { return NULL; priority = GetPriorityClass(hProcess); CloseHandle(hProcess); - if (priority == 0) { - PyErr_SetFromWindowsErr(0); - return NULL; - } + if (priority == 0) + return PyErr_SetFromWindowsErr(0); return Py_BuildValue("i", priority); } @@ -2004,10 +1992,8 @@ psutil_proc_priority_set(PyObject *self, PyObject *args) { return NULL; retval = SetPriorityClass(hProcess, priority); CloseHandle(hProcess); - if (retval == 0) { - PyErr_SetFromWindowsErr(0); - return NULL; - } + if (retval == 0) + return PyErr_SetFromWindowsErr(0); Py_RETURN_NONE; } @@ -3456,10 +3442,8 @@ psutil_cpu_freq(PyObject *self, PyObject *args) { // Allocate size. size = num_cpus * sizeof(PROCESSOR_POWER_INFORMATION); pBuffer = (BYTE*)LocalAlloc(LPTR, size); - if (! pBuffer) { - PyErr_SetFromWindowsErr(0); - return NULL; - } + if (! pBuffer) + return PyErr_SetFromWindowsErr(0); // Syscall. ret = CallNtPowerInformation( @@ -3492,10 +3476,8 @@ static PyObject * psutil_sensors_battery(PyObject *self, PyObject *args) { SYSTEM_POWER_STATUS sps; - if (GetSystemPowerStatus(&sps) == 0) { - PyErr_SetFromWindowsErr(0); - return NULL; - } + if (GetSystemPowerStatus(&sps) == 0) + return PyErr_SetFromWindowsErr(0); return Py_BuildValue( "iiiI", sps.ACLineStatus, // whether AC is connected: 0=no, 1=yes, 255=unknown diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index bfcbe6246..a871282cf 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -93,8 +93,7 @@ psutil_get_pids(DWORD *numberOfReturnedPIDs) { } if (! EnumProcesses(procArray, procArrayByteSz, &enumReturnSz)) { free(procArray); - PyErr_SetFromWindowsErr(0); - return NULL; + return PyErr_SetFromWindowsErr(0); } } while (enumReturnSz == procArraySz * sizeof(DWORD)); @@ -138,8 +137,7 @@ psutil_pid_is_running(DWORD pid) { // Be strict and raise an exception; the caller is supposed // to take -1 into account. else { - PyErr_SetFromWindowsErr(0); - return -1; + return PyErr_SetFromWindowsErr(err); } } @@ -154,13 +152,10 @@ psutil_pid_is_running(DWORD pid) { CloseHandle(hProcess); // Same as for OpenProcess, assume access denied means there's // a process to deny access to. - if (err == ERROR_ACCESS_DENIED) { + if (err == ERROR_ACCESS_DENIED) return 1; - } - else { - PyErr_SetFromWindowsErr(err); - return -1; - } + else + return PyErr_SetFromWindowsErr(err); } } From d2efc245f6885d00aa3cd442a18af829d8f1114e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 21:22:36 +0200 Subject: [PATCH 345/922] linux / c: small refactoring --- psutil/_psutil_linux.c | 3 +-- psutil/_psutil_posix.c | 9 +++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index a7eb789bc..22fb49863 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -534,8 +534,7 @@ psutil_net_if_duplex_speed(PyObject* self, PyObject* args) { error: if (sock != -1) close(sock); - PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return PyErr_SetFromErrno(PyExc_OSError); } diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index 707c55a1c..5823e61eb 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -295,8 +295,7 @@ psutil_net_if_mtu(PyObject *self, PyObject *args) { error: if (sock != 0) close(sock); - PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return PyErr_SetFromErrno(PyExc_OSError); } @@ -333,8 +332,7 @@ psutil_net_if_flags(PyObject *self, PyObject *args) { error: if (sock != 0) close(sock); - PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return PyErr_SetFromErrno(PyExc_OSError); } @@ -526,8 +524,7 @@ psutil_net_if_duplex_speed(PyObject *self, PyObject *args) { error: if (sock != 0) close(sock); - PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return PyErr_SetFromErrno(PyExc_OSError); } #endif // net_if_stats() OSX/BSD implementation From aba170493b17c0a49d9deee7d7918ec9aab511ae Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 21:25:57 +0200 Subject: [PATCH 346/922] osx / c: small refactoring --- psutil/_psutil_osx.c | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 1e7a5ac58..726e5a3ba 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -575,10 +575,8 @@ psutil_proc_memory_uss(PyObject *self, PyObject *args) { } len = sizeof(cpu_type); - if (sysctlbyname("sysctl.proc_cputype", &cpu_type, &len, NULL, 0) != 0) { - PyErr_SetFromErrno(PyExc_OSError); - return NULL; - } + if (sysctlbyname("sysctl.proc_cputype", &cpu_type, &len, NULL, 0) != 0) + return PyErr_SetFromErrno(PyExc_OSError); // Roughly based on libtop_update_vm_regions in // http://www.opensource.apple.com/source/top/top-100.1.2/libtop.c @@ -819,21 +817,17 @@ psutil_cpu_freq(PyObject *self, PyObject *args) { size_t size = sizeof(int64_t); if (sysctlbyname("hw.cpufrequency", &curr, &size, NULL, 0)) - goto error; + return PyErr_SetFromErrno(PyExc_OSError); if (sysctlbyname("hw.cpufrequency_min", &min, &size, NULL, 0)) - goto error; + return PyErr_SetFromErrno(PyExc_OSError); if (sysctlbyname("hw.cpufrequency_max", &max, &size, NULL, 0)) - goto error; + return PyErr_SetFromErrno(PyExc_OSError); return Py_BuildValue( "KKK", curr / 1000 / 1000, min / 1000 / 1000, max / 1000 / 1000); - -error: - PyErr_SetFromErrno(PyExc_OSError); - return NULL; } @@ -849,10 +843,8 @@ psutil_boot_time(PyObject *self, PyObject *args) { size_t result_len = sizeof result; time_t boot_time = 0; - if (sysctl(request, 2, &result, &result_len, NULL, 0) == -1) { - PyErr_SetFromErrno(PyExc_OSError); - return NULL; - } + if (sysctl(request, 2, &result, &result_len, NULL, 0) == -1) + return PyErr_SetFromErrno(PyExc_OSError); boot_time = result.tv_sec; return Py_BuildValue("f", (float)boot_time); } From a75fcc817f197636f4c94616cf57c85a01296eb3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 21:33:04 +0200 Subject: [PATCH 347/922] 1022: fix users() on freebsd --- psutil/_psutil_bsd.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 9c8c6a292..32366220f 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -826,11 +826,12 @@ psutil_users(PyObject *self, PyObject *args) { if (utx->ut_type != USER_PROCESS) continue; py_tuple = Py_BuildValue( - "(sssf)", + "(sssfi)", utx->ut_user, // username utx->ut_line, // tty utx->ut_host, // hostname - (float)utx->ut_tv.tv_sec // start time + (float)utx->ut_tv.tv_sec, // start time + utx->ut_pid // process id ); if (!py_tuple) { From 5e543ef1c83a105067b2ae8cbbbb0cd3aa6aa882 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 21:34:45 +0200 Subject: [PATCH 348/922] freebsd / c: small refactoring --- psutil/_psutil_bsd.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 32366220f..d37fc5da2 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -178,10 +178,8 @@ psutil_boot_time(PyObject *self, PyObject *args) { struct timeval boottime; size_t len = sizeof(boottime); - if (sysctl(request, 2, &boottime, &len, NULL, 0) == -1) { - PyErr_SetFromErrno(PyExc_OSError); - return NULL; - } + if (sysctl(request, 2, &boottime, &len, NULL, 0) == -1) + return PyErr_SetFromErrno(PyExc_OSError); return Py_BuildValue("d", (double)boottime.tv_sec); } @@ -442,11 +440,8 @@ psutil_cpu_times(PyObject *self, PyObject *args) { int mib[] = {CTL_KERN, KERN_CPTIME}; ret = sysctl(mib, 2, &cpu_time, &size, NULL, 0); #endif - if (ret == -1) { - PyErr_SetFromErrno(PyExc_OSError); - return NULL; - } - + if (ret == -1) + return PyErr_SetFromErrno(PyExc_OSError); return Py_BuildValue("(ddddd)", (double)cpu_time[CP_USER] / CLOCKS_PER_SEC, (double)cpu_time[CP_NICE] / CLOCKS_PER_SEC, From c1eb4dff36f3b7b9b6f02d72c938907792e7a2b3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 21:37:47 +0200 Subject: [PATCH 349/922] freebsd / c: small refactoring --- psutil/arch/bsd/freebsd.c | 42 +++++++++++++++------------------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index 4aac5a616..446042e2b 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -290,14 +290,11 @@ psutil_proc_exe(PyObject *self, PyObject *args) { size = sizeof(pathname); error = sysctl(mib, 4, pathname, &size, NULL, 0); if (error == -1) { - if (errno == ENOENT) { - // see: https://github.com/giampaolo/psutil/issues/907 + // see: https://github.com/giampaolo/psutil/issues/907 + if (errno == ENOENT) return Py_BuildValue("s", ""); - } - else { - PyErr_SetFromErrno(PyExc_OSError); - return NULL; - } + else + return PyErr_SetFromErrno(PyExc_OSError); } if (size == 0 || strlen(pathname) == 0) { ret = psutil_pid_exists(pid); @@ -492,8 +489,7 @@ psutil_virtual_mem(PyObject *self, PyObject *args) { ); error: - PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return PyErr_SetFromErrno(PyExc_OSError); } @@ -521,13 +517,13 @@ psutil_swap_mem(PyObject *self, PyObject *args) { kvm_close(kd); if (sysctlbyname("vm.stats.vm.v_swapin", &swapin, &size, NULL, 0) == -1) - goto sbn_error; + goto error; if (sysctlbyname("vm.stats.vm.v_swapout", &swapout, &size, NULL, 0) == -1) - goto sbn_error; + goto error; if (sysctlbyname("vm.stats.vm.v_vnodein", &nodein, &size, NULL, 0) == -1) - goto sbn_error; + goto error; if (sysctlbyname("vm.stats.vm.v_vnodeout", &nodeout, &size, NULL, 0) == -1) - goto sbn_error; + goto error; return Py_BuildValue("(iiiII)", kvmsw[0].ksw_total, // total @@ -536,9 +532,8 @@ psutil_swap_mem(PyObject *self, PyObject *args) { swapin + swapout, // swap in nodein + nodeout); // swap out -sbn_error: - PyErr_SetFromErrno(PyExc_OSError); - return NULL; +error: + return PyErr_SetFromErrno(PyExc_OSError); } @@ -643,8 +638,7 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { size = sizeof(maxcpus); if (sysctlbyname("kern.smp.maxcpus", &maxcpus, &size, NULL, 0) < 0) { Py_DECREF(py_retlist); - PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return PyErr_SetFromErrno(PyExc_OSError); } long cpu_time[maxcpus][CPUSTATES]; @@ -881,10 +875,8 @@ psutil_proc_cpu_affinity_get(PyObject* self, PyObject* args) { return NULL; ret = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid, sizeof(mask), &mask); - if (ret != 0) { - PyErr_SetFromErrno(PyExc_OSError); - return NULL; - } + if (ret != 0) + return PyErr_SetFromErrno(PyExc_OSError); py_retlist = PyList_New(0); if (py_retlist == NULL) @@ -992,8 +984,7 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { ); error: - PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return PyErr_SetFromErrno(PyExc_OSError); } @@ -1016,6 +1007,5 @@ psutil_sensors_battery(PyObject *self, PyObject *args) { return Py_BuildValue("iii", percent, minsleft, power_plugged); error: - PyErr_SetFromErrno(PyExc_OSError); - return NULL; + return PyErr_SetFromErrno(PyExc_OSError); } From 6aebd0f1c60db8d516343e3c00735cf3e6e23db6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 21:45:16 +0200 Subject: [PATCH 350/922] fix unicode test on ASCII machines --- psutil/tests/test_system.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index d72baa7bf..aa5fbbd46 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -451,10 +451,14 @@ def test_disk_usage(self): def test_disk_usage_unicode(self): # See: https://github.com/giampaolo/psutil/issues/416 - safe_rmpath(TESTFN_UNICODE) - self.addCleanup(safe_rmpath, TESTFN_UNICODE) - os.mkdir(TESTFN_UNICODE) - psutil.disk_usage(TESTFN_UNICODE) + if sys.getfilesystemencoding().lower() in ('ascii', 'us-ascii'): + with self.assertRaises(UnicodeEncodeError): + psutil.disk_usage(TESTFN_UNICODE) + else: + safe_rmpath(TESTFN_UNICODE) + self.addCleanup(safe_rmpath, TESTFN_UNICODE) + os.mkdir(TESTFN_UNICODE) + psutil.disk_usage(TESTFN_UNICODE) def test_disk_partitions(self): # all = False From 48f5ff308fd770e1a59ef1a730408e95f7c75124 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 23:01:23 +0200 Subject: [PATCH 351/922] fix encoding errors on filesystems where encoding == 'ascii' --- psutil/tests/test_misc.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index ea488b23b..42083c492 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -28,6 +28,7 @@ from psutil._common import memoize from psutil._common import memoize_when_activated from psutil._common import supports_ipv6 +from psutil._compat import PY3 from psutil.tests import APPVEYOR from psutil.tests import chdir from psutil.tests import create_proc_children_pair @@ -221,6 +222,7 @@ def foo(*args, **kwargs): def test_memoize_when_activated(self): class Foo: + @memoize_when_activated def foo(self): calls.append(None) @@ -387,7 +389,11 @@ def assert_stdout(exe, args=None): @staticmethod def assert_syntax(exe, args=None): exe = os.path.join(SCRIPTS_DIR, exe) - with open(exe, 'r') as f: + if PY3: + f = open(exe, 'rt', encoding='utf8') + else: + f = open(exe, 'rt') + with f: src = f.read() ast.parse(src) From c1a9ff6c598eebb3d58fdbee6320f9381477def4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 23:41:25 +0200 Subject: [PATCH 352/922] refine unicode tests (maybe will cause failures) --- psutil/tests/__init__.py | 2 +- psutil/tests/test_process.py | 29 +++++++++++++++-------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 60e409355..3b829bbea 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -221,7 +221,7 @@ def get_test_subprocess(cmd=None, **kwds): safe_rmpath(_TESTFN) pyline = "from time import sleep;" pyline += "open(r'%s', 'w').close();" % _TESTFN - pyline += "sleep(60)" + pyline += "sleep(60);" cmd = [PYTHON, "-c", pyline] sproc = subprocess.Popen(cmd, **kwds) wait_for_file(_TESTFN, delete=True, empty=True) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index ad92ed3a5..3e1cadb70 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -68,6 +68,7 @@ from psutil.tests import skip_on_not_implemented from psutil.tests import TESTFILE_PREFIX from psutil.tests import TESTFN +from psutil.tests import TESTFN_UNICODE from psutil.tests import ThreadTask from psutil.tests import TOX from psutil.tests import TRAVIS @@ -2029,8 +2030,8 @@ class TestUnicode(unittest.TestCase): Make sure that APIs returning a string are able to handle unicode, see: https://github.com/giampaolo/psutil/issues/655 """ - uexe = TESTFN + 'èfile' - udir = TESTFN + 'èdir' + uexe = TESTFN_UNICODE + udir = TESTFN_UNICODE + '-dir' @classmethod def setUpClass(cls): @@ -2104,24 +2105,24 @@ def test_proc_open_files(self): self.assertEqual(os.path.normcase(path), os.path.normcase(self.uexe)) + def test_disk_usage(self): + psutil.disk_usage(self.udir) + @unittest.skipUnless(hasattr(psutil.Process, "environ"), "platform not supported") def test_proc_environ(self): + # Note: differently from others, this test does not deal + # with fs paths. On Python 2 subprocess module is broken as + # it's not able to handle with non-ASCII env vars, so + # we use "è", which is part of the extended ASCII table + # (unicode point <= 255). env = os.environ.copy() - env['FUNNY_ARG'] = self.uexe + funny_str = TESTFN_UNICODE if PY3 else 'è' + env['FUNNY_ARG'] = funny_str sproc = get_test_subprocess(env=env) p = psutil.Process(sproc.pid) - if WINDOWS and not PY3: - uexe = self.uexe.decode(sys.getfilesystemencoding()) - else: - uexe = self.uexe - if not OSX and TRAVIS: - self.assertEqual(p.environ()['FUNNY_ARG'], uexe) - else: - p.environ() - - def test_disk_usage(self): - psutil.disk_usage(self.udir) + env = p.environ() + self.assertEqual(env['FUNNY_ARG'], funny_str) class TestInvalidUnicode(TestUnicode): From 131495d71f730e1cf128c738e8f2d9d1ceddda0e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 00:01:34 +0200 Subject: [PATCH 353/922] move unicode tests into test_misc.py --- psutil/tests/test_misc.py | 123 ++++++++++++++++++++++++++++++++++- psutil/tests/test_process.py | 120 ---------------------------------- 2 files changed, 122 insertions(+), 121 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 42083c492..bf83ddd60 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -19,6 +20,7 @@ import stat import sys +from psutil import BSD from psutil import LINUX from psutil import NETBSD from psutil import OPENBSD @@ -31,6 +33,7 @@ from psutil._compat import PY3 from psutil.tests import APPVEYOR from psutil.tests import chdir +from psutil.tests import create_exe from psutil.tests import create_proc_children_pair from psutil.tests import get_test_subprocess from psutil.tests import importlib @@ -43,6 +46,7 @@ from psutil.tests import SCRIPTS_DIR from psutil.tests import sh from psutil.tests import TESTFN +from psutil.tests import TESTFN_UNICODE from psutil.tests import TOX from psutil.tests import TRAVIS from psutil.tests import unittest @@ -645,7 +649,7 @@ def test_chdir(self): self.assertEqual(os.getcwd(), base) -class TestTestUtils(unittest.TestCase): +class TestProcessUtils(unittest.TestCase): def test_reap_children(self): subp = get_test_subprocess() @@ -676,5 +680,122 @@ def test_create_proc_children_pair(self): assert not psutil.tests._subprocesses_started +# =================================================================== +# --- Unicode tests +# =================================================================== + + +class TestUnicode(unittest.TestCase): + """ + Make sure that APIs returning a string are able to handle unicode, + see: https://github.com/giampaolo/psutil/issues/655 + """ + uexe = TESTFN_UNICODE + udir = TESTFN_UNICODE + '-dir' + + @classmethod + def setUpClass(cls): + safe_rmpath(cls.uexe) + safe_rmpath(cls.udir) + create_exe(cls.uexe) + os.mkdir(cls.udir) + + @classmethod + def tearDownClass(cls): + if not APPVEYOR: + safe_rmpath(cls.uexe) + safe_rmpath(cls.udir) + + def setUp(self): + reap_children() + + tearDown = setUp + + def test_proc_exe(self): + subp = get_test_subprocess(cmd=[self.uexe]) + p = psutil.Process(subp.pid) + self.assertIsInstance(p.name(), str) + if not OSX and TRAVIS: + self.assertEqual(p.exe(), self.uexe) + else: + p.exe() + + def test_proc_name(self): + subp = get_test_subprocess(cmd=[self.uexe]) + if WINDOWS: + # XXX: why is this like this? + from psutil._pswindows import py2_strencode + name = py2_strencode(psutil._psplatform.cext.proc_name(subp.pid)) + else: + name = psutil.Process(subp.pid).name() + if not OSX and TRAVIS: + self.assertEqual(name, os.path.basename(self.uexe)) + + def test_proc_cmdline(self): + subp = get_test_subprocess(cmd=[self.uexe]) + p = psutil.Process(subp.pid) + self.assertIsInstance("".join(p.cmdline()), str) + if not OSX and TRAVIS: + self.assertEqual(p.cmdline(), [self.uexe]) + else: + p.cmdline() + + def test_proc_cwd(self): + with chdir(self.udir): + p = psutil.Process() + self.assertIsInstance(p.cwd(), str) + if not OSX and TRAVIS: + self.assertEqual(p.cwd(), self.udir) + else: + p.cwd() + + # @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") + def test_proc_open_files(self): + p = psutil.Process() + start = set(p.open_files()) + with open(self.uexe, 'rb'): + new = set(p.open_files()) + path = (new - start).pop().path + if BSD and not path: + # XXX + # see https://github.com/giampaolo/psutil/issues/595 + self.skipTest("open_files on BSD is broken") + self.assertIsInstance(path, str) + if not OSX and TRAVIS: + self.assertEqual(os.path.normcase(path), + os.path.normcase(self.uexe)) + + def test_disk_usage(self): + psutil.disk_usage(self.udir) + + @unittest.skipUnless(hasattr(psutil.Process, "environ"), + "platform not supported") + def test_proc_environ(self): + # Note: differently from others, this test does not deal + # with fs paths. On Python 2 subprocess module is broken as + # it's not able to handle with non-ASCII env vars, so + # we use "è", which is part of the extended ASCII table + # (unicode point <= 255). + env = os.environ.copy() + funny_str = TESTFN_UNICODE if PY3 else 'è' + env['FUNNY_ARG'] = funny_str + sproc = get_test_subprocess(env=env) + p = psutil.Process(sproc.pid) + env = p.environ() + self.assertEqual(env['FUNNY_ARG'], funny_str) + + +class TestInvalidUnicode(TestUnicode): + """Test handling of invalid utf8 data.""" + if PY3: + uexe = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( + 'utf8', 'surrogateescape') + udir = (TESTFN.encode('utf8') + b"d\xc0\x80").decode( + 'utf8', 'surrogateescape') + else: + uexe = TESTFN + b"f\xc0\x80" + udir = TESTFN + b"d\xc0\x80" + + if __name__ == '__main__': run_test_module_by_name(__file__) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 3e1cadb70..d6b9087b5 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be @@ -46,7 +45,6 @@ from psutil.tests import AF_UNIX from psutil.tests import APPVEYOR from psutil.tests import call_until -from psutil.tests import chdir from psutil.tests import check_connection_ntuple from psutil.tests import create_exe from psutil.tests import create_proc_children_pair @@ -68,7 +66,6 @@ from psutil.tests import skip_on_not_implemented from psutil.tests import TESTFILE_PREFIX from psutil.tests import TESTFN -from psutil.tests import TESTFN_UNICODE from psutil.tests import ThreadTask from psutil.tests import TOX from psutil.tests import TRAVIS @@ -2020,122 +2017,5 @@ def test_zombie_process(self): pass -# =================================================================== -# --- Unicode tests -# =================================================================== - - -class TestUnicode(unittest.TestCase): - """ - Make sure that APIs returning a string are able to handle unicode, - see: https://github.com/giampaolo/psutil/issues/655 - """ - uexe = TESTFN_UNICODE - udir = TESTFN_UNICODE + '-dir' - - @classmethod - def setUpClass(cls): - safe_rmpath(cls.uexe) - safe_rmpath(cls.udir) - create_exe(cls.uexe) - os.mkdir(cls.udir) - - @classmethod - def tearDownClass(cls): - if not APPVEYOR: - safe_rmpath(cls.uexe) - safe_rmpath(cls.udir) - - def setUp(self): - reap_children() - - tearDown = setUp - - def test_proc_exe(self): - subp = get_test_subprocess(cmd=[self.uexe]) - p = psutil.Process(subp.pid) - self.assertIsInstance(p.name(), str) - if not OSX and TRAVIS: - self.assertEqual(p.exe(), self.uexe) - else: - p.exe() - - def test_proc_name(self): - subp = get_test_subprocess(cmd=[self.uexe]) - if WINDOWS: - # XXX: why is this like this? - from psutil._pswindows import py2_strencode - name = py2_strencode(psutil._psplatform.cext.proc_name(subp.pid)) - else: - name = psutil.Process(subp.pid).name() - if not OSX and TRAVIS: - self.assertEqual(name, os.path.basename(self.uexe)) - - def test_proc_cmdline(self): - subp = get_test_subprocess(cmd=[self.uexe]) - p = psutil.Process(subp.pid) - self.assertIsInstance("".join(p.cmdline()), str) - if not OSX and TRAVIS: - self.assertEqual(p.cmdline(), [self.uexe]) - else: - p.cmdline() - - def test_proc_cwd(self): - with chdir(self.udir): - p = psutil.Process() - self.assertIsInstance(p.cwd(), str) - if not OSX and TRAVIS: - self.assertEqual(p.cwd(), self.udir) - else: - p.cwd() - - # @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") - def test_proc_open_files(self): - p = psutil.Process() - start = set(p.open_files()) - with open(self.uexe, 'rb'): - new = set(p.open_files()) - path = (new - start).pop().path - if BSD and not path: - # XXX - # see https://github.com/giampaolo/psutil/issues/595 - self.skipTest("open_files on BSD is broken") - self.assertIsInstance(path, str) - if not OSX and TRAVIS: - self.assertEqual(os.path.normcase(path), - os.path.normcase(self.uexe)) - - def test_disk_usage(self): - psutil.disk_usage(self.udir) - - @unittest.skipUnless(hasattr(psutil.Process, "environ"), - "platform not supported") - def test_proc_environ(self): - # Note: differently from others, this test does not deal - # with fs paths. On Python 2 subprocess module is broken as - # it's not able to handle with non-ASCII env vars, so - # we use "è", which is part of the extended ASCII table - # (unicode point <= 255). - env = os.environ.copy() - funny_str = TESTFN_UNICODE if PY3 else 'è' - env['FUNNY_ARG'] = funny_str - sproc = get_test_subprocess(env=env) - p = psutil.Process(sproc.pid) - env = p.environ() - self.assertEqual(env['FUNNY_ARG'], funny_str) - - -class TestInvalidUnicode(TestUnicode): - """Test handling of invalid utf8 data.""" - if PY3: - uexe = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( - 'utf8', 'surrogateescape') - udir = (TESTFN.encode('utf8') + b"d\xc0\x80").decode( - 'utf8', 'surrogateescape') - else: - uexe = TESTFN + b"f\xc0\x80" - udir = TESTFN + b"d\xc0\x80" - - if __name__ == '__main__': run_test_module_by_name(__file__) From 9062f44119834efa091041acb1d8eebc6333444f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 00:17:15 +0200 Subject: [PATCH 354/922] skip unicode tests on system with ASCII fs encoding --- psutil/tests/__init__.py | 1 + psutil/tests/test_misc.py | 38 ++++++++++++++++++++++-------------- psutil/tests/test_process.py | 1 + psutil/tests/test_system.py | 3 ++- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 3b829bbea..7e5dfe949 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -133,6 +133,7 @@ TESTFN = os.path.join(os.path.realpath(os.getcwd()), TESTFILE_PREFIX) _TESTFN = TESTFN + '-internal' TESTFN_UNICODE = TESTFN + u("-ƒőő") +ASCII_FS = sys.getfilesystemencoding().lower() in ('ascii', 'us-ascii') # --- paths diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index bf83ddd60..02d827518 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -32,6 +32,7 @@ from psutil._common import supports_ipv6 from psutil._compat import PY3 from psutil.tests import APPVEYOR +from psutil.tests import ASCII_FS from psutil.tests import chdir from psutil.tests import create_exe from psutil.tests import create_proc_children_pair @@ -685,10 +686,11 @@ def test_create_proc_children_pair(self): # =================================================================== -class TestUnicode(unittest.TestCase): +@unittest.skipIf(ASCII_FS, "ASCII fs") +class TestUnicodeFilesystemAPIS(unittest.TestCase): """ - Make sure that APIs returning a string are able to handle unicode, - see: https://github.com/giampaolo/psutil/issues/655 + Make sure that fs-related APIs returning a string are able to + handle unicode, see: https://github.com/giampaolo/psutil/issues/655 """ uexe = TESTFN_UNICODE udir = TESTFN_UNICODE + '-dir' @@ -768,6 +770,24 @@ def test_proc_open_files(self): def test_disk_usage(self): psutil.disk_usage(self.udir) + +@unittest.skipIf(ASCII_FS, "ASCII fs") +class TestInvalidUnicodeFilesystemAPIS(TestUnicodeFilesystemAPIS): + """Like above but uses an invalid UTF8 file name.""" + # XXX: maybe this doesn't work as intended and should be removed. + if PY3: + uexe = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( + 'utf8', 'surrogateescape') + udir = (TESTFN.encode('utf8') + b"d\xc0\x80").decode( + 'utf8', 'surrogateescape') + else: + uexe = TESTFN + b"f\xc0\x80" + udir = TESTFN + b"d\xc0\x80" + + +class TestUnicodeNonFsAPIS(unittest.TestCase): + """Unicode tests for non fs-related APIs.""" + @unittest.skipUnless(hasattr(psutil.Process, "environ"), "platform not supported") def test_proc_environ(self): @@ -785,17 +805,5 @@ def test_proc_environ(self): self.assertEqual(env['FUNNY_ARG'], funny_str) -class TestInvalidUnicode(TestUnicode): - """Test handling of invalid utf8 data.""" - if PY3: - uexe = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( - 'utf8', 'surrogateescape') - udir = (TESTFN.encode('utf8') + b"d\xc0\x80").decode( - 'utf8', 'surrogateescape') - else: - uexe = TESTFN + b"f\xc0\x80" - udir = TESTFN + b"d\xc0\x80" - - if __name__ == '__main__': run_test_module_by_name(__file__) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index d6b9087b5..70404fac2 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1661,6 +1661,7 @@ def test_weird_environ(self): # --- Featch all processes test # =================================================================== + class TestFetchAllProcesses(unittest.TestCase): """Test which iterates over all running processes and performs some sanity checks against Process API's returned values. diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index aa5fbbd46..b30d905e9 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -31,6 +31,7 @@ from psutil._compat import long from psutil._compat import unicode from psutil.tests import APPVEYOR +from psutil.tests import ASCII_FS from psutil.tests import check_net_address from psutil.tests import DEVNULL from psutil.tests import enum @@ -451,7 +452,7 @@ def test_disk_usage(self): def test_disk_usage_unicode(self): # See: https://github.com/giampaolo/psutil/issues/416 - if sys.getfilesystemencoding().lower() in ('ascii', 'us-ascii'): + if ASCII_FS: with self.assertRaises(UnicodeEncodeError): psutil.disk_usage(TESTFN_UNICODE) else: From c930cc8be839236053837c65b7e9243c7daa41e3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 01:04:54 +0200 Subject: [PATCH 355/922] refactor unicode tests --- psutil/tests/test_misc.py | 43 ++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 02d827518..42ff3a4b2 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -692,38 +692,32 @@ class TestUnicodeFilesystemAPIS(unittest.TestCase): Make sure that fs-related APIs returning a string are able to handle unicode, see: https://github.com/giampaolo/psutil/issues/655 """ - uexe = TESTFN_UNICODE - udir = TESTFN_UNICODE + '-dir' + funky_name = TESTFN_UNICODE @classmethod def setUpClass(cls): - safe_rmpath(cls.uexe) - safe_rmpath(cls.udir) - create_exe(cls.uexe) - os.mkdir(cls.udir) - - @classmethod - def tearDownClass(cls): - if not APPVEYOR: - safe_rmpath(cls.uexe) - safe_rmpath(cls.udir) + safe_rmpath(cls.funky_name) def setUp(self): + safe_rmpath(self.funky_name) reap_children() + tearDownClass = setUpClass tearDown = setUp def test_proc_exe(self): - subp = get_test_subprocess(cmd=[self.uexe]) + create_exe(self.funky_name) + subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) self.assertIsInstance(p.name(), str) if not OSX and TRAVIS: - self.assertEqual(p.exe(), self.uexe) + self.assertEqual(p.exe(), self.funky_name) else: p.exe() def test_proc_name(self): - subp = get_test_subprocess(cmd=[self.uexe]) + create_exe(self.funky_name) + subp = get_test_subprocess(cmd=[self.funky_name]) if WINDOWS: # XXX: why is this like this? from psutil._pswindows import py2_strencode @@ -731,23 +725,25 @@ def test_proc_name(self): else: name = psutil.Process(subp.pid).name() if not OSX and TRAVIS: - self.assertEqual(name, os.path.basename(self.uexe)) + self.assertEqual(name, os.path.basename(self.funky_name)) def test_proc_cmdline(self): - subp = get_test_subprocess(cmd=[self.uexe]) + create_exe(self.funky_name) + subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) self.assertIsInstance("".join(p.cmdline()), str) if not OSX and TRAVIS: - self.assertEqual(p.cmdline(), [self.uexe]) + self.assertEqual(p.cmdline(), [self.funky_name]) else: p.cmdline() def test_proc_cwd(self): - with chdir(self.udir): + os.mkdir(self.funky_name) + with chdir(self.funky_name): p = psutil.Process() self.assertIsInstance(p.cwd(), str) if not OSX and TRAVIS: - self.assertEqual(p.cwd(), self.udir) + self.assertEqual(p.cwd(), self.funky_name) else: p.cwd() @@ -755,7 +751,7 @@ def test_proc_cwd(self): def test_proc_open_files(self): p = psutil.Process() start = set(p.open_files()) - with open(self.uexe, 'rb'): + with open(self.funky_name, 'wb'): new = set(p.open_files()) path = (new - start).pop().path if BSD and not path: @@ -765,10 +761,11 @@ def test_proc_open_files(self): self.assertIsInstance(path, str) if not OSX and TRAVIS: self.assertEqual(os.path.normcase(path), - os.path.normcase(self.uexe)) + os.path.normcase(self.funky_name)) def test_disk_usage(self): - psutil.disk_usage(self.udir) + os.mkdir(self.funky_name) + psutil.disk_usage(self.funky_name) @unittest.skipIf(ASCII_FS, "ASCII fs") From 5b2b8f6da7a5f902ce48dde4e1c4484e1c45df41 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 01:12:23 +0200 Subject: [PATCH 356/922] refactor unicode tests --- psutil/tests/test_misc.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 42ff3a4b2..e5bd7a700 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -686,11 +686,11 @@ def test_create_proc_children_pair(self): # =================================================================== -@unittest.skipIf(ASCII_FS, "ASCII fs") -class TestUnicodeFilesystemAPIS(unittest.TestCase): +class _UnicodeFilesystemAPIS(unittest.TestCase): """ Make sure that fs-related APIs returning a string are able to handle unicode, see: https://github.com/giampaolo/psutil/issues/655 + This is a base class which is tested by the mixins below. """ funky_name = TESTFN_UNICODE @@ -769,17 +769,17 @@ def test_disk_usage(self): @unittest.skipIf(ASCII_FS, "ASCII fs") -class TestInvalidUnicodeFilesystemAPIS(TestUnicodeFilesystemAPIS): +class TestUnicodeFilesystemAPISMixin(_UnicodeFilesystemAPIS): + funky_name = TESTFN_UNICODE + + +class TestInvalidUnicodeFilesystemAPISMixin(_UnicodeFilesystemAPIS): """Like above but uses an invalid UTF8 file name.""" - # XXX: maybe this doesn't work as intended and should be removed. if PY3: - uexe = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( - 'utf8', 'surrogateescape') - udir = (TESTFN.encode('utf8') + b"d\xc0\x80").decode( + funky_name = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( 'utf8', 'surrogateescape') else: - uexe = TESTFN + b"f\xc0\x80" - udir = TESTFN + b"d\xc0\x80" + funky_name = TESTFN + b"f\xc0\x80" class TestUnicodeNonFsAPIS(unittest.TestCase): From 52ed10cef5daf9781c05255e837a2872286b056c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 02:13:55 +0200 Subject: [PATCH 357/922] #655 / unicode tests: assume APIs handling with unicode paths are broken on Python 2 so disable tests which check for exact path match --- psutil/tests/test_misc.py | 55 ++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index e5bd7a700..cfcf89d7c 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -689,8 +689,14 @@ def test_create_proc_children_pair(self): class _UnicodeFilesystemAPIS(unittest.TestCase): """ Make sure that fs-related APIs returning a string are able to - handle unicode, see: https://github.com/giampaolo/psutil/issues/655 + handle unicode, see: + https://github.com/giampaolo/psutil/issues/655 This is a base class which is tested by the mixins below. + Note that on Python 2 we do not check whether the returned paths + match in case os.* functions are not able to so. + We just assume correct path handling on Python 2 is broken. In fact + it is broken for most os.* functions, see: + http://bugs.python.org/issue18695 """ funky_name = TESTFN_UNICODE @@ -698,54 +704,61 @@ class _UnicodeFilesystemAPIS(unittest.TestCase): def setUpClass(cls): safe_rmpath(cls.funky_name) + tearDownClass = setUpClass + def setUp(self): safe_rmpath(self.funky_name) + + def tearDown(self): reap_children() + safe_rmpath(self.funky_name) - tearDownClass = setUpClass - tearDown = setUp + @classmethod + def expect_exact_path_match(cls): + # Do not expect psutil to correctly handle unicode paths on + # Python 2 if os.listdir() is not able either. + return PY3 or cls.funky_name in os.listdir('.') def test_proc_exe(self): create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) - self.assertIsInstance(p.name(), str) - if not OSX and TRAVIS: - self.assertEqual(p.exe(), self.funky_name) - else: - p.exe() + exe = p.exe() + self.assertIsInstance(exe, str) + if self.expect_exact_path_match(): + self.assertEqual(exe, self.funky_name) def test_proc_name(self): create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) if WINDOWS: - # XXX: why is this like this? + # On Windows name() is determined from exe() first, because + # it's faster; we want to overcome the internal optimization + # and test name() instead of exe(). from psutil._pswindows import py2_strencode name = py2_strencode(psutil._psplatform.cext.proc_name(subp.pid)) else: name = psutil.Process(subp.pid).name() - if not OSX and TRAVIS: + self.assertIsInstance(name, str) + if self.expect_exact_path_match(): self.assertEqual(name, os.path.basename(self.funky_name)) def test_proc_cmdline(self): create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) - self.assertIsInstance("".join(p.cmdline()), str) - if not OSX and TRAVIS: - self.assertEqual(p.cmdline(), [self.funky_name]) - else: - p.cmdline() + cmdline = p.cmdline() + if self.expect_exact_path_match(): + self.assertEqual(cmdline, [self.funky_name]) def test_proc_cwd(self): os.mkdir(self.funky_name) with chdir(self.funky_name): p = psutil.Process() + cwd = p.cwd() self.assertIsInstance(p.cwd(), str) - if not OSX and TRAVIS: - self.assertEqual(p.cwd(), self.funky_name) - else: - p.cwd() + if self.expect_exact_path_match(): + self.assertEqual(cwd, self.funky_name) # @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") def test_proc_open_files(self): @@ -757,9 +770,9 @@ def test_proc_open_files(self): if BSD and not path: # XXX # see https://github.com/giampaolo/psutil/issues/595 - self.skipTest("open_files on BSD is broken") + return self.skipTest("open_files on BSD is broken") self.assertIsInstance(path, str) - if not OSX and TRAVIS: + if self.expect_exact_path_match(): self.assertEqual(os.path.normcase(path), os.path.normcase(self.funky_name)) From 0cdf85733efa2a4c729170658a8b0d86971ee558 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 03:20:38 +0200 Subject: [PATCH 358/922] move unicode tests in their own file --- Makefile | 5 + psutil/tests/test_misc.py | 140 +----------------------- psutil/tests/test_unicode.py | 202 +++++++++++++++++++++++++++++++++++ scripts/internal/winmake.py | 7 ++ 4 files changed, 215 insertions(+), 139 deletions(-) create mode 100644 psutil/tests/test_unicode.py diff --git a/Makefile b/Makefile index e2d27c949..5d41df56d 100644 --- a/Makefile +++ b/Makefile @@ -134,6 +134,11 @@ test-misc: ${MAKE} install $(PYTHON) psutil/tests/test_misc.py +# Test misc. +test-unicode: + ${MAKE} install + $(PYTHON) psutil/tests/test_unicode.py + # Test POSIX. test-posix: ${MAKE} install diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index cfcf89d7c..ee796d811 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -15,12 +15,10 @@ import json import os import pickle -import psutil import socket import stat import sys -from psutil import BSD from psutil import LINUX from psutil import NETBSD from psutil import OPENBSD @@ -32,9 +30,7 @@ from psutil._common import supports_ipv6 from psutil._compat import PY3 from psutil.tests import APPVEYOR -from psutil.tests import ASCII_FS from psutil.tests import chdir -from psutil.tests import create_exe from psutil.tests import create_proc_children_pair from psutil.tests import get_test_subprocess from psutil.tests import importlib @@ -47,12 +43,12 @@ from psutil.tests import SCRIPTS_DIR from psutil.tests import sh from psutil.tests import TESTFN -from psutil.tests import TESTFN_UNICODE from psutil.tests import TOX from psutil.tests import TRAVIS from psutil.tests import unittest from psutil.tests import wait_for_file from psutil.tests import wait_for_pid +import psutil import psutil.tests @@ -681,139 +677,5 @@ def test_create_proc_children_pair(self): assert not psutil.tests._subprocesses_started -# =================================================================== -# --- Unicode tests -# =================================================================== - - -class _UnicodeFilesystemAPIS(unittest.TestCase): - """ - Make sure that fs-related APIs returning a string are able to - handle unicode, see: - https://github.com/giampaolo/psutil/issues/655 - This is a base class which is tested by the mixins below. - Note that on Python 2 we do not check whether the returned paths - match in case os.* functions are not able to so. - We just assume correct path handling on Python 2 is broken. In fact - it is broken for most os.* functions, see: - http://bugs.python.org/issue18695 - """ - funky_name = TESTFN_UNICODE - - @classmethod - def setUpClass(cls): - safe_rmpath(cls.funky_name) - - tearDownClass = setUpClass - - def setUp(self): - safe_rmpath(self.funky_name) - - def tearDown(self): - reap_children() - safe_rmpath(self.funky_name) - - @classmethod - def expect_exact_path_match(cls): - # Do not expect psutil to correctly handle unicode paths on - # Python 2 if os.listdir() is not able either. - return PY3 or cls.funky_name in os.listdir('.') - - def test_proc_exe(self): - create_exe(self.funky_name) - subp = get_test_subprocess(cmd=[self.funky_name]) - p = psutil.Process(subp.pid) - exe = p.exe() - self.assertIsInstance(exe, str) - if self.expect_exact_path_match(): - self.assertEqual(exe, self.funky_name) - - def test_proc_name(self): - create_exe(self.funky_name) - subp = get_test_subprocess(cmd=[self.funky_name]) - if WINDOWS: - # On Windows name() is determined from exe() first, because - # it's faster; we want to overcome the internal optimization - # and test name() instead of exe(). - from psutil._pswindows import py2_strencode - name = py2_strencode(psutil._psplatform.cext.proc_name(subp.pid)) - else: - name = psutil.Process(subp.pid).name() - self.assertIsInstance(name, str) - if self.expect_exact_path_match(): - self.assertEqual(name, os.path.basename(self.funky_name)) - - def test_proc_cmdline(self): - create_exe(self.funky_name) - subp = get_test_subprocess(cmd=[self.funky_name]) - p = psutil.Process(subp.pid) - cmdline = p.cmdline() - if self.expect_exact_path_match(): - self.assertEqual(cmdline, [self.funky_name]) - - def test_proc_cwd(self): - os.mkdir(self.funky_name) - with chdir(self.funky_name): - p = psutil.Process() - cwd = p.cwd() - self.assertIsInstance(p.cwd(), str) - if self.expect_exact_path_match(): - self.assertEqual(cwd, self.funky_name) - - # @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") - def test_proc_open_files(self): - p = psutil.Process() - start = set(p.open_files()) - with open(self.funky_name, 'wb'): - new = set(p.open_files()) - path = (new - start).pop().path - if BSD and not path: - # XXX - # see https://github.com/giampaolo/psutil/issues/595 - return self.skipTest("open_files on BSD is broken") - self.assertIsInstance(path, str) - if self.expect_exact_path_match(): - self.assertEqual(os.path.normcase(path), - os.path.normcase(self.funky_name)) - - def test_disk_usage(self): - os.mkdir(self.funky_name) - psutil.disk_usage(self.funky_name) - - -@unittest.skipIf(ASCII_FS, "ASCII fs") -class TestUnicodeFilesystemAPISMixin(_UnicodeFilesystemAPIS): - funky_name = TESTFN_UNICODE - - -class TestInvalidUnicodeFilesystemAPISMixin(_UnicodeFilesystemAPIS): - """Like above but uses an invalid UTF8 file name.""" - if PY3: - funky_name = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( - 'utf8', 'surrogateescape') - else: - funky_name = TESTFN + b"f\xc0\x80" - - -class TestUnicodeNonFsAPIS(unittest.TestCase): - """Unicode tests for non fs-related APIs.""" - - @unittest.skipUnless(hasattr(psutil.Process, "environ"), - "platform not supported") - def test_proc_environ(self): - # Note: differently from others, this test does not deal - # with fs paths. On Python 2 subprocess module is broken as - # it's not able to handle with non-ASCII env vars, so - # we use "è", which is part of the extended ASCII table - # (unicode point <= 255). - env = os.environ.copy() - funny_str = TESTFN_UNICODE if PY3 else 'è' - env['FUNNY_ARG'] = funny_str - sproc = get_test_subprocess(env=env) - p = psutil.Process(sproc.pid) - env = p.environ() - self.assertEqual(env['FUNNY_ARG'], funny_str) - - if __name__ == '__main__': run_test_module_by_name(__file__) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py new file mode 100644 index 000000000..882cc8a21 --- /dev/null +++ b/psutil/tests/test_unicode.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Notes about unicode handling in psutil +====================================== + +In psutil these are the APIs returning or dealing with a string: + +- disk_io_counters() (not tested) +- disk_partitions() (not tested) +- disk_usage(str) +- net_if_addrs() (not tested) +- net_if_stats() (not tested) +- net_io_counters() (not tested) +- Process.cmdline() +- Process.connections('unix') (not tested) +- Process.cwd() +- Process.environ() +- Process.exe() +- Process.memory_maps() (not tested) +- Process.name() +- Process.open_files() +- Process.username() (not tested) +- sensors_fans() +- sensors_temperatures() +- users() (not tested) +- WindowsService (not tested) + +In here we create a unicode path with a funky non-ASCII name and (where +possible) make psutil return it back (e.g. on name(), exe(), +open_files(), etc.) and make sure it doesn't crash with +UnicodeDecodeError. + +On Python 3 the returned path is supposed to match 100% (and this +is tested). +Not on Python 2 though, where we assume correct unicode path handling +is broken. In fact it is broken for most os.* functions, see: +http://bugs.python.org/issue18695 +There really is no way for psutil to handle unicode correctly on +Python 2 unless we make such APIs return a unicode type in certain +circumstances. +I'd rather have unicode support broken on Python 2 than having APIs +returning variable str/unicode types, see: +https://github.com/giampaolo/psutil/issues/655#issuecomment-136131180 +""" + +import os + +from psutil import BSD +from psutil import WINDOWS +from psutil._compat import PY3 +from psutil.tests import ASCII_FS +from psutil.tests import chdir +from psutil.tests import create_exe +from psutil.tests import get_test_subprocess +from psutil.tests import reap_children +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_rmpath +from psutil.tests import TESTFN +from psutil.tests import TESTFN_UNICODE +from psutil.tests import unittest +import psutil +import psutil.tests + + +# =================================================================== +# FS APIs +# =================================================================== + + +class _BaseFSAPIsTests(object): + + funky_name = None + + @classmethod + def setUpClass(cls): + safe_rmpath(cls.funky_name) + + tearDownClass = setUpClass + + def setUp(self): + safe_rmpath(self.funky_name) + + def tearDown(self): + reap_children() + safe_rmpath(self.funky_name) + + @classmethod + def expect_exact_path_match(cls): + # Do not expect psutil to correctly handle unicode paths on + # Python 2 if os.listdir() is not able either. + return PY3 or cls.funky_name in os.listdir('.') + + def test_proc_exe(self): + create_exe(self.funky_name) + subp = get_test_subprocess(cmd=[self.funky_name]) + p = psutil.Process(subp.pid) + exe = p.exe() + self.assertIsInstance(exe, str) + if self.expect_exact_path_match(): + self.assertEqual(exe, self.funky_name) + + def test_proc_name(self): + create_exe(self.funky_name) + subp = get_test_subprocess(cmd=[self.funky_name]) + if WINDOWS: + # On Windows name() is determined from exe() first, because + # it's faster; we want to overcome the internal optimization + # and test name() instead of exe(). + from psutil._pswindows import py2_strencode + name = py2_strencode(psutil._psplatform.cext.proc_name(subp.pid)) + else: + name = psutil.Process(subp.pid).name() + self.assertIsInstance(name, str) + if self.expect_exact_path_match(): + self.assertEqual(name, os.path.basename(self.funky_name)) + + def test_proc_cmdline(self): + create_exe(self.funky_name) + subp = get_test_subprocess(cmd=[self.funky_name]) + p = psutil.Process(subp.pid) + cmdline = p.cmdline() + if self.expect_exact_path_match(): + self.assertEqual(cmdline, [self.funky_name]) + + def test_proc_cwd(self): + os.mkdir(self.funky_name) + with chdir(self.funky_name): + p = psutil.Process() + cwd = p.cwd() + self.assertIsInstance(p.cwd(), str) + if self.expect_exact_path_match(): + self.assertEqual(cwd, self.funky_name) + + # @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") + def test_proc_open_files(self): + p = psutil.Process() + start = set(p.open_files()) + with open(self.funky_name, 'wb'): + new = set(p.open_files()) + path = (new - start).pop().path + if BSD and not path: + # XXX + # see https://github.com/giampaolo/psutil/issues/595 + return self.skipTest("open_files on BSD is broken") + self.assertIsInstance(path, str) + if self.expect_exact_path_match(): + self.assertEqual(os.path.normcase(path), + os.path.normcase(self.funky_name)) + + def test_disk_usage(self): + os.mkdir(self.funky_name) + psutil.disk_usage(self.funky_name) + + +@unittest.skipIf(ASCII_FS, "ASCII fs") +class TestFSAPIs(_BaseFSAPIsTests, unittest.TestCase): + """Test FS APIs with a funky, valid, UTF8 path name.""" + funky_name = TESTFN_UNICODE + + +class TestFSAPIsWithInvalidPath(_BaseFSAPIsTests, unittest.TestCase): + """Test FS APIs with a funky, invalid path name.""" + if PY3: + funky_name = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( + 'utf8', 'surrogateescape') + else: + funky_name = TESTFN + "f\xc0\x80" + + +# =================================================================== +# FS APIs +# =================================================================== + + +class TestOtherAPIS(unittest.TestCase): + """Unicode tests for non fs-related APIs.""" + + @unittest.skipUnless(hasattr(psutil.Process, "environ"), + "platform not supported") + def test_proc_environ(self): + # Note: differently from others, this test does not deal + # with fs paths. On Python 2 subprocess module is broken as + # it's not able to handle with non-ASCII env vars, so + # we use "è", which is part of the extended ASCII table + # (unicode point <= 255). + env = os.environ.copy() + funny_str = TESTFN_UNICODE if PY3 else 'è' + env['FUNNY_ARG'] = funny_str + sproc = get_test_subprocess(env=env) + p = psutil.Process(sproc.pid) + env = p.environ() + self.assertEqual(env['FUNNY_ARG'], funny_str) + + +if __name__ == '__main__': + run_test_module_by_name(__file__) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 7215560d5..c2db0fe3e 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -345,6 +345,13 @@ def test_misc(): sh("%s -m unittest -v psutil.tests.test_misc" % PYTHON) +@cmd +def test_unicode(): + """Run unicode tests""" + install() + sh("%s -m unittest -v psutil.tests.test_unicode" % PYTHON) + + @cmd def test_by_name(): """Run test by name""" From 68f8215a344b48963bfdb543c058b00418f9f971 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 03:35:26 +0200 Subject: [PATCH 359/922] speed up unicode tests --- psutil/tests/test_unicode.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 882cc8a21..3d21861b7 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -60,6 +60,7 @@ from psutil.tests import get_test_subprocess from psutil.tests import reap_children from psutil.tests import run_test_module_by_name +from psutil.tests import safe_mkdir from psutil.tests import safe_rmpath from psutil.tests import TESTFN from psutil.tests import TESTFN_UNICODE @@ -79,16 +80,18 @@ class _BaseFSAPIsTests(object): @classmethod def setUpClass(cls): + cls.funky_dirname = cls.funky_name + '2' safe_rmpath(cls.funky_name) + safe_mkdir(cls.funky_dirname) + create_exe(cls.funky_name) - tearDownClass = setUpClass - - def setUp(self): - safe_rmpath(self.funky_name) + @classmethod + def tearDownClass(cls): + safe_rmpath(cls.funky_name) + safe_rmpath(cls.funky_dirname) def tearDown(self): reap_children() - safe_rmpath(self.funky_name) @classmethod def expect_exact_path_match(cls): @@ -97,7 +100,6 @@ def expect_exact_path_match(cls): return PY3 or cls.funky_name in os.listdir('.') def test_proc_exe(self): - create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) exe = p.exe() @@ -106,7 +108,6 @@ def test_proc_exe(self): self.assertEqual(exe, self.funky_name) def test_proc_name(self): - create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) if WINDOWS: # On Windows name() is determined from exe() first, because @@ -121,7 +122,6 @@ def test_proc_name(self): self.assertEqual(name, os.path.basename(self.funky_name)) def test_proc_cmdline(self): - create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) cmdline = p.cmdline() @@ -129,13 +129,12 @@ def test_proc_cmdline(self): self.assertEqual(cmdline, [self.funky_name]) def test_proc_cwd(self): - os.mkdir(self.funky_name) - with chdir(self.funky_name): + with chdir(self.funky_dirname): p = psutil.Process() cwd = p.cwd() self.assertIsInstance(p.cwd(), str) if self.expect_exact_path_match(): - self.assertEqual(cwd, self.funky_name) + self.assertEqual(cwd, self.funky_dirname) # @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") def test_proc_open_files(self): @@ -154,8 +153,7 @@ def test_proc_open_files(self): os.path.normcase(self.funky_name)) def test_disk_usage(self): - os.mkdir(self.funky_name) - psutil.disk_usage(self.funky_name) + psutil.disk_usage(self.funky_dirname) @unittest.skipIf(ASCII_FS, "ASCII fs") From 1c081d20998a0d9990981a35dabf8190bfc8dbfa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 03:43:31 +0200 Subject: [PATCH 360/922] update docstring --- psutil/tests/test_unicode.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 3d21861b7..15d8abf60 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -11,12 +11,6 @@ In psutil these are the APIs returning or dealing with a string: -- disk_io_counters() (not tested) -- disk_partitions() (not tested) -- disk_usage(str) -- net_if_addrs() (not tested) -- net_if_stats() (not tested) -- net_io_counters() (not tested) - Process.cmdline() - Process.connections('unix') (not tested) - Process.cwd() @@ -26,6 +20,13 @@ - Process.name() - Process.open_files() - Process.username() (not tested) + +- disk_io_counters() (not tested) +- disk_partitions() (not tested) +- disk_usage(str) +- net_if_addrs() (not tested) +- net_if_stats() (not tested) +- net_io_counters() (not tested) - sensors_fans() - sensors_temperatures() - users() (not tested) @@ -33,7 +34,7 @@ In here we create a unicode path with a funky non-ASCII name and (where possible) make psutil return it back (e.g. on name(), exe(), -open_files(), etc.) and make sure it doesn't crash with +open_files(), etc.) and make sure psutil never crashes with UnicodeDecodeError. On Python 3 the returned path is supposed to match 100% (and this From 62c299e6fc70361e3b91bd520c70cee114b99ac6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 04:02:37 +0200 Subject: [PATCH 361/922] make create_exe() utility function a lot faster by copying the python exe by default instead of compiling a C dummy code --- psutil/tests/__init__.py | 9 ++++----- psutil/tests/test_unicode.py | 29 ++++++++++++----------------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 7e5dfe949..5f0db11f2 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -559,7 +559,9 @@ def chdir(dirname): def create_exe(outpath, c_code=None): """Creates an executable file in the given location.""" assert not os.path.exists(outpath), outpath - if which("gcc"): + if c_code: + if not which("gcc"): + raise ValueError("gcc is not installed") if c_code is None: c_code = textwrap.dedent( """ @@ -577,10 +579,7 @@ def create_exe(outpath, c_code=None): finally: safe_rmpath(f.name) else: - # fallback - use python's executable - if c_code is not None: - raise ValueError( - "can't specify c_code arg as gcc is not installed") + # copy python executable shutil.copyfile(sys.executable, outpath) if POSIX: st = os.stat(outpath) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 15d8abf60..1eba1291d 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -79,20 +79,12 @@ class _BaseFSAPIsTests(object): funky_name = None - @classmethod - def setUpClass(cls): - cls.funky_dirname = cls.funky_name + '2' - safe_rmpath(cls.funky_name) - safe_mkdir(cls.funky_dirname) - create_exe(cls.funky_name) - - @classmethod - def tearDownClass(cls): - safe_rmpath(cls.funky_name) - safe_rmpath(cls.funky_dirname) + def setUp(self): + safe_rmpath(self.funky_name) def tearDown(self): reap_children() + safe_rmpath(self.funky_name) @classmethod def expect_exact_path_match(cls): @@ -101,6 +93,7 @@ def expect_exact_path_match(cls): return PY3 or cls.funky_name in os.listdir('.') def test_proc_exe(self): + create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) exe = p.exe() @@ -109,6 +102,7 @@ def test_proc_exe(self): self.assertEqual(exe, self.funky_name) def test_proc_name(self): + create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) if WINDOWS: # On Windows name() is determined from exe() first, because @@ -123,6 +117,7 @@ def test_proc_name(self): self.assertEqual(name, os.path.basename(self.funky_name)) def test_proc_cmdline(self): + create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) cmdline = p.cmdline() @@ -130,14 +125,14 @@ def test_proc_cmdline(self): self.assertEqual(cmdline, [self.funky_name]) def test_proc_cwd(self): - with chdir(self.funky_dirname): + safe_mkdir(self.funky_name) + with chdir(self.funky_name): p = psutil.Process() cwd = p.cwd() self.assertIsInstance(p.cwd(), str) if self.expect_exact_path_match(): - self.assertEqual(cwd, self.funky_dirname) + self.assertEqual(cwd, self.funky_name) - # @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") def test_proc_open_files(self): p = psutil.Process() start = set(p.open_files()) @@ -145,8 +140,7 @@ def test_proc_open_files(self): new = set(p.open_files()) path = (new - start).pop().path if BSD and not path: - # XXX - # see https://github.com/giampaolo/psutil/issues/595 + # XXX - see https://github.com/giampaolo/psutil/issues/595 return self.skipTest("open_files on BSD is broken") self.assertIsInstance(path, str) if self.expect_exact_path_match(): @@ -154,7 +148,8 @@ def test_proc_open_files(self): os.path.normcase(self.funky_name)) def test_disk_usage(self): - psutil.disk_usage(self.funky_dirname) + safe_mkdir(self.funky_name) + psutil.disk_usage(self.funky_name) @unittest.skipIf(ASCII_FS, "ASCII fs") From e822d6f93f0a98a59252d95ef33b20a506d2b417 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 04:32:29 +0200 Subject: [PATCH 362/922] make clean: delete the correct test file name --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5d41df56d..e5d8a52fc 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ clean: rm -rf \ *.core \ *.egg-info \ - *\$testfile* \ + *\$testfn* \ .coverage \ .tox \ build/ \ From d65ab6edb5a73918c2574201ae6887467f549e02 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 04:43:21 +0200 Subject: [PATCH 363/922] add OSX note about UNIX sockets which cannot be deleted --- psutil/tests/test_process.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 70404fac2..8d25aee86 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1101,6 +1101,9 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): def test_connections_unix(self): def check(type): safe_rmpath(TESTFN) + # TODO: for some reason on OSX a UNIX socket cannot be + # deleted once created (EACCES) so we create a temp file + # which will remain around. :-\ tfile = tempfile.mktemp(prefix=TESTFILE_PREFIX) if OSX else TESTFN sock = socket.socket(AF_UNIX, type) with contextlib.closing(sock): From 161f4fb751c89a11a843871ccf949e092d68e782 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 04:56:20 +0200 Subject: [PATCH 364/922] add unicode test for Process.connections('unix') --- psutil/tests/test_unicode.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 1eba1291d..58d01d4e5 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -12,7 +12,7 @@ In psutil these are the APIs returning or dealing with a string: - Process.cmdline() -- Process.connections('unix') (not tested) +- Process.connections('unix') - Process.cwd() - Process.environ() - Process.exe() @@ -51,11 +51,16 @@ """ import os +import tempfile +import contextlib +import socket from psutil import BSD +from psutil import OSX from psutil import WINDOWS from psutil._compat import PY3 from psutil.tests import ASCII_FS +from psutil.tests import TESTFILE_PREFIX from psutil.tests import chdir from psutil.tests import create_exe from psutil.tests import get_test_subprocess @@ -147,6 +152,28 @@ def test_proc_open_files(self): self.assertEqual(os.path.normcase(path), os.path.normcase(self.funky_name)) + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") + def test_connections(self): + safe_rmpath(TESTFN) + # TODO: for some reason on OSX a UNIX socket cannot be + # deleted once created (EACCES) so we create a temp file + # which will remain around. :-\ + if OSX: + tfile = tempfile.mktemp( + prefix=TESTFILE_PREFIX + self.funky_name) + else: + tfile = self.funky_name + self.addCleanup(safe_rmpath, tfile) + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + with contextlib.closing(sock): + try: + sock.bind(tfile) + except (socket.error, UnicodeEncodeError): + raise unittest.SkipTest("not supported") + conn = psutil.Process().connections(kind='unix')[0] + self.assertEqual(conn.laddr, tfile) + def test_disk_usage(self): safe_mkdir(self.funky_name) psutil.disk_usage(self.funky_name) From 5562b8344c6520b2680c1935424531800ce51c34 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 05:09:41 +0200 Subject: [PATCH 365/922] refactor tests --- psutil/tests/test_unicode.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 58d01d4e5..a885668be 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -50,17 +50,16 @@ https://github.com/giampaolo/psutil/issues/655#issuecomment-136131180 """ -import os -import tempfile import contextlib +import os import socket +import tempfile from psutil import BSD from psutil import OSX from psutil import WINDOWS from psutil._compat import PY3 from psutil.tests import ASCII_FS -from psutil.tests import TESTFILE_PREFIX from psutil.tests import chdir from psutil.tests import create_exe from psutil.tests import get_test_subprocess @@ -68,6 +67,7 @@ from psutil.tests import run_test_module_by_name from psutil.tests import safe_mkdir from psutil.tests import safe_rmpath +from psutil.tests import TESTFILE_PREFIX from psutil.tests import TESTFN from psutil.tests import TESTFN_UNICODE from psutil.tests import unittest @@ -81,7 +81,6 @@ class _BaseFSAPIsTests(object): - funky_name = None def setUp(self): @@ -91,11 +90,8 @@ def tearDown(self): reap_children() safe_rmpath(self.funky_name) - @classmethod - def expect_exact_path_match(cls): - # Do not expect psutil to correctly handle unicode paths on - # Python 2 if os.listdir() is not able either. - return PY3 or cls.funky_name in os.listdir('.') + def expect_exact_path_match(self): + raise NotImplementedError("must be implemented in subclass") def test_proc_exe(self): create_exe(self.funky_name) @@ -184,6 +180,12 @@ class TestFSAPIs(_BaseFSAPIsTests, unittest.TestCase): """Test FS APIs with a funky, valid, UTF8 path name.""" funky_name = TESTFN_UNICODE + @classmethod + def expect_exact_path_match(cls): + # Do not expect psutil to correctly handle unicode paths on + # Python 2 if os.listdir() is not able either. + return PY3 or cls.funky_name in os.listdir('.') + class TestFSAPIsWithInvalidPath(_BaseFSAPIsTests, unittest.TestCase): """Test FS APIs with a funky, invalid path name.""" @@ -193,6 +195,11 @@ class TestFSAPIsWithInvalidPath(_BaseFSAPIsTests, unittest.TestCase): else: funky_name = TESTFN + "f\xc0\x80" + @classmethod + def expect_exact_path_match(cls): + # Invalid unicode names are supposed to work on Python 2. + return True + # =================================================================== # FS APIs From 32ec0b97213019cea91b0c1d7a960ec5e5040e3a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 05:48:08 +0200 Subject: [PATCH 366/922] C / BSD: refactor open_files() code --- psutil/_psutil_bsd.c | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index d37fc5da2..2c7118d12 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -462,12 +462,16 @@ psutil_cpu_times(PyObject *self, PyObject *args) { static PyObject * psutil_proc_open_files(PyObject *self, PyObject *args) { long pid; - int i, cnt; + int i; + int cnt; + int regular; + int fd; + char *path; struct kinfo_file *freep = NULL; struct kinfo_file *kif; kinfo_proc kipp; - PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; @@ -485,22 +489,25 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { for (i = 0; i < cnt; i++) { kif = &freep[i]; + #ifdef PSUTIL_FREEBSD - if ((kif->kf_type == KF_TYPE_VNODE) && - (kif->kf_vnode_type == KF_VTYPE_VREG)) - { - py_tuple = Py_BuildValue("(si)", kif->kf_path, kif->kf_fd); + regular = (kif->kf_type == KF_TYPE_VNODE) && \ + (kif->kf_vnode_type == KF_VTYPE_VREG); + fd = kif->kf_fd; + path = kif->kf_path; #elif PSUTIL_OPENBSD - if ((kif->f_type == DTYPE_VNODE) && - (kif->v_type == VREG)) - { - py_tuple = Py_BuildValue("(si)", "", kif->fd_fd); + regular = (kif->f_type == DTYPE_VNODE) && (kif->v_type == VREG); + fd = kif->fd_fd; + // XXX - it appears path is not exposed in the kinfo_file struct. + path = ""; #elif PSUTIL_NETBSD - if ((kif->ki_ftype == DTYPE_VNODE) && - (kif->ki_vtype == VREG)) - { - py_tuple = Py_BuildValue("(si)", "", kif->ki_fd); + regular = (kif->ki_ftype == DTYPE_VNODE) && (kif->ki_vtype == VREG); + fd = kif->ki_fd; + // XXX - it appears path is not exposed in the kinfo_file struct. + path = ""; #endif + if (regular == 1) { + py_tuple = Py_BuildValue("(si)", path, fd); if (py_tuple == NULL) goto error; if (PyList_Append(py_retlist, py_tuple)) From 6f9f948e96d16d4979fc5d85e3fa282d1acb991d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 06:10:33 +0200 Subject: [PATCH 367/922] #1029: [FreeBSD] Process.connections('unix') on Python 3 doesn't properly handle unicode paths and may raise UnicodeDecodeError. --- HISTORY.rst | 2 ++ psutil/arch/bsd/freebsd_socks.c | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 309397ef1..3762bd88a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -25,6 +25,8 @@ cards installed. - 1021_: [Linux] open_files() may erroneously raise NoSuchProcess instead of skipping a file which gets deleted while open files are retrieved. +- 1029_: [FreeBSD] Process.connections('unix') on Python 3 doesn't properly + handle unicode paths and may raise UnicodeDecodeError. *2017-04-10* diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 826b27f77..2a8a5440a 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -487,6 +487,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { PyObject *py_type_filter = NULL; PyObject *py_family = NULL; PyObject *py_type = NULL; + PyObject *py_unix_path = NULL; if (py_retlist == NULL) return NULL; @@ -596,12 +597,20 @@ psutil_proc_connections(PyObject *self, PyObject *args) { (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), sun->sun_path); +#if PY_MAJOR_VERSION >= 3 + py_unix_path = PyUnicode_DecodeFSDefault(path); +#else + py_unix_path = Py_BuildValue("s", path); +#endif + if (! py_unix_path) + goto error; + py_tuple = Py_BuildValue( - "(iiisOi)", + "(iiiOOi)", kif->kf_fd, kif->kf_sock_domain, kif->kf_sock_type, - path, + py_unix_path, Py_None, PSUTIL_CONN_NONE ); @@ -622,6 +631,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { Py_XDECREF(py_tuple); Py_XDECREF(py_laddr); Py_XDECREF(py_raddr); + Py_XDECREF(py_unix_path); Py_DECREF(py_retlist); if (freep != NULL) free(freep); From 1ca4b8cd875647bef7f5aaec72776a42a7c2361d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 06:25:44 +0200 Subject: [PATCH 368/922] move stuff around --- psutil/tests/__init__.py | 126 ++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 60 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 5f0db11f2..42db5e1de 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -77,9 +77,8 @@ # classes 'ThreadTask' # test utils - 'check_connection_ntuple', 'check_net_address', 'unittest', 'cleanup', - 'skip_on_access_denied', 'skip_on_not_implemented', 'retry_before_failing', - 'run_test_module_by_name', + 'unittest', 'cleanup', 'skip_on_access_denied', 'skip_on_not_implemented', + 'retry_before_failing', 'run_test_module_by_name', # install utils 'install_pip', 'install_test_deps', # fs utils @@ -91,6 +90,8 @@ 'get_winver', 'get_kernel_version', # sync primitives 'call_until', 'wait_for_pid', 'wait_for_file', + # network + 'check_connection_ntuple', 'check_net_address', # others 'warn', ] @@ -701,6 +702,68 @@ def wrapper(*args, **kwargs): return decorator +def cleanup(): + for name in os.listdir('.'): + if name.startswith(TESTFILE_PREFIX): + try: + safe_rmpath(name) + except UnicodeEncodeError as exc: + warn(exc) + for path in _testfiles: + safe_rmpath(path) + + +atexit.register(cleanup) +atexit.register(lambda: DEVNULL.close()) + + +# =================================================================== +# --- install +# =================================================================== + + +def install_pip(): + """Install pip. Returns the exit code of the subprocess.""" + try: + import pip # NOQA + except ImportError: + f = tempfile.NamedTemporaryFile(suffix='.py') + with contextlib.closing(f): + print("downloading %s to %s" % (GET_PIP_URL, f.name)) + if hasattr(ssl, '_create_unverified_context'): + ctx = ssl._create_unverified_context() + else: + ctx = None + kwargs = dict(context=ctx) if ctx else {} + req = urlopen(GET_PIP_URL, **kwargs) + data = req.read() + f.write(data) + f.flush() + + print("installing pip") + code = os.system('%s %s --user' % (sys.executable, f.name)) + return code + + +def install_test_deps(deps=None): + """Install test dependencies via pip.""" + if deps is None: + deps = TEST_DEPS + deps = set(deps) + if deps: + is_venv = hasattr(sys, 'real_prefix') + opts = "--user" if not is_venv else "" + install_pip() + code = os.system('%s -m pip install %s --upgrade %s' % ( + sys.executable, opts, " ".join(deps))) + return code + + +# =================================================================== +# --- network +# =================================================================== + + def check_net_address(addr, family): """Check a net address validity. Supported families are IPv4, IPv6 and MAC addresses. @@ -787,63 +850,6 @@ def check_connection_ntuple(conn): assert dupsock.type == conn.type -def cleanup(): - for name in os.listdir('.'): - if name.startswith(TESTFILE_PREFIX): - try: - safe_rmpath(name) - except UnicodeEncodeError as exc: - warn(exc) - for path in _testfiles: - safe_rmpath(path) - - -atexit.register(cleanup) -atexit.register(lambda: DEVNULL.close()) - - -# =================================================================== -# --- install -# =================================================================== - - -def install_pip(): - """Install pip. Returns the exit code of the subprocess.""" - try: - import pip # NOQA - except ImportError: - f = tempfile.NamedTemporaryFile(suffix='.py') - with contextlib.closing(f): - print("downloading %s to %s" % (GET_PIP_URL, f.name)) - if hasattr(ssl, '_create_unverified_context'): - ctx = ssl._create_unverified_context() - else: - ctx = None - kwargs = dict(context=ctx) if ctx else {} - req = urlopen(GET_PIP_URL, **kwargs) - data = req.read() - f.write(data) - f.flush() - - print("installing pip") - code = os.system('%s %s --user' % (sys.executable, f.name)) - return code - - -def install_test_deps(deps=None): - """Install test dependencies via pip.""" - if deps is None: - deps = TEST_DEPS - deps = set(deps) - if deps: - is_venv = hasattr(sys, 'real_prefix') - opts = "--user" if not is_venv else "" - install_pip() - code = os.system('%s -m pip install %s --upgrade %s' % ( - sys.executable, opts, " ".join(deps))) - return code - - # =================================================================== # --- others # =================================================================== From ca10f641a111a2390fdeca4aaecdc920c3ef816d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 06:45:48 +0200 Subject: [PATCH 369/922] define a reusable bind_unix_socket() test utility --- psutil/tests/__init__.py | 24 ++++++++++++++++++++++++ psutil/tests/test_process.py | 35 ++++++++++++++++------------------- psutil/tests/test_unicode.py | 32 +++++++++----------------------- 3 files changed, 49 insertions(+), 42 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 42db5e1de..04f434eec 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -43,6 +43,7 @@ import psutil from psutil import LINUX +from psutil import OSX from psutil import POSIX from psutil import WINDOWS from psutil._compat import PY3 @@ -850,6 +851,29 @@ def check_connection_ntuple(conn): assert dupsock.type == conn.type +def bind_unix_socket(type=socket.SOCK_STREAM, suffix="", mode=0o600): + """Creates a listening unix socket. + Return a (sock, filemame) tuple. + """ + # TODO: for some reason on OSX a UNIX socket cannot be + # deleted once created (EACCES) so we create a temp file + # which will remain around. :-\ + if OSX: + file = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) + else: + file = TESTFN + suffix + assert not os.path.exists(file), file + sock = socket.socket(socket.AF_UNIX, type) + try: + sock.bind(file) + except Exception: + sock.close() + raise + if mode is not None: + os.chmod(file, mode) + return (sock, file) + + # =================================================================== # --- others # =================================================================== diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 8d25aee86..3e971cc84 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -44,6 +44,7 @@ from psutil._compat import unicode from psutil.tests import AF_UNIX from psutil.tests import APPVEYOR +from psutil.tests import bind_unix_socket from psutil.tests import call_until from psutil.tests import check_connection_ntuple from psutil.tests import create_exe @@ -1101,25 +1102,21 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): def test_connections_unix(self): def check(type): safe_rmpath(TESTFN) - # TODO: for some reason on OSX a UNIX socket cannot be - # deleted once created (EACCES) so we create a temp file - # which will remain around. :-\ - tfile = tempfile.mktemp(prefix=TESTFILE_PREFIX) if OSX else TESTFN - sock = socket.socket(AF_UNIX, type) - with contextlib.closing(sock): - sock.bind(tfile) - cons = psutil.Process().connections(kind='unix') - conn = cons[0] - check_connection_ntuple(conn) - if conn.fd != -1: # != sunos and windows - self.assertEqual(conn.fd, sock.fileno()) - self.assertEqual(conn.family, AF_UNIX) - self.assertEqual(conn.type, type) - self.assertEqual(conn.laddr, tfile) - if not SUNOS: - # XXX Solaris can't retrieve system-wide UNIX - # sockets. - self.compare_proc_sys_cons(os.getpid(), cons) + sock, name = bind_unix_socket(type=type) + self.addCleanup(sock.close) + self.addCleanup(safe_rmpath, name) + cons = psutil.Process().connections(kind='unix') + conn = cons[0] + check_connection_ntuple(conn) + if conn.fd != -1: # != sunos and windows + self.assertEqual(conn.fd, sock.fileno()) + self.assertEqual(conn.family, AF_UNIX) + self.assertEqual(conn.type, type) + self.assertEqual(conn.laddr, name) + if not SUNOS: + # XXX Solaris can't retrieve system-wide UNIX + # sockets. + self.compare_proc_sys_cons(os.getpid(), cons) check(SOCK_STREAM) check(SOCK_DGRAM) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index a885668be..f79131bb6 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -50,16 +50,14 @@ https://github.com/giampaolo/psutil/issues/655#issuecomment-136131180 """ -import contextlib import os import socket -import tempfile from psutil import BSD -from psutil import OSX from psutil import WINDOWS from psutil._compat import PY3 from psutil.tests import ASCII_FS +from psutil.tests import bind_unix_socket from psutil.tests import chdir from psutil.tests import create_exe from psutil.tests import get_test_subprocess @@ -67,7 +65,6 @@ from psutil.tests import run_test_module_by_name from psutil.tests import safe_mkdir from psutil.tests import safe_rmpath -from psutil.tests import TESTFILE_PREFIX from psutil.tests import TESTFN from psutil.tests import TESTFN_UNICODE from psutil.tests import unittest @@ -150,25 +147,14 @@ def test_proc_open_files(self): @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") def test_connections(self): - safe_rmpath(TESTFN) - # TODO: for some reason on OSX a UNIX socket cannot be - # deleted once created (EACCES) so we create a temp file - # which will remain around. :-\ - if OSX: - tfile = tempfile.mktemp( - prefix=TESTFILE_PREFIX + self.funky_name) - else: - tfile = self.funky_name - self.addCleanup(safe_rmpath, tfile) - - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - with contextlib.closing(sock): - try: - sock.bind(tfile) - except (socket.error, UnicodeEncodeError): - raise unittest.SkipTest("not supported") - conn = psutil.Process().connections(kind='unix')[0] - self.assertEqual(conn.laddr, tfile) + try: + sock, name = bind_unix_socket(suffix=self.funky_name) + except (socket.error, UnicodeEncodeError): + raise unittest.SkipTest("not supported") + self.addCleanup(safe_rmpath, name) + self.addCleanup(sock.close) + conn = psutil.Process().connections(kind='unix')[0] + self.assertEqual(conn.laddr, name) def test_disk_usage(self): safe_mkdir(self.funky_name) From 60e11eb60b499e1485a4eea90e032ec9a688f9e1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 06:52:47 +0200 Subject: [PATCH 370/922] bind_unix_socket() change signature --- psutil/tests/__init__.py | 21 +++++++++++++-------- psutil/tests/test_process.py | 3 ++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 04f434eec..ea8e2e82b 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -851,27 +851,32 @@ def check_connection_ntuple(conn): assert dupsock.type == conn.type -def bind_unix_socket(type=socket.SOCK_STREAM, suffix="", mode=0o600): +def bind_unix_socket(type=socket.SOCK_STREAM, name=None, suffix="", + mode=0o600): """Creates a listening unix socket. Return a (sock, filemame) tuple. """ # TODO: for some reason on OSX a UNIX socket cannot be # deleted once created (EACCES) so we create a temp file # which will remain around. :-\ - if OSX: - file = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) + if not name: + if OSX: + name = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) + else: + name = TESTFN + suffix else: - file = TESTFN + suffix - assert not os.path.exists(file), file + if suffix: + raise ValueError("name and suffix aregs are mutually exclusive") + assert not os.path.exists(name), name sock = socket.socket(socket.AF_UNIX, type) try: - sock.bind(file) + sock.bind(name) except Exception: sock.close() raise if mode is not None: - os.chmod(file, mode) - return (sock, file) + os.chmod(name, mode) + return (sock, name) # =================================================================== diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 3e971cc84..683ac57ca 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1102,7 +1102,8 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): def test_connections_unix(self): def check(type): safe_rmpath(TESTFN) - sock, name = bind_unix_socket(type=type) + sock, name = bind_unix_socket( + type=type, name=None if OSX else TESTFN) self.addCleanup(sock.close) self.addCleanup(safe_rmpath, name) cons = psutil.Process().connections(kind='unix') From 76dd68eecbcdce608dfc46b37a82f2ab83927b72 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 06:56:59 +0200 Subject: [PATCH 371/922] reuse bind_unix_socket() --- psutil/tests/__init__.py | 3 ++- psutil/tests/test_process.py | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index ea8e2e82b..69ef2ba6a 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -145,7 +145,6 @@ # --- misc -AF_UNIX = getattr(socket, "AF_UNIX", None) PYTHON = os.path.realpath(sys.executable) DEVNULL = open(os.devnull, 'r+') VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) @@ -793,6 +792,7 @@ def check_net_address(addr, family): def check_connection_ntuple(conn): """Check validity of a connection namedtuple.""" + AF_UNIX = getattr(socket, "AF_UNIX", object()) valid_conn_states = [getattr(psutil, x) for x in dir(psutil) if x.startswith('CONN_')] assert conn[0] == conn.fd @@ -869,6 +869,7 @@ def bind_unix_socket(type=socket.SOCK_STREAM, name=None, suffix="", raise ValueError("name and suffix aregs are mutually exclusive") assert not os.path.exists(name), name sock = socket.socket(socket.AF_UNIX, type) + sock.settimeout(GLOBAL_TIMEOUT) try: sock.bind(name) except Exception: diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 683ac57ca..78d01ae6a 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -42,7 +42,6 @@ from psutil._compat import long from psutil._compat import PY3 from psutil._compat import unicode -from psutil.tests import AF_UNIX from psutil.tests import APPVEYOR from psutil.tests import bind_unix_socket from psutil.tests import call_until @@ -1111,7 +1110,7 @@ def check(type): check_connection_ntuple(conn) if conn.fd != -1: # != sunos and windows self.assertEqual(conn.fd, sock.fileno()) - self.assertEqual(conn.family, AF_UNIX) + self.assertEqual(conn.family, socket.AF_UNIX) self.assertEqual(conn.type, type) self.assertEqual(conn.laddr, name) if not SUNOS: @@ -1448,10 +1447,9 @@ def succeed_or_zombie_p_exc(fun, *args, **kwargs): pid = bytes(str(os.getpid()), 'ascii') s.sendall(pid) """ % unix_file) - with contextlib.closing(socket.socket(socket.AF_UNIX)) as sock: + sock, _ = bind_unix_socket(name=unix_file) + with contextlib.closing(sock): try: - sock.settimeout(GLOBAL_TIMEOUT) - sock.bind(unix_file) sock.listen(1) pyrun(src) conn, _ = sock.accept() From 4ec6c5431f936c31af43d4d2f637e06d2c3b7619 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 07:01:25 +0200 Subject: [PATCH 372/922] minor refactoring --- psutil/tests/test_system.py | 3 ++- psutil/tests/test_unicode.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index b30d905e9..a4d48b8c7 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -526,9 +526,10 @@ def find_mount_point(path): @skip_on_access_denied() def test_net_connections(self): def check(cons, families, types_): + AF_UNIX = getattr(socket, 'AF_UNIX', object()) for conn in cons: self.assertIn(conn.family, families, msg=conn) - if conn.family != getattr(socket, 'AF_UNIX', object()): + if conn.family != AF_UNIX: self.assertIn(conn.type, types_, msg=conn) self.assertIsInstance(conn.status, (str, unicode)) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index f79131bb6..31e29b170 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -24,6 +24,7 @@ - disk_io_counters() (not tested) - disk_partitions() (not tested) - disk_usage(str) +- net_connections('unix') (not tested) - net_if_addrs() (not tested) - net_if_stats() (not tested) - net_io_counters() (not tested) @@ -146,7 +147,7 @@ def test_proc_open_files(self): os.path.normcase(self.funky_name)) @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") - def test_connections(self): + def test_proc_connections(self): try: sock, name = bind_unix_socket(suffix=self.funky_name) except (socket.error, UnicodeEncodeError): From 9f34016db45a960286c5a3d842d260d5076b04f3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 07:40:29 +0200 Subject: [PATCH 373/922] fix test; C: reuse variable --- psutil/arch/bsd/freebsd_socks.c | 10 ++++------ psutil/tests/test_process.py | 27 ++++++++++++++------------- psutil/tests/test_unicode.py | 10 +++++++--- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 2a8a5440a..b30fa8f81 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -487,7 +487,6 @@ psutil_proc_connections(PyObject *self, PyObject *args) { PyObject *py_type_filter = NULL; PyObject *py_family = NULL; PyObject *py_type = NULL; - PyObject *py_unix_path = NULL; if (py_retlist == NULL) return NULL; @@ -598,11 +597,11 @@ psutil_proc_connections(PyObject *self, PyObject *args) { sun->sun_path); #if PY_MAJOR_VERSION >= 3 - py_unix_path = PyUnicode_DecodeFSDefault(path); + py_laddr = PyUnicode_DecodeFSDefault(path); #else - py_unix_path = Py_BuildValue("s", path); + py_laddr = Py_BuildValue("s", path); #endif - if (! py_unix_path) + if (! py_laddr) goto error; py_tuple = Py_BuildValue( @@ -610,7 +609,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { kif->kf_fd, kif->kf_sock_domain, kif->kf_sock_type, - py_unix_path, + py_laddr, Py_None, PSUTIL_CONN_NONE ); @@ -631,7 +630,6 @@ psutil_proc_connections(PyObject *self, PyObject *args) { Py_XDECREF(py_tuple); Py_XDECREF(py_laddr); Py_XDECREF(py_raddr); - Py_XDECREF(py_unix_path); Py_DECREF(py_retlist); if (freep != NULL) free(freep); diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 78d01ae6a..75b92767b 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1103,20 +1103,21 @@ def check(type): safe_rmpath(TESTFN) sock, name = bind_unix_socket( type=type, name=None if OSX else TESTFN) - self.addCleanup(sock.close) self.addCleanup(safe_rmpath, name) - cons = psutil.Process().connections(kind='unix') - conn = cons[0] - check_connection_ntuple(conn) - if conn.fd != -1: # != sunos and windows - self.assertEqual(conn.fd, sock.fileno()) - self.assertEqual(conn.family, socket.AF_UNIX) - self.assertEqual(conn.type, type) - self.assertEqual(conn.laddr, name) - if not SUNOS: - # XXX Solaris can't retrieve system-wide UNIX - # sockets. - self.compare_proc_sys_cons(os.getpid(), cons) + with contextlib.closing(sock): + cons = psutil.Process().connections(kind='unix') + self.assertEqual(len(cons), 1) + conn = cons[0] + check_connection_ntuple(conn) + if conn.fd != -1: # != sunos and windows + self.assertEqual(conn.fd, sock.fileno()) + self.assertEqual(conn.family, socket.AF_UNIX) + self.assertEqual(conn.type, type) + self.assertEqual(conn.laddr, name) + if not SUNOS: + # XXX Solaris can't retrieve system-wide UNIX + # sockets. + self.compare_proc_sys_cons(os.getpid(), cons) check(SOCK_STREAM) check(SOCK_DGRAM) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 31e29b170..e036507d2 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -149,9 +149,13 @@ def test_proc_open_files(self): @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") def test_proc_connections(self): try: - sock, name = bind_unix_socket(suffix=self.funky_name) - except (socket.error, UnicodeEncodeError): - raise unittest.SkipTest("not supported") + sock, name = bind_unix_socket( + suffix=os.path.basename(self.funky_name)) + except UnicodeDecodeError: + if PY3: + raise + else: + raise unittest.SkipTest("not supported") self.addCleanup(safe_rmpath, name) self.addCleanup(sock.close) conn = psutil.Process().connections(kind='unix')[0] From 8f7ac93eec042799e6f866cc8b816dab3ebf9ac6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 07:47:12 +0200 Subject: [PATCH 374/922] #1029: fix encoding error for proc.econnections('unix') on OSX --- HISTORY.rst | 4 ++-- psutil/_psutil_osx.c | 28 +++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3762bd88a..dc49d65fa 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -25,8 +25,8 @@ cards installed. - 1021_: [Linux] open_files() may erroneously raise NoSuchProcess instead of skipping a file which gets deleted while open files are retrieved. -- 1029_: [FreeBSD] Process.connections('unix') on Python 3 doesn't properly - handle unicode paths and may raise UnicodeDecodeError. +- 1029_: [OSX, FreeBSD] Process.connections('unix') on Python 3 doesn't + properly handle unicode paths and may raise UnicodeDecodeError. *2017-04-10* diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 726e5a3ba..e7642421b 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1350,12 +1350,34 @@ psutil_proc_connections(PyObject *self, PyObject *args) { Py_DECREF(py_tuple); } else if (family == AF_UNIX) { + // decode laddr + #if PY_MAJOR_VERSION >= 3 + py_laddr = PyUnicode_DecodeFSDefault( + si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path); + #else + py_laddr = Py_BuildValue("s", + si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path); + #endif + if (!py_laddr) + goto error; + + // decode raddr + #if PY_MAJOR_VERSION >= 3 + py_raddr = PyUnicode_DecodeFSDefault( + si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path); + #else + py_raddr = Py_BuildValue("s", + si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path); + #endif + if (!py_raddr) + goto error; + // construct the python list py_tuple = Py_BuildValue( - "(iiissi)", + "(iiiOOi)", fd, family, type, - si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path, - si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path, + py_laddr, + py_raddr, PSUTIL_CONN_NONE); if (!py_tuple) goto error; From bd66132ff41350c176b480821daeedec93b13301 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 07:55:49 +0200 Subject: [PATCH 375/922] refactor C code --- psutil/_psutil_osx.c | 20 ++++++++++---------- psutil/tests/test_unicode.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index e7642421b..90391ddc0 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1351,24 +1351,24 @@ psutil_proc_connections(PyObject *self, PyObject *args) { } else if (family == AF_UNIX) { // decode laddr - #if PY_MAJOR_VERSION >= 3 +#if PY_MAJOR_VERSION >= 3 py_laddr = PyUnicode_DecodeFSDefault( - si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path); - #else +#else py_laddr = Py_BuildValue("s", - si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path); - #endif +#endif + si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path + ); if (!py_laddr) goto error; // decode raddr - #if PY_MAJOR_VERSION >= 3 +#if PY_MAJOR_VERSION >= 3 py_raddr = PyUnicode_DecodeFSDefault( - si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path); - #else +#else py_raddr = Py_BuildValue("s", - si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path); - #endif +#endif + si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path + ); if (!py_raddr) goto error; diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index e036507d2..27b0232e7 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -151,7 +151,7 @@ def test_proc_connections(self): try: sock, name = bind_unix_socket( suffix=os.path.basename(self.funky_name)) - except UnicodeDecodeError: + except UnicodeEncodeError: if PY3: raise else: From adbeb13b4fdc6d324aa35395ebd9285c3f3d3282 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 27 Apr 2017 23:08:36 -0700 Subject: [PATCH 376/922] add unicode test for net_connections() --- psutil/tests/test_unicode.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 27b0232e7..90f0e9d9f 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -24,12 +24,12 @@ - disk_io_counters() (not tested) - disk_partitions() (not tested) - disk_usage(str) -- net_connections('unix') (not tested) +- net_connections('unix') - net_if_addrs() (not tested) - net_if_stats() (not tested) - net_io_counters() (not tested) -- sensors_fans() -- sensors_temperatures() +- sensors_fans() (not tested) +- sensors_temperatures() (not tested) - users() (not tested) - WindowsService (not tested) @@ -66,6 +66,8 @@ from psutil.tests import run_test_module_by_name from psutil.tests import safe_mkdir from psutil.tests import safe_rmpath +from psutil.tests import skip_on_access_denied +from psutil.tests import TESTFILE_PREFIX from psutil.tests import TESTFN from psutil.tests import TESTFN_UNICODE from psutil.tests import unittest @@ -161,6 +163,29 @@ def test_proc_connections(self): conn = psutil.Process().connections(kind='unix')[0] self.assertEqual(conn.laddr, name) + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") + @skip_on_access_denied() + def test_net_connections(self): + def find_sock(cons): + for conn in cons: + if os.path.basename(conn.laddr).startswith(TESTFILE_PREFIX): + return conn + raise ValueError("connection not found") + + try: + sock, name = bind_unix_socket( + suffix=os.path.basename(self.funky_name)) + except UnicodeEncodeError: + if PY3: + raise + else: + raise unittest.SkipTest("not supported") + self.addCleanup(safe_rmpath, name) + self.addCleanup(sock.close) + cons = psutil.net_connections(kind='unix') + conn = find_sock(cons) + self.assertEqual(conn.laddr, name) + def test_disk_usage(self): safe_mkdir(self.funky_name) psutil.disk_usage(self.funky_name) From a5c59e18707fb19dc1e838421daea34b51f2838f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 09:41:02 +0200 Subject: [PATCH 377/922] 1032: add utiliy test function to bind 2 unix sockets --- psutil/tests/__init__.py | 27 ++++++++++++++++++++++----- psutil/tests/test_misc.py | 16 ++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 69ef2ba6a..f96bfbf00 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -851,14 +851,14 @@ def check_connection_ntuple(conn): assert dupsock.type == conn.type -def bind_unix_socket(type=socket.SOCK_STREAM, name=None, suffix="", - mode=0o600): +def bind_unix_socket(type=socket.SOCK_STREAM, name=None, suffix=""): """Creates a listening unix socket. Return a (sock, filemame) tuple. """ # TODO: for some reason on OSX a UNIX socket cannot be # deleted once created (EACCES) so we create a temp file # which will remain around. :-\ + assert psutil.POSIX, "not a POSIX system" if not name: if OSX: name = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) @@ -867,19 +867,36 @@ def bind_unix_socket(type=socket.SOCK_STREAM, name=None, suffix="", else: if suffix: raise ValueError("name and suffix aregs are mutually exclusive") + safe_rmpath(name) assert not os.path.exists(name), name sock = socket.socket(socket.AF_UNIX, type) - sock.settimeout(GLOBAL_TIMEOUT) try: sock.bind(name) except Exception: sock.close() raise - if mode is not None: - os.chmod(name, mode) + os.chmod(name, 0o600) return (sock, name) +def unix_socketpair(name=None, suffix=""): + """Build a pair of UNIX sockets connected to each other through + the same UNIX file name. + Return a (server_sock, client_sock, filename) tuple. + """ + assert psutil.POSIX, "not a POSIX system" + listener, name = bind_unix_socket(name=name, suffix=suffix) + listener.setblocking(0) + listener.listen(1) + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.setblocking(0) + # XXX - for some reason I don't have to select() even if they + # are non-blocking sockets. Why doesn't this raise EAGAIN? + client.connect(name) + # new = listener.accept() + return (listener, client, name) + + # =================================================================== # --- others # =================================================================== diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index ee796d811..5b5f365a2 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -46,6 +46,7 @@ from psutil.tests import TOX from psutil.tests import TRAVIS from psutil.tests import unittest +from psutil.tests import unix_socketpair from psutil.tests import wait_for_file from psutil.tests import wait_for_pid import psutil @@ -677,5 +678,20 @@ def test_create_proc_children_pair(self): assert not psutil.tests._subprocesses_started +class TestNetUtils(unittest.TestCase): + + @unittest.skipUnless(POSIX, "POSIX only") + def test_unix_socketpair(self): + p = psutil.Process() + num_fds = p.num_fds() + assert not p.connections(kind='unix') + ssock, csock, name = unix_socketpair() + self.addCleanup(safe_rmpath, name) + assert os.path.exists(name) + assert stat.S_ISSOCK(os.stat(name).st_mode) + self.assertEqual(p.num_fds() - num_fds, 2) + self.assertEqual(len(p.connections(kind='unix')), 2) + + if __name__ == '__main__': run_test_module_by_name(__file__) From 009ed08a51b8f42c2ba188b12c32bad5d0441a31 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 10:02:15 +0200 Subject: [PATCH 378/922] 1032: test process connections against 2 unix sockets --- psutil/tests/test_unicode.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 90f0e9d9f..263de1de8 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -71,6 +71,7 @@ from psutil.tests import TESTFN from psutil.tests import TESTFN_UNICODE from psutil.tests import unittest +from psutil.tests import unix_socketpair import psutil import psutil.tests @@ -151,17 +152,23 @@ def test_proc_open_files(self): @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") def test_proc_connections(self): try: - sock, name = bind_unix_socket( + server, client, name = unix_socketpair( suffix=os.path.basename(self.funky_name)) except UnicodeEncodeError: if PY3: raise else: raise unittest.SkipTest("not supported") + self.addCleanup(safe_rmpath, name) - self.addCleanup(sock.close) - conn = psutil.Process().connections(kind='unix')[0] - self.assertEqual(conn.laddr, name) + self.addCleanup(client.close) + self.addCleanup(server.close) + cons = psutil.Process().connections(kind='unix') + self.assertEqual(len(cons), 2) + cmap = dict([(x.fd, x) for x in cons]) + self.assertEqual(cmap[server.fileno()].laddr, name) + if cmap[client.fileno()].laddr: + self.assertEqual(cmap[client.fileno()].laddr, name) @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") @skip_on_access_denied() From c8f9c054551e3cbfdeb540927feae606f3a0d702 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 23:26:41 +0200 Subject: [PATCH 379/922] refactor UNIX tests --- psutil/tests/__init__.py | 45 ++++++++++++------------- psutil/tests/test_process.py | 40 +++++++++++----------- psutil/tests/test_unicode.py | 65 +++++++++++++++++++----------------- 3 files changed, 76 insertions(+), 74 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index f96bfbf00..c129224d6 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -43,7 +43,6 @@ import psutil from psutil import LINUX -from psutil import OSX from psutil import POSIX from psutil import WINDOWS from psutil._compat import PY3 @@ -851,23 +850,24 @@ def check_connection_ntuple(conn): assert dupsock.type == conn.type -def bind_unix_socket(type=socket.SOCK_STREAM, name=None, suffix=""): +@contextlib.contextmanager +def unix_socket_path(suffix=""): + assert psutil.POSIX, "not a POSIX system" + path = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) + try: + yield path + finally: + try: + os.unlink(path) + except OSError: + pass + + +def bind_unix_socket(name, type=socket.SOCK_STREAM): """Creates a listening unix socket. Return a (sock, filemame) tuple. """ - # TODO: for some reason on OSX a UNIX socket cannot be - # deleted once created (EACCES) so we create a temp file - # which will remain around. :-\ assert psutil.POSIX, "not a POSIX system" - if not name: - if OSX: - name = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) - else: - name = TESTFN + suffix - else: - if suffix: - raise ValueError("name and suffix aregs are mutually exclusive") - safe_rmpath(name) assert not os.path.exists(name), name sock = socket.socket(socket.AF_UNIX, type) try: @@ -875,26 +875,23 @@ def bind_unix_socket(type=socket.SOCK_STREAM, name=None, suffix=""): except Exception: sock.close() raise - os.chmod(name, 0o600) - return (sock, name) + return sock -def unix_socketpair(name=None, suffix=""): +def unix_socketpair(name): """Build a pair of UNIX sockets connected to each other through the same UNIX file name. Return a (server_sock, client_sock, filename) tuple. """ assert psutil.POSIX, "not a POSIX system" - listener, name = bind_unix_socket(name=name, suffix=suffix) - listener.setblocking(0) - listener.listen(1) + server = bind_unix_socket(name, type=socket.SOCK_STREAM) + server.setblocking(0) + server.listen(1) client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) client.setblocking(0) - # XXX - for some reason I don't have to select() even if they - # are non-blocking sockets. Why doesn't this raise EAGAIN? client.connect(name) - # new = listener.accept() - return (listener, client, name) + # new = server.accept() + return (server, client) # =================================================================== diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 75b92767b..b3c77464d 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -70,6 +70,7 @@ from psutil.tests import TOX from psutil.tests import TRAVIS from psutil.tests import unittest +from psutil.tests import unix_socket_path from psutil.tests import VALID_PROC_STATUSES from psutil.tests import wait_for_file from psutil.tests import wait_for_pid @@ -1101,23 +1102,23 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): def test_connections_unix(self): def check(type): safe_rmpath(TESTFN) - sock, name = bind_unix_socket( - type=type, name=None if OSX else TESTFN) - self.addCleanup(safe_rmpath, name) - with contextlib.closing(sock): - cons = psutil.Process().connections(kind='unix') - self.assertEqual(len(cons), 1) - conn = cons[0] - check_connection_ntuple(conn) - if conn.fd != -1: # != sunos and windows - self.assertEqual(conn.fd, sock.fileno()) - self.assertEqual(conn.family, socket.AF_UNIX) - self.assertEqual(conn.type, type) - self.assertEqual(conn.laddr, name) - if not SUNOS: - # XXX Solaris can't retrieve system-wide UNIX - # sockets. - self.compare_proc_sys_cons(os.getpid(), cons) + with unix_socket_path() as name: + sock = bind_unix_socket(name, type=type) + self.addCleanup(safe_rmpath, name) + with contextlib.closing(sock): + cons = psutil.Process().connections(kind='unix') + self.assertEqual(len(cons), 1) + conn = cons[0] + check_connection_ntuple(conn) + if conn.fd != -1: # != sunos and windows + self.assertEqual(conn.fd, sock.fileno()) + self.assertEqual(conn.family, socket.AF_UNIX) + self.assertEqual(conn.type, type) + self.assertEqual(conn.laddr, name) + if not SUNOS: + # XXX Solaris can't retrieve system-wide UNIX + # sockets. + self.compare_proc_sys_cons(os.getpid(), cons) check(SOCK_STREAM) check(SOCK_DGRAM) @@ -1448,9 +1449,10 @@ def succeed_or_zombie_p_exc(fun, *args, **kwargs): pid = bytes(str(os.getpid()), 'ascii') s.sendall(pid) """ % unix_file) - sock, _ = bind_unix_socket(name=unix_file) - with contextlib.closing(sock): + with contextlib.closing(socket.socket(socket.AF_UNIX)) as sock: try: + sock.settimeout(GLOBAL_TIMEOUT) + sock.bind(unix_file) sock.listen(1) pyrun(src) conn, _ = sock.accept() diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 263de1de8..c7e915ce2 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -72,6 +72,7 @@ from psutil.tests import TESTFN_UNICODE from psutil.tests import unittest from psutil.tests import unix_socketpair +from psutil.tests import unix_socket_path import psutil import psutil.tests @@ -151,24 +152,25 @@ def test_proc_open_files(self): @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") def test_proc_connections(self): - try: - server, client, name = unix_socketpair( - suffix=os.path.basename(self.funky_name)) - except UnicodeEncodeError: - if PY3: - raise - else: - raise unittest.SkipTest("not supported") - - self.addCleanup(safe_rmpath, name) - self.addCleanup(client.close) - self.addCleanup(server.close) - cons = psutil.Process().connections(kind='unix') - self.assertEqual(len(cons), 2) - cmap = dict([(x.fd, x) for x in cons]) - self.assertEqual(cmap[server.fileno()].laddr, name) - if cmap[client.fileno()].laddr: - self.assertEqual(cmap[client.fileno()].laddr, name) + with unix_socket_path( + suffix=os.path.basename(self.funky_name)) as name: + try: + server, client = unix_socketpair(name) + except UnicodeEncodeError: + if PY3: + raise + else: + raise unittest.SkipTest("not supported") + + self.addCleanup(safe_rmpath, name) + self.addCleanup(client.close) + self.addCleanup(server.close) + cons = psutil.Process().connections(kind='unix') + self.assertEqual(len(cons), 2) + cmap = dict([(x.fd, x) for x in cons]) + self.assertEqual(cmap[server.fileno()].laddr, name) + if cmap[client.fileno()].laddr: + self.assertEqual(cmap[client.fileno()].laddr, name) @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") @skip_on_access_denied() @@ -179,19 +181,20 @@ def find_sock(cons): return conn raise ValueError("connection not found") - try: - sock, name = bind_unix_socket( - suffix=os.path.basename(self.funky_name)) - except UnicodeEncodeError: - if PY3: - raise - else: - raise unittest.SkipTest("not supported") - self.addCleanup(safe_rmpath, name) - self.addCleanup(sock.close) - cons = psutil.net_connections(kind='unix') - conn = find_sock(cons) - self.assertEqual(conn.laddr, name) + with unix_socket_path( + suffix=os.path.basename(self.funky_name)) as name: + try: + sock = bind_unix_socket(name) + except UnicodeEncodeError: + if PY3: + raise + else: + raise unittest.SkipTest("not supported") + self.addCleanup(safe_rmpath, name) + self.addCleanup(sock.close) + cons = psutil.net_connections(kind='unix') + conn = find_sock(cons) + self.assertEqual(conn.laddr, name) def test_disk_usage(self): safe_mkdir(self.funky_name) From 22077c5755e5897306be92071885300238127100 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 23:44:55 +0200 Subject: [PATCH 380/922] fix misc tests --- psutil/tests/test_misc.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 5b5f365a2..2dea53aee 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -10,6 +10,7 @@ """ import ast +import contextlib import errno import imp import json @@ -30,6 +31,7 @@ from psutil._common import supports_ipv6 from psutil._compat import PY3 from psutil.tests import APPVEYOR +from psutil.tests import bind_unix_socket from psutil.tests import chdir from psutil.tests import create_proc_children_pair from psutil.tests import get_test_subprocess @@ -46,6 +48,7 @@ from psutil.tests import TOX from psutil.tests import TRAVIS from psutil.tests import unittest +from psutil.tests import unix_socket_path from psutil.tests import unix_socketpair from psutil.tests import wait_for_file from psutil.tests import wait_for_pid @@ -680,17 +683,39 @@ def test_create_proc_children_pair(self): class TestNetUtils(unittest.TestCase): + @unittest.skipUnless(POSIX, "POSIX only") + def test_bind_unix_socket(self): + with unix_socket_path() as name: + sock = bind_unix_socket(name) + with contextlib.closing(sock): + self.assertEqual(sock.family, socket.AF_UNIX) + self.assertEqual(sock.type, socket.SOCK_STREAM) + self.assertEqual(sock.getsockname(), name) + assert os.path.exists(name) + assert stat.S_ISSOCK(os.stat(name).st_mode) + # UDP + with unix_socket_path() as name: + sock = bind_unix_socket(name, type=socket.SOCK_DGRAM) + with contextlib.closing(sock): + self.assertEqual(sock.type, socket.SOCK_DGRAM) + @unittest.skipUnless(POSIX, "POSIX only") def test_unix_socketpair(self): p = psutil.Process() num_fds = p.num_fds() assert not p.connections(kind='unix') - ssock, csock, name = unix_socketpair() - self.addCleanup(safe_rmpath, name) - assert os.path.exists(name) - assert stat.S_ISSOCK(os.stat(name).st_mode) - self.assertEqual(p.num_fds() - num_fds, 2) - self.assertEqual(len(p.connections(kind='unix')), 2) + with unix_socket_path() as name: + server, client = unix_socketpair(name) + try: + assert os.path.exists(name) + assert stat.S_ISSOCK(os.stat(name).st_mode) + self.assertEqual(p.num_fds() - num_fds, 2) + self.assertEqual(len(p.connections(kind='unix')), 2) + self.assertEqual(server.getsockname(), name) + self.assertEqual(client.getpeername(), name) + finally: + client.close() + server.close() if __name__ == '__main__': From a8bf2e04f641e624e5dee11b3f5c45c29c94863a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 15:02:58 -0700 Subject: [PATCH 381/922] osx: it seems bind()ing on local host leaves a DNS-related UNIX socket around --- psutil/tests/test_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index b3c77464d..0e96d89a8 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1129,7 +1129,7 @@ def check(type): 'connection fd not available on this platform') def test_connection_fromfd(self): with contextlib.closing(socket.socket()) as sock: - sock.bind(('localhost', 0)) + sock.bind(('127.0.0.1', 0)) sock.listen(1) p = psutil.Process() for conn in p.connections(): From 54c93081a713146fb917f7fbacc3981651e86fff Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 01:40:33 +0200 Subject: [PATCH 382/922] disable failing tests on travis --- psutil/tests/test_linux.py | 2 ++ psutil/tests/test_unicode.py | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 6ceb5f848..eb37db00f 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -191,6 +191,8 @@ def test_buffers(self): self.assertAlmostEqual( vmstat_value, psutil_value, delta=MEMORY_TOLERANCE) + # https://travis-ci.org/giampaolo/psutil/jobs/226719664 + @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") @retry_before_failing() def test_active(self): vmstat_value = vmstat('active memory') * 1024 diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index c7e915ce2..76f2db1d8 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -55,6 +55,7 @@ import socket from psutil import BSD +from psutil import OSX from psutil import WINDOWS from psutil._compat import PY3 from psutil.tests import ASCII_FS @@ -70,9 +71,10 @@ from psutil.tests import TESTFILE_PREFIX from psutil.tests import TESTFN from psutil.tests import TESTFN_UNICODE +from psutil.tests import TRAVIS from psutil.tests import unittest -from psutil.tests import unix_socketpair from psutil.tests import unix_socket_path +from psutil.tests import unix_socketpair import psutil import psutil.tests @@ -201,6 +203,7 @@ def test_disk_usage(self): psutil.disk_usage(self.funky_name) +@unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # XXX @unittest.skipIf(ASCII_FS, "ASCII fs") class TestFSAPIs(_BaseFSAPIsTests, unittest.TestCase): """Test FS APIs with a funky, valid, UTF8 path name.""" @@ -213,6 +216,7 @@ def expect_exact_path_match(cls): return PY3 or cls.funky_name in os.listdir('.') +@unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # XXX class TestFSAPIsWithInvalidPath(_BaseFSAPIsTests, unittest.TestCase): """Test FS APIs with a funky, invalid path name.""" if PY3: From b999c0e765eef789d1917123cd46b11d79743dc0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 02:23:40 +0200 Subject: [PATCH 383/922] move connections-related tests in their own file --- Makefile | 26 ++-- psutil/tests/__init__.py | 92 ++++++------ psutil/tests/test_connections.py | 243 +++++++++++++++++++++++++++++++ psutil/tests/test_process.py | 178 ---------------------- psutil/tests/test_system.py | 22 --- 5 files changed, 305 insertions(+), 256 deletions(-) create mode 100644 psutil/tests/test_connections.py diff --git a/Makefile b/Makefile index e5d8a52fc..64484177d 100644 --- a/Makefile +++ b/Makefile @@ -119,47 +119,53 @@ test: ${MAKE} install $(PYTHON) $(TSCRIPT) -# Test psutil process-related APIs. +# Run process-related API tests. test-process: ${MAKE} install $(PYTHON) -m unittest -v psutil.tests.test_process -# Test psutil system-related APIs. +# Run system-related API tests. test-system: ${MAKE} install $(PYTHON) -m unittest -v psutil.tests.test_system -# Test misc. +# Run miscellaneous tests. test-misc: ${MAKE} install $(PYTHON) psutil/tests/test_misc.py -# Test misc. +# Test APIs dealing with strings. test-unicode: ${MAKE} install $(PYTHON) psutil/tests/test_unicode.py -# Test POSIX. -test-posix: +# Test net_connections() and Process.connections(). +test-connections: ${MAKE} install - $(PYTHON) psutil/tests/test_posix.py + $(PYTHON) psutil/tests/test_connections.py -# Test memory leaks. -test-memleaks: +# POSIX specific tests. +test-posix: ${MAKE} install - $(PYTHON) psutil/tests/test_memory_leaks.py + $(PYTHON) psutil/tests/test_posix.py # Run specific platform tests only. test-platform: ${MAKE} install $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py +# Memory leak tests. +test-memleaks: + ${MAKE} install + $(PYTHON) psutil/tests/test_memory_leaks.py + # Run a specific test by name, e.g. # make test-by-name psutil.tests.test_system.TestSystemAPIs.test_cpu_times test-by-name: ${MAKE} install @$(PYTHON) -m unittest -v $(ARGS) +# Run test coverage. coverage: ${MAKE} install # Note: coverage options are controlled by .coveragerc file diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index c129224d6..7bc777fee 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -763,6 +763,50 @@ def install_test_deps(deps=None): # =================================================================== +@contextlib.contextmanager +def unix_socket_path(suffix=""): + assert psutil.POSIX, "not a POSIX system" + path = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) + try: + yield path + finally: + try: + os.unlink(path) + except OSError: + pass + + +def bind_unix_socket(name, type=socket.SOCK_STREAM): + """Creates a listening unix socket. + Return a (sock, filemame) tuple. + """ + assert psutil.POSIX, "not a POSIX system" + assert not os.path.exists(name), name + sock = socket.socket(socket.AF_UNIX, type) + try: + sock.bind(name) + except Exception: + sock.close() + raise + return sock + + +def unix_socketpair(name): + """Build a pair of UNIX sockets connected to each other through + the same UNIX file name. + Return a (server_sock, client_sock, filename) tuple. + """ + assert psutil.POSIX, "not a POSIX system" + server = bind_unix_socket(name, type=socket.SOCK_STREAM) + server.setblocking(0) + server.listen(1) + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.setblocking(0) + client.connect(name) + # new = server.accept() + return (server, client) + + def check_net_address(addr, family): """Check a net address validity. Supported families are IPv4, IPv6 and MAC addresses. @@ -770,7 +814,7 @@ def check_net_address(addr, family): import ipaddress # python >= 3.3 / requires "pip install ipaddress" if enum and PY3: assert isinstance(family, enum.IntEnum), family - if family == AF_INET: + if family == socket.AF_INET: octs = [int(x) for x in addr.split('.')] assert len(octs) == 4, addr for num in octs: @@ -778,7 +822,7 @@ def check_net_address(addr, family): if not PY3: addr = unicode(addr) ipaddress.IPv4Address(addr) - elif family == AF_INET6: + elif family == socket.AF_INET6: assert isinstance(addr, str), addr if not PY3: addr = unicode(addr) @@ -850,50 +894,6 @@ def check_connection_ntuple(conn): assert dupsock.type == conn.type -@contextlib.contextmanager -def unix_socket_path(suffix=""): - assert psutil.POSIX, "not a POSIX system" - path = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) - try: - yield path - finally: - try: - os.unlink(path) - except OSError: - pass - - -def bind_unix_socket(name, type=socket.SOCK_STREAM): - """Creates a listening unix socket. - Return a (sock, filemame) tuple. - """ - assert psutil.POSIX, "not a POSIX system" - assert not os.path.exists(name), name - sock = socket.socket(socket.AF_UNIX, type) - try: - sock.bind(name) - except Exception: - sock.close() - raise - return sock - - -def unix_socketpair(name): - """Build a pair of UNIX sockets connected to each other through - the same UNIX file name. - Return a (server_sock, client_sock, filename) tuple. - """ - assert psutil.POSIX, "not a POSIX system" - server = bind_unix_socket(name, type=socket.SOCK_STREAM) - server.setblocking(0) - server.listen(1) - client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - client.setblocking(0) - client.connect(name) - # new = server.accept() - return (server, client) - - # =================================================================== # --- others # =================================================================== diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py new file mode 100644 index 000000000..bb4931847 --- /dev/null +++ b/psutil/tests/test_connections.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for net_connections() and Process.connections() APIs.""" + +import contextlib +import os +import socket +import textwrap +import unittest +from socket import AF_INET +from socket import AF_INET6 +from socket import SOCK_DGRAM +from socket import SOCK_STREAM + +import psutil +from psutil import FREEBSD +from psutil import OSX +from psutil import SUNOS +from psutil import WINDOWS +from psutil._common import supports_ipv6 +from psutil._compat import unicode +from psutil.tests import bind_unix_socket +from psutil.tests import check_connection_ntuple +from psutil.tests import pyrun +from psutil.tests import reap_children +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_rmpath +from psutil.tests import skip_on_access_denied +from psutil.tests import TESTFN +from psutil.tests import unix_socket_path +from psutil.tests import wait_for_file + + +AF_UNIX = getattr(socket, "AF_UNIX", object()) + + +class TestProcessConnections(unittest.TestCase): + """Tests for Process.connections().""" + + def tearDown(self): + safe_rmpath(TESTFN) + reap_children() + + def compare_proc_sys_cons(self, pid, proc_cons): + from psutil._common import pconn + sys_cons = [c[:-1] for c in psutil.net_connections(kind='all') + if c.pid == pid] + if FREEBSD: + # on FreeBSD all fds are set to -1 + proc_cons = [pconn(*[-1] + list(x[1:])) for x in proc_cons] + self.assertEqual(sorted(proc_cons), sorted(sys_cons)) + + @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'AF_UNIX not supported') + @skip_on_access_denied(only_if=OSX) + def test_connections_unix(self): + def check(type): + with unix_socket_path() as name: + sock = bind_unix_socket(name, type=type) + with contextlib.closing(sock): + cons = psutil.Process().connections(kind='unix') + self.assertEqual(len(cons), 1) + conn = cons[0] + check_connection_ntuple(conn) + if conn.fd != -1: # != sunos and windows + self.assertEqual(conn.fd, sock.fileno()) + self.assertEqual(conn.family, AF_UNIX) + self.assertEqual(conn.type, type) + self.assertEqual(conn.laddr, name) + if not SUNOS: + # XXX Solaris can't retrieve system-wide UNIX + # sockets. + self.compare_proc_sys_cons(os.getpid(), cons) + + check(SOCK_STREAM) + check(SOCK_DGRAM) + + @unittest.skipUnless(hasattr(socket, "fromfd"), + 'socket.fromfd() not supported') + @unittest.skipIf(WINDOWS or SUNOS, + 'connection fd not available on this platform') + def test_connection_fromfd(self): + with contextlib.closing(socket.socket()) as sock: + sock.bind(('127.0.0.1', 0)) + sock.listen(1) + p = psutil.Process() + for conn in p.connections(): + if conn.fd == sock.fileno(): + break + else: + self.fail("couldn't find socket fd") + dupsock = socket.fromfd(conn.fd, conn.family, conn.type) + with contextlib.closing(dupsock): + self.assertEqual(dupsock.getsockname(), conn.laddr) + self.assertNotEqual(sock.fileno(), dupsock.fileno()) + + def test_connection_constants(self): + ints = [] + strs = [] + for name in dir(psutil): + if name.startswith('CONN_'): + num = getattr(psutil, name) + str_ = str(num) + assert str_.isupper(), str_ + assert str_ not in strs, str_ + assert num not in ints, num + ints.append(num) + strs.append(str_) + if SUNOS: + psutil.CONN_IDLE + psutil.CONN_BOUND + if WINDOWS: + psutil.CONN_DELETE_TCB + + @skip_on_access_denied(only_if=OSX) + def test_connections(self): + def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): + all_kinds = ("all", "inet", "inet4", "inet6", "tcp", "tcp4", + "tcp6", "udp", "udp4", "udp6") + check_connection_ntuple(conn) + self.assertEqual(conn.family, family) + self.assertEqual(conn.type, type) + self.assertEqual(conn.laddr, laddr) + self.assertEqual(conn.raddr, raddr) + self.assertEqual(conn.status, status) + for kind in all_kinds: + cons = proc.connections(kind=kind) + if kind in kinds: + self.assertNotEqual(cons, []) + else: + self.assertEqual(cons, []) + # compare against system-wide connections + # XXX Solaris can't retrieve system-wide UNIX + # sockets. + if not SUNOS: + self.compare_proc_sys_cons(proc.pid, [conn]) + + tcp_template = textwrap.dedent(""" + import socket, time + s = socket.socket($family, socket.SOCK_STREAM) + s.bind(('$addr', 0)) + s.listen(1) + with open('$testfn', 'w') as f: + f.write(str(s.getsockname()[:2])) + time.sleep(60) + """) + + udp_template = textwrap.dedent(""" + import socket, time + s = socket.socket($family, socket.SOCK_DGRAM) + s.bind(('$addr', 0)) + with open('$testfn', 'w') as f: + f.write(str(s.getsockname()[:2])) + time.sleep(60) + """) + + from string import Template + testfile = os.path.basename(TESTFN) + tcp4_template = Template(tcp_template).substitute( + family=int(AF_INET), addr="127.0.0.1", testfn=testfile) + udp4_template = Template(udp_template).substitute( + family=int(AF_INET), addr="127.0.0.1", testfn=testfile) + tcp6_template = Template(tcp_template).substitute( + family=int(AF_INET6), addr="::1", testfn=testfile) + udp6_template = Template(udp_template).substitute( + family=int(AF_INET6), addr="::1", testfn=testfile) + + # launch various subprocess instantiating a socket of various + # families and types to enrich psutil results + tcp4_proc = pyrun(tcp4_template) + tcp4_addr = eval(wait_for_file(testfile)) + udp4_proc = pyrun(udp4_template) + udp4_addr = eval(wait_for_file(testfile)) + if supports_ipv6(): + tcp6_proc = pyrun(tcp6_template) + tcp6_addr = eval(wait_for_file(testfile)) + udp6_proc = pyrun(udp6_template) + udp6_addr = eval(wait_for_file(testfile)) + else: + tcp6_proc = None + udp6_proc = None + tcp6_addr = None + udp6_addr = None + + for p in psutil.Process().children(): + cons = p.connections() + self.assertEqual(len(cons), 1) + for conn in cons: + # TCP v4 + if p.pid == tcp4_proc.pid: + check_conn(p, conn, AF_INET, SOCK_STREAM, tcp4_addr, (), + psutil.CONN_LISTEN, + ("all", "inet", "inet4", "tcp", "tcp4")) + # UDP v4 + elif p.pid == udp4_proc.pid: + check_conn(p, conn, AF_INET, SOCK_DGRAM, udp4_addr, (), + psutil.CONN_NONE, + ("all", "inet", "inet4", "udp", "udp4")) + # TCP v6 + elif p.pid == getattr(tcp6_proc, "pid", None): + check_conn(p, conn, AF_INET6, SOCK_STREAM, tcp6_addr, (), + psutil.CONN_LISTEN, + ("all", "inet", "inet6", "tcp", "tcp6")) + # UDP v6 + elif p.pid == getattr(udp6_proc, "pid", None): + check_conn(p, conn, AF_INET6, SOCK_DGRAM, udp6_addr, (), + psutil.CONN_NONE, + ("all", "inet", "inet6", "udp", "udp6")) + + # err + self.assertRaises(ValueError, p.connections, kind='???') + + +class TestSystemConnections(unittest.TestCase): + """Tests for net_connections().""" + + @skip_on_access_denied() + def test_net_connections(self): + def check(cons, families, types_): + AF_UNIX = getattr(socket, 'AF_UNIX', object()) + for conn in cons: + self.assertIn(conn.family, families, msg=conn) + if conn.family != AF_UNIX: + self.assertIn(conn.type, types_, msg=conn) + self.assertIsInstance(conn.status, (str, unicode)) + + from psutil._common import conn_tmap + for kind, groups in conn_tmap.items(): + if SUNOS and kind == 'unix': + continue + families, types_ = groups + cons = psutil.net_connections(kind) + self.assertEqual(len(cons), len(set(cons))) + check(cons, families, types_) + + self.assertRaises(ValueError, psutil.net_connections, kind='???') + + +if __name__ == '__main__': + run_test_module_by_name(__file__) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 0e96d89a8..e531358c3 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -21,10 +21,6 @@ import time import traceback import types -from socket import AF_INET -from socket import AF_INET6 -from socket import SOCK_DGRAM -from socket import SOCK_STREAM import psutil @@ -37,13 +33,11 @@ from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS -from psutil._common import supports_ipv6 from psutil._compat import callable from psutil._compat import long from psutil._compat import PY3 from psutil._compat import unicode from psutil.tests import APPVEYOR -from psutil.tests import bind_unix_socket from psutil.tests import call_until from psutil.tests import check_connection_ntuple from psutil.tests import create_exe @@ -70,9 +64,7 @@ from psutil.tests import TOX from psutil.tests import TRAVIS from psutil.tests import unittest -from psutil.tests import unix_socket_path from psutil.tests import VALID_PROC_STATUSES -from psutil.tests import wait_for_file from psutil.tests import wait_for_pid from psutil.tests import warn from psutil.tests import WIN_VISTA @@ -990,176 +982,6 @@ def test_open_files_2(self): # test file is gone self.assertTrue(fileobj.name not in p.open_files()) - def compare_proc_sys_cons(self, pid, proc_cons): - from psutil._common import pconn - sys_cons = [c[:-1] for c in psutil.net_connections(kind='all') - if c.pid == pid] - if FREEBSD: - # on FreeBSD all fds are set to -1 - proc_cons = [pconn(*[-1] + list(x[1:])) for x in proc_cons] - self.assertEqual(sorted(proc_cons), sorted(sys_cons)) - - @skip_on_access_denied(only_if=OSX) - def test_connections(self): - def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): - all_kinds = ("all", "inet", "inet4", "inet6", "tcp", "tcp4", - "tcp6", "udp", "udp4", "udp6") - check_connection_ntuple(conn) - self.assertEqual(conn.family, family) - self.assertEqual(conn.type, type) - self.assertEqual(conn.laddr, laddr) - self.assertEqual(conn.raddr, raddr) - self.assertEqual(conn.status, status) - for kind in all_kinds: - cons = proc.connections(kind=kind) - if kind in kinds: - self.assertNotEqual(cons, []) - else: - self.assertEqual(cons, []) - # compare against system-wide connections - # XXX Solaris can't retrieve system-wide UNIX - # sockets. - if not SUNOS: - self.compare_proc_sys_cons(proc.pid, [conn]) - - tcp_template = textwrap.dedent(""" - import socket, time - s = socket.socket($family, socket.SOCK_STREAM) - s.bind(('$addr', 0)) - s.listen(1) - with open('$testfn', 'w') as f: - f.write(str(s.getsockname()[:2])) - time.sleep(60) - """) - - udp_template = textwrap.dedent(""" - import socket, time - s = socket.socket($family, socket.SOCK_DGRAM) - s.bind(('$addr', 0)) - with open('$testfn', 'w') as f: - f.write(str(s.getsockname()[:2])) - time.sleep(60) - """) - - from string import Template - testfile = os.path.basename(TESTFN) - tcp4_template = Template(tcp_template).substitute( - family=int(AF_INET), addr="127.0.0.1", testfn=testfile) - udp4_template = Template(udp_template).substitute( - family=int(AF_INET), addr="127.0.0.1", testfn=testfile) - tcp6_template = Template(tcp_template).substitute( - family=int(AF_INET6), addr="::1", testfn=testfile) - udp6_template = Template(udp_template).substitute( - family=int(AF_INET6), addr="::1", testfn=testfile) - - # launch various subprocess instantiating a socket of various - # families and types to enrich psutil results - tcp4_proc = pyrun(tcp4_template) - tcp4_addr = eval(wait_for_file(testfile)) - udp4_proc = pyrun(udp4_template) - udp4_addr = eval(wait_for_file(testfile)) - if supports_ipv6(): - tcp6_proc = pyrun(tcp6_template) - tcp6_addr = eval(wait_for_file(testfile)) - udp6_proc = pyrun(udp6_template) - udp6_addr = eval(wait_for_file(testfile)) - else: - tcp6_proc = None - udp6_proc = None - tcp6_addr = None - udp6_addr = None - - for p in psutil.Process().children(): - cons = p.connections() - self.assertEqual(len(cons), 1) - for conn in cons: - # TCP v4 - if p.pid == tcp4_proc.pid: - check_conn(p, conn, AF_INET, SOCK_STREAM, tcp4_addr, (), - psutil.CONN_LISTEN, - ("all", "inet", "inet4", "tcp", "tcp4")) - # UDP v4 - elif p.pid == udp4_proc.pid: - check_conn(p, conn, AF_INET, SOCK_DGRAM, udp4_addr, (), - psutil.CONN_NONE, - ("all", "inet", "inet4", "udp", "udp4")) - # TCP v6 - elif p.pid == getattr(tcp6_proc, "pid", None): - check_conn(p, conn, AF_INET6, SOCK_STREAM, tcp6_addr, (), - psutil.CONN_LISTEN, - ("all", "inet", "inet6", "tcp", "tcp6")) - # UDP v6 - elif p.pid == getattr(udp6_proc, "pid", None): - check_conn(p, conn, AF_INET6, SOCK_DGRAM, udp6_addr, (), - psutil.CONN_NONE, - ("all", "inet", "inet6", "udp", "udp6")) - - # err - self.assertRaises(ValueError, p.connections, kind='???') - - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'AF_UNIX not supported') - @skip_on_access_denied(only_if=OSX) - def test_connections_unix(self): - def check(type): - safe_rmpath(TESTFN) - with unix_socket_path() as name: - sock = bind_unix_socket(name, type=type) - self.addCleanup(safe_rmpath, name) - with contextlib.closing(sock): - cons = psutil.Process().connections(kind='unix') - self.assertEqual(len(cons), 1) - conn = cons[0] - check_connection_ntuple(conn) - if conn.fd != -1: # != sunos and windows - self.assertEqual(conn.fd, sock.fileno()) - self.assertEqual(conn.family, socket.AF_UNIX) - self.assertEqual(conn.type, type) - self.assertEqual(conn.laddr, name) - if not SUNOS: - # XXX Solaris can't retrieve system-wide UNIX - # sockets. - self.compare_proc_sys_cons(os.getpid(), cons) - - check(SOCK_STREAM) - check(SOCK_DGRAM) - - @unittest.skipUnless(hasattr(socket, "fromfd"), - 'socket.fromfd() not supported') - @unittest.skipIf(WINDOWS or SUNOS, - 'connection fd not available on this platform') - def test_connection_fromfd(self): - with contextlib.closing(socket.socket()) as sock: - sock.bind(('127.0.0.1', 0)) - sock.listen(1) - p = psutil.Process() - for conn in p.connections(): - if conn.fd == sock.fileno(): - break - else: - self.fail("couldn't find socket fd") - dupsock = socket.fromfd(conn.fd, conn.family, conn.type) - with contextlib.closing(dupsock): - self.assertEqual(dupsock.getsockname(), conn.laddr) - self.assertNotEqual(sock.fileno(), dupsock.fileno()) - - def test_connection_constants(self): - ints = [] - strs = [] - for name in dir(psutil): - if name.startswith('CONN_'): - num = getattr(psutil, name) - str_ = str(num) - assert str_.isupper(), str_ - assert str_ not in strs, str_ - assert num not in ints, num - ints.append(num) - strs.append(str_) - if SUNOS: - psutil.CONN_IDLE - psutil.CONN_BOUND - if WINDOWS: - psutil.CONN_DELETE_TCB - @unittest.skipUnless(POSIX, 'POSIX only') def test_num_fds(self): p = psutil.Process() diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index a4d48b8c7..4488a216b 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -41,7 +41,6 @@ from psutil.tests import retry_before_failing from psutil.tests import run_test_module_by_name from psutil.tests import safe_rmpath -from psutil.tests import skip_on_access_denied from psutil.tests import TESTFN from psutil.tests import TESTFN_UNICODE from psutil.tests import TRAVIS @@ -523,27 +522,6 @@ def find_mount_point(path): self.assertIn(mount, mounts) psutil.disk_usage(mount) - @skip_on_access_denied() - def test_net_connections(self): - def check(cons, families, types_): - AF_UNIX = getattr(socket, 'AF_UNIX', object()) - for conn in cons: - self.assertIn(conn.family, families, msg=conn) - if conn.family != AF_UNIX: - self.assertIn(conn.type, types_, msg=conn) - self.assertIsInstance(conn.status, (str, unicode)) - - from psutil._common import conn_tmap - for kind, groups in conn_tmap.items(): - if SUNOS and kind == 'unix': - continue - families, types_ = groups - cons = psutil.net_connections(kind) - self.assertEqual(len(cons), len(set(cons))) - check(cons, families, types_) - - self.assertRaises(ValueError, psutil.net_connections, kind='???') - def test_net_io_counters(self): def check_ntuple(nt): self.assertEqual(nt[0], nt.bytes_sent) From 5534cf711a414410672211751ae1b46a4769d3e9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 02:40:01 +0200 Subject: [PATCH 384/922] refactoring --- docs/index.rst | 7 ++++--- psutil/tests/test_connections.py | 21 ++++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 655791da9..3031ce581 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -462,10 +462,11 @@ Network Return system-wide socket connections as a list of named tuples. Every named tuple provides 7 attributes: - - **fd**: the socket file descriptor, if retrievable, else ``-1``. - If the connection refers to the current process this may be passed to + - **fd**: the socket file descriptor. If the connection refers to the current + process this may be passed to `socket.fromfd() `__ to obtain a usable socket object. + On Windows, FreeBSD and SunOS this is always set to ``-1``. - **family**: the address family, either `AF_INET `__, `AF_INET6 `__ @@ -1747,7 +1748,7 @@ Process class - **fd**: the socket file descriptor. This can be passed to `socket.fromfd() `__ to obtain a usable socket object. - This is only available on UNIX; on Windows ``-1`` is always returned. + On Windows, FreeBSD and SunOS this is always set to ``-1``. - **family**: the address family, either `AF_INET `__, `AF_INET6 `__ diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index bb4931847..28670f25c 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -10,7 +10,6 @@ import os import socket import textwrap -import unittest from socket import AF_INET from socket import AF_INET6 from socket import SOCK_DGRAM @@ -19,6 +18,7 @@ import psutil from psutil import FREEBSD from psutil import OSX +from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS from psutil._common import supports_ipv6 @@ -33,6 +33,7 @@ from psutil.tests import TESTFN from psutil.tests import unix_socket_path from psutil.tests import wait_for_file +from psutil.tests import unittest AF_UNIX = getattr(socket, "AF_UNIX", object()) @@ -47,15 +48,21 @@ def tearDown(self): def compare_proc_sys_cons(self, pid, proc_cons): from psutil._common import pconn - sys_cons = [c[:-1] for c in psutil.net_connections(kind='all') - if c.pid == pid] + try: + syscons = psutil.net_connections(kind='all') + except psutil.AccessDenied: + # On OSX, system-wide connections are retrieved by iterating + # over all processes + if not OSX: + raise + # exclude PIDs from syscons + syscons = [c[:-1] for c in syscons if c.pid == pid] if FREEBSD: - # on FreeBSD all fds are set to -1 + # on FreeBSD all fds are set to -1 so exclude them proc_cons = [pconn(*[-1] + list(x[1:])) for x in proc_cons] - self.assertEqual(sorted(proc_cons), sorted(sys_cons)) + self.assertEqual(sorted(proc_cons), sorted(syscons)) - @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'AF_UNIX not supported') - @skip_on_access_denied(only_if=OSX) + @unittest.skipUnless(POSIX, 'POSIX only') def test_connections_unix(self): def check(type): with unix_socket_path() as name: From c268d3cea17c51627ad44945d39d7716fd441a63 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 03:27:34 +0200 Subject: [PATCH 385/922] memleak / connections: create also UDP sockets in order to excercise more C code sections --- psutil/tests/__init__.py | 5 ++++- psutil/tests/test_memory_leaks.py | 28 +++++++++++++++++++--------- psutil/tests/test_unicode.py | 4 ++-- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 7bc777fee..1c4817dbc 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -765,6 +765,9 @@ def install_test_deps(deps=None): @contextlib.contextmanager def unix_socket_path(suffix=""): + """A context manager which returns a non-existent file name + and tries to delete it on exit. + """ assert psutil.POSIX, "not a POSIX system" path = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) try: @@ -794,7 +797,7 @@ def bind_unix_socket(name, type=socket.SOCK_STREAM): def unix_socketpair(name): """Build a pair of UNIX sockets connected to each other through the same UNIX file name. - Return a (server_sock, client_sock, filename) tuple. + Return a (server, client) tuple. """ assert psutil.POSIX, "not a POSIX system" server = bind_unix_socket(name, type=socket.SOCK_STREAM) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index a55008e52..402e7b8bb 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -32,6 +32,7 @@ from psutil import WINDOWS from psutil._common import supports_ipv6 from psutil._compat import xrange +from psutil.tests import bind_unix_socket from psutil.tests import get_test_subprocess from psutil.tests import reap_children from psutil.tests import RLIMIT_SUPPORT @@ -40,6 +41,8 @@ from psutil.tests import TESTFN from psutil.tests import TRAVIS from psutil.tests import unittest +from psutil.tests import unix_socket_path +from psutil.tests import unix_socketpair LOOPS = 1000 @@ -363,24 +366,31 @@ def create_socket(family, type): sock.listen(1) return sock + # Open as many socket types as possible so that we excercise + # as much C code sections as possible. socks = [] socks.append(create_socket(socket.AF_INET, socket.SOCK_STREAM)) socks.append(create_socket(socket.AF_INET, socket.SOCK_DGRAM)) if supports_ipv6(): socks.append(create_socket(socket.AF_INET6, socket.SOCK_STREAM)) socks.append(create_socket(socket.AF_INET6, socket.SOCK_DGRAM)) - if hasattr(socket, 'AF_UNIX'): - safe_rmpath(TESTFN) - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - s.bind(TESTFN) - s.listen(1) - socks.append(s) - kind = 'all' + if POSIX and not SUNOS: # TODO: SunOS + name1 = unix_socket_path().__enter__() + name2 = unix_socket_path().__enter__() + s1, s2 = unix_socketpair(name1) + s3 = bind_unix_socket(name2, type=socket.SOCK_DGRAM) + self.addCleanup(safe_rmpath, name1) + self.addCleanup(safe_rmpath, name2) + for s in (s1, s2, s3): + socks.append(s) + # TODO: UNIX sockets are temporarily implemented by parsing # 'pfiles' cmd output; we don't want that part of the code to # be executed. - if SUNOS: - kind = 'inet' + kind = 'inet' if SUNOS else 'all' + # Make sure we did a proper setup. + self.assertEqual( + len(psutil.Process().connections(kind=kind)), len(socks)) try: self.execute(self.proc.connections, kind) finally: diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 76f2db1d8..083f48141 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -203,7 +203,7 @@ def test_disk_usage(self): psutil.disk_usage(self.funky_name) -@unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # XXX +@unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # TODO @unittest.skipIf(ASCII_FS, "ASCII fs") class TestFSAPIs(_BaseFSAPIsTests, unittest.TestCase): """Test FS APIs with a funky, valid, UTF8 path name.""" @@ -216,7 +216,7 @@ def expect_exact_path_match(cls): return PY3 or cls.funky_name in os.listdir('.') -@unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # XXX +@unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # TODO class TestFSAPIsWithInvalidPath(_BaseFSAPIsTests, unittest.TestCase): """Test FS APIs with a funky, invalid path name.""" if PY3: From a759a044123c88a856aedb1e1994e8944981ecdf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 03:54:10 +0200 Subject: [PATCH 386/922] #1033 / net|proc connections / FreeBSD / OSX / memory leak: Py_DECREF object when retrieving UNIX sockets --- HISTORY.rst | 2 ++ psutil/_psutil_osx.c | 2 ++ psutil/arch/bsd/freebsd_socks.c | 1 + psutil/tests/test_connections.py | 4 +++- psutil/tests/test_memory_leaks.py | 2 +- 5 files changed, 9 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index dc49d65fa..35185d62e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,8 @@ skipping a file which gets deleted while open files are retrieved. - 1029_: [OSX, FreeBSD] Process.connections('unix') on Python 3 doesn't properly handle unicode paths and may raise UnicodeDecodeError. +- 1033_: [OSX, FreeBSD] memory leak for net_connections() and + Process.connections() when retrieving UNIX sockets (kind='unix'). *2017-04-10* diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 90391ddc0..559ffab9f 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1384,6 +1384,8 @@ psutil_proc_connections(PyObject *self, PyObject *args) { if (PyList_Append(py_retlist, py_tuple)) goto error; Py_DECREF(py_tuple); + Py_DECREF(py_laddr); + Py_DECREF(py_raddr); } } } diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index b30fa8f81..187a93de0 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -618,6 +618,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { if (PyList_Append(py_retlist, py_tuple)) goto error; Py_DECREF(py_tuple); + Py_DECREF(py_laddr); Py_INCREF(Py_None); } } diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 28670f25c..e1e9fca8a 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -53,7 +53,9 @@ def compare_proc_sys_cons(self, pid, proc_cons): except psutil.AccessDenied: # On OSX, system-wide connections are retrieved by iterating # over all processes - if not OSX: + if OSX: + return + else: raise # exclude PIDs from syscons syscons = [c[:-1] for c in syscons if c.pid == pid] diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 402e7b8bb..2bf7882ce 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -367,7 +367,7 @@ def create_socket(family, type): return sock # Open as many socket types as possible so that we excercise - # as much C code sections as possible. + # as many C code sections as possible. socks = [] socks.append(create_socket(socket.AF_INET, socket.SOCK_STREAM)) socks.append(create_socket(socket.AF_INET, socket.SOCK_DGRAM)) From c71d2bec27a85aaf161e389f7b40bce5adc375eb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 04:07:58 +0200 Subject: [PATCH 387/922] try to fix py 2.6 failure on travis --- psutil/tests/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 1c4817dbc..9ffbc68ca 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -36,11 +36,6 @@ except ImportError: from urllib2 import urlopen -try: - from unittest import mock # py3 -except ImportError: - import mock # NOQA - requires "pip install mock" - import psutil from psutil import LINUX from psutil import POSIX @@ -54,6 +49,12 @@ import unittest2 as unittest # requires "pip install unittest2" else: import unittest + +try: + from unittest import mock # py3 +except ImportError: + import mock # NOQA - requires "pip install mock" + if sys.version_info >= (3, 4): import enum else: From 8fb004a1242e1de0f07d22aa960d0ae761a7f4ab Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 05:46:11 +0200 Subject: [PATCH 388/922] write unit tests for all network types and families --- psutil/tests/__init__.py | 101 ++++++++++++++++--------- psutil/tests/test_connections.py | 124 ++++++++++++++++++++----------- 2 files changed, 145 insertions(+), 80 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 9ffbc68ca..1143a93c7 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -150,6 +150,7 @@ VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) if x.startswith('STATUS_')] GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" +AF_UNIX = getattr(socket, "AF_UNIX", object()) TEST_DEPS = [] if sys.version_info[:2] == (2, 6): TEST_DEPS.extend(["ipaddress", "unittest2", "argparse", "mock==1.0.1"]) @@ -764,6 +765,24 @@ def install_test_deps(deps=None): # =================================================================== +def get_free_port(host='127.0.0.1'): + """Return an unused TCP port.""" + with contextlib.closing(socket.socket()) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((host, 0)) + return sock.getsockname()[1] + + +def bind_socket(addr, family, type): + """Binds a generic socket.""" + sock = socket.socket(family, type) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(addr) + if type == socket.SOCK_STREAM: + sock.listen(1) + return sock + + @contextlib.contextmanager def unix_socket_path(suffix=""): """A context manager which returns a non-existent file name @@ -781,9 +800,7 @@ def unix_socket_path(suffix=""): def bind_unix_socket(name, type=socket.SOCK_STREAM): - """Creates a listening unix socket. - Return a (sock, filemame) tuple. - """ + """Bind a UNIX socket.""" assert psutil.POSIX, "not a POSIX system" assert not os.path.exists(name), name sock = socket.socket(socket.AF_UNIX, type) @@ -792,6 +809,7 @@ def bind_unix_socket(name, type=socket.SOCK_STREAM): except Exception: sock.close() raise + sock.listen(1) return sock @@ -839,34 +857,35 @@ def check_net_address(addr, family): def check_connection_ntuple(conn): """Check validity of a connection namedtuple.""" - AF_UNIX = getattr(socket, "AF_UNIX", object()) - valid_conn_states = [getattr(psutil, x) for x in dir(psutil) if - x.startswith('CONN_')] + # check ntuple + assert len(conn) in (6, 7), conn + has_pid = len(conn) == 7 + has_fd = getattr(conn, 'fd', -1) != -1 assert conn[0] == conn.fd assert conn[1] == conn.family assert conn[2] == conn.type assert conn[3] == conn.laddr assert conn[4] == conn.raddr assert conn[5] == conn.status - assert conn.type in (SOCK_STREAM, SOCK_DGRAM), repr(conn.type) - assert conn.family in (AF_INET, AF_INET6, AF_UNIX), repr(conn.family) - assert conn.status in valid_conn_states, conn.status + if has_pid: + assert conn[6] == conn.pid - # check IP address and port sanity - for addr in (conn.laddr, conn.raddr): - if not addr: - continue - if conn.family in (AF_INET, AF_INET6): - assert isinstance(addr, tuple), addr - ip, port = addr - assert isinstance(port, int), port - assert 0 <= port <= 65535, port - check_net_address(ip, conn.family) - elif conn.family == AF_UNIX: - assert isinstance(addr, (str, None)), addr - else: - raise ValueError("unknown family %r", conn.family) + # check fd + if has_fd: + assert conn.fd > 0, conn + if hasattr(socket, 'fromfd') and not WINDOWS: + try: + dupsock = socket.fromfd(conn.fd, conn.family, conn.type) + except (socket.error, OSError) as err: + if err.args[0] != errno.EBADF: + raise + else: + with contextlib.closing(dupsock): + assert dupsock.family == conn.family + assert dupsock.type == conn.type + # check family + assert conn.family in (AF_INET, AF_INET6, AF_UNIX), repr(conn.family) if conn.family in (AF_INET, AF_INET6): # actually try to bind the local socket; ignore IPv6 # sockets as their address might be represented as @@ -884,18 +903,30 @@ def check_connection_ntuple(conn): assert not conn.raddr, repr(conn.raddr) assert conn.status == psutil.CONN_NONE, conn.status - if getattr(conn, 'fd', -1) != -1: - assert conn.fd > 0, conn - if hasattr(socket, 'fromfd') and not WINDOWS: - try: - dupsock = socket.fromfd(conn.fd, conn.family, conn.type) - except (socket.error, OSError) as err: - if err.args[0] != errno.EBADF: - raise - else: - with contextlib.closing(dupsock): - assert dupsock.family == conn.family - assert dupsock.type == conn.type + # check type + assert conn.type in (SOCK_STREAM, SOCK_DGRAM), repr(conn.type) + if conn.type == SOCK_DGRAM: + assert conn.status == psutil.CONN_NONE, conn.status + + # check laddr (IP address and port sanity) + for addr in (conn.laddr, conn.raddr): + if not addr: + continue + if conn.family in (AF_INET, AF_INET6): + assert isinstance(addr, tuple), addr + ip, port = addr + assert isinstance(port, int), port + assert 0 <= port <= 65535, port + check_net_address(ip, conn.family) + elif conn.family == AF_UNIX: + assert isinstance(addr, (str, None)), addr + + # check raddr + assert isinstance(conn.status, (tuple, str, None)), repr(conn.status) + + # check status + valids = [getattr(psutil, x) for x in dir(psutil) if x.startswith('CONN_')] + assert conn.status in valids, conn.status # =================================================================== diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index e1e9fca8a..30f15a45b 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -6,10 +6,10 @@ """Tests for net_connections() and Process.connections() APIs.""" -import contextlib import os import socket import textwrap +from contextlib import closing from socket import AF_INET from socket import AF_INET6 from socket import SOCK_DGRAM @@ -23,28 +23,34 @@ from psutil import WINDOWS from psutil._common import supports_ipv6 from psutil._compat import unicode +from psutil.tests import AF_UNIX +from psutil.tests import bind_socket from psutil.tests import bind_unix_socket from psutil.tests import check_connection_ntuple +from psutil.tests import get_free_port from psutil.tests import pyrun from psutil.tests import reap_children from psutil.tests import run_test_module_by_name from psutil.tests import safe_rmpath from psutil.tests import skip_on_access_denied from psutil.tests import TESTFN +from psutil.tests import unittest from psutil.tests import unix_socket_path from psutil.tests import wait_for_file -from psutil.tests import unittest - - -AF_UNIX = getattr(socket, "AF_UNIX", object()) class TestProcessConnections(unittest.TestCase): """Tests for Process.connections().""" + def setUp(self): + cons = psutil.Process().connections(kind='all') + assert not cons, cons + def tearDown(self): safe_rmpath(TESTFN) reap_children() + cons = psutil.Process().connections(kind='all') + assert not cons, cons def compare_proc_sys_cons(self, pid, proc_cons): from psutil._common import pconn @@ -64,47 +70,75 @@ def compare_proc_sys_cons(self, pid, proc_cons): proc_cons = [pconn(*[-1] + list(x[1:])) for x in proc_cons] self.assertEqual(sorted(proc_cons), sorted(syscons)) + def check_socket(self, sock): + cons = psutil.Process().connections(kind='all') + self.assertEqual(len(cons), 1) + conn = cons[0] + + check_connection_ntuple(conn) + + # fd, family, type + if conn.fd != -1: + self.assertEqual(conn.fd, sock.fileno()) + self.assertEqual(conn.family, sock.family) + self.assertEqual(conn.type, sock.type) + + # local address + laddr = sock.getsockname() + if sock.family == AF_INET6: + laddr = laddr[:2] + self.assertEqual(conn.laddr, laddr) + + # XXX Solaris can't retrieve system-wide UNIX sockets + if not (SUNOS and sock.family == AF_UNIX): + self.compare_proc_sys_cons(os.getpid(), cons) + return conn + + # --- non connected sockets + + def test_tcp_v4(self): + addr = ("127.0.0.1", get_free_port()) + with closing(bind_socket(addr, AF_INET, SOCK_STREAM)) as sock: + conn = self.check_socket(sock) + assert not conn.raddr + self.assertEqual(conn.status, psutil.CONN_LISTEN) + + def test_tcp_v6(self): + addr = ("::1", get_free_port()) + with closing(bind_socket(addr, AF_INET6, SOCK_STREAM)) as sock: + conn = self.check_socket(sock) + assert not conn.raddr + self.assertEqual(conn.status, psutil.CONN_LISTEN) + + def test_udp_v4(self): + addr = ("127.0.0.1", get_free_port()) + with closing(bind_socket(addr, AF_INET, SOCK_DGRAM)) as sock: + conn = self.check_socket(sock) + assert not conn.raddr + self.assertEqual(conn.status, psutil.CONN_NONE) + + def test_udp_v6(self): + addr = ("127.0.0.1", get_free_port()) + with closing(bind_socket(addr, AF_INET, SOCK_DGRAM)) as sock: + conn = self.check_socket(sock) + assert not conn.raddr + self.assertEqual(conn.status, psutil.CONN_NONE) + @unittest.skipUnless(POSIX, 'POSIX only') - def test_connections_unix(self): - def check(type): - with unix_socket_path() as name: - sock = bind_unix_socket(name, type=type) - with contextlib.closing(sock): - cons = psutil.Process().connections(kind='unix') - self.assertEqual(len(cons), 1) - conn = cons[0] - check_connection_ntuple(conn) - if conn.fd != -1: # != sunos and windows - self.assertEqual(conn.fd, sock.fileno()) - self.assertEqual(conn.family, AF_UNIX) - self.assertEqual(conn.type, type) - self.assertEqual(conn.laddr, name) - if not SUNOS: - # XXX Solaris can't retrieve system-wide UNIX - # sockets. - self.compare_proc_sys_cons(os.getpid(), cons) - - check(SOCK_STREAM) - check(SOCK_DGRAM) - - @unittest.skipUnless(hasattr(socket, "fromfd"), - 'socket.fromfd() not supported') - @unittest.skipIf(WINDOWS or SUNOS, - 'connection fd not available on this platform') - def test_connection_fromfd(self): - with contextlib.closing(socket.socket()) as sock: - sock.bind(('127.0.0.1', 0)) - sock.listen(1) - p = psutil.Process() - for conn in p.connections(): - if conn.fd == sock.fileno(): - break - else: - self.fail("couldn't find socket fd") - dupsock = socket.fromfd(conn.fd, conn.family, conn.type) - with contextlib.closing(dupsock): - self.assertEqual(dupsock.getsockname(), conn.laddr) - self.assertNotEqual(sock.fileno(), dupsock.fileno()) + def test_unix_tcp(self): + with unix_socket_path() as name: + with closing(bind_unix_socket(name, type=SOCK_STREAM)) as sock: + conn = self.check_socket(sock) + assert not conn.raddr + self.assertEqual(conn.status, psutil.CONN_NONE) + + @unittest.skipUnless(POSIX, 'POSIX only') + def test_unix_udp(self): + with unix_socket_path() as name: + with closing(bind_unix_socket(name, type=SOCK_STREAM)) as sock: + conn = self.check_socket(sock) + assert not conn.raddr + self.assertEqual(conn.status, psutil.CONN_NONE) def test_connection_constants(self): ints = [] From 81e3578b688783923109fffed789b96876b23377 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 06:08:05 +0200 Subject: [PATCH 389/922] refactoring --- psutil/tests/test_connections.py | 57 ++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 30f15a45b..b0ccbb9ad 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -39,6 +39,32 @@ from psutil.tests import wait_for_file +def compare_procsys_connections(pid, proc_cons, kind='all'): + """Given a process PID and its list of connections compare + those against system-wide connections retrieved via + psutil.net_connections. + """ + from psutil._common import pconn + try: + sys_cons = psutil.net_connections(kind=kind) + except psutil.AccessDenied: + # On OSX, system-wide connections are retrieved by iterating + # over all processes + if OSX: + return + else: + raise + # exclude PIDs from syscons + sys_cons = [c[:-1] for c in sys_cons if c.pid == pid] + if FREEBSD: + # On FreeBSD all fds are set to -1 so exclude them + # for comparison. + proc_cons = [pconn(*[-1] + list(x[1:])) for x in proc_cons] + proc_cons.sort() + sys_cons.sort() + assert proc_cons == sys_cons, (proc_cons, sys_cons) + + class TestProcessConnections(unittest.TestCase): """Tests for Process.connections().""" @@ -49,28 +75,15 @@ def setUp(self): def tearDown(self): safe_rmpath(TESTFN) reap_children() + # make sure we closed all resources cons = psutil.Process().connections(kind='all') assert not cons, cons - def compare_proc_sys_cons(self, pid, proc_cons): - from psutil._common import pconn - try: - syscons = psutil.net_connections(kind='all') - except psutil.AccessDenied: - # On OSX, system-wide connections are retrieved by iterating - # over all processes - if OSX: - return - else: - raise - # exclude PIDs from syscons - syscons = [c[:-1] for c in syscons if c.pid == pid] - if FREEBSD: - # on FreeBSD all fds are set to -1 so exclude them - proc_cons = [pconn(*[-1] + list(x[1:])) for x in proc_cons] - self.assertEqual(sorted(proc_cons), sorted(syscons)) - def check_socket(self, sock): + """Given a socket, makes sure it matches the one obtained + via psutil. It assumes this process created one connection + only (the one supposed to be checked). + """ cons = psutil.Process().connections(kind='all') self.assertEqual(len(cons), 1) conn = cons[0] @@ -91,7 +104,7 @@ def check_socket(self, sock): # XXX Solaris can't retrieve system-wide UNIX sockets if not (SUNOS and sock.family == AF_UNIX): - self.compare_proc_sys_cons(os.getpid(), cons) + compare_procsys_connections(os.getpid(), cons) return conn # --- non connected sockets @@ -172,14 +185,14 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): for kind in all_kinds: cons = proc.connections(kind=kind) if kind in kinds: - self.assertNotEqual(cons, []) + assert cons else: - self.assertEqual(cons, []) + assert not cons, cons # compare against system-wide connections # XXX Solaris can't retrieve system-wide UNIX # sockets. if not SUNOS: - self.compare_proc_sys_cons(proc.pid, [conn]) + compare_procsys_connections(proc.pid, [conn]) tcp_template = textwrap.dedent(""" import socket, time From 0009de58af93deb25402bea0745310f8f6884398 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 06:21:17 +0200 Subject: [PATCH 390/922] refactor tests --- psutil/tests/test_connections.py | 68 +++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index b0ccbb9ad..4f34e902b 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -65,8 +65,7 @@ def compare_procsys_connections(pid, proc_cons, kind='all'): assert proc_cons == sys_cons, (proc_cons, sys_cons) -class TestProcessConnections(unittest.TestCase): - """Tests for Process.connections().""" +class Base(object): def setUp(self): cons = psutil.Process().connections(kind='all') @@ -79,7 +78,16 @@ def tearDown(self): cons = psutil.Process().connections(kind='all') assert not cons, cons - def check_socket(self, sock): + +# ===================================================================== +# --- Test unconnected sockets +# ===================================================================== + + +class TestUnconnectedSockets(Base, unittest.TestCase): + """Tests sockets which are open but not connected to anything.""" + + def check_socket(self, sock, conn=None): """Given a socket, makes sure it matches the one obtained via psutil. It assumes this process created one connection only (the one supposed to be checked). @@ -153,26 +161,19 @@ def test_unix_udp(self): assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_NONE) - def test_connection_constants(self): - ints = [] - strs = [] - for name in dir(psutil): - if name.startswith('CONN_'): - num = getattr(psutil, name) - str_ = str(num) - assert str_.isupper(), str_ - assert str_ not in strs, str_ - assert num not in ints, num - ints.append(num) - strs.append(str_) - if SUNOS: - psutil.CONN_IDLE - psutil.CONN_BOUND - if WINDOWS: - psutil.CONN_DELETE_TCB + +# ===================================================================== +# --- Test connected sockets +# ===================================================================== + + +class TestConnectedSocketPairs(Base, unittest.TestCase): + """Test socket pairs which are are actually connected to + each other. + """ @skip_on_access_denied(only_if=OSX) - def test_connections(self): + def test_combos(self): def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): all_kinds = ("all", "inet", "inet4", "inet6", "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6") @@ -270,9 +271,32 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): self.assertRaises(ValueError, p.connections, kind='???') -class TestSystemConnections(unittest.TestCase): +# ===================================================================== +# --- Miscellaneous tests +# ===================================================================== + + +class TestMisc(Base, unittest.TestCase): """Tests for net_connections().""" + def test_connection_constants(self): + ints = [] + strs = [] + for name in dir(psutil): + if name.startswith('CONN_'): + num = getattr(psutil, name) + str_ = str(num) + assert str_.isupper(), str_ + assert str_ not in strs, str_ + assert num not in ints, num + ints.append(num) + strs.append(str_) + if SUNOS: + psutil.CONN_IDLE + psutil.CONN_BOUND + if WINDOWS: + psutil.CONN_DELETE_TCB + @skip_on_access_denied() def test_net_connections(self): def check(cons, families, types_): From bec9b1725d2d71c415eaf70bba5cafa2bb5b2435 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 07:50:53 +0200 Subject: [PATCH 391/922] add inet_peername() utility --- psutil/tests/__init__.py | 47 +++++++++++++++++++++++++++++---------- psutil/tests/test_misc.py | 9 ++++++++ 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 1143a93c7..fad0c3e7e 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -773,16 +773,6 @@ def get_free_port(host='127.0.0.1'): return sock.getsockname()[1] -def bind_socket(addr, family, type): - """Binds a generic socket.""" - sock = socket.socket(family, type) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(addr) - if type == socket.SOCK_STREAM: - sock.listen(1) - return sock - - @contextlib.contextmanager def unix_socket_path(suffix=""): """A context manager which returns a non-existent file name @@ -799,6 +789,16 @@ def unix_socket_path(suffix=""): pass +def bind_socket(addr, family, type): + """Binds a generic socket.""" + sock = socket.socket(family, type) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(addr) + if type == socket.SOCK_STREAM: + sock.listen(10) + return sock + + def bind_unix_socket(name, type=socket.SOCK_STREAM): """Bind a UNIX socket.""" assert psutil.POSIX, "not a POSIX system" @@ -809,10 +809,34 @@ def bind_unix_socket(name, type=socket.SOCK_STREAM): except Exception: sock.close() raise - sock.listen(1) + if type == socket.SOCK_STREAM: + sock.listen(10) return sock +def inet_socketpair(family, type, addr=("", 0)): + """Build a pair of INET sockets connected to each other. + Return a (server, client) tuple. + """ + with contextlib.closing(socket.socket(family, type)) as ll: + ll.bind(addr) + ll.listen(10) + addr = ll.getsockname() + c = socket.socket(family, type) + try: + c.connect(addr) + caddr = c.getsockname() + while True: + a, addr = ll.accept() + # check that we've got the correct client + if addr == caddr: + return c, a + a.close() + except OSError: + c.close() + raise + + def unix_socketpair(name): """Build a pair of UNIX sockets connected to each other through the same UNIX file name. @@ -821,7 +845,6 @@ def unix_socketpair(name): assert psutil.POSIX, "not a POSIX system" server = bind_unix_socket(name, type=socket.SOCK_STREAM) server.setblocking(0) - server.listen(1) client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) client.setblocking(0) client.connect(name) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 2dea53aee..358fa7958 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -36,6 +36,7 @@ from psutil.tests import create_proc_children_pair from psutil.tests import get_test_subprocess from psutil.tests import importlib +from psutil.tests import inet_socketpair from psutil.tests import mock from psutil.tests import reap_children from psutil.tests import retry @@ -699,6 +700,14 @@ def test_bind_unix_socket(self): with contextlib.closing(sock): self.assertEqual(sock.type, socket.SOCK_DGRAM) + def test_inet_socketpair(self): + server, client = inet_socketpair(socket.AF_INET, socket.SOCK_STREAM) + with contextlib.closing(server): + with contextlib.closing(client): + # ensure they are connected + self.assertEqual(server.getsockname(), client.getpeername()) + self.assertEqual(server.getpeername(), client.getsockname()) + @unittest.skipUnless(POSIX, "POSIX only") def test_unix_socketpair(self): p = psutil.Process() From c9fa529862d8997bfa0fbe8a704f40e187dc883d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 08:05:00 +0200 Subject: [PATCH 392/922] add test for TCP socket --- psutil/tests/test_connections.py | 41 ++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 4f34e902b..597ead019 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -78,23 +78,15 @@ def tearDown(self): cons = psutil.Process().connections(kind='all') assert not cons, cons - -# ===================================================================== -# --- Test unconnected sockets -# ===================================================================== - - -class TestUnconnectedSockets(Base, unittest.TestCase): - """Tests sockets which are open but not connected to anything.""" - def check_socket(self, sock, conn=None): """Given a socket, makes sure it matches the one obtained via psutil. It assumes this process created one connection only (the one supposed to be checked). """ cons = psutil.Process().connections(kind='all') - self.assertEqual(len(cons), 1) - conn = cons[0] + if not conn: + self.assertEqual(len(cons), 1) + conn = cons[0] check_connection_ntuple(conn) @@ -115,7 +107,14 @@ def check_socket(self, sock, conn=None): compare_procsys_connections(os.getpid(), cons) return conn - # --- non connected sockets + +# ===================================================================== +# --- Test unconnected sockets +# ===================================================================== + + +class TestUnconnectedSockets(Base, unittest.TestCase): + """Tests sockets which are open but not connected to anything.""" def test_tcp_v4(self): addr = ("127.0.0.1", get_free_port()) @@ -172,6 +171,24 @@ class TestConnectedSocketPairs(Base, unittest.TestCase): each other. """ + @staticmethod + def differentiate_tcp_socks(cons, server_addr): + if cons[0].raddr == server_addr: + return (cons[0], cons[1]) + else: + assert cons[1].raddr == server_addr + return (cons[1], cons[0]) + + def test_tcp(self): + from psutil.tests import inet_socketpair + addr = ("127.0.0.1", get_free_port()) + s_sock, c_sock = inet_socketpair(AF_INET, SOCK_STREAM, addr=addr) + cons = psutil.Process().connections(kind='all') + s_conn, c_conn = self.differentiate_tcp_socks(cons, addr) + self.check_socket(s_sock, conn=s_conn) + self.assertEqual(s_conn.status, psutil.CONN_ESTABLISHED) + self.assertEqual(c_conn.status, psutil.CONN_ESTABLISHED) + @skip_on_access_denied(only_if=OSX) def test_combos(self): def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): From 78a7658b19ad1c745e9cc6b8033bf5f23bbd946a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 08:13:05 +0200 Subject: [PATCH 393/922] fix connection tests --- psutil/tests/__init__.py | 2 +- psutil/tests/test_connections.py | 12 ++++++++---- psutil/tests/test_misc.py | 13 +++++++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index fad0c3e7e..99cc30a97 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -830,7 +830,7 @@ def inet_socketpair(family, type, addr=("", 0)): a, addr = ll.accept() # check that we've got the correct client if addr == caddr: - return c, a + return (a, c) a.close() except OSError: c.close() diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 597ead019..aec9a5e50 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -173,19 +173,23 @@ class TestConnectedSocketPairs(Base, unittest.TestCase): @staticmethod def differentiate_tcp_socks(cons, server_addr): - if cons[0].raddr == server_addr: + """Given a list of connections return a (server, client) + tuple. + """ + if cons[0].laddr == server_addr: return (cons[0], cons[1]) else: - assert cons[1].raddr == server_addr + assert cons[1].laddr == server_addr return (cons[1], cons[0]) def test_tcp(self): from psutil.tests import inet_socketpair addr = ("127.0.0.1", get_free_port()) - s_sock, c_sock = inet_socketpair(AF_INET, SOCK_STREAM, addr=addr) + server, c_sock = inet_socketpair(AF_INET, SOCK_STREAM, addr=addr) cons = psutil.Process().connections(kind='all') s_conn, c_conn = self.differentiate_tcp_socks(cons, addr) - self.check_socket(s_sock, conn=s_conn) + self.check_socket(server, conn=s_conn) + self.check_socket(c_sock, conn=c_conn) self.assertEqual(s_conn.status, psutil.CONN_ESTABLISHED) self.assertEqual(c_conn.status, psutil.CONN_ESTABLISHED) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 358fa7958..22dfc7d2e 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -34,6 +34,7 @@ from psutil.tests import bind_unix_socket from psutil.tests import chdir from psutil.tests import create_proc_children_pair +from psutil.tests import get_free_port from psutil.tests import get_test_subprocess from psutil.tests import importlib from psutil.tests import inet_socketpair @@ -701,12 +702,16 @@ def test_bind_unix_socket(self): self.assertEqual(sock.type, socket.SOCK_DGRAM) def test_inet_socketpair(self): - server, client = inet_socketpair(socket.AF_INET, socket.SOCK_STREAM) + addr = ("127.0.0.1", get_free_port()) + server, client = inet_socketpair( + socket.AF_INET, socket.SOCK_STREAM, addr=addr) with contextlib.closing(server): with contextlib.closing(client): - # ensure they are connected - self.assertEqual(server.getsockname(), client.getpeername()) - self.assertEqual(server.getpeername(), client.getsockname()) + # Ensure they are connected and the positions are + # correct. + self.assertEqual(server.getsockname(), addr) + self.assertEqual(client.getpeername(), addr) + self.assertNotEqual(client.getsockname(), addr) @unittest.skipUnless(POSIX, "POSIX only") def test_unix_socketpair(self): From ee7d6dd4b49a91d45581fe4575dd2e9aa39b63fe Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 28 Apr 2017 23:19:53 -0700 Subject: [PATCH 394/922] refactoring --- psutil/tests/test_connections.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index aec9a5e50..18759c0dd 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -10,6 +10,7 @@ import socket import textwrap from contextlib import closing +from contextlib import nested from socket import AF_INET from socket import AF_INET6 from socket import SOCK_DGRAM @@ -28,6 +29,7 @@ from psutil.tests import bind_unix_socket from psutil.tests import check_connection_ntuple from psutil.tests import get_free_port +from psutil.tests import inet_socketpair from psutil.tests import pyrun from psutil.tests import reap_children from psutil.tests import run_test_module_by_name @@ -183,15 +185,21 @@ def differentiate_tcp_socks(cons, server_addr): return (cons[1], cons[0]) def test_tcp(self): - from psutil.tests import inet_socketpair addr = ("127.0.0.1", get_free_port()) - server, c_sock = inet_socketpair(AF_INET, SOCK_STREAM, addr=addr) - cons = psutil.Process().connections(kind='all') - s_conn, c_conn = self.differentiate_tcp_socks(cons, addr) - self.check_socket(server, conn=s_conn) - self.check_socket(c_sock, conn=c_conn) - self.assertEqual(s_conn.status, psutil.CONN_ESTABLISHED) - self.assertEqual(c_conn.status, psutil.CONN_ESTABLISHED) + server, client = inet_socketpair(AF_INET, SOCK_STREAM, addr=addr) + with nested(closing(server), closing(client)): + cons = psutil.Process().connections(kind='all') + server_conn, client_conn = self.differentiate_tcp_socks(cons, addr) + self.check_socket(server, conn=server_conn) + self.check_socket(client, conn=client_conn) + self.assertEqual(server_conn.status, psutil.CONN_ESTABLISHED) + self.assertEqual(client_conn.status, psutil.CONN_ESTABLISHED) + # May not be fast enough to change state so it stays + # commenteed. + # client.close() + # cons = psutil.Process().connections(kind='all') + # self.assertEqual(len(cons), 1) + # self.assertEqual(cons[0].status, psutil.CONN_CLOSE_WAIT) @skip_on_access_denied(only_if=OSX) def test_combos(self): From 7e6afb20d5d7b0fe2028125ba9924dab28e9976c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 08:24:20 +0200 Subject: [PATCH 395/922] refactoring --- psutil/tests/test_unicode.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 083f48141..c212ccf66 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -53,6 +53,8 @@ import os import socket +from contextlib import closing +from contextlib import nested from psutil import BSD from psutil import OSX @@ -163,16 +165,13 @@ def test_proc_connections(self): raise else: raise unittest.SkipTest("not supported") - - self.addCleanup(safe_rmpath, name) - self.addCleanup(client.close) - self.addCleanup(server.close) - cons = psutil.Process().connections(kind='unix') - self.assertEqual(len(cons), 2) - cmap = dict([(x.fd, x) for x in cons]) - self.assertEqual(cmap[server.fileno()].laddr, name) - if cmap[client.fileno()].laddr: - self.assertEqual(cmap[client.fileno()].laddr, name) + with nested(closing(server), closing(client)): + cons = psutil.Process().connections(kind='unix') + self.assertEqual(len(cons), 2) + cmap = dict([(x.fd, x) for x in cons]) + self.assertEqual(cmap[server.fileno()].laddr, name) + if cmap[client.fileno()].laddr: + self.assertEqual(cmap[client.fileno()].laddr, name) @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") @skip_on_access_denied() From 319e2fd4f54c6cc1bc2f9f9c8990d8b18593f82e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 09:28:13 +0200 Subject: [PATCH 396/922] add a test case which exposes #1013 on FreeBSD --- psutil/_compat.py | 49 ++++++++++++++++++++++++++++++++ psutil/tests/__init__.py | 1 - psutil/tests/test_connections.py | 37 ++++++++++++++++++++---- psutil/tests/test_unicode.py | 31 ++++++++------------ 4 files changed, 93 insertions(+), 25 deletions(-) diff --git a/psutil/_compat.py b/psutil/_compat.py index de91638f6..5777c38dd 100644 --- a/psutil/_compat.py +++ b/psutil/_compat.py @@ -5,6 +5,7 @@ """Module which provides compatibility with older Python versions.""" import collections +import contextlib import functools import os import sys @@ -247,3 +248,51 @@ def _access_check(fn, mode): if _access_check(name, mode): return name return None + + +# A backport of contextlib.nested for Python 3. +nested = getattr(contextlib, "nested") +if nested is None: + @contextlib.contextmanager + def nested(*managers): + """Support multiple context managers in a single with-statement. + + Code like this: + + with nested(A, B, C) as (X, Y, Z): + + + is equivalent to this: + + with A as X: + with B as Y: + with C as Z: + + + """ + exits = [] + vars = [] + exc = (None, None, None) + try: + for mgr in managers: + exit = mgr.__exit__ + enter = mgr.__enter__ + vars.append(enter()) + exits.append(exit) + yield vars + except: # NOQA + exc = sys.exc_info() + finally: + while exits: + exit = exits.pop() + try: + if exit(*exc): + exc = (None, None, None) + except: # NOQA + exc = sys.exc_info() + if exc != (None, None, None): + # Don't rely on sys.exc_info() still containing + # the right information. Another exception may + # have been raised and caught by an exit method + # exc[1] already has the __traceback__ attribute populated + raise exc[1] diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 99cc30a97..36ac713ce 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -923,7 +923,6 @@ def check_connection_ntuple(conn): if err.errno != errno.EADDRNOTAVAIL: raise elif conn.family == AF_UNIX: - assert not conn.raddr, repr(conn.raddr) assert conn.status == psutil.CONN_NONE, conn.status # check type diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 18759c0dd..8b67768e1 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -10,7 +10,6 @@ import socket import textwrap from contextlib import closing -from contextlib import nested from socket import AF_INET from socket import AF_INET6 from socket import SOCK_DGRAM @@ -23,6 +22,7 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._common import supports_ipv6 +from psutil._compat import nested from psutil._compat import unicode from psutil.tests import AF_UNIX from psutil.tests import bind_socket @@ -38,6 +38,7 @@ from psutil.tests import TESTFN from psutil.tests import unittest from psutil.tests import unix_socket_path +from psutil.tests import unix_socketpair from psutil.tests import wait_for_file @@ -174,14 +175,25 @@ class TestConnectedSocketPairs(Base, unittest.TestCase): """ @staticmethod - def differentiate_tcp_socks(cons, server_addr): + def distinguish_tcp_socks(cons, server_addr): """Given a list of connections return a (server, client) - tuple. + connection ntuple. """ if cons[0].laddr == server_addr: return (cons[0], cons[1]) else: - assert cons[1].laddr == server_addr + assert cons[1].laddr == server_addr, (cons, server_addr) + return (cons[1], cons[0]) + + @staticmethod + def distinguish_unix_socks(cons): + """Given a list of connections and 2 sockets return a + (server, client) connection ntuple. + """ + if cons[0].laddr: + return (cons[0], cons[1]) + else: + assert cons[1].laddr, cons return (cons[1], cons[0]) def test_tcp(self): @@ -189,7 +201,7 @@ def test_tcp(self): server, client = inet_socketpair(AF_INET, SOCK_STREAM, addr=addr) with nested(closing(server), closing(client)): cons = psutil.Process().connections(kind='all') - server_conn, client_conn = self.differentiate_tcp_socks(cons, addr) + server_conn, client_conn = self.distinguish_tcp_socks(cons, addr) self.check_socket(server, conn=server_conn) self.check_socket(client, conn=client_conn) self.assertEqual(server_conn.status, psutil.CONN_ESTABLISHED) @@ -201,6 +213,21 @@ def test_tcp(self): # self.assertEqual(len(cons), 1) # self.assertEqual(cons[0].status, psutil.CONN_CLOSE_WAIT) + def test_unix(self): + with unix_socket_path() as name: + server, client = unix_socketpair(name) + with nested(closing(server), closing(client)): + cons = psutil.Process().connections(kind='unix') + self.assertEqual(len(cons), 2) + server_conn, client_conn = self.distinguish_unix_socks(cons) + self.check_socket(server, conn=server_conn) + self.check_socket(client, conn=client_conn) + self.assertEqual(server_conn.laddr, name) + # TODO: https://github.com/giampaolo/psutil/issues/1035 + self.assertIn(server_conn.raddr, ("", None)) + # TODO: https://github.com/giampaolo/psutil/issues/1035 + self.assertIn(client_conn.laddr, ("", None)) + @skip_on_access_denied(only_if=OSX) def test_combos(self): def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index c212ccf66..571f4cc95 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -54,7 +54,6 @@ import os import socket from contextlib import closing -from contextlib import nested from psutil import BSD from psutil import OSX @@ -76,7 +75,6 @@ from psutil.tests import TRAVIS from psutil.tests import unittest from psutil.tests import unix_socket_path -from psutil.tests import unix_socketpair import psutil import psutil.tests @@ -156,22 +154,18 @@ def test_proc_open_files(self): @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") def test_proc_connections(self): - with unix_socket_path( - suffix=os.path.basename(self.funky_name)) as name: + suffix = os.path.basename(self.funky_name) + with unix_socket_path(suffix=suffix) as name: try: - server, client = unix_socketpair(name) + sock = bind_unix_socket(name) except UnicodeEncodeError: if PY3: raise else: raise unittest.SkipTest("not supported") - with nested(closing(server), closing(client)): - cons = psutil.Process().connections(kind='unix') - self.assertEqual(len(cons), 2) - cmap = dict([(x.fd, x) for x in cons]) - self.assertEqual(cmap[server.fileno()].laddr, name) - if cmap[client.fileno()].laddr: - self.assertEqual(cmap[client.fileno()].laddr, name) + with closing(sock): + conn = psutil.Process().connections('unix')[0] + self.assertEqual(conn.laddr, name) @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") @skip_on_access_denied() @@ -182,8 +176,8 @@ def find_sock(cons): return conn raise ValueError("connection not found") - with unix_socket_path( - suffix=os.path.basename(self.funky_name)) as name: + suffix = os.path.basename(self.funky_name) + with unix_socket_path(suffix=suffix) as name: try: sock = bind_unix_socket(name) except UnicodeEncodeError: @@ -191,11 +185,10 @@ def find_sock(cons): raise else: raise unittest.SkipTest("not supported") - self.addCleanup(safe_rmpath, name) - self.addCleanup(sock.close) - cons = psutil.net_connections(kind='unix') - conn = find_sock(cons) - self.assertEqual(conn.laddr, name) + with closing(sock): + cons = psutil.net_connections(kind='unix') + conn = find_sock(cons) + self.assertEqual(conn.laddr, name) def test_disk_usage(self): safe_mkdir(self.funky_name) From 2a40c610c9716d16fdd93cb9453e01b70def3672 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 10:50:21 +0200 Subject: [PATCH 397/922] fix python bug of socket.setblocking() which changes socket.type value - http://bugs.python.org/issue30204 --- psutil/_compat.py | 2 +- psutil/tests/__init__.py | 12 +++++------- psutil/tests/test_connections.py | 8 +++++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/psutil/_compat.py b/psutil/_compat.py index 5777c38dd..9f2182c22 100644 --- a/psutil/_compat.py +++ b/psutil/_compat.py @@ -251,7 +251,7 @@ def _access_check(fn, mode): # A backport of contextlib.nested for Python 3. -nested = getattr(contextlib, "nested") +nested = getattr(contextlib, "nested", None) if nested is None: @contextlib.contextmanager def nested(*managers): diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 36ac713ce..3dc8292ec 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -932,23 +932,21 @@ def check_connection_ntuple(conn): # check laddr (IP address and port sanity) for addr in (conn.laddr, conn.raddr): - if not addr: - continue if conn.family in (AF_INET, AF_INET6): assert isinstance(addr, tuple), addr + if not addr: + continue ip, port = addr assert isinstance(port, int), port assert 0 <= port <= 65535, port check_net_address(ip, conn.family) elif conn.family == AF_UNIX: - assert isinstance(addr, (str, None)), addr - - # check raddr - assert isinstance(conn.status, (tuple, str, None)), repr(conn.status) + assert isinstance(addr, (str, type(None))), addr # check status + assert isinstance(conn.status, str), conn valids = [getattr(psutil, x) for x in dir(psutil) if x.startswith('CONN_')] - assert conn.status in valids, conn.status + assert conn.status in valids, conn # =================================================================== diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 8b67768e1..da83ad5b5 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -23,7 +23,6 @@ from psutil import WINDOWS from psutil._common import supports_ipv6 from psutil._compat import nested -from psutil._compat import unicode from psutil.tests import AF_UNIX from psutil.tests import bind_socket from psutil.tests import bind_unix_socket @@ -97,7 +96,9 @@ def check_socket(self, sock, conn=None): if conn.fd != -1: self.assertEqual(conn.fd, sock.fileno()) self.assertEqual(conn.family, sock.family) - self.assertEqual(conn.type, sock.type) + # see: http://bugs.python.org/issue30204 + self.assertEqual( + conn.type, sock.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE)) # local address laddr = sock.getsockname() @@ -221,6 +222,7 @@ def test_unix(self): self.assertEqual(len(cons), 2) server_conn, client_conn = self.distinguish_unix_socks(cons) self.check_socket(server, conn=server_conn) + self.check_socket(client, conn=client_conn) self.assertEqual(server_conn.laddr, name) # TODO: https://github.com/giampaolo/psutil/issues/1035 @@ -361,7 +363,7 @@ def check(cons, families, types_): self.assertIn(conn.family, families, msg=conn) if conn.family != AF_UNIX: self.assertIn(conn.type, types_, msg=conn) - self.assertIsInstance(conn.status, (str, unicode)) + check_connection_ntuple(conn) from psutil._common import conn_tmap for kind, groups in conn_tmap.items(): From cbefa12f3dc279624c8985b8aa46b6bb03da2e89 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 11:07:43 +0200 Subject: [PATCH 398/922] fix python bug #30205: socket.getsockname() for a UNIX socket may return bytes instead of str --- psutil/tests/test_connections.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index da83ad5b5..cb5807c45 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -23,6 +23,7 @@ from psutil import WINDOWS from psutil._common import supports_ipv6 from psutil._compat import nested +from psutil._compat import PY3 from psutil.tests import AF_UNIX from psutil.tests import bind_socket from psutil.tests import bind_unix_socket @@ -102,6 +103,9 @@ def check_socket(self, sock, conn=None): # local address laddr = sock.getsockname() + if not laddr and PY3 and isinstance(laddr, bytes): + # See: http://bugs.python.org/issue30205 + laddr = laddr.decode() if sock.family == AF_INET6: laddr = laddr[:2] self.assertEqual(conn.laddr, laddr) From 19514419f13526135394a95f6960169d9968fa84 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 11:36:04 +0200 Subject: [PATCH 399/922] linux / connectins: it appears it exist a UNIX socket with SOCK_SEQPACKET type --- psutil/tests/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 3dc8292ec..fa01ea4c8 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -151,6 +151,8 @@ if x.startswith('STATUS_')] GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" AF_UNIX = getattr(socket, "AF_UNIX", object()) +SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) + TEST_DEPS = [] if sys.version_info[:2] == (2, 6): TEST_DEPS.extend(["ipaddress", "unittest2", "argparse", "mock==1.0.1"]) @@ -778,7 +780,7 @@ def unix_socket_path(suffix=""): """A context manager which returns a non-existent file name and tries to delete it on exit. """ - assert psutil.POSIX, "not a POSIX system" + assert psutil.POSIX path = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) try: yield path @@ -801,7 +803,7 @@ def bind_socket(addr, family, type): def bind_unix_socket(name, type=socket.SOCK_STREAM): """Bind a UNIX socket.""" - assert psutil.POSIX, "not a POSIX system" + assert psutil.POSIX assert not os.path.exists(name), name sock = socket.socket(socket.AF_UNIX, type) try: @@ -925,8 +927,9 @@ def check_connection_ntuple(conn): elif conn.family == AF_UNIX: assert conn.status == psutil.CONN_NONE, conn.status - # check type - assert conn.type in (SOCK_STREAM, SOCK_DGRAM), repr(conn.type) + # check type (SOCK_SEQPACKET may happen in case of AF_UNIX socks) + assert conn.type in (SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET), \ + repr(conn.type) if conn.type == SOCK_DGRAM: assert conn.status == psutil.CONN_NONE, conn.status From d647fda22709e1e969df59258325b78afee6f2a4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Apr 2017 11:42:15 +0200 Subject: [PATCH 400/922] rename function --- psutil/tests/__init__.py | 10 +++++----- psutil/tests/test_connections.py | 4 ++-- psutil/tests/test_misc.py | 7 +++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index fa01ea4c8..0faecc86b 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -816,15 +816,15 @@ def bind_unix_socket(name, type=socket.SOCK_STREAM): return sock -def inet_socketpair(family, type, addr=("", 0)): - """Build a pair of INET sockets connected to each other. +def tcp_socketpair(family, addr=("", 0)): + """Build a pair of TCP sockets connected to each other. Return a (server, client) tuple. """ - with contextlib.closing(socket.socket(family, type)) as ll: + with contextlib.closing(socket.socket(family, SOCK_STREAM)) as ll: ll.bind(addr) ll.listen(10) addr = ll.getsockname() - c = socket.socket(family, type) + c = socket.socket(family, SOCK_STREAM) try: c.connect(addr) caddr = c.getsockname() @@ -844,7 +844,7 @@ def unix_socketpair(name): the same UNIX file name. Return a (server, client) tuple. """ - assert psutil.POSIX, "not a POSIX system" + assert psutil.POSIX server = bind_unix_socket(name, type=socket.SOCK_STREAM) server.setblocking(0) client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index cb5807c45..b30c65e23 100644 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -29,12 +29,12 @@ from psutil.tests import bind_unix_socket from psutil.tests import check_connection_ntuple from psutil.tests import get_free_port -from psutil.tests import inet_socketpair from psutil.tests import pyrun from psutil.tests import reap_children from psutil.tests import run_test_module_by_name from psutil.tests import safe_rmpath from psutil.tests import skip_on_access_denied +from psutil.tests import tcp_socketpair from psutil.tests import TESTFN from psutil.tests import unittest from psutil.tests import unix_socket_path @@ -203,7 +203,7 @@ def distinguish_unix_socks(cons): def test_tcp(self): addr = ("127.0.0.1", get_free_port()) - server, client = inet_socketpair(AF_INET, SOCK_STREAM, addr=addr) + server, client = tcp_socketpair(AF_INET, addr=addr) with nested(closing(server), closing(client)): cons = psutil.Process().connections(kind='all') server_conn, client_conn = self.distinguish_tcp_socks(cons, addr) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 22dfc7d2e..a0a7a0b1b 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -37,7 +37,6 @@ from psutil.tests import get_free_port from psutil.tests import get_test_subprocess from psutil.tests import importlib -from psutil.tests import inet_socketpair from psutil.tests import mock from psutil.tests import reap_children from psutil.tests import retry @@ -46,6 +45,7 @@ from psutil.tests import safe_rmpath from psutil.tests import SCRIPTS_DIR from psutil.tests import sh +from psutil.tests import tcp_socketpair from psutil.tests import TESTFN from psutil.tests import TOX from psutil.tests import TRAVIS @@ -701,10 +701,9 @@ def test_bind_unix_socket(self): with contextlib.closing(sock): self.assertEqual(sock.type, socket.SOCK_DGRAM) - def test_inet_socketpair(self): + def tcp_tcp_socketpair(self): addr = ("127.0.0.1", get_free_port()) - server, client = inet_socketpair( - socket.AF_INET, socket.SOCK_STREAM, addr=addr) + server, client = tcp_socketpair(socket.AF_INET, addr=addr) with contextlib.closing(server): with contextlib.closing(client): # Ensure they are connected and the positions are From 9cd786691a12a4db3da3ba02e2c2772ca348966e Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 21:36:26 +0530 Subject: [PATCH 401/922] create script to check broken links --- scripts/internal/check_broken_links.py | 115 +++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100755 scripts/internal/check_broken_links.py diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py new file mode 100755 index 000000000..7a5b69837 --- /dev/null +++ b/scripts/internal/check_broken_links.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python + +# Author : Himanshu Shekhar < https://github.com/himanshub16 > (2017) +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Checks for broken links in file names specified as command line parameters. + +There are a ton of a solutions available for validating URLs in string using +regex, but less for searching, of which very few are accurate. +This snippet is intended to just do the required work, and avoid complexities. +Django Validator has pretty good regex for validation, but we have to find +urls instead of validating them. (REFERENCES [7]) +There's always room for improvement. + +Method: +* Match URLs using regex (REFERENCES [1]]) +* Some URLs need to be fixed, as they have < (or) > due to inefficient regex. +* Remove duplicates (because regex is not 100% efficient as of now). +* Check validity of URL, using HEAD request. (HEAD to save bandwidth) + Uses requests module for others are painful to use. REFERENCES[9] + +REFERENCES: +Using [1] with some modificatons for including ftp +[1] http://stackoverflow.com/questions/6883049/regex-to-find-urls-in-string-in-python +[2] http://stackoverflow.com/a/31952097/5163807 +[3] http://daringfireball.net/2010/07/improved_regex_for_matching_urls +[4] https://mathiasbynens.be/demo/url-regex +[5] https://github.com/django/django/blob/master/django/core/validators.py +[6] https://data.iana.org/TLD/tlds-alpha-by-domain.txt +[7] https://codereview.stackexchange.com/questions/19663/http-url-validating +[8] https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD +[9] http://docs.python-requests.org/ + +""" + +from __future__ import print_function + +import os +import re +import sys +import requests + + +HERE = os.path.abspath(os.path.dirname(__file__)) +print (HERE) + +URL_REGEX = '(?:http|ftp|https)?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' + + +def get_urls(filename): + """Extracts all URLs available in specified filename + """ + fname = os.path.abspath(os.path.join(HERE, filename)) + print (fname) + text = '' + with open(fname) as f: + text = f.read() + + urls = re.findall(URL_REGEX, text) + # remove duplicates, list for sets are not iterable + urls = list(set(urls)) + # correct urls which are between < and/or > + i = 0 + while i < len(urls): + urls[i] = re.sub("[<>]", '', urls[i]) + i += 1 + + return urls + + +def validate_url(url): + """Validate the URL by attempting an HTTP connection. + Makes an HTTP-HEAD request for each URL. + Uses requests module. + """ + try: + res = requests.head(url) + return res.ok + except Exception as e: + return False + + +def main(): + """Main function + """ + files = sys.argv[1:] + fails = [] + for fname in files: + urls = get_urls(fname) + i = 0 + last = len(urls) + for url in urls: + i += 1 + if not validate_url(url): + fails.append((url, fname)) + sys.stdout.write("\r " + fname + " : " + str(i) + " / " + str(last)) + sys.stdout.flush() + + print() + if len(fails) == 0: + print("All URLs are valid. Cheers!") + else: + print ("Total :", len(fails), "fails!") + print ("Writing failed urls to fails.txt") + with open("../../fails.txt", 'w') as f: + for fail in fails: + f.write(fail[1] + ' : ' + fail[0] + os.linesep) + f.write('-' * 20) + f.write(os.linesep*2) + + +if __name__ == '__main__': + main() From f210a54057669aa57b042372fe6604d5cbde9cd6 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 21:39:05 +0530 Subject: [PATCH 402/922] add description for requests --- scripts/internal/check_broken_links.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 7a5b69837..d22192bc7 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -20,6 +20,7 @@ * Remove duplicates (because regex is not 100% efficient as of now). * Check validity of URL, using HEAD request. (HEAD to save bandwidth) Uses requests module for others are painful to use. REFERENCES[9] + Handles redirects, http, https, ftp as well. REFERENCES: Using [1] with some modificatons for including ftp @@ -44,7 +45,6 @@ HERE = os.path.abspath(os.path.dirname(__file__)) -print (HERE) URL_REGEX = '(?:http|ftp|https)?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' From 6ca38cb13411da88f6027e13b7ceaae487c0a3d6 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 21:46:06 +0530 Subject: [PATCH 403/922] expecting absolute path in script --- scripts/internal/check_broken_links.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index d22192bc7..8a5072037 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -52,7 +52,9 @@ def get_urls(filename): """Extracts all URLs available in specified filename """ - fname = os.path.abspath(os.path.join(HERE, filename)) + # fname = os.path.abspath(os.path.join(HERE, filename)) + # expecting absolute path + fname = os.path.abspath(filename) print (fname) text = '' with open(fname) as f: From 3527c7eb42d543aa2427be23886647c954699608 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 21:46:23 +0530 Subject: [PATCH 404/922] update makefile to include check-broken-links --- Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Makefile b/Makefile index 64484177d..9a55cbab3 100644 --- a/Makefile +++ b/Makefile @@ -273,3 +273,10 @@ doc: cd docs && make html && cd _build/html/ && zip doc.zip -r . mv docs/_build/html/doc.zip . @echo "done; now manually upload doc.zip from here: https://pypi.python.org/pypi?:action=pkg_edit&name=psutil" + +# check whether the links mentioned in some files are valid. +FILES_TO_CHECK_FOR_BROKEN_LINKS = $(PWD)/docs/index.rst \ + $(PWD)/DEVGUIDE.rst + +check-broken-links: + $(PYTHON) scripts/internal/check_broken_links.py $(FILES_TO_CHECK_FOR_BROKEN_LINKS) From 5128374c015a77bb6fe596ac29a99fdce2ae90a4 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 22:45:04 +0530 Subject: [PATCH 405/922] fix linting for ci-tests --- scripts/internal/check_broken_links.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 8a5072037..ac62b05d7 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -24,7 +24,7 @@ REFERENCES: Using [1] with some modificatons for including ftp -[1] http://stackoverflow.com/questions/6883049/regex-to-find-urls-in-string-in-python +[1] http://stackoverflow.com/a/6883094/5163807 [2] http://stackoverflow.com/a/31952097/5163807 [3] http://daringfireball.net/2010/07/improved_regex_for_matching_urls [4] https://mathiasbynens.be/demo/url-regex @@ -46,7 +46,8 @@ HERE = os.path.abspath(os.path.dirname(__file__)) -URL_REGEX = '(?:http|ftp|https)?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' +REGEX = r'(?:http|ftp|https)?://' +REGEX += r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' def get_urls(filename): @@ -60,7 +61,7 @@ def get_urls(filename): with open(fname) as f: text = f.read() - urls = re.findall(URL_REGEX, text) + urls = re.findall(REGEX, text) # remove duplicates, list for sets are not iterable urls = list(set(urls)) # correct urls which are between < and/or > @@ -80,7 +81,7 @@ def validate_url(url): try: res = requests.head(url) return res.ok - except Exception as e: + except requests.exceptions.RequestException: return False @@ -97,7 +98,8 @@ def main(): i += 1 if not validate_url(url): fails.append((url, fname)) - sys.stdout.write("\r " + fname + " : " + str(i) + " / " + str(last)) + sys.stdout.write("\r " + + fname + " : " + str(i) + " / " + str(last)) sys.stdout.flush() print() From 8aa1bd7ad75b4d7812704f89724113bde83d4650 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 23:02:22 +0530 Subject: [PATCH 406/922] remove space before print --- scripts/internal/check_broken_links.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index ac62b05d7..038c6f4e4 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -56,7 +56,7 @@ def get_urls(filename): # fname = os.path.abspath(os.path.join(HERE, filename)) # expecting absolute path fname = os.path.abspath(filename) - print (fname) + print(fname) text = '' with open(fname) as f: text = f.read() @@ -106,8 +106,8 @@ def main(): if len(fails) == 0: print("All URLs are valid. Cheers!") else: - print ("Total :", len(fails), "fails!") - print ("Writing failed urls to fails.txt") + print("Total :", len(fails), "fails!") + print("Writing failed urls to fails.txt") with open("../../fails.txt", 'w') as f: for fail in fails: f.write(fail[1] + ' : ' + fail[0] + os.linesep) From 83d7e252694d7da9c57d40e3d83ef180894e803e Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 23:38:37 +0530 Subject: [PATCH 407/922] differentiate stdlib and requests import --- scripts/internal/check_broken_links.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 038c6f4e4..1bf502f91 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -41,6 +41,7 @@ import os import re import sys + import requests From 866a326aaddbcc1903e2d1eebe065b5d4be3fc36 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 23:39:35 +0530 Subject: [PATCH 408/922] break regex to two lines by backslash --- scripts/internal/check_broken_links.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 1bf502f91..3f178bbd5 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -47,8 +47,8 @@ HERE = os.path.abspath(os.path.dirname(__file__)) -REGEX = r'(?:http|ftp|https)?://' -REGEX += r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' +REGEX = r'(?:http|ftp|https)?://' \ + r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' def get_urls(filename): From 0647e9c6b8c6cf20e4e54f48682eb1ddb87f0b2d Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 23:40:12 +0530 Subject: [PATCH 409/922] remove print statement from utility function --- scripts/internal/check_broken_links.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 3f178bbd5..b1074dd34 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -57,7 +57,6 @@ def get_urls(filename): # fname = os.path.abspath(os.path.join(HERE, filename)) # expecting absolute path fname = os.path.abspath(filename) - print(fname) text = '' with open(fname) as f: text = f.read() From 9f639ad83c166aab1ea2f6bb047692d9678e764d Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 23:44:46 +0530 Subject: [PATCH 410/922] add message if no files are provided --- scripts/internal/check_broken_links.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index b1074dd34..783d62637 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -89,6 +89,9 @@ def main(): """Main function """ files = sys.argv[1:] + + if not files: + return sys.exit("usage: %s " % __name__) fails = [] for fname in files: urls = get_urls(fname) @@ -104,7 +107,7 @@ def main(): print() if len(fails) == 0: - print("All URLs are valid. Cheers!") + print("All links are valid. Cheers!") else: print("Total :", len(fails), "fails!") print("Writing failed urls to fails.txt") From fbeaf949f50aee7b7e8a14d15aab0ae9c1ad975b Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sat, 29 Apr 2017 23:50:52 +0530 Subject: [PATCH 411/922] change output target from file to stdout --- scripts/internal/check_broken_links.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 783d62637..bab8f3c29 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -107,15 +107,12 @@ def main(): print() if len(fails) == 0: - print("All links are valid. Cheers!") + print("all links are valid. cheers!") else: - print("Total :", len(fails), "fails!") - print("Writing failed urls to fails.txt") - with open("../../fails.txt", 'w') as f: - for fail in fails: - f.write(fail[1] + ' : ' + fail[0] + os.linesep) - f.write('-' * 20) - f.write(os.linesep*2) + print("total :", len(fails), "fails!") + for fail in fails: + print(fail[1] + ' : ' + fail[0] + os.linesep) + print('-' * 20) if __name__ == '__main__': From 6d5c62f40e7daa9903335974cde0c28f047bec39 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sun, 30 Apr 2017 01:45:02 +0530 Subject: [PATCH 412/922] add concurrency to validations --- scripts/internal/check_broken_links.py | 43 ++++++++++++++++++++------ 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index bab8f3c29..81a0dd7d4 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -41,7 +41,9 @@ import os import re import sys +import time +from concurrent.futures import ThreadPoolExecutor import requests @@ -85,6 +87,34 @@ def validate_url(url): return False +def parallel_validator(urls): + """validates all urls in parallel + urls: tuple(filename, url) + """ + fails = [] # list of tuples (filename, url) + completed = 0 + total = len(urls) + threads = [] + + with ThreadPoolExecutor() as executor: + for url in urls: + fut = executor.submit(validate_url, url[1]) + threads.append((url, fut)) + + # wait for threads to progress a little + time.sleep(2) + for thr in threads: + url = thr[0] + fut = thr[1] + if not fut.result(): + fails.append((url[0], url[1])) + completed += 1 + sys.stdout.write("\r" + str(completed)+' / '+str(total)) + sys.stdout.flush() + print() + return fails + + def main(): """Main function """ @@ -92,20 +122,13 @@ def main(): if not files: return sys.exit("usage: %s " % __name__) - fails = [] + all_urls = [] for fname in files: urls = get_urls(fname) - i = 0 - last = len(urls) for url in urls: - i += 1 - if not validate_url(url): - fails.append((url, fname)) - sys.stdout.write("\r " + - fname + " : " + str(i) + " / " + str(last)) - sys.stdout.flush() + all_urls.append((fname, url)) - print() + fails = parallel_validator(all_urls) if len(fails) == 0: print("all links are valid. cheers!") else: From b220c3bcad5e674146a8e5257b666d1699d61259 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sun, 30 Apr 2017 02:04:17 +0530 Subject: [PATCH 413/922] add * to regex exclusion (case with *boldtext* in markdown) --- scripts/internal/check_broken_links.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 81a0dd7d4..8690981dd 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -69,7 +69,7 @@ def get_urls(filename): # correct urls which are between < and/or > i = 0 while i < len(urls): - urls[i] = re.sub("[<>]", '', urls[i]) + urls[i] = re.sub("[\*<>]", '', urls[i]) i += 1 return urls From 4592acd7f2bce250814af960e8c6d6f0c5b1368a Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sun, 30 Apr 2017 02:22:22 +0530 Subject: [PATCH 414/922] handle some special error codes if not 200 --- scripts/internal/check_broken_links.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 8690981dd..7e54c4cd4 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -52,6 +52,11 @@ REGEX = r'(?:http|ftp|https)?://' \ r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' +# There are some status codes sent by websites on HEAD request. +# Like 503 by Microsoft, and 401 by Apple +# They need to be sent GET request +RETRY_STATUSES = [503, 401, 403] + def get_urls(filename): """Extracts all URLs available in specified filename @@ -69,7 +74,7 @@ def get_urls(filename): # correct urls which are between < and/or > i = 0 while i < len(urls): - urls[i] = re.sub("[\*<>]", '', urls[i]) + urls[i] = re.sub("[\*<>\(\)\)]", '', urls[i]) i += 1 return urls @@ -82,6 +87,10 @@ def validate_url(url): """ try: res = requests.head(url) + # some websites deny 503, like Microsoft + # and some send 401, like Apple, observations + if (not res.ok) and (res.status_code in RETRY_STATUSES): + res = requests.get(url) return res.ok except requests.exceptions.RequestException: return False From 89193ac8e7b208c3ed75b92eb138b1dde40710b9 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sun, 30 Apr 2017 02:23:28 +0530 Subject: [PATCH 415/922] update argument passing method to script --- Makefile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9a55cbab3..ff9bc7782 100644 --- a/Makefile +++ b/Makefile @@ -275,8 +275,7 @@ doc: @echo "done; now manually upload doc.zip from here: https://pypi.python.org/pypi?:action=pkg_edit&name=psutil" # check whether the links mentioned in some files are valid. -FILES_TO_CHECK_FOR_BROKEN_LINKS = $(PWD)/docs/index.rst \ - $(PWD)/DEVGUIDE.rst - check-broken-links: - $(PYTHON) scripts/internal/check_broken_links.py $(FILES_TO_CHECK_FOR_BROKEN_LINKS) + git ls-files | grep \\.rst$ | xargs $(PYTHON) scripts/internal/check_broken_links.py +# Alternate method, DOCFILES need to be defined +# $(PYTHON) scripts/internal/check_broken_links.py $(DOCFILES) From 31f960cefc9eae7d8d55a9877c3d4ceb3ec481d9 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sun, 30 Apr 2017 02:26:12 +0530 Subject: [PATCH 416/922] add requests as dependency --- Makefile | 3 ++- scripts/internal/winmake.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ff9bc7782..0fa418df2 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,8 @@ DEPS = \ setuptools \ sphinx \ twine \ - unittest2 + unittest2 \ + requests # In not in a virtualenv, add --user options for install commands. INSTALL_OPTS = `$(PYTHON) -c "import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"` diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index c2db0fe3e..27f2d2591 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -41,6 +41,7 @@ "unittest2", "wheel", "wmi", + "requests" ] _cmds = {} From c58e00fa43a581456d3314d4fa13f42297decff1 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sun, 30 Apr 2017 02:26:57 +0530 Subject: [PATCH 417/922] exit with non-zero exit code on failure --- scripts/internal/check_broken_links.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 7e54c4cd4..7f69e9ee3 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -145,6 +145,7 @@ def main(): for fail in fails: print(fail[1] + ' : ' + fail[0] + os.linesep) print('-' * 20) + sys.exit(1) if __name__ == '__main__': From a267c772eef1b762430e6aa831f7ce0d97dd9c9a Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sun, 30 Apr 2017 03:10:05 +0530 Subject: [PATCH 418/922] fix license --- scripts/internal/check_broken_links.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 7f69e9ee3..f628bc4d8 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -1,6 +1,8 @@ #!/usr/bin/env python # Author : Himanshu Shekhar < https://github.com/himanshub16 > (2017) + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. From 96eda726cfcfea4efe8b5c7e6ce83ef30bc04166 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 00:25:36 +0200 Subject: [PATCH 419/922] refactoring --- .git-pre-commit | 5 +++-- scripts/internal/download_exes.py | 33 ++++++++++++++++--------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.git-pre-commit b/.git-pre-commit index 23ed51ab1..a2f2d18e4 100755 --- a/.git-pre-commit +++ b/.git-pre-commit @@ -37,6 +37,8 @@ def term_supports_colors(): def hilite(s, ok=True, bold=False): """Return an highlighted version of 'string'.""" + if not term_supports_colors(): + return s attr = [] if ok is None: # no color pass @@ -50,8 +52,7 @@ def hilite(s, ok=True, bold=False): def exit(msg): - if term_supports_colors(): - msg = hilite(msg, ok=False) + msg = hilite(msg, ok=False) print(msg, file=sys.stderr) sys.exit(1) diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_exes.py index e607b87d6..249b86899 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_exes.py @@ -20,7 +20,6 @@ import requests import shutil import sys - from concurrent.futures import ThreadPoolExecutor from psutil import __version__ as PSUTIL_VERSION @@ -28,6 +27,7 @@ BASE_URL = 'https://ci.appveyor.com/api' PY_VERSIONS = ['2.7', '3.3', '3.4', '3.5', '3.6'] +COLORS = True def exit(msg): @@ -47,22 +47,23 @@ def term_supports_colors(file=sys.stdout): return True -if term_supports_colors(): - def hilite(s, ok=True, bold=False): - """Return an highlighted version of 'string'.""" - attr = [] - if ok is None: # no color - pass - elif ok: # green - attr.append('32') - else: # red - attr.append('31') - if bold: - attr.append('1') - return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s) -else: - def hilite(s, *a, **k): +COLORS = term_supports_colors() + + +def hilite(s, ok=True, bold=False): + """Return an highlighted version of 'string'.""" + if not COLORS: return s + attr = [] + if ok is None: # no color + pass + elif ok: # green + attr.append('32') + else: # red + attr.append('31') + if bold: + attr.append('1') + return '\x1b[%sm%s\x1b[0m' % (';'.join(attr), s) def safe_makedirs(path): From 6e16db5d49dbe5c93885e4950e43c6c1465e6a61 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 00:52:02 +0200 Subject: [PATCH 420/922] do not pass max_workers=cpus to ThreadPoolExecutor: by default it does cpus * 5 which is better --- scripts/internal/download_exes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_exes.py index 249b86899..62f3e94d6 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_exes.py @@ -15,7 +15,6 @@ from __future__ import print_function import argparse import errno -import multiprocessing import os import requests import shutil @@ -154,7 +153,7 @@ def rename_27_wheels(): def main(options): files = [] safe_rmtree('dist') - with ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as e: + with ThreadPoolExecutor() as e: for url in get_file_urls(options): fut = e.submit(download_file, url) files.append(fut.result()) From 50028b71cd026636718ff5af8ea8339bd377513d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 02:00:16 +0200 Subject: [PATCH 421/922] skip all unicode tests if subprocess module is not able to deal with the funky named exe --- psutil/tests/__init__.py | 3 ++- psutil/tests/test_unicode.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 0faecc86b..9b0530781 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -229,11 +229,12 @@ def get_test_subprocess(cmd=None, **kwds): pyline += "sleep(60);" cmd = [PYTHON, "-c", pyline] sproc = subprocess.Popen(cmd, **kwds) + _subprocesses_started.add(sproc) wait_for_file(_TESTFN, delete=True, empty=True) else: sproc = subprocess.Popen(cmd, **kwds) + _subprocesses_started.add(sproc) wait_for_pid(sproc.pid) - _subprocesses_started.add(sproc) return sproc diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 571f4cc95..33cf5653f 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -87,6 +87,20 @@ class _BaseFSAPIsTests(object): funky_name = None + @classmethod + def setUpClass(cls): + if not PY3: + create_exe(cls.funky_name) + try: + get_test_subprocess(cmd=[cls.funky_name]) + except UnicodeEncodeError: + # We skip all tests if subprocess module is not able to + # deal with such an exe. + raise unittest.SkipTest( + "subprocess module bumped into encoding error") + else: + reap_children() + def setUp(self): safe_rmpath(self.funky_name) From 13b66a74cd6b39e75781fdc8e77a7b4c03489c24 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 02:14:42 +0200 Subject: [PATCH 422/922] disable test occasionally failing on travis --- psutil/tests/test_linux.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index eb37db00f..ccef60ae6 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -200,6 +200,8 @@ def test_active(self): self.assertAlmostEqual( vmstat_value, psutil_value, delta=MEMORY_TOLERANCE) + # https://travis-ci.org/giampaolo/psutil/jobs/227242952 + @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") @retry_before_failing() def test_inactive(self): vmstat_value = vmstat('inactive memory') * 1024 From 35ebdfb172f4d27678c6d33d1a6f74ca2771e303 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 03:05:13 +0200 Subject: [PATCH 423/922] add test which checks all str-related APIs return a str type and never unicode --- psutil/tests/test_unicode.py | 139 ++++++++++++++++++++++++++++++++--- 1 file changed, 130 insertions(+), 9 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 33cf5653f..e011c2376 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -49,14 +49,17 @@ I'd rather have unicode support broken on Python 2 than having APIs returning variable str/unicode types, see: https://github.com/giampaolo/psutil/issues/655#issuecomment-136131180 + +As such we also test that all APIs on Python 2 always return str and +never unicode. """ import os -import socket from contextlib import closing from psutil import BSD from psutil import OSX +from psutil import POSIX from psutil import WINDOWS from psutil._compat import PY3 from psutil.tests import ASCII_FS @@ -117,6 +120,7 @@ def test_proc_exe(self): p = psutil.Process(subp.pid) exe = p.exe() self.assertIsInstance(exe, str) + self.assertIsInstance(exe, str) if self.expect_exact_path_match(): self.assertEqual(exe, self.funky_name) @@ -140,6 +144,8 @@ def test_proc_cmdline(self): subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) cmdline = p.cmdline() + for part in cmdline: + self.assertIsInstance(part, str) if self.expect_exact_path_match(): self.assertEqual(cmdline, [self.funky_name]) @@ -158,15 +164,15 @@ def test_proc_open_files(self): with open(self.funky_name, 'wb'): new = set(p.open_files()) path = (new - start).pop().path + self.assertIsInstance(path, str) if BSD and not path: # XXX - see https://github.com/giampaolo/psutil/issues/595 return self.skipTest("open_files on BSD is broken") - self.assertIsInstance(path, str) if self.expect_exact_path_match(): self.assertEqual(os.path.normcase(path), os.path.normcase(self.funky_name)) - @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") + @unittest.skipUnless(POSIX, "POSIX only") def test_proc_connections(self): suffix = os.path.basename(self.funky_name) with unix_socket_path(suffix=suffix) as name: @@ -179,9 +185,10 @@ def test_proc_connections(self): raise unittest.SkipTest("not supported") with closing(sock): conn = psutil.Process().connections('unix')[0] + self.assertIsInstance(conn.laddr, str) self.assertEqual(conn.laddr, name) - @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "AF_UNIX not supported") + @unittest.skipUnless(POSIX, "POSIX only") @skip_on_access_denied() def test_net_connections(self): def find_sock(cons): @@ -202,6 +209,7 @@ def find_sock(cons): with closing(sock): cons = psutil.net_connections(kind='unix') conn = find_sock(cons) + self.assertIsInstance(conn.laddr, str) self.assertEqual(conn.laddr, name) def test_disk_usage(self): @@ -238,11 +246,11 @@ def expect_exact_path_match(cls): # =================================================================== -# FS APIs +# Non fs APIs # =================================================================== -class TestOtherAPIS(unittest.TestCase): +class TestNonFSAPIS(unittest.TestCase): """Unicode tests for non fs-related APIs.""" @unittest.skipUnless(hasattr(psutil.Process, "environ"), @@ -254,12 +262,125 @@ def test_proc_environ(self): # we use "è", which is part of the extended ASCII table # (unicode point <= 255). env = os.environ.copy() - funny_str = TESTFN_UNICODE if PY3 else 'è' - env['FUNNY_ARG'] = funny_str + funky_str = TESTFN_UNICODE if PY3 else 'è' + env['FUNNY_ARG'] = funky_str sproc = get_test_subprocess(env=env) p = psutil.Process(sproc.pid) env = p.environ() - self.assertEqual(env['FUNNY_ARG'], funny_str) + for k, v in env.items(): + self.assertIsInstance(k, str) + self.assertIsInstance(v, str) + self.assertEqual(env['FUNNY_ARG'], funky_str) + + +# =================================================================== +# Base str types +# =================================================================== + + +class TestAlwaysStrType(unittest.TestCase): + """Make sure all str-related APIs on Python 2 return a str type + and never unicode. + """ + + @classmethod + def setUpClass(cls): + cls.proc = psutil.Process() + + def tearDown(self): + safe_rmpath(TESTFN) + + def test_proc_cmdline(self): + for bit in self.proc.cmdline(): + self.assertIsInstance(bit, str) + + @unittest.skipUnless(POSIX, 'POSIX only') + def test_proc_connections(self): + with unix_socket_path() as name: + with closing(bind_unix_socket(name)): + conn = self.proc.connections(kind='unix')[0] + self.assertIsInstance(conn.laddr, str) + + def test_proc_cwd(self): + self.assertIsInstance(self.proc.cwd(), str) + + def test_proc_environ(self): + for k, v in self.proc.environ().items(): + self.assertIsInstance(k, str) + self.assertIsInstance(v, str) + + def test_proc_exe(self): + self.assertIsInstance(self.proc.exe(), str) + + def test_proc_memory_maps(self): + for region in self.proc.memory_maps(grouped=False): + self.assertIsInstance(region.addr, str) + self.assertIsInstance(region.path, str) + + def test_proc_name(self): + self.assertIsInstance(self.proc.name(), str) + + def test_proc_open_files(self): + with open(TESTFN, 'w'): + self.assertIsInstance(self.proc.open_files()[0].path, str) + + def test_proc_username(self): + self.assertIsInstance(self.proc.username(), str) + + def test_io_counters(self): + for k in psutil.disk_io_counters(perdisk=True): + self.assertIsInstance(k, str) + + def test_disk_partitions(self): + for disk in psutil.disk_partitions(): + self.assertIsInstance(disk.device, str) + self.assertIsInstance(disk.mountpoint, str) + self.assertIsInstance(disk.fstype, str) + self.assertIsInstance(disk.opts, str) + + @unittest.skipUnless(POSIX, 'POSIX only') + @skip_on_access_denied(only_if=OSX) + def test_net_connections(self): + with unix_socket_path() as name: + with closing(bind_unix_socket(name)): + cons = psutil.net_connections(kind='unix') + assert cons + for conn in cons: + self.assertIsInstance(conn.laddr, str) + + def test_net_if_addrs(self): + for ifname, addrs in psutil.net_if_addrs().items(): + self.assertIsInstance(ifname, str) + for addr in addrs: + self.assertIsInstance(addr.address, str) + self.assertIsInstance(addr.netmask, (str, type(None))) + self.assertIsInstance(addr.broadcast, (str, type(None))) + + def test_net_if_stats(self): + for ifname, _ in psutil.net_if_stats().items(): + self.assertIsInstance(ifname, str) + + def test_net_io_counters(self): + for ifname, _ in psutil.net_io_counters(pernic=True).items(): + self.assertIsInstance(ifname, str) + + def test_sensors_fans(self): + for name, units in psutil.sensors_fans().items(): + self.assertIsInstance(name, str) + for unit in units: + self.assertIsInstance(unit.label, str) + + def test_sensors_temperatures(self): + for name, units in psutil.sensors_temperatures().items(): + self.assertIsInstance(name, str) + for unit in units: + self.assertIsInstance(unit.label, str) + + def test_users(self): + for user in psutil.users(): + self.assertIsInstance(user.name, str) + self.assertIsInstance(user.terminal, str) + self.assertIsInstance(user.host, (str, type(None))) if __name__ == '__main__': From d67154d8a067526eaff953ae95561380b8f9b62a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 03:43:27 +0200 Subject: [PATCH 424/922] add tests for types --- psutil/tests/__init__.py | 12 ++++ psutil/tests/test_process.py | 104 +++++++++++++++++++++++++---------- 2 files changed, 87 insertions(+), 29 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 9b0530781..5c7ea9fb5 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -961,3 +961,15 @@ def check_connection_ntuple(conn): def warn(msg): """Raise a warning msg.""" warnings.warn(msg, UserWarning) + + +def is_namedtuple(x): + """Check if object is an instance of namedtuple.""" + t = type(x) + b = t.__bases__ + if len(b) != 1 or b[0] != tuple: + return False + f = getattr(t, '_fields', None) + if not isinstance(f, tuple): + return False + return all(type(n) == str for n in f) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index e531358c3..26e5cd3e4 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -36,7 +36,6 @@ from psutil._compat import callable from psutil._compat import long from psutil._compat import PY3 -from psutil._compat import unicode from psutil.tests import APPVEYOR from psutil.tests import call_until from psutil.tests import check_connection_ntuple @@ -46,6 +45,7 @@ from psutil.tests import get_test_subprocess from psutil.tests import get_winver from psutil.tests import GLOBAL_TIMEOUT +from psutil.tests import is_namedtuple from psutil.tests import mock from psutil.tests import PYPY from psutil.tests import pyrun @@ -1526,11 +1526,14 @@ def test_fetch_all(self): ret = default try: args = () + kwargs = {} attr = getattr(p, name, None) if attr is not None and callable(attr): if name == 'rlimit': args = (psutil.RLIMIT_NOFILE,) - ret = attr(*args) + elif name == 'memory_maps': + kwargs = {'grouped': False} + ret = attr(*args, **kwargs) else: ret = attr valid_procs += 1 @@ -1572,9 +1575,12 @@ def test_fetch_all(self): self.assertTrue(valid_procs > 0) def cmdline(self, ret, proc): - pass + self.assertIsInstance(ret, list) + for part in ret: + self.assertIsInstance(part, str) def exe(self, ret, proc): + self.assertIsInstance(ret, (str, type(None))) if not ret: self.assertEqual(ret, '') else: @@ -1588,13 +1594,15 @@ def exe(self, ret, proc): self.assertTrue(os.access(ret, os.X_OK)) def ppid(self, ret, proc): + self.assertIsInstance(ret, int) self.assertTrue(ret >= 0) def name(self, ret, proc): - self.assertIsInstance(ret, (str, unicode)) + self.assertIsInstance(ret, str) self.assertTrue(ret) def create_time(self, ret, proc): + self.assertIsInstance(ret, float) try: self.assertGreaterEqual(ret, 0) except AssertionError: @@ -1609,34 +1617,45 @@ def create_time(self, ret, proc): time.strftime("%Y %m %d %H:%M:%S", time.localtime(ret)) def uids(self, ret, proc): + assert is_namedtuple(ret) for uid in ret: + self.assertIsInstance(uid, int) self.assertGreaterEqual(uid, 0) self.assertIn(uid, self.all_uids) def gids(self, ret, proc): + assert is_namedtuple(ret) # note: testing all gids as above seems not to be reliable for # gid == 30 (nodoby); not sure why. for gid in ret: + self.assertIsInstance(gid, int) if not OSX and not NETBSD: self.assertGreaterEqual(gid, 0) self.assertIn(gid, self.all_gids) def username(self, ret, proc): + self.assertIsInstance(ret, str) self.assertTrue(ret) if POSIX: self.assertIn(ret, self.all_usernames) def status(self, ret, proc): + self.assertIsInstance(ret, str) self.assertTrue(ret != "") self.assertTrue(ret != '?') self.assertIn(ret, VALID_PROC_STATUSES) def io_counters(self, ret, proc): + assert is_namedtuple(ret) for field in ret: + self.assertIsInstance(field, (int, long)) if field != -1: self.assertTrue(field >= 0) def ionice(self, ret, proc): + assert is_namedtuple(ret) + for field in ret: + self.assertIsInstance(field, int) if LINUX: self.assertTrue(ret.ioclass >= 0) self.assertTrue(ret.value >= 0) @@ -1645,43 +1664,56 @@ def ionice(self, ret, proc): self.assertIn(ret, (0, 1, 2)) def num_threads(self, ret, proc): - self.assertTrue(ret >= 1) + self.assertIsInstance(ret, int) + self.assertGreaterEqual(ret, 1) def threads(self, ret, proc): + self.assertIsInstance(ret, list) for t in ret: - self.assertTrue(t.id >= 0) - self.assertTrue(t.user_time >= 0) - self.assertTrue(t.system_time >= 0) + assert is_namedtuple(t) + self.assertGreaterEqual(t.id, 0) + self.assertGreaterEqual(t.user_time, 0) + self.assertGreaterEqual(t.system_time, 0) + for field in t: + self.assertIsInstance(field, (int, float)) def cpu_times(self, ret, proc): - self.assertTrue(ret.user >= 0) - self.assertTrue(ret.system >= 0) + assert is_namedtuple(ret) + self.assertGreaterEqual(ret.user, 0) + self.assertGreaterEqual(ret.system, 0) + for field in ret: + self.assertIsInstance(field, float) def cpu_num(self, ret, proc): + self.assertIsInstance(ret, int) self.assertGreaterEqual(ret, 0) if psutil.cpu_count() == 1: self.assertEqual(ret, 0) self.assertIn(ret, range(psutil.cpu_count())) def memory_info(self, ret, proc): - for name in ret._fields: - self.assertGreaterEqual(getattr(ret, name), 0) + assert is_namedtuple(ret) + for value in ret: + self.assertIsInstance(value, (int, long)) + self.assertGreaterEqual(value, 0) if POSIX and ret.vms != 0: # VMS is always supposed to be the highest for name in ret._fields: if name != 'vms': value = getattr(ret, name) - assert ret.vms > value, ret + self.assertGreater(ret.vms, value, msg=ret) elif WINDOWS: - assert ret.peak_wset >= ret.wset, ret - assert ret.peak_paged_pool >= ret.paged_pool, ret - assert ret.peak_nonpaged_pool >= ret.nonpaged_pool, ret - assert ret.peak_pagefile >= ret.pagefile, ret + self.assertGreaterEqual(ret.peak_wset, ret.wset) + self.assertGreaterEqual(ret.peak_paged_pool, ret.paged_pool) + self.assertGreaterEqual(ret.peak_nonpaged_pool, ret.nonpaged_pool) + self.assertGreaterEqual(ret.peak_pagefile, ret.pagefile) def memory_full_info(self, ret, proc): + assert is_namedtuple(ret) total = psutil.virtual_memory().total for name in ret._fields: value = getattr(ret, name) + self.assertIsInstance(value, (int, long)) self.assertGreaterEqual(value, 0, msg=(name, value)) self.assertLessEqual(value, total, msg=(name, value, total)) @@ -1689,24 +1721,28 @@ def memory_full_info(self, ret, proc): self.assertGreaterEqual(ret.pss, ret.uss) def open_files(self, ret, proc): + self.assertIsInstance(ret, list) for f in ret: + self.assertIsInstance(f.fd, int) + self.assertIsInstance(f.path, str) if WINDOWS: - assert f.fd == -1, f - else: - self.assertIsInstance(f.fd, int) - if LINUX: + self.assertEqual(f.fd, -1) + elif LINUX: self.assertIsInstance(f.position, int) + self.assertIsInstance(f.mode, str) + self.assertIsInstance(f.flags, int) self.assertGreaterEqual(f.position, 0) self.assertIn(f.mode, ('r', 'w', 'a', 'r+', 'a+')) self.assertGreater(f.flags, 0) - if BSD and not f.path: + elif BSD and not f.path: # XXX see: https://github.com/giampaolo/psutil/issues/595 continue assert os.path.isabs(f.path), f assert os.path.isfile(f.path), f def num_fds(self, ret, proc): - self.assertTrue(ret >= 0) + self.assertIsInstance(ret, int) + self.assertGreaterEqual(ret, 0) def connections(self, ret, proc): self.assertEqual(len(ret), len(set(ret))) @@ -1714,6 +1750,7 @@ def connections(self, ret, proc): check_connection_ntuple(conn) def cwd(self, ret, proc): + self.assertIsInstance(ret, str) if ret is not None: # BSD may return None assert os.path.isabs(ret), ret try: @@ -1729,24 +1766,31 @@ def cwd(self, ret, proc): self.assertTrue(stat.S_ISDIR(st.st_mode)) def memory_percent(self, ret, proc): + self.assertIsInstance(ret, float) assert 0 <= ret <= 100, ret def is_running(self, ret, proc): + self.assertIsInstance(ret, bool) self.assertTrue(ret) def cpu_affinity(self, ret, proc): + self.assertIsInstance(ret, list) assert ret != [], ret cpus = range(psutil.cpu_count()) for n in ret: self.assertIn(n, cpus) def terminal(self, ret, proc): + self.assertIsInstance(ret, (str, type(None))) if ret is not None: assert os.path.isabs(ret), ret assert os.path.exists(ret), ret def memory_maps(self, ret, proc): for nt in ret: + self.assertIsInstance(nt.addr, str) + self.assertIsInstance(nt.perms, str) + self.assertIsInstance(nt.path, str) for fname in nt._fields: value = getattr(nt, fname) if fname == 'path': @@ -1762,12 +1806,11 @@ def memory_maps(self, ret, proc): assert value >= 0, value def num_handles(self, ret, proc): - if WINDOWS: - self.assertGreaterEqual(ret, 0) - else: - self.assertGreaterEqual(ret, 0) + self.assertIsInstance(ret, int) + self.assertGreaterEqual(ret, 0) def nice(self, ret, proc): + self.assertIsInstance(ret, int) if POSIX: assert -20 <= ret <= 20, ret else: @@ -1776,10 +1819,13 @@ def nice(self, ret, proc): self.assertIn(ret, priorities) def num_ctx_switches(self, ret, proc): - self.assertGreaterEqual(ret.voluntary, 0) - self.assertGreaterEqual(ret.involuntary, 0) + assert is_namedtuple(ret) + for value in ret: + self.assertIsInstance(value, int) + self.assertGreaterEqual(value, 0) def rlimit(self, ret, proc): + self.assertIsInstance(ret, tuple) self.assertEqual(len(ret), 2) self.assertGreaterEqual(ret[0], -1) self.assertGreaterEqual(ret[1], -1) From 5aa3af31aa35d904d358171ad1049b46205494cf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 05:06:49 +0200 Subject: [PATCH 425/922] enhance tests --- psutil/tests/test_connections.py | 0 psutil/tests/test_process.py | 37 +++++++++++++++++++++----------- 2 files changed, 25 insertions(+), 12 deletions(-) mode change 100644 => 100755 psutil/tests/test_connections.py diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py old mode 100644 new mode 100755 diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 26e5cd3e4..daf0712f7 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -619,10 +619,10 @@ def test_memory_full_info(self): self.assertGreaterEqual(value, 0, msg=(name, value)) self.assertLessEqual(value, total, msg=(name, value, total)) if LINUX or WINDOWS or OSX: - mem.uss + self.assertGreaterEqual(mem.uss, 0) if LINUX: - mem.pss - self.assertGreater(mem.pss, mem.uss) + self.assertGreaterEqual(mem.pss, 0) + self.assertGreaterEqual(mem.swap, 0) @unittest.skipIf(OPENBSD or NETBSD, "platfform not supported") def test_memory_maps(self): @@ -1505,8 +1505,7 @@ def test_fetch_all(self): valid_procs = 0 excluded_names = set([ 'send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', - 'as_dict', 'cpu_percent', 'parent', 'children', 'pid', - 'memory_info_ex', 'oneshot', + 'as_dict', 'parent', 'children', 'memory_info_ex', 'oneshot', ]) if LINUX and not RLIMIT_SUPPORT: excluded_names.add('rlimit') @@ -1593,9 +1592,13 @@ def exe(self, ret, proc): # XXX may fail on OSX self.assertTrue(os.access(ret, os.X_OK)) + def pid(self, ret, proc): + self.assertIsInstance(ret, int) + self.assertGreaterEqual(ret, 0) + def ppid(self, ret, proc): self.assertIsInstance(ret, int) - self.assertTrue(ret >= 0) + self.assertGreaterEqual(ret, 0) def name(self, ret, proc): self.assertIsInstance(ret, str) @@ -1606,7 +1609,8 @@ def create_time(self, ret, proc): try: self.assertGreaterEqual(ret, 0) except AssertionError: - if OPENBSD and proc.status == psutil.STATUS_ZOMBIE: + # XXX + if OPENBSD and proc.status() == psutil.STATUS_ZOMBIE: pass else: raise @@ -1679,10 +1683,14 @@ def threads(self, ret, proc): def cpu_times(self, ret, proc): assert is_namedtuple(ret) - self.assertGreaterEqual(ret.user, 0) - self.assertGreaterEqual(ret.system, 0) - for field in ret: - self.assertIsInstance(field, float) + for n in ret: + self.assertIsInstance(n, float) + self.assertGreaterEqual(n, 0) + # TODO: check ntuple fields + + def cpu_percent(self, ret, proc): + self.assertIsInstance(ret, float) + assert 0.0 <= ret <= 100.0, ret def cpu_num(self, ret, proc): self.assertIsInstance(ret, int) @@ -1771,6 +1779,7 @@ def memory_percent(self, ret, proc): def is_running(self, ret, proc): self.assertIsInstance(ret, bool) + # XXX: racy self.assertTrue(ret) def cpu_affinity(self, ret, proc): @@ -1778,6 +1787,7 @@ def cpu_affinity(self, ret, proc): assert ret != [], ret cpus = range(psutil.cpu_count()) for n in ret: + self.assertIsInstance(n, int) self.assertIn(n, cpus) def terminal(self, ret, proc): @@ -1803,7 +1813,7 @@ def memory_maps(self, ret, proc): self.assertTrue(value) else: self.assertIsInstance(value, (int, long)) - assert value >= 0, value + self.assertGreaterEqual(value, 0) def num_handles(self, ret, proc): self.assertIsInstance(ret, int) @@ -1832,6 +1842,9 @@ def rlimit(self, ret, proc): def environ(self, ret, proc): self.assertIsInstance(ret, dict) + for k, v in ret.items(): + self.assertIsInstance(k, str) + self.assertIsInstance(v, str) # =================================================================== From 163bcd190f5b369117c61eb405f74b2b96c54305 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 05:25:47 +0200 Subject: [PATCH 426/922] check named tuples --- psutil/tests/test_system.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 4488a216b..c14e4ec67 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -196,6 +196,9 @@ def test_virtual_memory(self): def test_swap_memory(self): mem = psutil.swap_memory() + self.assertEqual( + mem._fields, ('total', 'used', 'free', 'percent', 'sin', 'sout')) + assert mem.total >= 0, mem assert mem.used >= 0, mem if mem.total > 0: @@ -422,6 +425,8 @@ def test_per_cpu_times_percent_negative(self): def test_disk_usage(self): usage = psutil.disk_usage(os.getcwd()) + self.assertEqual(usage._fields, ('total', 'used', 'free', 'percent')) + assert usage.total > 0, usage assert usage.used > 0, usage assert usage.free > 0, usage @@ -702,6 +707,9 @@ def test_users(self): def test_cpu_stats(self): # Tested more extensively in per-platform test modules. infos = psutil.cpu_stats() + self.assertEqual( + infos._fields, + ('ctx_switches', 'interrupts', 'soft_interrupts', 'syscalls')) for name in infos._fields: value = getattr(infos, name) self.assertGreaterEqual(value, 0) @@ -713,6 +721,7 @@ def test_cpu_stats(self): def test_cpu_freq(self): def check_ls(ls): for nt in ls: + self.assertEqual(nt._fields, ('current', 'min', 'max')) self.assertLessEqual(nt.current, nt.max) for name in nt._fields: value = getattr(nt, name) From ccf0380e12b737adf1d9d94d05916118535dd003 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 05:28:08 +0200 Subject: [PATCH 427/922] skip unavailable tests --- psutil/tests/test_unicode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index e011c2376..7bdc5ddef 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -364,12 +364,15 @@ def test_net_io_counters(self): for ifname, _ in psutil.net_io_counters(pernic=True).items(): self.assertIsInstance(ifname, str) + @unittest.skipUnless(hasattr(psutil, "sensors_fans"), "not supported") def test_sensors_fans(self): for name, units in psutil.sensors_fans().items(): self.assertIsInstance(name, str) for unit in units: self.assertIsInstance(unit.label, str) + @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), + "not supported") def test_sensors_temperatures(self): for name, units in psutil.sensors_temperatures().items(): self.assertIsInstance(name, str) From bdaedcaa6589a582ae34366c64376ba2fe2eab39 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 06:09:27 +0200 Subject: [PATCH 428/922] add a new test_contracts.py test suite which checks API sanity, mainly in terms of returned types and API availability --- Makefile | 5 + psutil/tests/test_process.py | 373 ----------------------------------- psutil/tests/test_unicode.py | 115 +---------- 3 files changed, 6 insertions(+), 487 deletions(-) diff --git a/Makefile b/Makefile index 64484177d..fba1f93dd 100644 --- a/Makefile +++ b/Makefile @@ -139,6 +139,11 @@ test-unicode: ${MAKE} install $(PYTHON) psutil/tests/test_unicode.py +# APIs sanity tests. +test-contracts: + ${MAKE} install + $(PYTHON) psutil/tests/test_contracts.py + # Test net_connections() and Process.connections(). test-connections: ${MAKE} install diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index daf0712f7..cc94caf2d 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -13,13 +13,11 @@ import select import signal import socket -import stat import subprocess import sys import tempfile import textwrap import time -import traceback import types import psutil @@ -33,19 +31,16 @@ from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS -from psutil._compat import callable from psutil._compat import long from psutil._compat import PY3 from psutil.tests import APPVEYOR from psutil.tests import call_until -from psutil.tests import check_connection_ntuple from psutil.tests import create_exe from psutil.tests import create_proc_children_pair from psutil.tests import enum from psutil.tests import get_test_subprocess from psutil.tests import get_winver from psutil.tests import GLOBAL_TIMEOUT -from psutil.tests import is_namedtuple from psutil.tests import mock from psutil.tests import PYPY from psutil.tests import pyrun @@ -64,9 +59,7 @@ from psutil.tests import TOX from psutil.tests import TRAVIS from psutil.tests import unittest -from psutil.tests import VALID_PROC_STATUSES from psutil.tests import wait_for_pid -from psutil.tests import warn from psutil.tests import WIN_VISTA @@ -1481,372 +1474,6 @@ def test_weird_environ(self): self.assertEqual(sproc.returncode, 0) -# =================================================================== -# --- Featch all processes test -# =================================================================== - - -class TestFetchAllProcesses(unittest.TestCase): - """Test which iterates over all running processes and performs - some sanity checks against Process API's returned values. - """ - - def setUp(self): - if POSIX: - import pwd - import grp - users = pwd.getpwall() - groups = grp.getgrall() - self.all_uids = set([x.pw_uid for x in users]) - self.all_usernames = set([x.pw_name for x in users]) - self.all_gids = set([x.gr_gid for x in groups]) - - def test_fetch_all(self): - valid_procs = 0 - excluded_names = set([ - 'send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', - 'as_dict', 'parent', 'children', 'memory_info_ex', 'oneshot', - ]) - if LINUX and not RLIMIT_SUPPORT: - excluded_names.add('rlimit') - attrs = [] - for name in dir(psutil.Process): - if name.startswith("_"): - continue - if name in excluded_names: - continue - attrs.append(name) - - default = object() - failures = [] - for p in psutil.process_iter(): - with p.oneshot(): - for name in attrs: - ret = default - try: - args = () - kwargs = {} - attr = getattr(p, name, None) - if attr is not None and callable(attr): - if name == 'rlimit': - args = (psutil.RLIMIT_NOFILE,) - elif name == 'memory_maps': - kwargs = {'grouped': False} - ret = attr(*args, **kwargs) - else: - ret = attr - valid_procs += 1 - except NotImplementedError: - msg = "%r was skipped because not implemented" % ( - self.__class__.__name__ + '.test_' + name) - warn(msg) - except (psutil.NoSuchProcess, psutil.AccessDenied) as err: - self.assertEqual(err.pid, p.pid) - if err.name: - # make sure exception's name attr is set - # with the actual process name - self.assertEqual(err.name, p.name()) - self.assertTrue(str(err)) - self.assertTrue(err.msg) - except Exception as err: - s = '\n' + '=' * 70 + '\n' - s += "FAIL: test_%s (proc=%s" % (name, p) - if ret != default: - s += ", ret=%s)" % repr(ret) - s += ')\n' - s += '-' * 70 - s += "\n%s" % traceback.format_exc() - s = "\n".join((" " * 4) + i for i in s.splitlines()) - s += '\n' - failures.append(s) - break - else: - if ret not in (0, 0.0, [], None, '', {}): - assert ret, ret - meth = getattr(self, name) - meth(ret, p) - - if failures: - self.fail(''.join(failures)) - - # we should always have a non-empty list, not including PID 0 etc. - # special cases. - self.assertTrue(valid_procs > 0) - - def cmdline(self, ret, proc): - self.assertIsInstance(ret, list) - for part in ret: - self.assertIsInstance(part, str) - - def exe(self, ret, proc): - self.assertIsInstance(ret, (str, type(None))) - if not ret: - self.assertEqual(ret, '') - else: - assert os.path.isabs(ret), ret - # Note: os.stat() may return False even if the file is there - # hence we skip the test, see: - # http://stackoverflow.com/questions/3112546/os-path-exists-lies - if POSIX and os.path.isfile(ret): - if hasattr(os, 'access') and hasattr(os, "X_OK"): - # XXX may fail on OSX - self.assertTrue(os.access(ret, os.X_OK)) - - def pid(self, ret, proc): - self.assertIsInstance(ret, int) - self.assertGreaterEqual(ret, 0) - - def ppid(self, ret, proc): - self.assertIsInstance(ret, int) - self.assertGreaterEqual(ret, 0) - - def name(self, ret, proc): - self.assertIsInstance(ret, str) - self.assertTrue(ret) - - def create_time(self, ret, proc): - self.assertIsInstance(ret, float) - try: - self.assertGreaterEqual(ret, 0) - except AssertionError: - # XXX - if OPENBSD and proc.status() == psutil.STATUS_ZOMBIE: - pass - else: - raise - # this can't be taken for granted on all platforms - # self.assertGreaterEqual(ret, psutil.boot_time()) - # make sure returned value can be pretty printed - # with strftime - time.strftime("%Y %m %d %H:%M:%S", time.localtime(ret)) - - def uids(self, ret, proc): - assert is_namedtuple(ret) - for uid in ret: - self.assertIsInstance(uid, int) - self.assertGreaterEqual(uid, 0) - self.assertIn(uid, self.all_uids) - - def gids(self, ret, proc): - assert is_namedtuple(ret) - # note: testing all gids as above seems not to be reliable for - # gid == 30 (nodoby); not sure why. - for gid in ret: - self.assertIsInstance(gid, int) - if not OSX and not NETBSD: - self.assertGreaterEqual(gid, 0) - self.assertIn(gid, self.all_gids) - - def username(self, ret, proc): - self.assertIsInstance(ret, str) - self.assertTrue(ret) - if POSIX: - self.assertIn(ret, self.all_usernames) - - def status(self, ret, proc): - self.assertIsInstance(ret, str) - self.assertTrue(ret != "") - self.assertTrue(ret != '?') - self.assertIn(ret, VALID_PROC_STATUSES) - - def io_counters(self, ret, proc): - assert is_namedtuple(ret) - for field in ret: - self.assertIsInstance(field, (int, long)) - if field != -1: - self.assertTrue(field >= 0) - - def ionice(self, ret, proc): - assert is_namedtuple(ret) - for field in ret: - self.assertIsInstance(field, int) - if LINUX: - self.assertTrue(ret.ioclass >= 0) - self.assertTrue(ret.value >= 0) - else: - self.assertTrue(ret >= 0) - self.assertIn(ret, (0, 1, 2)) - - def num_threads(self, ret, proc): - self.assertIsInstance(ret, int) - self.assertGreaterEqual(ret, 1) - - def threads(self, ret, proc): - self.assertIsInstance(ret, list) - for t in ret: - assert is_namedtuple(t) - self.assertGreaterEqual(t.id, 0) - self.assertGreaterEqual(t.user_time, 0) - self.assertGreaterEqual(t.system_time, 0) - for field in t: - self.assertIsInstance(field, (int, float)) - - def cpu_times(self, ret, proc): - assert is_namedtuple(ret) - for n in ret: - self.assertIsInstance(n, float) - self.assertGreaterEqual(n, 0) - # TODO: check ntuple fields - - def cpu_percent(self, ret, proc): - self.assertIsInstance(ret, float) - assert 0.0 <= ret <= 100.0, ret - - def cpu_num(self, ret, proc): - self.assertIsInstance(ret, int) - self.assertGreaterEqual(ret, 0) - if psutil.cpu_count() == 1: - self.assertEqual(ret, 0) - self.assertIn(ret, range(psutil.cpu_count())) - - def memory_info(self, ret, proc): - assert is_namedtuple(ret) - for value in ret: - self.assertIsInstance(value, (int, long)) - self.assertGreaterEqual(value, 0) - if POSIX and ret.vms != 0: - # VMS is always supposed to be the highest - for name in ret._fields: - if name != 'vms': - value = getattr(ret, name) - self.assertGreater(ret.vms, value, msg=ret) - elif WINDOWS: - self.assertGreaterEqual(ret.peak_wset, ret.wset) - self.assertGreaterEqual(ret.peak_paged_pool, ret.paged_pool) - self.assertGreaterEqual(ret.peak_nonpaged_pool, ret.nonpaged_pool) - self.assertGreaterEqual(ret.peak_pagefile, ret.pagefile) - - def memory_full_info(self, ret, proc): - assert is_namedtuple(ret) - total = psutil.virtual_memory().total - for name in ret._fields: - value = getattr(ret, name) - self.assertIsInstance(value, (int, long)) - self.assertGreaterEqual(value, 0, msg=(name, value)) - self.assertLessEqual(value, total, msg=(name, value, total)) - - if LINUX: - self.assertGreaterEqual(ret.pss, ret.uss) - - def open_files(self, ret, proc): - self.assertIsInstance(ret, list) - for f in ret: - self.assertIsInstance(f.fd, int) - self.assertIsInstance(f.path, str) - if WINDOWS: - self.assertEqual(f.fd, -1) - elif LINUX: - self.assertIsInstance(f.position, int) - self.assertIsInstance(f.mode, str) - self.assertIsInstance(f.flags, int) - self.assertGreaterEqual(f.position, 0) - self.assertIn(f.mode, ('r', 'w', 'a', 'r+', 'a+')) - self.assertGreater(f.flags, 0) - elif BSD and not f.path: - # XXX see: https://github.com/giampaolo/psutil/issues/595 - continue - assert os.path.isabs(f.path), f - assert os.path.isfile(f.path), f - - def num_fds(self, ret, proc): - self.assertIsInstance(ret, int) - self.assertGreaterEqual(ret, 0) - - def connections(self, ret, proc): - self.assertEqual(len(ret), len(set(ret))) - for conn in ret: - check_connection_ntuple(conn) - - def cwd(self, ret, proc): - self.assertIsInstance(ret, str) - if ret is not None: # BSD may return None - assert os.path.isabs(ret), ret - try: - st = os.stat(ret) - except OSError as err: - if WINDOWS and err.errno in \ - psutil._psplatform.ACCESS_DENIED_SET: - pass - # directory has been removed in mean time - elif err.errno != errno.ENOENT: - raise - else: - self.assertTrue(stat.S_ISDIR(st.st_mode)) - - def memory_percent(self, ret, proc): - self.assertIsInstance(ret, float) - assert 0 <= ret <= 100, ret - - def is_running(self, ret, proc): - self.assertIsInstance(ret, bool) - # XXX: racy - self.assertTrue(ret) - - def cpu_affinity(self, ret, proc): - self.assertIsInstance(ret, list) - assert ret != [], ret - cpus = range(psutil.cpu_count()) - for n in ret: - self.assertIsInstance(n, int) - self.assertIn(n, cpus) - - def terminal(self, ret, proc): - self.assertIsInstance(ret, (str, type(None))) - if ret is not None: - assert os.path.isabs(ret), ret - assert os.path.exists(ret), ret - - def memory_maps(self, ret, proc): - for nt in ret: - self.assertIsInstance(nt.addr, str) - self.assertIsInstance(nt.perms, str) - self.assertIsInstance(nt.path, str) - for fname in nt._fields: - value = getattr(nt, fname) - if fname == 'path': - if not value.startswith('['): - assert os.path.isabs(nt.path), nt.path - # commented as on Linux we might get - # '/foo/bar (deleted)' - # assert os.path.exists(nt.path), nt.path - elif fname in ('addr', 'perms'): - self.assertTrue(value) - else: - self.assertIsInstance(value, (int, long)) - self.assertGreaterEqual(value, 0) - - def num_handles(self, ret, proc): - self.assertIsInstance(ret, int) - self.assertGreaterEqual(ret, 0) - - def nice(self, ret, proc): - self.assertIsInstance(ret, int) - if POSIX: - assert -20 <= ret <= 20, ret - else: - priorities = [getattr(psutil, x) for x in dir(psutil) - if x.endswith('_PRIORITY_CLASS')] - self.assertIn(ret, priorities) - - def num_ctx_switches(self, ret, proc): - assert is_namedtuple(ret) - for value in ret: - self.assertIsInstance(value, int) - self.assertGreaterEqual(value, 0) - - def rlimit(self, ret, proc): - self.assertIsInstance(ret, tuple) - self.assertEqual(len(ret), 2) - self.assertGreaterEqual(ret[0], -1) - self.assertGreaterEqual(ret[1], -1) - - def environ(self, ret, proc): - self.assertIsInstance(ret, dict) - for k, v in ret.items(): - self.assertIsInstance(k, str) - self.assertIsInstance(v, str) - - # =================================================================== # --- Limited user tests # =================================================================== diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 7bdc5ddef..68ba847aa 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -51,7 +51,7 @@ https://github.com/giampaolo/psutil/issues/655#issuecomment-136131180 As such we also test that all APIs on Python 2 always return str and -never unicode. +never unicode (in test_contracts.py). """ import os @@ -273,118 +273,5 @@ def test_proc_environ(self): self.assertEqual(env['FUNNY_ARG'], funky_str) -# =================================================================== -# Base str types -# =================================================================== - - -class TestAlwaysStrType(unittest.TestCase): - """Make sure all str-related APIs on Python 2 return a str type - and never unicode. - """ - - @classmethod - def setUpClass(cls): - cls.proc = psutil.Process() - - def tearDown(self): - safe_rmpath(TESTFN) - - def test_proc_cmdline(self): - for bit in self.proc.cmdline(): - self.assertIsInstance(bit, str) - - @unittest.skipUnless(POSIX, 'POSIX only') - def test_proc_connections(self): - with unix_socket_path() as name: - with closing(bind_unix_socket(name)): - conn = self.proc.connections(kind='unix')[0] - self.assertIsInstance(conn.laddr, str) - - def test_proc_cwd(self): - self.assertIsInstance(self.proc.cwd(), str) - - def test_proc_environ(self): - for k, v in self.proc.environ().items(): - self.assertIsInstance(k, str) - self.assertIsInstance(v, str) - - def test_proc_exe(self): - self.assertIsInstance(self.proc.exe(), str) - - def test_proc_memory_maps(self): - for region in self.proc.memory_maps(grouped=False): - self.assertIsInstance(region.addr, str) - self.assertIsInstance(region.path, str) - - def test_proc_name(self): - self.assertIsInstance(self.proc.name(), str) - - def test_proc_open_files(self): - with open(TESTFN, 'w'): - self.assertIsInstance(self.proc.open_files()[0].path, str) - - def test_proc_username(self): - self.assertIsInstance(self.proc.username(), str) - - def test_io_counters(self): - for k in psutil.disk_io_counters(perdisk=True): - self.assertIsInstance(k, str) - - def test_disk_partitions(self): - for disk in psutil.disk_partitions(): - self.assertIsInstance(disk.device, str) - self.assertIsInstance(disk.mountpoint, str) - self.assertIsInstance(disk.fstype, str) - self.assertIsInstance(disk.opts, str) - - @unittest.skipUnless(POSIX, 'POSIX only') - @skip_on_access_denied(only_if=OSX) - def test_net_connections(self): - with unix_socket_path() as name: - with closing(bind_unix_socket(name)): - cons = psutil.net_connections(kind='unix') - assert cons - for conn in cons: - self.assertIsInstance(conn.laddr, str) - - def test_net_if_addrs(self): - for ifname, addrs in psutil.net_if_addrs().items(): - self.assertIsInstance(ifname, str) - for addr in addrs: - self.assertIsInstance(addr.address, str) - self.assertIsInstance(addr.netmask, (str, type(None))) - self.assertIsInstance(addr.broadcast, (str, type(None))) - - def test_net_if_stats(self): - for ifname, _ in psutil.net_if_stats().items(): - self.assertIsInstance(ifname, str) - - def test_net_io_counters(self): - for ifname, _ in psutil.net_io_counters(pernic=True).items(): - self.assertIsInstance(ifname, str) - - @unittest.skipUnless(hasattr(psutil, "sensors_fans"), "not supported") - def test_sensors_fans(self): - for name, units in psutil.sensors_fans().items(): - self.assertIsInstance(name, str) - for unit in units: - self.assertIsInstance(unit.label, str) - - @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), - "not supported") - def test_sensors_temperatures(self): - for name, units in psutil.sensors_temperatures().items(): - self.assertIsInstance(name, str) - for unit in units: - self.assertIsInstance(unit.label, str) - - def test_users(self): - for user in psutil.users(): - self.assertIsInstance(user.name, str) - self.assertIsInstance(user.terminal, str) - self.assertIsInstance(user.host, (str, type(None))) - - if __name__ == '__main__': run_test_module_by_name(__file__) From 7bc4a31acf8383c66ab60aa14f975ccf839f5c4b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 06:24:25 +0200 Subject: [PATCH 429/922] #1039 add a new test_contracts.py test suite which checks API sanity, mainly in terms of returned types and API availability --- psutil/tests/test_contracts.py | 604 +++++++++++++++++++++++++++++++++ 1 file changed, 604 insertions(+) create mode 100755 psutil/tests/test_contracts.py diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py new file mode 100755 index 000000000..ef3cb231e --- /dev/null +++ b/psutil/tests/test_contracts.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Contracts tests. These tests mainly check API sanity in terms of +returned types and APIs availability. +""" + +import errno +import os +import stat +import time +import traceback +from contextlib import closing + +from psutil import BSD +from psutil import FREEBSD +from psutil import LINUX +from psutil import NETBSD +from psutil import OPENBSD +from psutil import OSX +from psutil import POSIX +from psutil import SUNOS +from psutil import WINDOWS +from psutil._compat import callable +from psutil._compat import long +from psutil.tests import bind_unix_socket +from psutil.tests import check_connection_ntuple +from psutil.tests import is_namedtuple +from psutil.tests import RLIMIT_SUPPORT +from psutil.tests import run_test_module_by_name +from psutil.tests import safe_rmpath +from psutil.tests import skip_on_access_denied +from psutil.tests import TESTFN +from psutil.tests import unittest +from psutil.tests import unix_socket_path +from psutil.tests import VALID_PROC_STATUSES +from psutil.tests import warn +import psutil + + +# =================================================================== +# --- APIs availability +# =================================================================== + + +class TestAvailability(unittest.TestCase): + """Make sure code reflects what doc promises in terms of APIs + availability. + """ + + def test_cpu_affinity(self): + hasit = LINUX or WINDOWS or BSD + self.assertEqual(hasattr(psutil.Process, "cpu_affinity"), hasit) + + def test_win_service(self): + self.assertEqual(hasattr(psutil, "win_service_iter"), WINDOWS) + self.assertEqual(hasattr(psutil, "win_service_get"), WINDOWS) + + def test_PROCFS_PATH(self): + self.assertEqual(hasattr(psutil, "PROCFS_PATH"), LINUX or SUNOS) + + def test_win_priority(self): + ae = self.assertEqual + ae(hasattr(psutil, "ABOVE_NORMAL_PRIORITY_CLASS"), WINDOWS) + ae(hasattr(psutil, "BELOW_NORMAL_PRIORITY_CLASS"), WINDOWS) + ae(hasattr(psutil, "HIGH_PRIORITY_CLASS"), WINDOWS) + ae(hasattr(psutil, "IDLE_PRIORITY_CLASS"), WINDOWS) + ae(hasattr(psutil, "NORMAL_PRIORITY_CLASS"), WINDOWS) + ae(hasattr(psutil, "REALTIME_PRIORITY_CLASS"), WINDOWS) + + def test_linux_ioprio(self): + ae = self.assertEqual + ae(hasattr(psutil, "IOPRIO_CLASS_NONE"), LINUX) + ae(hasattr(psutil, "IOPRIO_CLASS_RT"), LINUX) + ae(hasattr(psutil, "IOPRIO_CLASS_BE"), LINUX) + ae(hasattr(psutil, "IOPRIO_CLASS_IDLE"), LINUX) + + def test_linux_rlimit(self): + ae = self.assertEqual + ae(hasattr(psutil, "RLIM_INFINITY"), LINUX) + ae(hasattr(psutil, "RLIMIT_AS"), LINUX) + ae(hasattr(psutil, "RLIMIT_CORE"), LINUX) + ae(hasattr(psutil, "RLIMIT_CPU"), LINUX) + ae(hasattr(psutil, "RLIMIT_DATA"), LINUX) + ae(hasattr(psutil, "RLIMIT_FSIZE"), LINUX) + ae(hasattr(psutil, "RLIMIT_LOCKS"), LINUX) + ae(hasattr(psutil, "RLIMIT_MEMLOCK"), LINUX) + ae(hasattr(psutil, "RLIMIT_MSGQUEUE"), LINUX) + ae(hasattr(psutil, "RLIMIT_NICE"), LINUX) + ae(hasattr(psutil, "RLIMIT_NOFILE"), LINUX) + ae(hasattr(psutil, "RLIMIT_NPROC"), LINUX) + ae(hasattr(psutil, "RLIMIT_RSS"), LINUX) + ae(hasattr(psutil, "RLIMIT_RTPRIO"), LINUX) + ae(hasattr(psutil, "RLIMIT_RTTIME"), LINUX) + ae(hasattr(psutil, "RLIMIT_SIGPENDING"), LINUX) + ae(hasattr(psutil, "RLIMIT_STACK"), LINUX) + + def test_cpu_freq(self): + self.assertEqual(hasattr(psutil, "cpu_freq"), LINUX or OSX or WINDOWS) + + def test_sensors_temperatures(self): + self.assertEqual(hasattr(psutil, "sensors_temperatures"), LINUX) + + def test_sensors_fans(self): + self.assertEqual(hasattr(psutil, "sensors_fans"), LINUX) + + def test_battery(self): + self.assertEqual(hasattr(psutil, "sensors_battery"), + LINUX or WINDOWS or FREEBSD) + + def test_proc_environ(self): + self.assertEqual(hasattr(psutil.Process, "environ"), + LINUX or OSX or WINDOWS) + + def test_proc_uids(self): + self.assertEqual(hasattr(psutil.Process, "uids"), POSIX) + + def test_proc_gids(self): + self.assertEqual(hasattr(psutil.Process, "uids"), POSIX) + + def test_proc_terminal(self): + self.assertEqual(hasattr(psutil.Process, "terminal"), POSIX) + + def test_proc_ionice(self): + self.assertEqual(hasattr(psutil.Process, "ionice"), LINUX or WINDOWS) + + def test_proc_rlimit(self): + self.assertEqual(hasattr(psutil.Process, "rlimit"), LINUX) + + def test_proc_io_counters(self): + hasit = hasattr(psutil.Process, "io_counters") + self.assertEqual(hasit, False if OSX or SUNOS else True) + + def test_proc_num_fds(self): + self.assertEqual(hasattr(psutil.Process, "num_fds"), POSIX) + + def test_proc_num_handles(self): + self.assertEqual(hasattr(psutil.Process, "num_handles"), WINDOWS) + + def test_proc_cpu_affinity(self): + self.assertEqual(hasattr(psutil.Process, "cpu_affinity"), + LINUX or WINDOWS or FREEBSD) + + def test_proc_cpu_num(self): + self.assertEqual(hasattr(psutil.Process, "cpu_num"), + LINUX or FREEBSD or SUNOS) + + def test_proc_memory_maps(self): + hasit = hasattr(psutil.Process, "memory_maps") + self.assertEqual(hasit, False if OPENBSD or NETBSD else True) + + +# =================================================================== +# --- System API types +# =================================================================== + + +class TestSystem(unittest.TestCase): + """Check the return types of system related APIs.""" + + @classmethod + def setUpClass(cls): + cls.proc = psutil.Process() + + def tearDown(self): + safe_rmpath(TESTFN) + + def test_cpu_times(self): + ret = psutil.cpu_times() + assert is_namedtuple(ret) + for n in ret: + self.assertIsInstance(n, float) + self.assertGreaterEqual(n, 0) + + def test_io_counters(self): + for k in psutil.disk_io_counters(perdisk=True): + self.assertIsInstance(k, str) + + def test_disk_partitions(self): + for disk in psutil.disk_partitions(): + self.assertIsInstance(disk.device, str) + self.assertIsInstance(disk.mountpoint, str) + self.assertIsInstance(disk.fstype, str) + self.assertIsInstance(disk.opts, str) + + @unittest.skipUnless(POSIX, 'POSIX only') + @skip_on_access_denied(only_if=OSX) + def test_net_connections(self): + with unix_socket_path() as name: + with closing(bind_unix_socket(name)): + cons = psutil.net_connections(kind='unix') + assert cons + for conn in cons: + self.assertIsInstance(conn.laddr, str) + + def test_net_if_addrs(self): + for ifname, addrs in psutil.net_if_addrs().items(): + self.assertIsInstance(ifname, str) + for addr in addrs: + self.assertIsInstance(addr.address, str) + self.assertIsInstance(addr.netmask, (str, type(None))) + self.assertIsInstance(addr.broadcast, (str, type(None))) + + def test_net_if_stats(self): + for ifname, _ in psutil.net_if_stats().items(): + self.assertIsInstance(ifname, str) + + def test_net_io_counters(self): + for ifname, _ in psutil.net_io_counters(pernic=True).items(): + self.assertIsInstance(ifname, str) + + @unittest.skipUnless(hasattr(psutil, "sensors_fans"), "not supported") + def test_sensors_fans(self): + for name, units in psutil.sensors_fans().items(): + self.assertIsInstance(name, str) + for unit in units: + self.assertIsInstance(unit.label, str) + + @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), + "not supported") + def test_sensors_temperatures(self): + for name, units in psutil.sensors_temperatures().items(): + self.assertIsInstance(name, str) + for unit in units: + self.assertIsInstance(unit.label, str) + + def test_users(self): + for user in psutil.users(): + self.assertIsInstance(user.name, str) + self.assertIsInstance(user.terminal, str) + self.assertIsInstance(user.host, (str, type(None))) + + +# =================================================================== +# --- Featch all processes test +# =================================================================== + + +class TestFetchAllProcesses(unittest.TestCase): + """Test which iterates over all running processes and performs + some sanity checks against Process API's returned values. + """ + + def setUp(self): + if POSIX: + import pwd + import grp + users = pwd.getpwall() + groups = grp.getgrall() + self.all_uids = set([x.pw_uid for x in users]) + self.all_usernames = set([x.pw_name for x in users]) + self.all_gids = set([x.gr_gid for x in groups]) + + def test_fetch_all(self): + valid_procs = 0 + excluded_names = set([ + 'send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', + 'as_dict', 'parent', 'children', 'memory_info_ex', 'oneshot', + ]) + if LINUX and not RLIMIT_SUPPORT: + excluded_names.add('rlimit') + attrs = [] + for name in dir(psutil.Process): + if name.startswith("_"): + continue + if name in excluded_names: + continue + attrs.append(name) + + default = object() + failures = [] + for p in psutil.process_iter(): + with p.oneshot(): + for name in attrs: + ret = default + try: + args = () + kwargs = {} + attr = getattr(p, name, None) + if attr is not None and callable(attr): + if name == 'rlimit': + args = (psutil.RLIMIT_NOFILE,) + elif name == 'memory_maps': + kwargs = {'grouped': False} + ret = attr(*args, **kwargs) + else: + ret = attr + valid_procs += 1 + except NotImplementedError: + msg = "%r was skipped because not implemented" % ( + self.__class__.__name__ + '.test_' + name) + warn(msg) + except (psutil.NoSuchProcess, psutil.AccessDenied) as err: + self.assertEqual(err.pid, p.pid) + if err.name: + # make sure exception's name attr is set + # with the actual process name + self.assertEqual(err.name, p.name()) + self.assertTrue(str(err)) + self.assertTrue(err.msg) + except Exception as err: + s = '\n' + '=' * 70 + '\n' + s += "FAIL: test_%s (proc=%s" % (name, p) + if ret != default: + s += ", ret=%s)" % repr(ret) + s += ')\n' + s += '-' * 70 + s += "\n%s" % traceback.format_exc() + s = "\n".join((" " * 4) + i for i in s.splitlines()) + s += '\n' + failures.append(s) + break + else: + if ret not in (0, 0.0, [], None, '', {}): + assert ret, ret + meth = getattr(self, name) + meth(ret, p) + + if failures: + self.fail(''.join(failures)) + + # we should always have a non-empty list, not including PID 0 etc. + # special cases. + self.assertTrue(valid_procs > 0) + + def cmdline(self, ret, proc): + self.assertIsInstance(ret, list) + for part in ret: + self.assertIsInstance(part, str) + + def exe(self, ret, proc): + self.assertIsInstance(ret, (str, type(None))) + if not ret: + self.assertEqual(ret, '') + else: + assert os.path.isabs(ret), ret + # Note: os.stat() may return False even if the file is there + # hence we skip the test, see: + # http://stackoverflow.com/questions/3112546/os-path-exists-lies + if POSIX and os.path.isfile(ret): + if hasattr(os, 'access') and hasattr(os, "X_OK"): + # XXX may fail on OSX + self.assertTrue(os.access(ret, os.X_OK)) + + def pid(self, ret, proc): + self.assertIsInstance(ret, int) + self.assertGreaterEqual(ret, 0) + + def ppid(self, ret, proc): + self.assertIsInstance(ret, int) + self.assertGreaterEqual(ret, 0) + + def name(self, ret, proc): + self.assertIsInstance(ret, str) + self.assertTrue(ret) + + def create_time(self, ret, proc): + self.assertIsInstance(ret, float) + try: + self.assertGreaterEqual(ret, 0) + except AssertionError: + # XXX + if OPENBSD and proc.status() == psutil.STATUS_ZOMBIE: + pass + else: + raise + # this can't be taken for granted on all platforms + # self.assertGreaterEqual(ret, psutil.boot_time()) + # make sure returned value can be pretty printed + # with strftime + time.strftime("%Y %m %d %H:%M:%S", time.localtime(ret)) + + def uids(self, ret, proc): + assert is_namedtuple(ret) + for uid in ret: + self.assertIsInstance(uid, int) + self.assertGreaterEqual(uid, 0) + self.assertIn(uid, self.all_uids) + + def gids(self, ret, proc): + assert is_namedtuple(ret) + # note: testing all gids as above seems not to be reliable for + # gid == 30 (nodoby); not sure why. + for gid in ret: + self.assertIsInstance(gid, int) + if not OSX and not NETBSD: + self.assertGreaterEqual(gid, 0) + self.assertIn(gid, self.all_gids) + + def username(self, ret, proc): + self.assertIsInstance(ret, str) + self.assertTrue(ret) + if POSIX: + self.assertIn(ret, self.all_usernames) + + def status(self, ret, proc): + self.assertIsInstance(ret, str) + self.assertTrue(ret != "") + self.assertTrue(ret != '?') + self.assertIn(ret, VALID_PROC_STATUSES) + + def io_counters(self, ret, proc): + assert is_namedtuple(ret) + for field in ret: + self.assertIsInstance(field, (int, long)) + if field != -1: + self.assertTrue(field >= 0) + + def ionice(self, ret, proc): + assert is_namedtuple(ret) + for field in ret: + self.assertIsInstance(field, int) + if LINUX: + self.assertTrue(ret.ioclass >= 0) + self.assertTrue(ret.value >= 0) + else: + self.assertTrue(ret >= 0) + self.assertIn(ret, (0, 1, 2)) + + def num_threads(self, ret, proc): + self.assertIsInstance(ret, int) + self.assertGreaterEqual(ret, 1) + + def threads(self, ret, proc): + self.assertIsInstance(ret, list) + for t in ret: + assert is_namedtuple(t) + self.assertGreaterEqual(t.id, 0) + self.assertGreaterEqual(t.user_time, 0) + self.assertGreaterEqual(t.system_time, 0) + for field in t: + self.assertIsInstance(field, (int, float)) + + def cpu_times(self, ret, proc): + assert is_namedtuple(ret) + for n in ret: + self.assertIsInstance(n, float) + self.assertGreaterEqual(n, 0) + # TODO: check ntuple fields + + def cpu_percent(self, ret, proc): + self.assertIsInstance(ret, float) + assert 0.0 <= ret <= 100.0, ret + + def cpu_num(self, ret, proc): + self.assertIsInstance(ret, int) + self.assertGreaterEqual(ret, 0) + if psutil.cpu_count() == 1: + self.assertEqual(ret, 0) + self.assertIn(ret, range(psutil.cpu_count())) + + def memory_info(self, ret, proc): + assert is_namedtuple(ret) + for value in ret: + self.assertIsInstance(value, (int, long)) + self.assertGreaterEqual(value, 0) + if POSIX and ret.vms != 0: + # VMS is always supposed to be the highest + for name in ret._fields: + if name != 'vms': + value = getattr(ret, name) + self.assertGreater(ret.vms, value, msg=ret) + elif WINDOWS: + self.assertGreaterEqual(ret.peak_wset, ret.wset) + self.assertGreaterEqual(ret.peak_paged_pool, ret.paged_pool) + self.assertGreaterEqual(ret.peak_nonpaged_pool, ret.nonpaged_pool) + self.assertGreaterEqual(ret.peak_pagefile, ret.pagefile) + + def memory_full_info(self, ret, proc): + assert is_namedtuple(ret) + total = psutil.virtual_memory().total + for name in ret._fields: + value = getattr(ret, name) + self.assertIsInstance(value, (int, long)) + self.assertGreaterEqual(value, 0, msg=(name, value)) + self.assertLessEqual(value, total, msg=(name, value, total)) + + if LINUX: + self.assertGreaterEqual(ret.pss, ret.uss) + + def open_files(self, ret, proc): + self.assertIsInstance(ret, list) + for f in ret: + self.assertIsInstance(f.fd, int) + self.assertIsInstance(f.path, str) + if WINDOWS: + self.assertEqual(f.fd, -1) + elif LINUX: + self.assertIsInstance(f.position, int) + self.assertIsInstance(f.mode, str) + self.assertIsInstance(f.flags, int) + self.assertGreaterEqual(f.position, 0) + self.assertIn(f.mode, ('r', 'w', 'a', 'r+', 'a+')) + self.assertGreater(f.flags, 0) + elif BSD and not f.path: + # XXX see: https://github.com/giampaolo/psutil/issues/595 + continue + assert os.path.isabs(f.path), f + assert os.path.isfile(f.path), f + + def num_fds(self, ret, proc): + self.assertIsInstance(ret, int) + self.assertGreaterEqual(ret, 0) + + def connections(self, ret, proc): + self.assertEqual(len(ret), len(set(ret))) + for conn in ret: + check_connection_ntuple(conn) + + def cwd(self, ret, proc): + self.assertIsInstance(ret, str) + if ret is not None: # BSD may return None + assert os.path.isabs(ret), ret + try: + st = os.stat(ret) + except OSError as err: + if WINDOWS and err.errno in \ + psutil._psplatform.ACCESS_DENIED_SET: + pass + # directory has been removed in mean time + elif err.errno != errno.ENOENT: + raise + else: + self.assertTrue(stat.S_ISDIR(st.st_mode)) + + def memory_percent(self, ret, proc): + self.assertIsInstance(ret, float) + assert 0 <= ret <= 100, ret + + def is_running(self, ret, proc): + self.assertIsInstance(ret, bool) + # XXX: racy + self.assertTrue(ret) + + def cpu_affinity(self, ret, proc): + self.assertIsInstance(ret, list) + assert ret != [], ret + cpus = range(psutil.cpu_count()) + for n in ret: + self.assertIsInstance(n, int) + self.assertIn(n, cpus) + + def terminal(self, ret, proc): + self.assertIsInstance(ret, (str, type(None))) + if ret is not None: + assert os.path.isabs(ret), ret + assert os.path.exists(ret), ret + + def memory_maps(self, ret, proc): + for nt in ret: + self.assertIsInstance(nt.addr, str) + self.assertIsInstance(nt.perms, str) + self.assertIsInstance(nt.path, str) + for fname in nt._fields: + value = getattr(nt, fname) + if fname == 'path': + if not value.startswith('['): + assert os.path.isabs(nt.path), nt.path + # commented as on Linux we might get + # '/foo/bar (deleted)' + # assert os.path.exists(nt.path), nt.path + elif fname in ('addr', 'perms'): + self.assertTrue(value) + else: + self.assertIsInstance(value, (int, long)) + self.assertGreaterEqual(value, 0) + + def num_handles(self, ret, proc): + self.assertIsInstance(ret, int) + self.assertGreaterEqual(ret, 0) + + def nice(self, ret, proc): + self.assertIsInstance(ret, int) + if POSIX: + assert -20 <= ret <= 20, ret + else: + priorities = [getattr(psutil, x) for x in dir(psutil) + if x.endswith('_PRIORITY_CLASS')] + self.assertIn(ret, priorities) + + def num_ctx_switches(self, ret, proc): + assert is_namedtuple(ret) + for value in ret: + self.assertIsInstance(value, int) + self.assertGreaterEqual(value, 0) + + def rlimit(self, ret, proc): + self.assertIsInstance(ret, tuple) + self.assertEqual(len(ret), 2) + self.assertGreaterEqual(ret[0], -1) + self.assertGreaterEqual(ret[1], -1) + + def environ(self, ret, proc): + self.assertIsInstance(ret, dict) + for k, v in ret.items(): + self.assertIsInstance(k, str) + self.assertIsInstance(v, str) + + +if __name__ == '__main__': + run_test_module_by_name(__file__) From a8f5694fa1381dd6fc929696939a5dab6bdbf158 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 06:46:44 +0200 Subject: [PATCH 430/922] #1039 make sure we never return unicode --- psutil/tests/test_contracts.py | 46 +++++++++++++++++++++------------- psutil/tests/test_linux.py | 42 +++++++++++++++---------------- psutil/tests/test_system.py | 30 +++++++++++----------- psutil/tests/test_unicode.py | 1 - 4 files changed, 65 insertions(+), 54 deletions(-) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index ef3cb231e..8f9f6eb6f 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -6,6 +6,7 @@ """Contracts tests. These tests mainly check API sanity in terms of returned types and APIs availability. +Some of these are duplicates of tests test_system.py and test_process.py """ import errno @@ -159,7 +160,10 @@ def test_proc_memory_maps(self): class TestSystem(unittest.TestCase): - """Check the return types of system related APIs.""" + """Check the return types of system related APIs. + Mainly we want to test we never return unicode on Python 2, see: + https://github.com/giampaolo/psutil/issues/1039 + """ @classmethod def setUpClass(cls): @@ -169,6 +173,7 @@ def tearDown(self): safe_rmpath(TESTFN) def test_cpu_times(self): + # Duplicate of test_system.py. Keep it anyway. ret = psutil.cpu_times() assert is_namedtuple(ret) for n in ret: @@ -176,10 +181,12 @@ def test_cpu_times(self): self.assertGreaterEqual(n, 0) def test_io_counters(self): + # Duplicate of test_system.py. Keep it anyway. for k in psutil.disk_io_counters(perdisk=True): self.assertIsInstance(k, str) def test_disk_partitions(self): + # Duplicate of test_system.py. Keep it anyway. for disk in psutil.disk_partitions(): self.assertIsInstance(disk.device, str) self.assertIsInstance(disk.mountpoint, str) @@ -197,6 +204,7 @@ def test_net_connections(self): self.assertIsInstance(conn.laddr, str) def test_net_if_addrs(self): + # Duplicate of test_system.py. Keep it anyway. for ifname, addrs in psutil.net_if_addrs().items(): self.assertIsInstance(ifname, str) for addr in addrs: @@ -205,15 +213,18 @@ def test_net_if_addrs(self): self.assertIsInstance(addr.broadcast, (str, type(None))) def test_net_if_stats(self): + # Duplicate of test_system.py. Keep it anyway. for ifname, _ in psutil.net_if_stats().items(): self.assertIsInstance(ifname, str) def test_net_io_counters(self): + # Duplicate of test_system.py. Keep it anyway. for ifname, _ in psutil.net_io_counters(pernic=True).items(): self.assertIsInstance(ifname, str) @unittest.skipUnless(hasattr(psutil, "sensors_fans"), "not supported") def test_sensors_fans(self): + # Duplicate of test_system.py. Keep it anyway. for name, units in psutil.sensors_fans().items(): self.assertIsInstance(name, str) for unit in units: @@ -222,12 +233,14 @@ def test_sensors_fans(self): @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), "not supported") def test_sensors_temperatures(self): + # Duplicate of test_system.py. Keep it anyway. for name, units in psutil.sensors_temperatures().items(): self.assertIsInstance(name, str) for unit in units: self.assertIsInstance(unit.label, str) def test_users(self): + # Duplicate of test_system.py. Keep it anyway. for user in psutil.users(): self.assertIsInstance(user.name, str) self.assertIsInstance(user.terminal, str) @@ -299,8 +312,8 @@ def test_fetch_all(self): # make sure exception's name attr is set # with the actual process name self.assertEqual(err.name, p.name()) - self.assertTrue(str(err)) - self.assertTrue(err.msg) + assert str(err) + assert err.msg except Exception as err: s = '\n' + '=' * 70 + '\n' s += "FAIL: test_%s (proc=%s" % (name, p) @@ -324,7 +337,7 @@ def test_fetch_all(self): # we should always have a non-empty list, not including PID 0 etc. # special cases. - self.assertTrue(valid_procs > 0) + assert valid_procs def cmdline(self, ret, proc): self.assertIsInstance(ret, list) @@ -343,7 +356,7 @@ def exe(self, ret, proc): if POSIX and os.path.isfile(ret): if hasattr(os, 'access') and hasattr(os, "X_OK"): # XXX may fail on OSX - self.assertTrue(os.access(ret, os.X_OK)) + assert os.access(ret, os.X_OK) def pid(self, ret, proc): self.assertIsInstance(ret, int) @@ -355,7 +368,7 @@ def ppid(self, ret, proc): def name(self, ret, proc): self.assertIsInstance(ret, str) - self.assertTrue(ret) + assert ret def create_time(self, ret, proc): self.assertIsInstance(ret, float) @@ -392,14 +405,14 @@ def gids(self, ret, proc): def username(self, ret, proc): self.assertIsInstance(ret, str) - self.assertTrue(ret) + assert ret if POSIX: self.assertIn(ret, self.all_usernames) def status(self, ret, proc): self.assertIsInstance(ret, str) - self.assertTrue(ret != "") - self.assertTrue(ret != '?') + assert ret + self.assertNotEqual(ret, '?') # XXX self.assertIn(ret, VALID_PROC_STATUSES) def io_counters(self, ret, proc): @@ -407,17 +420,17 @@ def io_counters(self, ret, proc): for field in ret: self.assertIsInstance(field, (int, long)) if field != -1: - self.assertTrue(field >= 0) + self.assertGreaterEqual(field, 0) def ionice(self, ret, proc): assert is_namedtuple(ret) for field in ret: self.assertIsInstance(field, int) if LINUX: - self.assertTrue(ret.ioclass >= 0) - self.assertTrue(ret.value >= 0) + self.assertGreaterEqual(ret.ioclass, 0) + self.assertGreaterEqual(ret.value, 0) else: - self.assertTrue(ret >= 0) + self.assertGreaterEqual(ret, 0) self.assertIn(ret, (0, 1, 2)) def num_threads(self, ret, proc): @@ -524,7 +537,7 @@ def cwd(self, ret, proc): elif err.errno != errno.ENOENT: raise else: - self.assertTrue(stat.S_ISDIR(st.st_mode)) + assert stat.S_ISDIR(st.st_mode) def memory_percent(self, ret, proc): self.assertIsInstance(ret, float) @@ -532,8 +545,7 @@ def memory_percent(self, ret, proc): def is_running(self, ret, proc): self.assertIsInstance(ret, bool) - # XXX: racy - self.assertTrue(ret) + assert ret # XXX: racy def cpu_affinity(self, ret, proc): self.assertIsInstance(ret, list) @@ -563,7 +575,7 @@ def memory_maps(self, ret, proc): # '/foo/bar (deleted)' # assert os.path.exists(nt.path), nt.path elif fname in ('addr', 'perms'): - self.assertTrue(value) + assert value else: self.assertIsInstance(value, (int, long)) self.assertGreaterEqual(value, 0) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index ccef60ae6..d521088a3 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -263,7 +263,7 @@ def open_mock(name, *args, **kwargs): assert m.called self.assertEqual(len(ws), 1) w = ws[0] - self.assertTrue(w.filename.endswith('psutil/_pslinux.py')) + assert w.filename.endswith('psutil/_pslinux.py') self.assertIn( "memory stats couldn't be determined", str(w.message)) self.assertIn("cached", str(w.message)) @@ -429,7 +429,7 @@ def test_missing_sin_sout(self): assert m.called self.assertEqual(len(ws), 1) w = ws[0] - self.assertTrue(w.filename.endswith('psutil/_pslinux.py')) + assert w.filename.endswith('psutil/_pslinux.py') self.assertIn( "'sin' and 'sout' swap memory stats couldn't " "be determined", str(w.message)) @@ -453,7 +453,7 @@ def open_mock(name, *args, **kwargs): assert m.called self.assertEqual(len(ws), 1) w = ws[0] - self.assertTrue(w.filename.endswith('psutil/_pslinux.py')) + assert w.filename.endswith('psutil/_pslinux.py') self.assertIn( "'sin' and 'sout' swap memory stats couldn't " "be determined and were set to 0", @@ -1037,29 +1037,29 @@ def test_prlimit_availability(self): p.rlimit(psutil.RLIMIT_NOFILE) # if prlimit() is supported *at least* these constants should # be available - self.assertTrue(hasattr(psutil, "RLIM_INFINITY")) - self.assertTrue(hasattr(psutil, "RLIMIT_AS")) - self.assertTrue(hasattr(psutil, "RLIMIT_CORE")) - self.assertTrue(hasattr(psutil, "RLIMIT_CPU")) - self.assertTrue(hasattr(psutil, "RLIMIT_DATA")) - self.assertTrue(hasattr(psutil, "RLIMIT_FSIZE")) - self.assertTrue(hasattr(psutil, "RLIMIT_LOCKS")) - self.assertTrue(hasattr(psutil, "RLIMIT_MEMLOCK")) - self.assertTrue(hasattr(psutil, "RLIMIT_NOFILE")) - self.assertTrue(hasattr(psutil, "RLIMIT_NPROC")) - self.assertTrue(hasattr(psutil, "RLIMIT_RSS")) - self.assertTrue(hasattr(psutil, "RLIMIT_STACK")) + assert hasattr(psutil, "RLIM_INFINITY") + assert hasattr(psutil, "RLIMIT_AS") + assert hasattr(psutil, "RLIMIT_CORE") + assert hasattr(psutil, "RLIMIT_CPU") + assert hasattr(psutil, "RLIMIT_DATA") + assert hasattr(psutil, "RLIMIT_FSIZE") + assert hasattr(psutil, "RLIMIT_LOCKS") + assert hasattr(psutil, "RLIMIT_MEMLOCK") + assert hasattr(psutil, "RLIMIT_NOFILE") + assert hasattr(psutil, "RLIMIT_NPROC") + assert hasattr(psutil, "RLIMIT_RSS") + assert hasattr(psutil, "RLIMIT_STACK") @unittest.skipUnless( get_kernel_version() >= (3, 0), "prlimit constants not available on this Linux kernel version") def test_resource_consts_kernel_v(self): # more recent constants - self.assertTrue(hasattr(psutil, "RLIMIT_MSGQUEUE")) - self.assertTrue(hasattr(psutil, "RLIMIT_NICE")) - self.assertTrue(hasattr(psutil, "RLIMIT_RTPRIO")) - self.assertTrue(hasattr(psutil, "RLIMIT_RTTIME")) - self.assertTrue(hasattr(psutil, "RLIMIT_SIGPENDING")) + assert hasattr(psutil, "RLIMIT_MSGQUEUE") + assert hasattr(psutil, "RLIMIT_NICE") + assert hasattr(psutil, "RLIMIT_RTPRIO") + assert hasattr(psutil, "RLIMIT_RTTIME") + assert hasattr(psutil, "RLIMIT_SIGPENDING") def test_boot_time_mocked(self): with mock.patch('psutil._pslinux.open', create=True) as m: @@ -1124,7 +1124,7 @@ def open_mock(name, *args, **kwargs): patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock): psutil.disk_io_counters() - self.assertTrue(flag) + assert flag def test_issue_687(self): # In case of thread ID: diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index c14e4ec67..41255e70d 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -29,7 +29,6 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._compat import long -from psutil._compat import unicode from psutil.tests import APPVEYOR from psutil.tests import ASCII_FS from psutil.tests import check_net_address @@ -473,10 +472,10 @@ def test_disk_partitions(self): # AssertionError: Lists differ: [0, 1, 2, 3, 4, 5, 6, 7,... != [0] self.assertTrue(ls, msg=ls) for disk in ls: - self.assertIsInstance(disk.device, (str, unicode)) - self.assertIsInstance(disk.mountpoint, (str, unicode)) - self.assertIsInstance(disk.fstype, (str, unicode)) - self.assertIsInstance(disk.opts, (str, unicode)) + self.assertIsInstance(disk.device, str) + self.assertIsInstance(disk.mountpoint, str) + self.assertIsInstance(disk.fstype, str) + self.assertIsInstance(disk.opts, str) if WINDOWS and 'cdrom' in disk.opts: continue if not POSIX: @@ -552,7 +551,7 @@ def check_ntuple(nt): self.assertNotEqual(ret, []) for key in ret: self.assertTrue(key) - self.assertIsInstance(key, (str, unicode)) + self.assertIsInstance(key, str) check_ntuple(ret[key]) def test_net_if_addrs(self): @@ -568,7 +567,7 @@ def test_net_if_addrs(self): families = set([socket.AF_INET, socket.AF_INET6, psutil.AF_LINK]) for nic, addrs in nics.items(): - self.assertIsInstance(nic, (str, unicode)) + self.assertIsInstance(nic, str) self.assertEqual(len(set(addrs)), len(addrs)) for addr in addrs: self.assertIsInstance(addr.family, int) @@ -639,7 +638,8 @@ def test_net_if_stats(self): all_duplexes = (psutil.NIC_DUPLEX_FULL, psutil.NIC_DUPLEX_HALF, psutil.NIC_DUPLEX_UNKNOWN) - for nic, stats in nics.items(): + for name, stats in nics.items(): + self.assertIsInstance(name, str) isup, duplex, speed, mtu = stats self.assertIsInstance(isup, bool) self.assertIn(duplex, all_duplexes) @@ -691,10 +691,10 @@ def test_users(self): self.assertNotEqual(users, []) for user in users: assert user.name, user - self.assertIsInstance(user.name, (str, unicode)) - self.assertIsInstance(user.terminal, (str, unicode, type(None))) + self.assertIsInstance(user.name, str) + self.assertIsInstance(user.terminal, (str, type(None))) if user.host is not None: - self.assertIsInstance(user.host, (str, unicode, type(None))) + self.assertIsInstance(user.host, (str, type(None))) user.terminal user.host assert user.started > 0.0, user @@ -780,9 +780,9 @@ def test_os_constants(self): def test_sensors_temperatures(self): temps = psutil.sensors_temperatures() for name, entries in temps.items(): - self.assertIsInstance(name, (str, unicode)) + self.assertIsInstance(name, str) for entry in entries: - self.assertIsInstance(entry.label, (str, unicode)) + self.assertIsInstance(entry.label, str) if entry.current is not None: self.assertGreaterEqual(entry.current, 0) if entry.high is not None: @@ -824,9 +824,9 @@ def test_sensors_battery(self): def test_sensors_fans(self): fans = psutil.sensors_fans() for name, entries in fans.items(): - self.assertIsInstance(name, (str, unicode)) + self.assertIsInstance(name, str) for entry in entries: - self.assertIsInstance(entry.label, (str, unicode)) + self.assertIsInstance(entry.label, str) self.assertIsInstance(entry.current, (int, long)) self.assertGreaterEqual(entry.current, 0) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 68ba847aa..f5ba9c462 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -120,7 +120,6 @@ def test_proc_exe(self): p = psutil.Process(subp.pid) exe = p.exe() self.assertIsInstance(exe, str) - self.assertIsInstance(exe, str) if self.expect_exact_path_match(): self.assertEqual(exe, self.funky_name) From d1fc7e2a8c4d51fd042da486b2262e997345aa90 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 06:54:04 +0200 Subject: [PATCH 431/922] move tests --- psutil/tests/test_contracts.py | 39 +++++++++++++++++++--------------- psutil/tests/test_linux.py | 33 ---------------------------- 2 files changed, 22 insertions(+), 50 deletions(-) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 8f9f6eb6f..033e180f1 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -29,6 +29,7 @@ from psutil._compat import long from psutil.tests import bind_unix_socket from psutil.tests import check_connection_ntuple +from psutil.tests import get_kernel_version from psutil.tests import is_namedtuple from psutil.tests import RLIMIT_SUPPORT from psutil.tests import run_test_module_by_name @@ -81,23 +82,27 @@ def test_linux_ioprio(self): def test_linux_rlimit(self): ae = self.assertEqual - ae(hasattr(psutil, "RLIM_INFINITY"), LINUX) - ae(hasattr(psutil, "RLIMIT_AS"), LINUX) - ae(hasattr(psutil, "RLIMIT_CORE"), LINUX) - ae(hasattr(psutil, "RLIMIT_CPU"), LINUX) - ae(hasattr(psutil, "RLIMIT_DATA"), LINUX) - ae(hasattr(psutil, "RLIMIT_FSIZE"), LINUX) - ae(hasattr(psutil, "RLIMIT_LOCKS"), LINUX) - ae(hasattr(psutil, "RLIMIT_MEMLOCK"), LINUX) - ae(hasattr(psutil, "RLIMIT_MSGQUEUE"), LINUX) - ae(hasattr(psutil, "RLIMIT_NICE"), LINUX) - ae(hasattr(psutil, "RLIMIT_NOFILE"), LINUX) - ae(hasattr(psutil, "RLIMIT_NPROC"), LINUX) - ae(hasattr(psutil, "RLIMIT_RSS"), LINUX) - ae(hasattr(psutil, "RLIMIT_RTPRIO"), LINUX) - ae(hasattr(psutil, "RLIMIT_RTTIME"), LINUX) - ae(hasattr(psutil, "RLIMIT_SIGPENDING"), LINUX) - ae(hasattr(psutil, "RLIMIT_STACK"), LINUX) + hasit = LINUX and get_kernel_version() >= (2, 6, 36) + ae(hasattr(psutil.Process, "rlimit"), hasit) + ae(hasattr(psutil, "RLIM_INFINITY"), hasit) + ae(hasattr(psutil, "RLIMIT_AS"), hasit) + ae(hasattr(psutil, "RLIMIT_CORE"), hasit) + ae(hasattr(psutil, "RLIMIT_CPU"), hasit) + ae(hasattr(psutil, "RLIMIT_DATA"), hasit) + ae(hasattr(psutil, "RLIMIT_FSIZE"), hasit) + ae(hasattr(psutil, "RLIMIT_LOCKS"), hasit) + ae(hasattr(psutil, "RLIMIT_MEMLOCK"), hasit) + ae(hasattr(psutil, "RLIMIT_NOFILE"), hasit) + ae(hasattr(psutil, "RLIMIT_NPROC"), hasit) + ae(hasattr(psutil, "RLIMIT_RSS"), hasit) + ae(hasattr(psutil, "RLIMIT_STACK"), hasit) + + hasit = LINUX and get_kernel_version() >= (3, 0) + ae(hasattr(psutil, "RLIMIT_MSGQUEUE"), hasit) + ae(hasattr(psutil, "RLIMIT_NICE"), hasit) + ae(hasattr(psutil, "RLIMIT_RTPRIO"), hasit) + ae(hasattr(psutil, "RLIMIT_RTTIME"), hasit) + ae(hasattr(psutil, "RLIMIT_SIGPENDING"), hasit) def test_cpu_freq(self): self.assertEqual(hasattr(psutil, "cpu_freq"), LINUX or OSX or WINDOWS) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index d521088a3..075024868 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1028,39 +1028,6 @@ def open_mock(name, *args, **kwargs): self.assertEqual(psutil.PROCFS_PATH, '/proc') - @unittest.skipUnless( - get_kernel_version() >= (2, 6, 36), - "prlimit() not available on this Linux kernel version") - def test_prlimit_availability(self): - # prlimit() should be available starting from kernel 2.6.36 - p = psutil.Process(os.getpid()) - p.rlimit(psutil.RLIMIT_NOFILE) - # if prlimit() is supported *at least* these constants should - # be available - assert hasattr(psutil, "RLIM_INFINITY") - assert hasattr(psutil, "RLIMIT_AS") - assert hasattr(psutil, "RLIMIT_CORE") - assert hasattr(psutil, "RLIMIT_CPU") - assert hasattr(psutil, "RLIMIT_DATA") - assert hasattr(psutil, "RLIMIT_FSIZE") - assert hasattr(psutil, "RLIMIT_LOCKS") - assert hasattr(psutil, "RLIMIT_MEMLOCK") - assert hasattr(psutil, "RLIMIT_NOFILE") - assert hasattr(psutil, "RLIMIT_NPROC") - assert hasattr(psutil, "RLIMIT_RSS") - assert hasattr(psutil, "RLIMIT_STACK") - - @unittest.skipUnless( - get_kernel_version() >= (3, 0), - "prlimit constants not available on this Linux kernel version") - def test_resource_consts_kernel_v(self): - # more recent constants - assert hasattr(psutil, "RLIMIT_MSGQUEUE") - assert hasattr(psutil, "RLIMIT_NICE") - assert hasattr(psutil, "RLIMIT_RTPRIO") - assert hasattr(psutil, "RLIMIT_RTTIME") - assert hasattr(psutil, "RLIMIT_SIGPENDING") - def test_boot_time_mocked(self): with mock.patch('psutil._pslinux.open', create=True) as m: self.assertRaises( From fe044f3d3ce3ba796ad26a71625f27f2a91bb091 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 07:05:42 +0200 Subject: [PATCH 432/922] skip test on win --- psutil/tests/test_connections.py | 1 + psutil/tests/test_process.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index b30c65e23..a8f19a28d 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -218,6 +218,7 @@ def test_tcp(self): # self.assertEqual(len(cons), 1) # self.assertEqual(cons[0].status, psutil.CONN_CLOSE_WAIT) + @unittest.skipUnless(POSIX, 'POSIX only') def test_unix(self): with unix_socket_path() as name: server, client = unix_socketpair(name) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index cc94caf2d..6e5ed9b65 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -973,7 +973,7 @@ def test_open_files_2(self): self.assertEqual(ntuple[0], ntuple.path) self.assertEqual(ntuple[1], ntuple.fd) # test file is gone - self.assertTrue(fileobj.name not in p.open_files()) + self.assertNotIn(fileobj.name, p.open_files()) @unittest.skipUnless(POSIX, 'POSIX only') def test_num_fds(self): From 0731087704fcae01e424651a05eda1e3eef40840 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 07:24:53 +0200 Subject: [PATCH 433/922] #1039 / proc.cpu_times / windows was returning int instead of float for children times --- HISTORY.rst | 2 ++ psutil/_pswindows.py | 2 +- psutil/tests/test_contracts.py | 2 +- scripts/internal/winmake.py | 7 +++++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 35185d62e..54e2d99da 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -29,6 +29,8 @@ properly handle unicode paths and may raise UnicodeDecodeError. - 1033_: [OSX, FreeBSD] memory leak for net_connections() and Process.connections() when retrieving UNIX sockets (kind='unix'). +- 1039_: returned types consolidation: + - Windows: Process.cpu_times()'s fields #3 and #4 were int instead of float *2017-04-10* diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index a5525df28..54b094b39 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -821,7 +821,7 @@ def cpu_times(self): else: raise # Children user/system times are not retrievable (set to 0). - return _common.pcputimes(user, system, 0, 0) + return _common.pcputimes(user, system, 0.0, 0.0) @wrap_exceptions def suspend(self): diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 033e180f1..6834135cf 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -248,7 +248,7 @@ def test_users(self): # Duplicate of test_system.py. Keep it anyway. for user in psutil.users(): self.assertIsInstance(user.name, str) - self.assertIsInstance(user.terminal, str) + self.assertIsInstance(user.terminal, (str, type(None))) self.assertIsInstance(user.host, (str, type(None))) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index c2db0fe3e..0c8f8fea7 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -352,6 +352,13 @@ def test_unicode(): sh("%s -m unittest -v psutil.tests.test_unicode" % PYTHON) +@cmd +def test_contracts(): + """Run contracts tests""" + install() + sh("%s -m unittest -v psutil.tests.test_contracts" % PYTHON) + + @cmd def test_by_name(): """Run test by name""" From 1fe72d482e77283ec17dcb77b70bfde417791dbb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 07:52:16 +0200 Subject: [PATCH 434/922] better way to skip unicode tests --- psutil/tests/test_unicode.py | 59 ++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index f5ba9c462..7bcd93d34 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -82,6 +82,30 @@ import psutil.tests +def can_deal_with_funky_name(name): + if PY3: + return True + + safe_rmpath(name) + create_exe(name) + try: + get_test_subprocess(cmd=[name]) + except UnicodeEncodeError: + return False + else: + reap_children() + return True + + +if PY3: + INVALID_NAME = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( + 'utf8', 'surrogateescape') +else: + INVALID_NAME = TESTFN + "f\xc0\x80" +UNICODE_OK = can_deal_with_funky_name(TESTFN_UNICODE) +INVALID_UNICODE_OK = can_deal_with_funky_name(INVALID_NAME) + + # =================================================================== # FS APIs # =================================================================== @@ -90,20 +114,6 @@ class _BaseFSAPIsTests(object): funky_name = None - @classmethod - def setUpClass(cls): - if not PY3: - create_exe(cls.funky_name) - try: - get_test_subprocess(cmd=[cls.funky_name]) - except UnicodeEncodeError: - # We skip all tests if subprocess module is not able to - # deal with such an exe. - raise unittest.SkipTest( - "subprocess module bumped into encoding error") - else: - reap_children() - def setUp(self): safe_rmpath(self.funky_name) @@ -218,6 +228,7 @@ def test_disk_usage(self): @unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # TODO @unittest.skipIf(ASCII_FS, "ASCII fs") +@unittest.skipIf(not UNICODE_OK, "subprocess can't deal with unicode") class TestFSAPIs(_BaseFSAPIsTests, unittest.TestCase): """Test FS APIs with a funky, valid, UTF8 path name.""" funky_name = TESTFN_UNICODE @@ -230,13 +241,11 @@ def expect_exact_path_match(cls): @unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # TODO +@unittest.skipIf(not INVALID_UNICODE_OK, + "subprocess can't deal with invalid unicode") class TestFSAPIsWithInvalidPath(_BaseFSAPIsTests, unittest.TestCase): """Test FS APIs with a funky, invalid path name.""" - if PY3: - funky_name = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( - 'utf8', 'surrogateescape') - else: - funky_name = TESTFN + "f\xc0\x80" + funky_name = INVALID_NAME @classmethod def expect_exact_path_match(cls): @@ -244,6 +253,18 @@ def expect_exact_path_match(cls): return True +@unittest.skipUnless(WINDOWS, "WINDOWS only") +class TestWinProcessName(unittest.TestCase): + + def test_name_type(self): + # On Windows name() is determined from exe() first, because + # it's faster; we want to overcome the internal optimization + # and test name() instead of exe(). + from psutil._pswindows import py2_strencode + name = py2_strencode(psutil._psplatform.cext.proc_name(os.getpid())) + self.assertIsInstance(name, str) + + # =================================================================== # Non fs APIs # =================================================================== From a42eb079a7275843ded575ce5f9f95816f2a9ed6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 08:48:23 +0200 Subject: [PATCH 435/922] avoid to use @skipUnless in tests; always use @skipIf (a lot clearer) --- psutil/tests/test_bsd.py | 28 +++++++------- psutil/tests/test_connections.py | 6 +-- psutil/tests/test_contracts.py | 8 ++-- psutil/tests/test_linux.py | 62 ++++++++++++++---------------- psutil/tests/test_memory_leaks.py | 64 ++++++++++++++----------------- psutil/tests/test_misc.py | 19 +++++---- psutil/tests/test_osx.py | 4 +- psutil/tests/test_posix.py | 8 ++-- psutil/tests/test_process.py | 62 +++++++++++++++--------------- psutil/tests/test_sunos.py | 2 +- psutil/tests/test_system.py | 22 +++++------ psutil/tests/test_unicode.py | 10 ++--- psutil/tests/test_windows.py | 22 +++++------ 13 files changed, 152 insertions(+), 165 deletions(-) diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index aeda555a9..2e6278869 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -71,7 +71,7 @@ def muse(field): # ===================================================================== -@unittest.skipUnless(BSD, "BSD only") +@unittest.skipIf(not BSD, "BSD only") class BSDSpecificTestCase(unittest.TestCase): """Generic tests common to all BSD variants.""" @@ -145,7 +145,7 @@ def test_net_if_stats(self): # ===================================================================== -@unittest.skipUnless(FREEBSD, "FREEBSD only") +@unittest.skipIf(not FREEBSD, "FREEBSD only") class FreeBSDSpecificTestCase(unittest.TestCase): @classmethod @@ -274,47 +274,47 @@ def test_vmem_buffers(self): # --- virtual_memory(); tests against muse - @unittest.skipUnless(MUSE_AVAILABLE, "muse not installed") + @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") def test_muse_vmem_total(self): num = muse('Total') self.assertEqual(psutil.virtual_memory().total, num) - @unittest.skipUnless(MUSE_AVAILABLE, "muse not installed") + @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_before_failing() def test_muse_vmem_active(self): num = muse('Active') self.assertAlmostEqual(psutil.virtual_memory().active, num, delta=MEMORY_TOLERANCE) - @unittest.skipUnless(MUSE_AVAILABLE, "muse not installed") + @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_before_failing() def test_muse_vmem_inactive(self): num = muse('Inactive') self.assertAlmostEqual(psutil.virtual_memory().inactive, num, delta=MEMORY_TOLERANCE) - @unittest.skipUnless(MUSE_AVAILABLE, "muse not installed") + @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_before_failing() def test_muse_vmem_wired(self): num = muse('Wired') self.assertAlmostEqual(psutil.virtual_memory().wired, num, delta=MEMORY_TOLERANCE) - @unittest.skipUnless(MUSE_AVAILABLE, "muse not installed") + @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_before_failing() def test_muse_vmem_cached(self): num = muse('Cache') self.assertAlmostEqual(psutil.virtual_memory().cached, num, delta=MEMORY_TOLERANCE) - @unittest.skipUnless(MUSE_AVAILABLE, "muse not installed") + @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_before_failing() def test_muse_vmem_free(self): num = muse('Free') self.assertAlmostEqual(psutil.virtual_memory().free, num, delta=MEMORY_TOLERANCE) - @unittest.skipUnless(MUSE_AVAILABLE, "muse not installed") + @unittest.skipIf(not MUSE_AVAILABLE, "muse not installed") @retry_before_failing() def test_muse_vmem_buffers(self): num = muse('Buffer') @@ -352,9 +352,9 @@ def test_boot_time(self): # --- sensors_battery - @unittest.skipUnless( - hasattr(psutil, "sensors_battery") and psutil.sensors_battery(), - "no battery") + @unittest.skipIf(not (hasattr(psutil, "sensors_battery") and + psutil.sensors_battery()), + "no battery") def test_sensors_battery(self): def secs2hours(secs): m, s = divmod(secs, 60) @@ -390,7 +390,7 @@ def test_sensors_battery_against_sysctl(self): # ===================================================================== -@unittest.skipUnless(OPENBSD, "OPENBSD only") +@unittest.skipIf(not OPENBSD, "OPENBSD only") class OpenBSDSpecificTestCase(unittest.TestCase): def test_boot_time(self): @@ -405,7 +405,7 @@ def test_boot_time(self): # ===================================================================== -@unittest.skipUnless(NETBSD, "NETBSD only") +@unittest.skipIf(not NETBSD, "NETBSD only") class NetBSDSpecificTestCase(unittest.TestCase): @staticmethod diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index a8f19a28d..8538fa511 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -152,7 +152,7 @@ def test_udp_v6(self): assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_NONE) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') def test_unix_tcp(self): with unix_socket_path() as name: with closing(bind_unix_socket(name, type=SOCK_STREAM)) as sock: @@ -160,7 +160,7 @@ def test_unix_tcp(self): assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_NONE) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') def test_unix_udp(self): with unix_socket_path() as name: with closing(bind_unix_socket(name, type=SOCK_STREAM)) as sock: @@ -218,7 +218,7 @@ def test_tcp(self): # self.assertEqual(len(cons), 1) # self.assertEqual(cons[0].status, psutil.CONN_CLOSE_WAIT) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') def test_unix(self): with unix_socket_path() as name: server, client = unix_socketpair(name) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 6834135cf..021f17bdd 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -198,7 +198,7 @@ def test_disk_partitions(self): self.assertIsInstance(disk.fstype, str) self.assertIsInstance(disk.opts, str) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') @skip_on_access_denied(only_if=OSX) def test_net_connections(self): with unix_socket_path() as name: @@ -227,7 +227,7 @@ def test_net_io_counters(self): for ifname, _ in psutil.net_io_counters(pernic=True).items(): self.assertIsInstance(ifname, str) - @unittest.skipUnless(hasattr(psutil, "sensors_fans"), "not supported") + @unittest.skipIf(not hasattr(psutil, "sensors_fans"), "not supported") def test_sensors_fans(self): # Duplicate of test_system.py. Keep it anyway. for name, units in psutil.sensors_fans().items(): @@ -235,8 +235,8 @@ def test_sensors_fans(self): for unit in units: self.assertIsInstance(unit.label, str) - @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), - "not supported") + @unittest.skipIf(not hasattr(psutil, "sensors_temperatures"), + "1not supported") def test_sensors_temperatures(self): # Duplicate of test_system.py. Keep it anyway. for name, units in psutil.sensors_temperatures().items(): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 075024868..f0c31b94d 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -28,7 +28,6 @@ from psutil._compat import PY3 from psutil._compat import u from psutil.tests import call_until -from psutil.tests import get_kernel_version from psutil.tests import importlib from psutil.tests import MEMORY_TOLERANCE from psutil.tests import mock @@ -36,6 +35,7 @@ from psutil.tests import pyrun from psutil.tests import reap_children from psutil.tests import retry_before_failing +from psutil.tests import RLIMIT_SUPPORT from psutil.tests import run_test_module_by_name from psutil.tests import safe_rmpath from psutil.tests import sh @@ -146,7 +146,7 @@ def get_free_version_info(): # ===================================================================== -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestSystemVirtualMemory(unittest.TestCase): def test_total(self): @@ -161,8 +161,8 @@ def test_total(self): # This got changed in: # https://gitlab.com/procps-ng/procps/commit/ # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e - @unittest.skipUnless( - LINUX and get_free_version_info() >= (3, 3, 12), "old free version") + @unittest.skipIf(LINUX and get_free_version_info() < (3, 3, 12), + "old free version") @retry_before_failing() def test_used(self): free = free_physmem() @@ -391,7 +391,7 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestSystemSwapMemory(unittest.TestCase): @staticmethod @@ -499,7 +499,7 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestSystemCPU(unittest.TestCase): @unittest.skipIf(TRAVIS, "unknown failure on travis") @@ -520,8 +520,8 @@ def test_cpu_times(self): else: self.assertNotIn('guest_nice', fields) - @unittest.skipUnless(os.path.exists("/sys/devices/system/cpu/online"), - "/sys/devices/system/cpu/online does not exist") + @unittest.skipIf(not os.path.exists("/sys/devices/system/cpu/online"), + "/sys/devices/system/cpu/online does not exist") def test_cpu_count_logical_w_sysdev_cpu_online(self): with open("/sys/devices/system/cpu/online") as f: value = f.read().strip() @@ -529,19 +529,19 @@ def test_cpu_count_logical_w_sysdev_cpu_online(self): value = int(value.split('-')[1]) + 1 self.assertEqual(psutil.cpu_count(), value) - @unittest.skipUnless(os.path.exists("/sys/devices/system/cpu"), - "/sys/devices/system/cpu does not exist") + @unittest.skipIf(not os.path.exists("/sys/devices/system/cpu"), + "/sys/devices/system/cpu does not exist") def test_cpu_count_logical_w_sysdev_cpu_num(self): ls = os.listdir("/sys/devices/system/cpu") count = len([x for x in ls if re.search("cpu\d+$", x) is not None]) self.assertEqual(psutil.cpu_count(), count) - @unittest.skipUnless(which("nproc"), "nproc utility not available") + @unittest.skipIf(not which("nproc"), "nproc utility not available") def test_cpu_count_logical_w_nproc(self): num = int(sh("nproc --all")) self.assertEqual(psutil.cpu_count(logical=True), num) - @unittest.skipUnless(which("lscpu"), "lscpu utility not available") + @unittest.skipIf(not which("lscpu"), "lscpu utility not available") def test_cpu_count_logical_w_lscpu(self): out = sh("lscpu -p") num = len([x for x in out.split('\n') if not x.startswith('#')]) @@ -667,7 +667,7 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestSystemCPUStats(unittest.TestCase): @unittest.skipIf(TRAVIS, "fails on Travis") @@ -688,7 +688,7 @@ def test_interrupts(self): # ===================================================================== -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestSystemNetwork(unittest.TestCase): def test_net_if_addrs_ips(self): @@ -749,7 +749,7 @@ def ifconfig(nic): self.assertAlmostEqual( stats.dropout, ifconfig_ret['dropout'], delta=10) - @unittest.skipUnless(which('ip'), "'ip' utility not available") + @unittest.skipIf(not which('ip'), "'ip' utility not available") @unittest.skipIf(TRAVIS, "skipped on Travis") def test_net_if_names(self): out = sh("ip addr").strip() @@ -800,11 +800,10 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestSystemDisks(unittest.TestCase): - @unittest.skipUnless( - hasattr(os, 'statvfs'), "os.statvfs() function not available") + @unittest.skipIf(not hasattr(os, 'statvfs'), "os.statvfs() not available") @skip_on_not_implemented() def test_disk_partitions_and_usage(self): # test psutil.disk_usage() and psutil.disk_partitions() @@ -958,7 +957,7 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestMisc(unittest.TestCase): def test_boot_time(self): @@ -1132,20 +1131,19 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -@unittest.skipUnless(LINUX, "LINUX only") -@unittest.skipUnless(hasattr(psutil, "sensors_battery") and - psutil.sensors_battery() is not None, - "no battery") +@unittest.skipIf(not LINUX, "LINUX only") +@unittest.skipIf(not getattr(psutil, "sensors_batterya", object)(), + "no battery") class TestSensorsBattery(unittest.TestCase): - @unittest.skipUnless(which("acpi"), "acpi utility not available") + @unittest.skipIf(not which("acpi"), "acpi utility not available") def test_percent(self): out = sh("acpi -b") acpi_value = int(out.split(",")[1].strip().replace('%', '')) psutil_value = psutil.sensors_battery().percent self.assertAlmostEqual(acpi_value, psutil_value, delta=1) - @unittest.skipUnless(which("acpi"), "acpi utility not available") + @unittest.skipIf(not which("acpi"), "acpi utility not available") def test_power_plugged(self): out = sh("acpi -b") if 'unknown' in out.lower(): @@ -1316,7 +1314,7 @@ def open_mock(name, *args, **kwargs): assert m.called -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestSensorsTemperatures(unittest.TestCase): @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") @@ -1362,7 +1360,7 @@ def open_mock(name, *args, **kwargs): self.assertEqual(temp.critical, 50.0) -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestSensorsFans(unittest.TestCase): def test_emulate_data(self): @@ -1391,7 +1389,7 @@ def open_mock(name, *args, **kwargs): # ===================================================================== -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestProcess(unittest.TestCase): def setUp(self): @@ -1596,9 +1594,7 @@ def open_mock(name, *args, **kwargs): self.assertEqual(err.exception.errno, errno.ENOENT) assert m.called - @unittest.skipUnless( - get_kernel_version() >= (2, 6, 36), - "prlimit() not available on this Linux kernel version") + @unittest.skipIf(not RLIMIT_SUPPORT, "not supported") def test_rlimit_zombie(self): # Emulate a case where rlimit() raises ENOSYS, which may # happen in case of zombie process: @@ -1625,7 +1621,7 @@ def test_cwd_zombie(self): self.assertEqual(exc.exception.name, p.name()) -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestProcessAgainstStatus(unittest.TestCase): """/proc/pid/stat and /proc/pid/status have many values in common. Whenever possible, psutil uses /proc/pid/stat (it's faster). @@ -1708,7 +1704,7 @@ def test_cpu_affinity_eligible_cpus(self): # ===================================================================== -@unittest.skipUnless(LINUX, "LINUX only") +@unittest.skipIf(not LINUX, "LINUX only") class TestUtils(unittest.TestCase): def test_open_text(self): diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 2bf7882ce..fce54a1bf 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -23,7 +23,6 @@ import psutil import psutil._common -from psutil import FREEBSD from psutil import LINUX from psutil import OPENBSD from psutil import OSX @@ -211,12 +210,12 @@ def test_exe(self): def test_ppid(self): self.execute(self.proc.ppid) - @unittest.skipUnless(POSIX, "POSIX only") + @unittest.skipIf(not POSIX, "POSIX only") @skip_if_linux() def test_uids(self): self.execute(self.proc.uids) - @unittest.skipUnless(POSIX, "POSIX only") + @unittest.skipIf(not POSIX, "POSIX only") @skip_if_linux() def test_gids(self): self.execute(self.proc.gids) @@ -232,13 +231,11 @@ def test_nice_set(self): niceness = thisproc.nice() self.execute(self.proc.nice, niceness) - @unittest.skipUnless(hasattr(psutil.Process, 'ionice'), - "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, 'ionice'), "not supported") def test_ionice_get(self): self.execute(self.proc.ionice) - @unittest.skipUnless(hasattr(psutil.Process, 'ionice'), - "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, 'ionice'), "not supported") def test_ionice_set(self): if WINDOWS: value = thisproc.ionice() @@ -248,7 +245,8 @@ def test_ionice_set(self): fun = functools.partial(cext.proc_ioprio_set, os.getpid(), -1, 0) self.execute_w_exc(OSError, fun) - @unittest.skipIf(OSX or SUNOS, "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, "io_counters"), + "not supported") @skip_if_linux() def test_io_counters(self): self.execute(self.proc.io_counters) @@ -265,11 +263,11 @@ def test_create_time(self): def test_num_threads(self): self.execute(self.proc.num_threads) - @unittest.skipUnless(WINDOWS, "WINDOWS only") + @unittest.skipIf(not WINDOWS, "WINDOWS only") def test_num_handles(self): self.execute(self.proc.num_handles) - @unittest.skipUnless(POSIX, "POSIX only") + @unittest.skipIf(not POSIX, "POSIX only") @skip_if_linux() def test_num_fds(self): self.execute(self.proc.num_fds) @@ -287,8 +285,7 @@ def test_cpu_times(self): self.execute(self.proc.cpu_times) @skip_if_linux() - @unittest.skipUnless(hasattr(psutil.Process, "cpu_num"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, "cpu_num"), "not supported") def test_cpu_num(self): self.execute(self.proc.cpu_num) @@ -296,13 +293,11 @@ def test_cpu_num(self): def test_memory_info(self): self.execute(self.proc.memory_info) - # also available on Linux but it's pure python - @unittest.skipUnless(OSX or WINDOWS, - "platform not supported") + @skip_if_linux() def test_memory_full_info(self): self.execute(self.proc.memory_full_info) - @unittest.skipUnless(POSIX, "POSIX only") + @unittest.skipIf(not POSIX, "POSIX only") @skip_if_linux() def test_terminal(self): self.execute(self.proc.terminal) @@ -316,13 +311,13 @@ def test_resume(self): def test_cwd(self): self.execute(self.proc.cwd) - @unittest.skipUnless(WINDOWS or LINUX or FREEBSD, - "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, "cpu_affinity"), + "not supported") def test_cpu_affinity_get(self): self.execute(self.proc.cpu_affinity) - @unittest.skipUnless(WINDOWS or LINUX or FREEBSD, - "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, "cpu_affinity"), + "not supported") def test_cpu_affinity_set(self): affinity = thisproc.cpu_affinity() self.execute(self.proc.cpu_affinity, affinity) @@ -337,18 +332,18 @@ def test_open_files(self): # OSX implementation is unbelievably slow @unittest.skipIf(OSX, "too slow on OSX") - @unittest.skipIf(OPENBSD, "platform not supported") + @unittest.skipIf(OPENBSD, "not supported") @skip_if_linux() def test_memory_maps(self): self.execute(self.proc.memory_maps) - @unittest.skipUnless(LINUX, "LINUX only") - @unittest.skipUnless(LINUX and RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not LINUX, "LINUX only") + @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") def test_rlimit_get(self): self.execute(self.proc.rlimit, psutil.RLIMIT_NOFILE) - @unittest.skipUnless(LINUX, "LINUX only") - @unittest.skipUnless(LINUX and RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not LINUX, "LINUX only") + @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") def test_rlimit_set(self): limit = thisproc.rlimit(psutil.RLIMIT_NOFILE) self.execute(self.proc.rlimit, psutil.RLIMIT_NOFILE, limit) @@ -397,12 +392,11 @@ def create_socket(family, type): for s in socks: s.close() - @unittest.skipUnless(hasattr(psutil.Process, 'environ'), - "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, 'environ'), "not supported") def test_environ(self): self.execute(self.proc.environ) - @unittest.skipUnless(WINDOWS, "WINDOWS only") + @unittest.skipIf(not WINDOWS, "WINDOWS only") def test_proc_info(self): self.execute(cext.proc_info, os.getpid()) @@ -503,7 +497,7 @@ def test_cpu_stats(self): self.execute(psutil.cpu_stats) @skip_if_linux() - @unittest.skipUnless(hasattr(psutil, "cpu_freq"), "platform not supported") + @unittest.skipIf(not hasattr(psutil, "cpu_freq"), "not supported") def test_cpu_freq(self): self.execute(psutil.cpu_freq) @@ -568,20 +562,20 @@ def test_net_if_stats(self): # --- sensors - @unittest.skipUnless(hasattr(psutil, "sensors_battery"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil, "sensors_battery"), + "not supported") @skip_if_linux() def test_sensors_battery(self): self.execute(psutil.sensors_battery) @skip_if_linux() - @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil, "sensors_temperatures"), + "not supported") def test_sensors_temperatures(self): self.execute(psutil.sensors_temperatures) - @unittest.skipUnless(hasattr(psutil, "sensors_fans"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil, "sensors_fans"), + "not supported") @skip_if_linux() def test_sensors_fans(self): self.execute(psutil.sensors_fans) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index a0a7a0b1b..b11a10d16 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -21,8 +21,6 @@ import sys from psutil import LINUX -from psutil import NETBSD -from psutil import OPENBSD from psutil import OSX from psutil import POSIX from psutil import WINDOWS @@ -414,7 +412,7 @@ def test_coverage(self): self.fail('no test defined for %r script' % os.path.join(SCRIPTS_DIR, name)) - @unittest.skipUnless(POSIX, "POSIX only") + @unittest.skipIf(not POSIX, "POSIX only") def test_executable(self): for name in os.listdir(SCRIPTS_DIR): if name.endswith('.py'): @@ -454,11 +452,12 @@ def test_netstat(self): def test_ifconfig(self): self.assert_stdout('ifconfig.py') - @unittest.skipIf(OPENBSD or NETBSD, "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, "memory_maps"), + "not supported") def test_pmap(self): self.assert_stdout('pmap.py', args=str(os.getpid())) - @unittest.skipUnless(OSX or WINDOWS or LINUX, "platform not supported") + @unittest.skipIf(not OSX or WINDOWS or LINUX, "platform not supported") def test_procsmem(self): self.assert_stdout('procsmem.py') @@ -478,14 +477,14 @@ def test_pidof(self): output = self.assert_stdout('pidof.py', args=psutil.Process().name()) self.assertIn(str(os.getpid()), output) - @unittest.skipUnless(WINDOWS, "WINDOWS only") + @unittest.skipIf(not WINDOWS, "WINDOWS only") def test_winservices(self): self.assert_stdout('winservices.py') def test_cpu_distribution(self): self.assert_syntax('cpu_distribution.py') - @unittest.skipIf(TRAVIS, "unreliable on travis") + @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_temperatures(self): if hasattr(psutil, "sensors_temperatures") and \ psutil.sensors_temperatures(): @@ -493,7 +492,7 @@ def test_temperatures(self): else: self.assert_syntax('temperatures.py') - @unittest.skipIf(TRAVIS, "unreliable on travis") + @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_fans(self): if hasattr(psutil, "sensors_fans") and psutil.sensors_fans(): self.assert_stdout('fans.py') @@ -685,7 +684,7 @@ def test_create_proc_children_pair(self): class TestNetUtils(unittest.TestCase): - @unittest.skipUnless(POSIX, "POSIX only") + @unittest.skipIf(not POSIX, "POSIX only") def test_bind_unix_socket(self): with unix_socket_path() as name: sock = bind_unix_socket(name) @@ -712,7 +711,7 @@ def tcp_tcp_socketpair(self): self.assertEqual(client.getpeername(), addr) self.assertNotEqual(client.getsockname(), addr) - @unittest.skipUnless(POSIX, "POSIX only") + @unittest.skipIf(not POSIX, "POSIX only") def test_unix_socketpair(self): p = psutil.Process() num_fds = p.num_fds() diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index db40efbc8..8ba949b0a 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -74,7 +74,7 @@ def human2bytes(s): return int(num * prefix[letter]) -@unittest.skipUnless(OSX, "OSX only") +@unittest.skipIf(not OSX, "OSX only") class TestProcess(unittest.TestCase): @classmethod @@ -99,7 +99,7 @@ def test_process_create_time(self): time.strftime("%Y", time.localtime(start_psutil))) -@unittest.skipUnless(OSX, "OSX only") +@unittest.skipIf(not OSX, "OSX only") class TestSystemAPIs(unittest.TestCase): # --- disk diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 654b16276..1b2dc5063 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -55,7 +55,7 @@ def ps(cmd): return output -@unittest.skipUnless(POSIX, "POSIX only") +@unittest.skipIf(not POSIX, "POSIX only") class TestProcess(unittest.TestCase): """Compare psutil results against 'ps' command line utility (mainly).""" @@ -238,14 +238,14 @@ def call(p, attr): if failures: self.fail('\n' + '\n'.join(failures)) - @unittest.skipUnless(os.path.islink("/proc/%s/cwd" % os.getpid()), - "/proc fs not available") + @unittest.skipIf(not os.path.islink("/proc/%s/cwd" % os.getpid()), + "/proc fs not available") def test_cwd(self): self.assertEqual(os.readlink("/proc/%s/cwd" % os.getpid()), psutil.Process().cwd()) -@unittest.skipUnless(POSIX, "POSIX only") +@unittest.skipIf(not POSIX, "POSIX only") class TestSystemAPIs(unittest.TestCase): """Test some system APIs.""" diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 6e5ed9b65..79ec4be4b 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -23,7 +23,6 @@ import psutil from psutil import BSD -from psutil import FREEBSD from psutil import LINUX from psutil import NETBSD from psutil import OPENBSD @@ -254,8 +253,8 @@ def test_cpu_times(self): # XXX fails on OSX: not sure if it's for os.times(). We should # try this with Python 2.7 and re-enable the test. - @unittest.skipUnless(sys.version_info > (2, 6, 1) and not OSX, - 'os.times() broken on OSX + PY2.6.1') + @unittest.skipIf(sys.version_info <= (2, 6, 1) and OSX, + 'os.times() broken on OSX + PY2.6.1') def test_cpu_times_2(self): user_time, kernel_time = psutil.Process().cpu_times()[:2] utime, ktime = os.times()[:2] @@ -269,8 +268,7 @@ def test_cpu_times_2(self): if (max([kernel_time, ktime]) - min([kernel_time, ktime])) > 0.1: self.fail("expected: %s, found: %s" % (ktime, kernel_time)) - @unittest.skipUnless(hasattr(psutil.Process, "cpu_num"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, "cpu_num"), "not supported") def test_cpu_num(self): p = psutil.Process() num = p.cpu_num() @@ -296,7 +294,7 @@ def test_create_time(self): # make sure returned value can be pretty printed with strftime time.strftime("%Y %m %d %H:%M:%S", time.localtime(p.create_time())) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') @unittest.skipIf(TRAVIS, 'not reliable on TRAVIS') def test_terminal(self): terminal = psutil.Process().terminal() @@ -306,8 +304,8 @@ def test_terminal(self): else: self.assertIsNone(terminal) - @unittest.skipUnless(LINUX or BSD or WINDOWS, - 'platform not supported') + @unittest.skipIf(not hasattr(psutil.Process, "io_counters"), + 'not supported') @skip_on_not_implemented(only_if=LINUX) def test_io_counters(self): p = psutil.Process() @@ -351,8 +349,8 @@ def test_io_counters(self): self.assertGreaterEqual(io2[i], 0) self.assertGreaterEqual(io2[i], 0) - @unittest.skipUnless(LINUX or (WINDOWS and get_winver() >= WIN_VISTA), - 'platform not supported') + @unittest.skipIf(not hasattr(psutil.Process, "ionice"), "not supported") + @unittest.skipIf(WINDOWS and get_winver() < WIN_VISTA, 'not supported') def test_ionice(self): if LINUX: from psutil import (IOPRIO_CLASS_NONE, IOPRIO_CLASS_RT, @@ -398,8 +396,8 @@ def test_ionice(self): finally: p.ionice(original) - @unittest.skipUnless(LINUX or (WINDOWS and get_winver() >= WIN_VISTA), - 'platform not supported') + @unittest.skipIf(not hasattr(psutil.Process, "ionice"), "not supported") + @unittest.skipIf(WINDOWS and get_winver() < WIN_VISTA, 'not supported') def test_ionice_errs(self): sproc = get_test_subprocess() p = psutil.Process(sproc.pid) @@ -421,7 +419,7 @@ def test_ionice_errs(self): self.assertRaises(ValueError, p.ionice, 3) self.assertRaises(TypeError, p.ionice, 2, 1) - @unittest.skipUnless(LINUX and RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") def test_rlimit_get(self): import resource p = psutil.Process(os.getpid()) @@ -444,7 +442,7 @@ def test_rlimit_get(self): self.assertGreaterEqual(ret[0], -1) self.assertGreaterEqual(ret[1], -1) - @unittest.skipUnless(LINUX and RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") def test_rlimit_set(self): sproc = get_test_subprocess() p = psutil.Process(sproc.pid) @@ -457,7 +455,7 @@ def test_rlimit_set(self): with self.assertRaises(ValueError): p.rlimit(psutil.RLIMIT_NOFILE, (5, 5, 5)) - @unittest.skipUnless(LINUX and RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") def test_rlimit(self): p = psutil.Process() soft, hard = p.rlimit(psutil.RLIMIT_FSIZE) @@ -476,7 +474,7 @@ def test_rlimit(self): p.rlimit(psutil.RLIMIT_FSIZE, (soft, hard)) self.assertEqual(p.rlimit(psutil.RLIMIT_FSIZE), (soft, hard)) - @unittest.skipUnless(LINUX and RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") def test_rlimit_infinity(self): # First set a limit, then re-set it by specifying INFINITY # and assume we overridden the previous limit. @@ -491,7 +489,7 @@ def test_rlimit_infinity(self): p.rlimit(psutil.RLIMIT_FSIZE, (soft, hard)) self.assertEqual(p.rlimit(psutil.RLIMIT_FSIZE), (soft, hard)) - @unittest.skipUnless(LINUX and RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") def test_rlimit_infinity_value(self): # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really # big number on a platform with large file support. On these @@ -524,7 +522,7 @@ def test_num_threads(self): finally: thread.stop() - @unittest.skipUnless(WINDOWS, 'WINDOWS only') + @unittest.skipIf(not WINDOWS, 'WINDOWS only') def test_num_handles(self): # a better test is done later into test/_windows.py p = psutil.Process() @@ -617,7 +615,7 @@ def test_memory_full_info(self): self.assertGreaterEqual(mem.pss, 0) self.assertGreaterEqual(mem.swap, 0) - @unittest.skipIf(OPENBSD or NETBSD, "platfform not supported") + @unittest.skipIf(OPENBSD or NETBSD, "not supported") def test_memory_maps(self): p = psutil.Process() maps = p.memory_maps() @@ -754,7 +752,7 @@ def test_prog_w_funky_name(self): self.assertEqual(os.path.normcase(p.exe()), os.path.normcase(funky_path)) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') def test_uids(self): p = psutil.Process() real, effective, saved = p.uids() @@ -768,7 +766,7 @@ def test_uids(self): if hasattr(os, "getresuid"): self.assertEqual(os.getresuid(), p.uids()) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') def test_gids(self): p = psutil.Process() real, effective, saved = p.gids() @@ -858,7 +856,8 @@ def test_cwd_2(self): p = psutil.Process(sproc.pid) call_until(p.cwd, "ret == os.path.dirname(os.getcwd())") - @unittest.skipUnless(WINDOWS or LINUX or FREEBSD, 'platform not supported') + @unittest.skipIf(not hasattr(psutil.Process, "cpu_affinity"), + 'not supported') def test_cpu_affinity(self): p = psutil.Process() initial = p.cpu_affinity() @@ -901,7 +900,8 @@ def test_cpu_affinity(self): p.cpu_affinity(set(all_cpus)) p.cpu_affinity(tuple(all_cpus)) - @unittest.skipUnless(WINDOWS or LINUX or FREEBSD, 'platform not supported') + @unittest.skipIf(not hasattr(psutil.Process, "cpu_affinity"), + 'not supported') def test_cpu_affinity_errs(self): sproc = get_test_subprocess() p = psutil.Process(sproc.pid) @@ -975,7 +975,7 @@ def test_open_files_2(self): # test file is gone self.assertNotIn(fileobj.name, p.open_files()) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') def test_num_fds(self): p = psutil.Process() start = p.num_fds() @@ -1235,7 +1235,7 @@ def test_halfway_terminated_process(self): "NoSuchProcess exception not raised for %r, retval=%s" % ( name, ret)) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') def test_zombie_process(self): def succeed_or_zombie_p_exc(fun, *args, **kwargs): try: @@ -1330,7 +1330,7 @@ def succeed_or_zombie_p_exc(fun, *args, **kwargs): finally: reap_children(recursive=True) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') def test_zombie_process_is_running_w_exc(self): # Emulate a case where internally is_running() raises # ZombieProcess. @@ -1340,7 +1340,7 @@ def test_zombie_process_is_running_w_exc(self): assert p.is_running() assert m.called - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') def test_zombie_process_status_w_exc(self): # Emulate a case where internally status() raises # ZombieProcess. @@ -1416,8 +1416,7 @@ def test_Popen_ctx_manager(self): assert proc.stderr.closed assert proc.stdin.closed - @unittest.skipUnless(hasattr(psutil.Process, "environ"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, "environ"), "not supported") def test_environ(self): self.maxDiff = None p = psutil.Process() @@ -1440,9 +1439,8 @@ def test_environ(self): self.assertEqual(d, d2) - @unittest.skipUnless(hasattr(psutil.Process, "environ"), - "platform not supported") - @unittest.skipUnless(POSIX, "posix only") + @unittest.skipIf(not hasattr(psutil.Process, "environ"), "not supported") + @unittest.skipIf(not POSIX, "POSIX only") def test_weird_environ(self): # environment variables can contain values without an equals sign code = textwrap.dedent(""" diff --git a/psutil/tests/test_sunos.py b/psutil/tests/test_sunos.py index 0e7704443..ea9afcde0 100755 --- a/psutil/tests/test_sunos.py +++ b/psutil/tests/test_sunos.py @@ -15,7 +15,7 @@ from psutil.tests import unittest -@unittest.skipUnless(SUNOS, "SUNOS only") +@unittest.skipIf(not SUNOS, "SUNOS only") class SunOSSpecificTestCase(unittest.TestCase): def test_swap_memory(self): diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 41255e70d..8118e9970 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -167,7 +167,7 @@ def test_boot_time(self): self.assertGreater(bt, 0) self.assertLess(bt, time.time()) - @unittest.skipUnless(POSIX, 'POSIX only') + @unittest.skipIf(not POSIX, 'POSIX only') def test_PAGESIZE(self): # pagesize is used internally to perform different calculations # and it's determined by using SC_PAGE_SIZE; make sure @@ -716,8 +716,8 @@ def test_cpu_stats(self): if name in ('ctx_switches', 'interrupts'): self.assertGreater(value, 0) - @unittest.skipUnless(hasattr(psutil, "cpu_freq"), - "platform not suported") + @unittest.skipIf(not hasattr(psutil, "cpu_freq"), + "platform not suported") def test_cpu_freq(self): def check_ls(ls): for nt in ls: @@ -775,8 +775,8 @@ def test_os_constants(self): for name in names: self.assertIs(getattr(psutil, name), False, msg=name) - @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil, "sensors_temperatures"), + "platform not supported") def test_sensors_temperatures(self): temps = psutil.sensors_temperatures() for name, entries in temps.items(): @@ -790,8 +790,8 @@ def test_sensors_temperatures(self): if entry.critical is not None: self.assertGreaterEqual(entry.critical, 0) - @unittest.skipUnless(hasattr(psutil, "sensors_temperatures"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil, "sensors_temperatures"), + "platform not supported") def test_sensors_temperatures_fahreneit(self): d = {'coretemp': [('label', 50.0, 60.0, 70.0)]} with mock.patch("psutil._psplatform.sensors_temperatures", @@ -803,8 +803,8 @@ def test_sensors_temperatures_fahreneit(self): self.assertEqual(temps.high, 140.0) self.assertEqual(temps.critical, 158.0) - @unittest.skipUnless(hasattr(psutil, "sensors_battery"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil, "sensors_battery"), + "platform not supported") def test_sensors_battery(self): ret = psutil.sensors_battery() if ret is None: @@ -819,8 +819,8 @@ def test_sensors_battery(self): self.assertTrue(ret.power_plugged) self.assertIsInstance(ret.power_plugged, bool) - @unittest.skipUnless(hasattr(psutil, "sensors_fans"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil, "sensors_fans"), + "platform not supported") def test_sensors_fans(self): fans = psutil.sensors_fans() for name, entries in fans.items(): diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 7bcd93d34..7240c6b28 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -181,7 +181,7 @@ def test_proc_open_files(self): self.assertEqual(os.path.normcase(path), os.path.normcase(self.funky_name)) - @unittest.skipUnless(POSIX, "POSIX only") + @unittest.skipIf(not POSIX, "POSIX only") def test_proc_connections(self): suffix = os.path.basename(self.funky_name) with unix_socket_path(suffix=suffix) as name: @@ -197,7 +197,7 @@ def test_proc_connections(self): self.assertIsInstance(conn.laddr, str) self.assertEqual(conn.laddr, name) - @unittest.skipUnless(POSIX, "POSIX only") + @unittest.skipIf(not POSIX, "POSIX only") @skip_on_access_denied() def test_net_connections(self): def find_sock(cons): @@ -253,7 +253,7 @@ def expect_exact_path_match(cls): return True -@unittest.skipUnless(WINDOWS, "WINDOWS only") +@unittest.skipIf(not WINDOWS, "WINDOWS only") class TestWinProcessName(unittest.TestCase): def test_name_type(self): @@ -273,8 +273,8 @@ def test_name_type(self): class TestNonFSAPIS(unittest.TestCase): """Unicode tests for non fs-related APIs.""" - @unittest.skipUnless(hasattr(psutil.Process, "environ"), - "platform not supported") + @unittest.skipIf(not hasattr(psutil.Process, "environ"), + "platform not supported") def test_proc_environ(self): # Note: differently from others, this test does not deal # with fs paths. On Python 2 subprocess module is broken as diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index b4982f047..1c28c4fe4 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -65,7 +65,7 @@ def wrapper(self, *args, **kwargs): # =================================================================== -@unittest.skipUnless(WINDOWS, "WINDOWS only") +@unittest.skipIf(not WINDOWS, "WINDOWS only") class TestSystemAPIs(unittest.TestCase): def test_nic_names(self): @@ -78,8 +78,8 @@ def test_nic_names(self): self.fail( "%r nic wasn't found in 'ipconfig /all' output" % nic) - @unittest.skipUnless('NUMBER_OF_PROCESSORS' in os.environ, - 'NUMBER_OF_PROCESSORS env var is not available') + @unittest.skipIf('NUMBER_OF_PROCESSORS' not in os.environ, + 'NUMBER_OF_PROCESSORS env var is not available') def test_cpu_count(self): num_cpus = int(os.environ['NUMBER_OF_PROCESSORS']) self.assertEqual(num_cpus, psutil.cpu_count()) @@ -185,7 +185,7 @@ def test_net_if_stats(self): # =================================================================== -@unittest.skipUnless(WINDOWS, "WINDOWS only") +@unittest.skipIf(not WINDOWS, "WINDOWS only") class TestSensorsBattery(unittest.TestCase): def test_percent(self): @@ -245,7 +245,7 @@ def test_emulate_secs_left_unknown(self): # =================================================================== -@unittest.skipUnless(WINDOWS, "WINDOWS only") +@unittest.skipIf(not WINDOWS, "WINDOWS only") class TestProcess(unittest.TestCase): @classmethod @@ -343,8 +343,8 @@ def test_name_always_available(self): except psutil.NoSuchProcess: pass - @unittest.skipUnless(sys.version_info >= (2, 7), - "CTRL_* signals not supported") + @unittest.skipIf(not sys.version_info >= (2, 7), + "CTRL_* signals not supported") def test_ctrl_signals(self): p = psutil.Process(get_test_subprocess().pid) p.send_signal(signal.CTRL_C_EVENT) @@ -483,7 +483,7 @@ def test_num_handles(self): self.assertEqual(psutil_value, sys_value + 1) -@unittest.skipUnless(WINDOWS, "WINDOWS only") +@unittest.skipIf(not WINDOWS, "WINDOWS only") class TestProcessWMI(unittest.TestCase): """Compare Process API results with WMI.""" @@ -549,7 +549,7 @@ def test_create_time(self): self.assertEqual(wmic_create, psutil_create) -@unittest.skipUnless(WINDOWS, "WINDOWS only") +@unittest.skipIf(not WINDOWS, "WINDOWS only") class TestDualProcessImplementation(unittest.TestCase): """ Certain APIs on Windows have 2 internal implementations, one @@ -628,7 +628,7 @@ def test_num_handles(self): assert fun.called -@unittest.skipUnless(WINDOWS, "WINDOWS only") +@unittest.skipIf(not WINDOWS, "WINDOWS only") class RemoteProcessTestCase(unittest.TestCase): """Certain functions require calling ReadProcessMemory. This trivially works when called on the current process. @@ -723,7 +723,7 @@ def test_environ_64(self): # =================================================================== -@unittest.skipUnless(WINDOWS, "WINDOWS only") +@unittest.skipIf(not WINDOWS, "WINDOWS only") class TestServices(unittest.TestCase): def test_win_service_iter(self): From 27ab5a2329923a36b12bae681f1c64ef8e5091f1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 09:11:23 +0200 Subject: [PATCH 436/922] define support constants to check availability of functionalities --- psutil/tests/__init__.py | 15 +++++++++++++ psutil/tests/test_contracts.py | 7 +++--- psutil/tests/test_memory_leaks.py | 37 +++++++++++++++++-------------- psutil/tests/test_misc.py | 4 ++-- psutil/tests/test_system.py | 20 ++++++++--------- psutil/tests/test_unicode.py | 4 ++-- 6 files changed, 53 insertions(+), 34 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 5c7ea9fb5..382b2e6a6 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -143,6 +143,21 @@ ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') +# --- support + +HAS_CPU_AFFINITY = hasattr(psutil.Process, "cpu_affinity") +HAS_CPU_FREQ = hasattr(psutil.Process, "cpu_affinity") +HAS_ENVIRON = hasattr(psutil.Process, "environ") +HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters") +HAS_IONICE = hasattr(psutil.Process, "ionice") +HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps") +HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") +HAS_RLIMIT = hasattr(psutil.Process, "rlimit") +HAS_SENSORS_BATTERY = hasattr(psutil, "sensors_battery") +HAS_BATTERY = HAS_SENSORS_BATTERY and psutil.sensors_battery() +HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans") +HAS_SENSORS_TEMPERATURES = hasattr(psutil, "sensors_temperatures") + # --- misc PYTHON = os.path.realpath(sys.executable) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 021f17bdd..48496dc96 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -30,6 +30,8 @@ from psutil.tests import bind_unix_socket from psutil.tests import check_connection_ntuple from psutil.tests import get_kernel_version +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import is_namedtuple from psutil.tests import RLIMIT_SUPPORT from psutil.tests import run_test_module_by_name @@ -227,7 +229,7 @@ def test_net_io_counters(self): for ifname, _ in psutil.net_io_counters(pernic=True).items(): self.assertIsInstance(ifname, str) - @unittest.skipIf(not hasattr(psutil, "sensors_fans"), "not supported") + @unittest.skipIf(not HAS_SENSORS_FANS, "not supported") def test_sensors_fans(self): # Duplicate of test_system.py. Keep it anyway. for name, units in psutil.sensors_fans().items(): @@ -235,8 +237,7 @@ def test_sensors_fans(self): for unit in units: self.assertIsInstance(unit.label, str) - @unittest.skipIf(not hasattr(psutil, "sensors_temperatures"), - "1not supported") + @unittest.skipIf(not HAS_SENSORS_TEMPERATURES, "not supported") def test_sensors_temperatures(self): # Duplicate of test_system.py. Keep it anyway. for name, units in psutil.sensors_temperatures().items(): diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index fce54a1bf..e049361ae 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -33,6 +33,15 @@ from psutil._compat import xrange from psutil.tests import bind_unix_socket from psutil.tests import get_test_subprocess +from psutil.tests import HAS_CPU_AFFINITY +from psutil.tests import HAS_CPU_FREQ +from psutil.tests import HAS_ENVIRON +from psutil.tests import HAS_IONICE +from psutil.tests import HAS_PROC_CPU_NUM +from psutil.tests import HAS_PROC_IO_COUNTERS +from psutil.tests import HAS_SENSORS_BATTERY +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import reap_children from psutil.tests import RLIMIT_SUPPORT from psutil.tests import run_test_module_by_name @@ -231,11 +240,11 @@ def test_nice_set(self): niceness = thisproc.nice() self.execute(self.proc.nice, niceness) - @unittest.skipIf(not hasattr(psutil.Process, 'ionice'), "not supported") + @unittest.skipIf(not HAS_IONICE, "not supported") def test_ionice_get(self): self.execute(self.proc.ionice) - @unittest.skipIf(not hasattr(psutil.Process, 'ionice'), "not supported") + @unittest.skipIf(not HAS_IONICE, "not supported") def test_ionice_set(self): if WINDOWS: value = thisproc.ionice() @@ -245,8 +254,7 @@ def test_ionice_set(self): fun = functools.partial(cext.proc_ioprio_set, os.getpid(), -1, 0) self.execute_w_exc(OSError, fun) - @unittest.skipIf(not hasattr(psutil.Process, "io_counters"), - "not supported") + @unittest.skipIf(not HAS_PROC_IO_COUNTERS, "not supported") @skip_if_linux() def test_io_counters(self): self.execute(self.proc.io_counters) @@ -285,7 +293,7 @@ def test_cpu_times(self): self.execute(self.proc.cpu_times) @skip_if_linux() - @unittest.skipIf(not hasattr(psutil.Process, "cpu_num"), "not supported") + @unittest.skipIf(not HAS_PROC_CPU_NUM, "not supported") def test_cpu_num(self): self.execute(self.proc.cpu_num) @@ -311,13 +319,11 @@ def test_resume(self): def test_cwd(self): self.execute(self.proc.cwd) - @unittest.skipIf(not hasattr(psutil.Process, "cpu_affinity"), - "not supported") + @unittest.skipIf(not HAS_CPU_AFFINITY, "not supported") def test_cpu_affinity_get(self): self.execute(self.proc.cpu_affinity) - @unittest.skipIf(not hasattr(psutil.Process, "cpu_affinity"), - "not supported") + @unittest.skipIf(not HAS_CPU_AFFINITY, "not supported") def test_cpu_affinity_set(self): affinity = thisproc.cpu_affinity() self.execute(self.proc.cpu_affinity, affinity) @@ -392,7 +398,7 @@ def create_socket(family, type): for s in socks: s.close() - @unittest.skipIf(not hasattr(psutil.Process, 'environ'), "not supported") + @unittest.skipIf(not HAS_ENVIRON, "not supported") def test_environ(self): self.execute(self.proc.environ) @@ -497,7 +503,7 @@ def test_cpu_stats(self): self.execute(psutil.cpu_stats) @skip_if_linux() - @unittest.skipIf(not hasattr(psutil, "cpu_freq"), "not supported") + @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq(self): self.execute(psutil.cpu_freq) @@ -562,20 +568,17 @@ def test_net_if_stats(self): # --- sensors - @unittest.skipIf(not hasattr(psutil, "sensors_battery"), - "not supported") + @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") @skip_if_linux() def test_sensors_battery(self): self.execute(psutil.sensors_battery) @skip_if_linux() - @unittest.skipIf(not hasattr(psutil, "sensors_temperatures"), - "not supported") + @unittest.skipIf(not HAS_SENSORS_TEMPERATURES, "not supported") def test_sensors_temperatures(self): self.execute(psutil.sensors_temperatures) - @unittest.skipIf(not hasattr(psutil, "sensors_fans"), - "not supported") + @unittest.skipIf(not HAS_SENSORS_FANS, "not supported") @skip_if_linux() def test_sensors_fans(self): self.execute(psutil.sensors_fans) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index b11a10d16..4f653cea3 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -34,6 +34,7 @@ from psutil.tests import create_proc_children_pair from psutil.tests import get_free_port from psutil.tests import get_test_subprocess +from psutil.tests import HAS_MEMORY_MAPS from psutil.tests import importlib from psutil.tests import mock from psutil.tests import reap_children @@ -452,8 +453,7 @@ def test_netstat(self): def test_ifconfig(self): self.assert_stdout('ifconfig.py') - @unittest.skipIf(not hasattr(psutil.Process, "memory_maps"), - "not supported") + @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") def test_pmap(self): self.assert_stdout('pmap.py', args=str(os.getpid())) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 8118e9970..88bdd093d 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -35,7 +35,12 @@ from psutil.tests import DEVNULL from psutil.tests import enum from psutil.tests import get_test_subprocess +from psutil.tests import HAS_CPU_FREQ +from psutil.tests import HAS_SENSORS_BATTERY +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import mock +from psutil.tests import NO_BATTERY from psutil.tests import reap_children from psutil.tests import retry_before_failing from psutil.tests import run_test_module_by_name @@ -716,8 +721,7 @@ def test_cpu_stats(self): if name in ('ctx_switches', 'interrupts'): self.assertGreater(value, 0) - @unittest.skipIf(not hasattr(psutil, "cpu_freq"), - "platform not suported") + @unittest.skipIf(not HAS_CPU_FREQ, "not suported") def test_cpu_freq(self): def check_ls(ls): for nt in ls: @@ -775,8 +779,7 @@ def test_os_constants(self): for name in names: self.assertIs(getattr(psutil, name), False, msg=name) - @unittest.skipIf(not hasattr(psutil, "sensors_temperatures"), - "platform not supported") + @unittest.skipIf(not HAS_SENSORS_TEMPERATURES, "not supported") def test_sensors_temperatures(self): temps = psutil.sensors_temperatures() for name, entries in temps.items(): @@ -790,8 +793,7 @@ def test_sensors_temperatures(self): if entry.critical is not None: self.assertGreaterEqual(entry.critical, 0) - @unittest.skipIf(not hasattr(psutil, "sensors_temperatures"), - "platform not supported") + @unittest.skipIf(not HAS_SENSORS_TEMPERATURES, "not supported") def test_sensors_temperatures_fahreneit(self): d = {'coretemp': [('label', 50.0, 60.0, 70.0)]} with mock.patch("psutil._psplatform.sensors_temperatures", @@ -803,8 +805,7 @@ def test_sensors_temperatures_fahreneit(self): self.assertEqual(temps.high, 140.0) self.assertEqual(temps.critical, 158.0) - @unittest.skipIf(not hasattr(psutil, "sensors_battery"), - "platform not supported") + @unittest.skipIf(not HAS_SENSORS_BATTERY or NO_BATTERY, "not supported") def test_sensors_battery(self): ret = psutil.sensors_battery() if ret is None: @@ -819,8 +820,7 @@ def test_sensors_battery(self): self.assertTrue(ret.power_plugged) self.assertIsInstance(ret.power_plugged, bool) - @unittest.skipIf(not hasattr(psutil, "sensors_fans"), - "platform not supported") + @unittest.skipIf(not HAS_SENSORS_FANS, "not supported") def test_sensors_fans(self): fans = psutil.sensors_fans() for name, entries in fans.items(): diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 7240c6b28..d7a107213 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -67,6 +67,7 @@ from psutil.tests import chdir from psutil.tests import create_exe from psutil.tests import get_test_subprocess +from psutil.tests import HAS_ENVIRON from psutil.tests import reap_children from psutil.tests import run_test_module_by_name from psutil.tests import safe_mkdir @@ -273,8 +274,7 @@ def test_name_type(self): class TestNonFSAPIS(unittest.TestCase): """Unicode tests for non fs-related APIs.""" - @unittest.skipIf(not hasattr(psutil.Process, "environ"), - "platform not supported") + @unittest.skipIf(not HAS_ENVIRON, "not supported") def test_proc_environ(self): # Note: differently from others, this test does not deal # with fs paths. On Python 2 subprocess module is broken as From 1223096701f8ec2a2518bfd813a96be1a51b483b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 09:21:03 +0200 Subject: [PATCH 437/922] use HAS_ support constants in tests --- psutil/tests/test_process.py | 22 ++++++++++++---------- psutil/tests/test_system.py | 7 +++---- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 79ec4be4b..9ac049861 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -40,6 +40,11 @@ from psutil.tests import get_test_subprocess from psutil.tests import get_winver from psutil.tests import GLOBAL_TIMEOUT +from psutil.tests import HAS_CPU_AFFINITY +from psutil.tests import HAS_ENVIRON +from psutil.tests import HAS_IONICE +from psutil.tests import HAS_PROC_CPU_NUM +from psutil.tests import HAS_PROC_IO_COUNTERS from psutil.tests import mock from psutil.tests import PYPY from psutil.tests import pyrun @@ -268,7 +273,7 @@ def test_cpu_times_2(self): if (max([kernel_time, ktime]) - min([kernel_time, ktime])) > 0.1: self.fail("expected: %s, found: %s" % (ktime, kernel_time)) - @unittest.skipIf(not hasattr(psutil.Process, "cpu_num"), "not supported") + @unittest.skipIf(not HAS_PROC_CPU_NUM, "not supported") def test_cpu_num(self): p = psutil.Process() num = p.cpu_num() @@ -304,8 +309,7 @@ def test_terminal(self): else: self.assertIsNone(terminal) - @unittest.skipIf(not hasattr(psutil.Process, "io_counters"), - 'not supported') + @unittest.skipIf(not HAS_PROC_IO_COUNTERS, 'not supported') @skip_on_not_implemented(only_if=LINUX) def test_io_counters(self): p = psutil.Process() @@ -349,7 +353,7 @@ def test_io_counters(self): self.assertGreaterEqual(io2[i], 0) self.assertGreaterEqual(io2[i], 0) - @unittest.skipIf(not hasattr(psutil.Process, "ionice"), "not supported") + @unittest.skipIf(not HAS_IONICE, "not supported") @unittest.skipIf(WINDOWS and get_winver() < WIN_VISTA, 'not supported') def test_ionice(self): if LINUX: @@ -396,7 +400,7 @@ def test_ionice(self): finally: p.ionice(original) - @unittest.skipIf(not hasattr(psutil.Process, "ionice"), "not supported") + @unittest.skipIf(not HAS_IONICE, "not supported") @unittest.skipIf(WINDOWS and get_winver() < WIN_VISTA, 'not supported') def test_ionice_errs(self): sproc = get_test_subprocess() @@ -856,8 +860,7 @@ def test_cwd_2(self): p = psutil.Process(sproc.pid) call_until(p.cwd, "ret == os.path.dirname(os.getcwd())") - @unittest.skipIf(not hasattr(psutil.Process, "cpu_affinity"), - 'not supported') + @unittest.skipIf(not HAS_CPU_AFFINITY, 'not supported') def test_cpu_affinity(self): p = psutil.Process() initial = p.cpu_affinity() @@ -900,8 +903,7 @@ def test_cpu_affinity(self): p.cpu_affinity(set(all_cpus)) p.cpu_affinity(tuple(all_cpus)) - @unittest.skipIf(not hasattr(psutil.Process, "cpu_affinity"), - 'not supported') + @unittest.skipIf(not HAS_CPU_AFFINITY, 'not supported') def test_cpu_affinity_errs(self): sproc = get_test_subprocess() p = psutil.Process(sproc.pid) @@ -1416,7 +1418,7 @@ def test_Popen_ctx_manager(self): assert proc.stderr.closed assert proc.stdin.closed - @unittest.skipIf(not hasattr(psutil.Process, "environ"), "not supported") + @unittest.skipIf(not HAS_ENVIRON, "not supported") def test_environ(self): self.maxDiff = None p = psutil.Process() diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 88bdd093d..4f1781b88 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -35,12 +35,12 @@ from psutil.tests import DEVNULL from psutil.tests import enum from psutil.tests import get_test_subprocess +from psutil.tests import HAS_BATTERY from psutil.tests import HAS_CPU_FREQ from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import mock -from psutil.tests import NO_BATTERY from psutil.tests import reap_children from psutil.tests import retry_before_failing from psutil.tests import run_test_module_by_name @@ -805,11 +805,10 @@ def test_sensors_temperatures_fahreneit(self): self.assertEqual(temps.high, 140.0) self.assertEqual(temps.critical, 158.0) - @unittest.skipIf(not HAS_SENSORS_BATTERY or NO_BATTERY, "not supported") + @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") + @unittest.skipIf(not HAS_BATTERY, "no battery") def test_sensors_battery(self): ret = psutil.sensors_battery() - if ret is None: - return # no battery self.assertGreaterEqual(ret.percent, 0) self.assertLessEqual(ret.percent, 100) if ret.secsleft not in (psutil.POWER_TIME_UNKNOWN, From 98b6fb5524d0915095e1e53e52af7089ecdbbf6d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 09:28:59 +0200 Subject: [PATCH 438/922] use HAS_ support constants in tests --- psutil/tests/test_bsd.py | 3 --- psutil/tests/test_linux.py | 2 -- psutil/tests/test_memory_leaks.py | 4 ++-- psutil/tests/test_windows.py | 6 ++++++ 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 2e6278869..0f98a3c6c 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -352,9 +352,6 @@ def test_boot_time(self): # --- sensors_battery - @unittest.skipIf(not (hasattr(psutil, "sensors_battery") and - psutil.sensors_battery()), - "no battery") def test_sensors_battery(self): def secs2hours(secs): m, s = divmod(secs, 60) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index f0c31b94d..d5d0f7b8e 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1132,8 +1132,6 @@ def open_mock(name, *args, **kwargs): @unittest.skipIf(not LINUX, "LINUX only") -@unittest.skipIf(not getattr(psutil, "sensors_batterya", object)(), - "no battery") class TestSensorsBattery(unittest.TestCase): @unittest.skipIf(not which("acpi"), "acpi utility not available") diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index e049361ae..b979ed179 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -568,8 +568,8 @@ def test_net_if_stats(self): # --- sensors - @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") @skip_if_linux() + @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") def test_sensors_battery(self): self.execute(psutil.sensors_battery) @@ -578,8 +578,8 @@ def test_sensors_battery(self): def test_sensors_temperatures(self): self.execute(psutil.sensors_temperatures) - @unittest.skipIf(not HAS_SENSORS_FANS, "not supported") @skip_if_linux() + @unittest.skipIf(not HAS_SENSORS_FANS, "not supported") def test_sensors_fans(self): self.execute(psutil.sensors_fans) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 1c28c4fe4..776476526 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -32,6 +32,8 @@ from psutil._compat import callable from psutil.tests import APPVEYOR from psutil.tests import get_test_subprocess +from psutil.tests import HAS_BATTERY +from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import mock from psutil.tests import reap_children from psutil.tests import retry_before_failing @@ -188,6 +190,8 @@ def test_net_if_stats(self): @unittest.skipIf(not WINDOWS, "WINDOWS only") class TestSensorsBattery(unittest.TestCase): + @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") + @unittest.skipIf(not HAS_BATTERY, "no battery") def test_percent(self): w = wmi.WMI() battery_psutil = psutil.sensors_battery() @@ -206,6 +210,8 @@ def test_percent(self): self.assertEqual( battery_psutil.power_plugged, battery_wmi.BatteryStatus == 1) + @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") + @unittest.skipIf(not HAS_BATTERY, "no battery") def test_battery_present(self): if win32api.GetPwrCapabilities()['SystemBatteriesPresent']: self.assertIsNotNone(psutil.sensors_battery()) From 77dd4784844b1e4f310f69bef065e5ad8af26712 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 09:41:47 +0200 Subject: [PATCH 439/922] use HAS_ support constants in tests --- psutil/tests/__init__.py | 13 +++---------- psutil/tests/test_contracts.py | 4 ++-- psutil/tests/test_linux.py | 4 ++-- psutil/tests/test_memory_leaks.py | 6 +++--- psutil/tests/test_process.py | 24 ++++++++---------------- 5 files changed, 18 insertions(+), 33 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 382b2e6a6..e93d032d1 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -37,7 +37,6 @@ from urllib2 import urlopen import psutil -from psutil import LINUX from psutil import POSIX from psutil import WINDOWS from psutil._compat import PY3 @@ -72,9 +71,9 @@ __all__ = [ # constants 'APPVEYOR', 'DEVNULL', 'GLOBAL_TIMEOUT', 'MEMORY_TOLERANCE', 'NO_RETRIES', - 'PYPY', 'PYTHON', 'RLIMIT_SUPPORT', 'ROOT_DIR', 'SCRIPTS_DIR', - 'TESTFILE_PREFIX', 'TESTFN', 'TESTFN_UNICODE', 'TOX', 'TRAVIS', - 'VALID_PROC_STATUSES', 'VERBOSITY', + 'PYPY', 'PYTHON', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFILE_PREFIX', + 'TESTFN', 'TESTFN_UNICODE', 'TOX', 'TRAVIS', 'VALID_PROC_STATUSES', + 'VERBOSITY', # classes 'ThreadTask' # test utils @@ -416,12 +415,6 @@ def get_kernel_version(): return (major, minor, micro) -if LINUX: - RLIMIT_SUPPORT = get_kernel_version() >= (2, 6, 36) -else: - RLIMIT_SUPPORT = False - - if not WINDOWS: def get_winver(): raise NotImplementedError("not a Windows OS") diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 48496dc96..34015b9e2 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -30,10 +30,10 @@ from psutil.tests import bind_unix_socket from psutil.tests import check_connection_ntuple from psutil.tests import get_kernel_version +from psutil.tests import HAS_RLIMIT from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import is_namedtuple -from psutil.tests import RLIMIT_SUPPORT from psutil.tests import run_test_module_by_name from psutil.tests import safe_rmpath from psutil.tests import skip_on_access_denied @@ -279,7 +279,7 @@ def test_fetch_all(self): 'send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', 'as_dict', 'parent', 'children', 'memory_info_ex', 'oneshot', ]) - if LINUX and not RLIMIT_SUPPORT: + if LINUX and not HAS_RLIMIT: excluded_names.add('rlimit') attrs = [] for name in dir(psutil.Process): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index d5d0f7b8e..519bf0041 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -28,6 +28,7 @@ from psutil._compat import PY3 from psutil._compat import u from psutil.tests import call_until +from psutil.tests import HAS_RLIMIT from psutil.tests import importlib from psutil.tests import MEMORY_TOLERANCE from psutil.tests import mock @@ -35,7 +36,6 @@ from psutil.tests import pyrun from psutil.tests import reap_children from psutil.tests import retry_before_failing -from psutil.tests import RLIMIT_SUPPORT from psutil.tests import run_test_module_by_name from psutil.tests import safe_rmpath from psutil.tests import sh @@ -1592,7 +1592,7 @@ def open_mock(name, *args, **kwargs): self.assertEqual(err.exception.errno, errno.ENOENT) assert m.called - @unittest.skipIf(not RLIMIT_SUPPORT, "not supported") + @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit_zombie(self): # Emulate a case where rlimit() raises ENOSYS, which may # happen in case of zombie process: diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index b979ed179..6e0992306 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -39,11 +39,11 @@ from psutil.tests import HAS_IONICE from psutil.tests import HAS_PROC_CPU_NUM from psutil.tests import HAS_PROC_IO_COUNTERS +from psutil.tests import HAS_RLIMIT from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import reap_children -from psutil.tests import RLIMIT_SUPPORT from psutil.tests import run_test_module_by_name from psutil.tests import safe_rmpath from psutil.tests import TESTFN @@ -344,12 +344,12 @@ def test_memory_maps(self): self.execute(self.proc.memory_maps) @unittest.skipIf(not LINUX, "LINUX only") - @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit_get(self): self.execute(self.proc.rlimit, psutil.RLIMIT_NOFILE) @unittest.skipIf(not LINUX, "LINUX only") - @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit_set(self): limit = thisproc.rlimit(psutil.RLIMIT_NOFILE) self.execute(self.proc.rlimit, psutil.RLIMIT_NOFILE, limit) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 9ac049861..06b7de129 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -45,13 +45,13 @@ from psutil.tests import HAS_IONICE from psutil.tests import HAS_PROC_CPU_NUM from psutil.tests import HAS_PROC_IO_COUNTERS +from psutil.tests import HAS_RLIMIT from psutil.tests import mock from psutil.tests import PYPY from psutil.tests import pyrun from psutil.tests import PYTHON from psutil.tests import reap_children from psutil.tests import retry_before_failing -from psutil.tests import RLIMIT_SUPPORT from psutil.tests import run_test_module_by_name from psutil.tests import safe_rmpath from psutil.tests import sh @@ -252,14 +252,6 @@ def test_cpu_times(self): for name in times._fields: time.strftime("%H:%M:%S", time.localtime(getattr(times, name))) - # Test Process.cpu_times() against os.times() - # os.times() is broken on Python 2.6 - # http://bugs.python.org/issue1040026 - # XXX fails on OSX: not sure if it's for os.times(). We should - # try this with Python 2.7 and re-enable the test. - - @unittest.skipIf(sys.version_info <= (2, 6, 1) and OSX, - 'os.times() broken on OSX + PY2.6.1') def test_cpu_times_2(self): user_time, kernel_time = psutil.Process().cpu_times()[:2] utime, ktime = os.times()[:2] @@ -423,7 +415,7 @@ def test_ionice_errs(self): self.assertRaises(ValueError, p.ionice, 3) self.assertRaises(TypeError, p.ionice, 2, 1) - @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit_get(self): import resource p = psutil.Process(os.getpid()) @@ -446,7 +438,7 @@ def test_rlimit_get(self): self.assertGreaterEqual(ret[0], -1) self.assertGreaterEqual(ret[1], -1) - @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit_set(self): sproc = get_test_subprocess() p = psutil.Process(sproc.pid) @@ -459,7 +451,7 @@ def test_rlimit_set(self): with self.assertRaises(ValueError): p.rlimit(psutil.RLIMIT_NOFILE, (5, 5, 5)) - @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit(self): p = psutil.Process() soft, hard = p.rlimit(psutil.RLIMIT_FSIZE) @@ -478,7 +470,7 @@ def test_rlimit(self): p.rlimit(psutil.RLIMIT_FSIZE, (soft, hard)) self.assertEqual(p.rlimit(psutil.RLIMIT_FSIZE), (soft, hard)) - @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit_infinity(self): # First set a limit, then re-set it by specifying INFINITY # and assume we overridden the previous limit. @@ -493,7 +485,7 @@ def test_rlimit_infinity(self): p.rlimit(psutil.RLIMIT_FSIZE, (soft, hard)) self.assertEqual(p.rlimit(psutil.RLIMIT_FSIZE), (soft, hard)) - @unittest.skipIf(not RLIMIT_SUPPORT, "LINUX >= 2.6.36 only") + @unittest.skipIf(not HAS_RLIMIT, "not supported") def test_rlimit_infinity_value(self): # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really # big number on a platform with large file support. On these @@ -1193,7 +1185,7 @@ def test_halfway_terminated_process(self): excluded_names = ['pid', 'is_running', 'wait', 'create_time', 'oneshot', 'memory_info_ex'] - if LINUX and not RLIMIT_SUPPORT: + if LINUX and not HAS_RLIMIT: excluded_names.append('rlimit') for name in dir(p): if (name.startswith('_') or @@ -1441,7 +1433,7 @@ def test_environ(self): self.assertEqual(d, d2) - @unittest.skipIf(not hasattr(psutil.Process, "environ"), "not supported") + @unittest.skipIf(not HAS_ENVIRON, "not supported") @unittest.skipIf(not POSIX, "POSIX only") def test_weird_environ(self): # environment variables can contain values without an equals sign From 265118eb52749e7e7e922fc620e766f5702d4732 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 15:17:59 +0200 Subject: [PATCH 440/922] fix osx / linux on travis --- psutil/_psosx.py | 2 +- psutil/tests/test_linux.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/psutil/_psosx.py b/psutil/_psosx.py index f5851b81a..872d3a144 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -418,7 +418,7 @@ def cpu_times(self): rawtuple[pidtaskinfo_map['cpuutime']], rawtuple[pidtaskinfo_map['cpustime']], # children user / system times are not retrievable (set to 0) - 0, 0) + 0.0, 0.0) @wrap_exceptions def create_time(self): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 519bf0041..053c5f692 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -28,6 +28,7 @@ from psutil._compat import PY3 from psutil._compat import u from psutil.tests import call_until +from psutil.tests import HAS_BATTERY from psutil.tests import HAS_RLIMIT from psutil.tests import importlib from psutil.tests import MEMORY_TOLERANCE @@ -1132,6 +1133,7 @@ def open_mock(name, *args, **kwargs): @unittest.skipIf(not LINUX, "LINUX only") +@unittest.skipIf(not HAS_BATTERY, "no battery") class TestSensorsBattery(unittest.TestCase): @unittest.skipIf(not which("acpi"), "acpi utility not available") From a1517915cf0386942d0d0fbc5b101d7ae1c4aa2a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 16:06:38 +0200 Subject: [PATCH 441/922] download_exe.py: use concurrent.futures.as_completed() --- docs/index.rst | 2 +- scripts/internal/download_exes.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 3031ce581..1ce8bee89 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -831,7 +831,7 @@ Functions structure:: >>> import psutil - >>> procs = dict([(p.pid, p.info) for p in psutil.process_iter(attrs=['name', 'username'])]) + >>> procs = {p.pid: p.info for p in psutil.process_iter(attrs=['name', 'username'])} >>> procs {1: {'name': 'systemd', 'username': 'root'}, 2: {'name': 'kthreadd', 'username': 'root'}, diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_exes.py index 62f3e94d6..ae2604b03 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_exes.py @@ -14,12 +14,12 @@ from __future__ import print_function import argparse +import concurrent.futures import errno import os import requests import shutil import sys -from concurrent.futures import ThreadPoolExecutor from psutil import __version__ as PSUTIL_VERSION @@ -114,7 +114,6 @@ def download_file(url): if chunk: # filter out keep-alive new chunks f.write(chunk) tot_bytes += len(chunk) - print("downloaded %-45s %s" % (local_fname, bytes2human(tot_bytes))) return local_fname @@ -151,15 +150,17 @@ def rename_27_wheels(): def main(options): - files = [] safe_rmtree('dist') - with ThreadPoolExecutor() as e: - for url in get_file_urls(options): - fut = e.submit(download_file, url) - files.append(fut.result()) + urls = get_file_urls(options) + with concurrent.futures.ThreadPoolExecutor() as e: + fut_to_url = {e.submit(download_file, url): url for url in urls} + for fut in concurrent.futures.as_completed(fut_to_url): + local_fname = fut.result() + print("downloaded %-45s %s" % ( + local_fname, bytes2human(os.path.getsize(local_fname)))) # 2 exes (32 and 64 bit) and 2 wheels (32 and 64 bit) for each ver. expected = len(PY_VERSIONS) * 4 - got = len(files) + got = len(fut_to_url) if expected != got: return exit("expected %s files, got %s" % (expected, got)) rename_27_wheels() From 130c62eb1f2e024d974deccbe7ad845663424d04 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sun, 30 Apr 2017 20:23:12 +0530 Subject: [PATCH 442/922] implement good coding styles --- scripts/internal/check_broken_links.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index f628bc4d8..76ed8da75 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -65,19 +65,15 @@ def get_urls(filename): """ # fname = os.path.abspath(os.path.join(HERE, filename)) # expecting absolute path - fname = os.path.abspath(filename) - text = '' - with open(fname) as f: - text = f.read() + with open(filename) as fs: + text = fs.read() urls = re.findall(REGEX, text) # remove duplicates, list for sets are not iterable urls = list(set(urls)) # correct urls which are between < and/or > - i = 0 - while i < len(urls): - urls[i] = re.sub("[\*<>\(\)\)]", '', urls[i]) - i += 1 + for i, url in enumerate(urls): + urls[i] = re.sub("[\*<>\(\)\)]", '', url) return urls @@ -140,7 +136,7 @@ def main(): all_urls.append((fname, url)) fails = parallel_validator(all_urls) - if len(fails) == 0: + if not fails: print("all links are valid. cheers!") else: print("total :", len(fails), "fails!") From 5dcc9607bd89cd7db09b69f7e71a963bfcf6c80a Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sun, 30 Apr 2017 20:24:25 +0530 Subject: [PATCH 443/922] implement timeout in http requests --- scripts/internal/check_broken_links.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 76ed8da75..9ae5e6440 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -54,6 +54,8 @@ REGEX = r'(?:http|ftp|https)?://' \ r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' +REQUEST_TIMEOUT = 30 + # There are some status codes sent by websites on HEAD request. # Like 503 by Microsoft, and 401 by Apple # They need to be sent GET request @@ -84,11 +86,11 @@ def validate_url(url): Uses requests module. """ try: - res = requests.head(url) + res = requests.head(url, timeout=REQUEST_TIMEOUT) # some websites deny 503, like Microsoft # and some send 401, like Apple, observations if (not res.ok) and (res.status_code in RETRY_STATUSES): - res = requests.get(url) + res = requests.get(url, timeout=REQUEST_TIMEOUT) return res.ok except requests.exceptions.RequestException: return False From 1eff05650de5f87fa448f498256451baa8a606c9 Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Sun, 30 Apr 2017 20:30:23 +0530 Subject: [PATCH 444/922] remove comment from Makefile --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 0fa418df2..650f35f1d 100644 --- a/Makefile +++ b/Makefile @@ -278,5 +278,3 @@ doc: # check whether the links mentioned in some files are valid. check-broken-links: git ls-files | grep \\.rst$ | xargs $(PYTHON) scripts/internal/check_broken_links.py -# Alternate method, DOCFILES need to be defined -# $(PYTHON) scripts/internal/check_broken_links.py $(DOCFILES) From e5f082e1410d319de124092a8507e8730b9c4270 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 17:11:38 +0200 Subject: [PATCH 445/922] futures: catch exception on result() --- scripts/internal/download_exes.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_exes.py index ae2604b03..f98399672 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_exes.py @@ -152,17 +152,27 @@ def rename_27_wheels(): def main(options): safe_rmtree('dist') urls = get_file_urls(options) + completed = 0 + exc = None with concurrent.futures.ThreadPoolExecutor() as e: fut_to_url = {e.submit(download_file, url): url for url in urls} for fut in concurrent.futures.as_completed(fut_to_url): - local_fname = fut.result() - print("downloaded %-45s %s" % ( - local_fname, bytes2human(os.path.getsize(local_fname)))) + url = fut_to_url[fut] + try: + local_fname = fut.result() + except Exception as _: + exc = _ + print("error while downloading %s: %s" % (url, exc)) + else: + completed += 1 + print("downloaded %-45s %s" % ( + local_fname, bytes2human(os.path.getsize(local_fname)))) # 2 exes (32 and 64 bit) and 2 wheels (32 and 64 bit) for each ver. expected = len(PY_VERSIONS) * 4 - got = len(fut_to_url) - if expected != got: - return exit("expected %s files, got %s" % (expected, got)) + if expected != completed: + return exit("expected %s files, got %s" % (expected, completed)) + if exc: + return exit(1) rename_27_wheels() From 1b79ea4bf8bc06a9cad346f23cebf140fd474d21 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 17:31:35 +0200 Subject: [PATCH 446/922] downalod_exes.py: set timeout for HTTP requests --- scripts/internal/download_exes.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_exes.py index f98399672..9688919bf 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_exes.py @@ -7,7 +7,7 @@ """ Script which downloads exe and wheel files hosted on AppVeyor: https://ci.appveyor.com/project/giampaolo/psutil -Copied and readapted from the original recipe of Ibarra Corretge' +Readapted from the original recipe of Ibarra Corretge' : http://code.saghul.net/index.php/2015/09/09/ """ @@ -26,11 +26,13 @@ BASE_URL = 'https://ci.appveyor.com/api' PY_VERSIONS = ['2.7', '3.3', '3.4', '3.5', '3.6'] +TIMEOUT = 30 COLORS = True -def exit(msg): - print(hilite(msg, ok=False), file=sys.stderr) +def exit(msg=""): + if msg: + print(hilite(msg, ok=False), file=sys.stderr) sys.exit(1) @@ -107,7 +109,7 @@ def download_file(url): local_fname = url.split('/')[-1] local_fname = os.path.join('dist', local_fname) safe_makedirs('dist') - r = requests.get(url, stream=True) + r = requests.get(url, stream=True, timeout=TIMEOUT) tot_bytes = 0 with open(local_fname, 'wb') as f: for chunk in r.iter_content(chunk_size=16384): @@ -120,13 +122,14 @@ def download_file(url): def get_file_urls(options): session = requests.Session() data = session.get( - BASE_URL + '/projects/' + options.user + '/' + options.project) + BASE_URL + '/projects/' + options.user + '/' + options.project, + timeout=TIMEOUT) data = data.json() urls = [] for job in (job['jobId'] for job in data['build']['jobs']): job_url = BASE_URL + '/buildjobs/' + job + '/artifacts' - data = session.get(job_url) + data = session.get(job_url, timeout=TIMEOUT) data = data.json() for item in data: file_url = job_url + '/' + item['fileName'] @@ -172,7 +175,7 @@ def main(options): if expected != completed: return exit("expected %s files, got %s" % (expected, completed)) if exc: - return exit(1) + return exit() rename_27_wheels() From e7f6d67bcca76740d3bb69ff41ebdea59585a3dd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 18:39:14 +0200 Subject: [PATCH 447/922] fix freebsd failure --- psutil/tests/__init__.py | 2 +- psutil/tests/test_unicode.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index e93d032d1..4a97dba8c 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -145,7 +145,7 @@ # --- support HAS_CPU_AFFINITY = hasattr(psutil.Process, "cpu_affinity") -HAS_CPU_FREQ = hasattr(psutil.Process, "cpu_affinity") +HAS_CPU_FREQ = hasattr(psutil, "cpu_freq") HAS_ENVIRON = hasattr(psutil.Process, "environ") HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters") HAS_IONICE = hasattr(psutil.Process, "ionice") diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index d7a107213..12cbd01d6 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -84,12 +84,14 @@ def can_deal_with_funky_name(name): + """Return True if both the fs and the subprocess module can + deal with a funky file name. + """ if PY3: return True - - safe_rmpath(name) - create_exe(name) try: + safe_rmpath(name) + create_exe(name) get_test_subprocess(cmd=[name]) except UnicodeEncodeError: return False From ef6727cd13e347741ed8f2c731fdb4f42768002a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 19:00:56 +0200 Subject: [PATCH 448/922] setup.py: always include _psutil_common.c and _psutil_posix.c in all C extensions --- setup.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 2fad970e3..d4fc45915 100755 --- a/setup.py +++ b/setup.py @@ -43,6 +43,10 @@ if BSD: macros.append(("PSUTIL_BSD", 1)) +sources = ['psutil/_psutil_common.c'] +if POSIX: + sources.append('psutil/_psutil_posix.c') + def get_version(): INIT = os.path.join(HERE, 'psutil/__init__.py') @@ -90,8 +94,8 @@ def get_winver(): return '0x0%s' % ((maj * 100) + min) if sys.getwindowsversion()[0] < 6: - msg = "Windows versions < Vista are no longer supported or maintained;" - msg = " latest supported version is psutil 3.4.2; " + msg = "warning: Windows versions < Vista are no longer supported or " + msg = "maintained; latest official supported version is psutil 3.4.2; " msg += "psutil may still be installed from sources if you have " msg += "Visual Studio and may also (kind of) work though" warnings.warn(msg, UserWarning) @@ -108,7 +112,7 @@ def get_winver(): ext = Extension( 'psutil._psutil_windows', - sources=[ + sources=sources + [ 'psutil/_psutil_windows.c', 'psutil/_psutil_common.c', 'psutil/arch/windows/process_info.c', @@ -131,9 +135,8 @@ def get_winver(): macros.append(("PSUTIL_OSX", 1)) ext = Extension( 'psutil._psutil_osx', - sources=[ + sources=sources + [ 'psutil/_psutil_osx.c', - 'psutil/_psutil_common.c', 'psutil/arch/osx/process_info.c', ], define_macros=macros, @@ -146,9 +149,8 @@ def get_winver(): macros.append(("PSUTIL_FREEBSD", 1)) ext = Extension( 'psutil._psutil_bsd', - sources=[ + sources=sources + [ 'psutil/_psutil_bsd.c', - 'psutil/_psutil_common.c', 'psutil/arch/bsd/freebsd.c', 'psutil/arch/bsd/freebsd_socks.c', ], @@ -160,9 +162,8 @@ def get_winver(): macros.append(("PSUTIL_OPENBSD", 1)) ext = Extension( 'psutil._psutil_bsd', - sources=[ + sources=sources + [ 'psutil/_psutil_bsd.c', - 'psutil/_psutil_common.c', 'psutil/arch/bsd/openbsd.c', ], define_macros=macros, @@ -173,9 +174,8 @@ def get_winver(): macros.append(("PSUTIL_NETBSD", 1)) ext = Extension( 'psutil._psutil_bsd', - sources=[ + sources=sources + [ 'psutil/_psutil_bsd.c', - 'psutil/_psutil_common.c', 'psutil/arch/bsd/netbsd.c', 'psutil/arch/bsd/netbsd_socks.c', ], @@ -215,7 +215,7 @@ def get_ethtool_macro(): macros.append(ETHTOOL_MACRO) ext = Extension( 'psutil._psutil_linux', - sources=['psutil/_psutil_linux.c'], + sources=sources + ['psutil/_psutil_linux.c'], define_macros=macros) # Solaris @@ -223,7 +223,7 @@ def get_ethtool_macro(): macros.append(("PSUTIL_SUNOS", 1)) ext = Extension( 'psutil._psutil_sunos', - sources=['psutil/_psutil_sunos.c'], + sources=sources + ['psutil/_psutil_sunos.c'], define_macros=macros, libraries=['kstat', 'nsl', 'socket']) @@ -235,7 +235,7 @@ def get_ethtool_macro(): posix_extension = Extension( 'psutil._psutil_posix', define_macros=macros, - sources=['psutil/_psutil_posix.c']) + sources=sources) if SUNOS: posix_extension.libraries.append('socket') if platform.release() == '5.10': From 89534a538f56353ece672e5dec209f29e1b5bbdf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 19:05:57 +0200 Subject: [PATCH 449/922] move psutil_pid_exists() and psutil_raise_for_pid() from _psutil_common.c to _psutil_posix.c --- psutil/_psutil_bsd.c | 1 + psutil/_psutil_common.c | 90 --------------------------- psutil/_psutil_common.h | 5 -- psutil/_psutil_linux.c | 2 + psutil/_psutil_osx.c | 1 + psutil/_psutil_posix.c | 84 +++++++++++++++++++++++++ psutil/_psutil_posix.h | 3 + psutil/_psutil_sunos.c | 2 + psutil/arch/bsd/freebsd.c | 2 +- psutil/arch/bsd/freebsd_socks.c | 2 +- psutil/arch/bsd/netbsd.c | 1 + psutil/arch/bsd/netbsd_socks.c | 3 + psutil/arch/bsd/openbsd.c | 2 +- psutil/arch/osx/process_info.c | 2 +- psutil/arch/windows/process_handles.c | 1 + psutil/arch/windows/services.c | 2 +- 16 files changed, 103 insertions(+), 100 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 2c7118d12..75de731de 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -60,6 +60,7 @@ #include #include "_psutil_common.h" +#include "_psutil_posix.h" #ifdef PSUTIL_FREEBSD #include "arch/bsd/freebsd.h" diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 5d025739f..c8d736e82 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -6,14 +6,8 @@ * Routines common to all platforms. */ -#ifdef PSUTIL_POSIX -#include -#include -#endif - #include - /* * Set OSError(errno=ESRCH, strerror="No such process") Python exception. */ @@ -40,87 +34,3 @@ AccessDenied(void) { Py_XDECREF(exc); return NULL; } - - -#ifdef PSUTIL_POSIX -/* - * Check if PID exists. Return values: - * 1: exists - * 0: does not exist - * -1: error (Python exception is set) - */ -int -psutil_pid_exists(long pid) { - int ret; - - // No negative PID exists, plus -1 is an alias for sending signal - // too all processes except system ones. Not what we want. - if (pid < 0) - return 0; - - // As per "man 2 kill" PID 0 is an alias for sending the signal to - // every process in the process group of the calling process. - // Not what we want. Some platforms have PID 0, some do not. - // We decide that at runtime. - if (pid == 0) { -#if defined(PSUTIL_LINUX) || defined(PSUTIL_FREEBSD) - return 0; -#else - return 1; -#endif - } - -#if defined(PSUTIL_OSX) - ret = kill((pid_t)pid , 0); -#else - ret = kill(pid , 0); -#endif - - if (ret == 0) - return 1; - else { - if (errno == ESRCH) { - // ESRCH == No such process - return 0; - } - else if (errno == EPERM) { - // EPERM clearly indicates there's a process to deny - // access to. - return 1; - } - else { - // According to "man 2 kill" possible error values are - // (EINVAL, EPERM, ESRCH) therefore we should never get - // here. If we do let's be explicit in considering this - // an error. - PyErr_SetFromErrno(PyExc_OSError); - return -1; - } - } -} - - -/* - * Utility used for those syscalls which do not return a meaningful - * error that we can translate into an exception which makes sense. - * As such, we'll have to guess. - * On UNIX, if errno is set, we return that one (OSError). - * Else, if PID does not exist we assume the syscall failed because - * of that so we raise NoSuchProcess. - * If none of this is true we giveup and raise RuntimeError(msg). - * This will always set a Python exception and return NULL. - */ -int -psutil_raise_for_pid(long pid, char *msg) { - // Set exception to AccessDenied if pid exists else NoSuchProcess. - if (errno != 0) { - PyErr_SetFromErrno(PyExc_OSError); - return 0; - } - if (psutil_pid_exists(pid) == 0) - NoSuchProcess(); - else - PyErr_SetString(PyExc_RuntimeError, msg); - return 0; -} -#endif diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 982c59c7c..43021a72d 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -8,8 +8,3 @@ PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); - -#ifdef PSUTIL_POSIX -int psutil_pid_exists(long pid); -void psutil_raise_for_pid(long pid, char *msg); -#endif diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 22fb49863..9abe44e09 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -54,6 +54,8 @@ static const int NCPUS_START = sizeof(unsigned long) * CHAR_BIT; #include #endif +#include "_psutil_common.h" +#include "_psutil_posix.h" // May happen on old RedHat versions, see: // https://github.com/giampaolo/psutil/issues/607 diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 559ffab9f..db1f997ab 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -40,6 +40,7 @@ #include #include "_psutil_common.h" +#include "_psutil_posix.h" #include "arch/osx/process_info.h" diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index 5823e61eb..80c1b8cba 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,89 @@ #include #endif +#include "_psutil_common.h" + +/* + * Check if PID exists. Return values: + * 1: exists + * 0: does not exist + * -1: error (Python exception is set) + */ +int +psutil_pid_exists(long pid) { + int ret; + + // No negative PID exists, plus -1 is an alias for sending signal + // too all processes except system ones. Not what we want. + if (pid < 0) + return 0; + + // As per "man 2 kill" PID 0 is an alias for sending the signal to + // every process in the process group of the calling process. + // Not what we want. Some platforms have PID 0, some do not. + // We decide that at runtime. + if (pid == 0) { +#if defined(PSUTIL_LINUX) || defined(PSUTIL_FREEBSD) + return 0; +#else + return 1; +#endif + } + +#if defined(PSUTIL_OSX) + ret = kill((pid_t)pid , 0); +#else + ret = kill(pid , 0); +#endif + + if (ret == 0) + return 1; + else { + if (errno == ESRCH) { + // ESRCH == No such process + return 0; + } + else if (errno == EPERM) { + // EPERM clearly indicates there's a process to deny + // access to. + return 1; + } + else { + // According to "man 2 kill" possible error values are + // (EINVAL, EPERM, ESRCH) therefore we should never get + // here. If we do let's be explicit in considering this + // an error. + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + } +} + + +/* + * Utility used for those syscalls which do not return a meaningful + * error that we can translate into an exception which makes sense. + * As such, we'll have to guess. + * On UNIX, if errno is set, we return that one (OSError). + * Else, if PID does not exist we assume the syscall failed because + * of that so we raise NoSuchProcess. + * If none of this is true we giveup and raise RuntimeError(msg). + * This will always set a Python exception and return NULL. + */ +int +psutil_raise_for_pid(long pid, char *msg) { + // Set exception to AccessDenied if pid exists else NoSuchProcess. + if (errno != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return 0; + } + if (psutil_pid_exists(pid) == 0) + NoSuchProcess(); + else + PyErr_SetString(PyExc_RuntimeError, msg); + return 0; +} + /* * Given a PID return process priority as a Python integer. diff --git a/psutil/_psutil_posix.h b/psutil/_psutil_posix.h index 86708f4b8..fe25b3669 100644 --- a/psutil/_psutil_posix.h +++ b/psutil/_psutil_posix.h @@ -3,3 +3,6 @@ * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. */ + +int psutil_pid_exists(long pid); +void psutil_raise_for_pid(long pid, char *msg); diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index e26c92d0f..3e2262ffb 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -42,6 +42,8 @@ #include #include +#include "_psutil_common.h" +#include "_psutil_posix.h" #define PSUTIL_TV2DOUBLE(t) (((t).tv_nsec * 0.000000001) + (t).tv_sec) #ifndef EXPER_IP_AND_ALL_IRES diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index 446042e2b..2188564f5 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -26,7 +26,7 @@ #include #include "../../_psutil_common.h" - +#include "../../_psutil_posix.h" #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) #define PSUTIL_BT2MSEC(bt) (bt.sec * 1000 + (((uint64_t) 1000000000 * (uint32_t) \ diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 187a93de0..7d216280d 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -26,7 +26,7 @@ #include #include "../../_psutil_common.h" - +#include "../../_psutil_posix.h" #define HASHSIZE 1009 // a signaler for connections without an actual status diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index d5c3e3b9e..361ab1f9b 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -42,6 +42,7 @@ #include "netbsd_socks.h" #include "netbsd.h" #include "../../_psutil_common.h" +#include "../../_psutil_posix.h" #define PSUTIL_KPT2DOUBLE(t) (t ## _sec + t ## _usec / 1000000.0) #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) diff --git a/psutil/arch/bsd/netbsd_socks.c b/psutil/arch/bsd/netbsd_socks.c index c782a4436..d05981d22 100644 --- a/psutil/arch/bsd/netbsd_socks.c +++ b/psutil/arch/bsd/netbsd_socks.c @@ -21,6 +21,9 @@ #include #include +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + // a signaler for connections without an actual status int PSUTIL_CONN_NONE = 128; diff --git a/psutil/arch/bsd/openbsd.c b/psutil/arch/bsd/openbsd.c index dfa8999bd..c86b003c1 100644 --- a/psutil/arch/bsd/openbsd.c +++ b/psutil/arch/bsd/openbsd.c @@ -35,8 +35,8 @@ #include // for inet_ntoa() #include // for warn() & err() - #include "../../_psutil_common.h" +#include "../../_psutil_posix.h" #define PSUTIL_KPT2DOUBLE(t) (t ## _sec + t ## _usec / 1000000.0) // #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/osx/process_info.c index 4b0b458ba..1c97b69f5 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -21,7 +21,7 @@ #include "process_info.h" #include "../../_psutil_common.h" - +#include "../../_psutil_posix.h" /* * Returns a list of all BSD processes on the system. This routine diff --git a/psutil/arch/windows/process_handles.c b/psutil/arch/windows/process_handles.c index 670d74f0b..356e23686 100644 --- a/psutil/arch/windows/process_handles.c +++ b/psutil/arch/windows/process_handles.c @@ -5,6 +5,7 @@ * */ #include "process_handles.h" +#include "../../_psutil_common.h" static _NtQuerySystemInformation __NtQuerySystemInformation = NULL; static _NtQueryObject __NtQueryObject = NULL; diff --git a/psutil/arch/windows/services.c b/psutil/arch/windows/services.c index 26e582255..85ea6ff42 100644 --- a/psutil/arch/windows/services.c +++ b/psutil/arch/windows/services.c @@ -9,7 +9,7 @@ #include #include "services.h" - +#include "../../_psutil_common.h" // ================================================================== // utils From a3e1b41238dc8382056a518a29b4c8984c3eed33 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 30 Apr 2017 21:05:22 +0200 Subject: [PATCH 450/922] update doc --- psutil/tests/test_unicode.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 12cbd01d6..4a293fd6a 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -9,29 +9,36 @@ Notes about unicode handling in psutil ====================================== -In psutil these are the APIs returning or dealing with a string: +In psutil these are the APIs returning or dealing with a string +('not tested' means they are not tested to deal with non-ASCII strings): - Process.cmdline() - Process.connections('unix') - Process.cwd() - Process.environ() - Process.exe() -- Process.memory_maps() (not tested) +- Process.memory_maps() (not tested) - Process.name() - Process.open_files() -- Process.username() (not tested) +- Process.username() (not tested) -- disk_io_counters() (not tested) -- disk_partitions() (not tested) +- disk_io_counters() (not tested) +- disk_partitions() (not tested) - disk_usage(str) - net_connections('unix') -- net_if_addrs() (not tested) -- net_if_stats() (not tested) -- net_io_counters() (not tested) -- sensors_fans() (not tested) -- sensors_temperatures() (not tested) -- users() (not tested) -- WindowsService (not tested) +- net_if_addrs() (not tested) +- net_if_stats() (not tested) +- net_io_counters() (not tested) +- sensors_fans() (not tested) +- sensors_temperatures() (not tested) +- users() (not tested) + +- WindowsService.binpath() (not tested) +- WindowsService.description() (not tested) +- WindowsService.display_name() (not tested) +- WindowsService.name() (not tested) +- WindowsService.status() (not tested) +- WindowsService.username() (not tested) In here we create a unicode path with a funky non-ASCII name and (where possible) make psutil return it back (e.g. on name(), exe(), From 8646be534fd5a2bb1653b3c05e05cf443c853249 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 00:04:45 +0200 Subject: [PATCH 451/922] #1040: provide an alias for PyUnicode_DecodeFSDefault which is not available on Python 2; also start to port the first C functions in FreeBSD --- psutil/_psutil_bsd.c | 13 ++----------- psutil/_psutil_common.c | 16 ++++++++++++++++ psutil/_psutil_common.h | 1 + psutil/arch/bsd/freebsd.c | 23 +++++------------------ psutil/arch/bsd/freebsd_socks.c | 6 +----- 5 files changed, 25 insertions(+), 34 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 75de731de..c158177f4 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -215,11 +215,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { #elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) sprintf(str, "%s", kp.p_comm); #endif -#if PY_MAJOR_VERSION >= 3 - py_name = PyUnicode_DecodeFSDefault(str); -#else - py_name = Py_BuildValue("s", str); -#endif + py_name = psutil_PyUnicode_DecodeFSDefault(str); if (! py_name) { // Likely a decoding error. We don't want to fail the whole // operation. The python module may retry with proc_name(). @@ -372,12 +368,7 @@ psutil_proc_name(PyObject *self, PyObject *args) { #elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) sprintf(str, "%s", kp.p_comm); #endif - -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_DecodeFSDefault(str); -#else - return Py_BuildValue("s", str); -#endif + return psutil_PyUnicode_DecodeFSDefault(str); } diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index c8d736e82..88b86202a 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -34,3 +34,19 @@ AccessDenied(void) { Py_XDECREF(exc); return NULL; } + + +/* + * Alias for PyUnicode_DecodeFSDefault which is not available + * on Python 2. On Python 2 we just return a plain byte string + * which is never supposed to raise decoding errors. + * See: https://github.com/giampaolo/psutil/issues/1040 + */ +PyObject * +psutil_PyUnicode_DecodeFSDefault(char *s) { +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_DecodeFSDefault(s); +#else + return Py_BuildValue("s", s); +#endif +} diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 43021a72d..31d93fbde 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -8,3 +8,4 @@ PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); +PyObject* psutil_PyUnicode_DecodeFSDefault(char *s); diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index 2188564f5..4e8ebdfac 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -238,11 +238,7 @@ psutil_get_cmdline(long pid) { // separator if (argsize > 0) { while (pos < argsize) { -#if PY_MAJOR_VERSION >= 3 - py_arg = PyUnicode_DecodeFSDefault(&argstr[pos]); -#else - py_arg = Py_BuildValue("s", &argstr[pos]); -#endif + py_arg = psutil_PyUnicode_DecodeFSDefault(&argstr[pos]); if (!py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) @@ -292,7 +288,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (error == -1) { // see: https://github.com/giampaolo/psutil/issues/907 if (errno == ENOENT) - return Py_BuildValue("s", ""); + return psutil_PyUnicode_DecodeFSDefault(""); else return PyErr_SetFromErrno(PyExc_OSError); } @@ -306,12 +302,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { strcpy(pathname, ""); } -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_DecodeFSDefault(pathname); -#else - return Py_BuildValue("s", pathname); -#endif - + return psutil_PyUnicode_DecodeFSDefault(pathname); } @@ -564,11 +555,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { for (i = 0; i < cnt; i++) { kif = &freep[i]; if (kif->kf_fd == KF_FD_TYPE_CWD) { -#if PY_MAJOR_VERSION >= 3 - py_path = PyUnicode_DecodeFSDefault(kif->kf_path); -#else - py_path = Py_BuildValue("s", kif->kf_path); -#endif + py_path = psutil_PyUnicode_DecodeFSDefault(kif->kf_path); if (!py_path) goto error; break; @@ -580,7 +567,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { * as root we return an empty string instead of AccessDenied. */ if (py_path == NULL) - py_path = Py_BuildValue("s", ""); + py_path = psutil_PyUnicode_DecodeFSDefault(""); free(freep); return py_path; diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 7d216280d..165acd74a 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -596,11 +596,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), sun->sun_path); -#if PY_MAJOR_VERSION >= 3 - py_laddr = PyUnicode_DecodeFSDefault(path); -#else - py_laddr = Py_BuildValue("s", path); -#endif + py_laddr = psutil_PyUnicode_DecodeFSDefault(path); if (! py_laddr) goto error; From cdc9b4fd9167e9cb2d97b83905b4896ecb99f49a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 00:14:58 +0200 Subject: [PATCH 452/922] #1040 / FreeBSD: fix net_connections('unix') which may raise decoding error on py3 --- psutil/arch/bsd/freebsd_socks.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 165acd74a..708ff893f 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -358,8 +358,7 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { char path[PATH_MAX]; PyObject *py_tuple = NULL; - PyObject *py_laddr = NULL; - PyObject *py_raddr = NULL; + PyObject *py_lpath = NULL; switch (proto) { case SOCK_STREAM: @@ -418,13 +417,23 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { snprintf(path, sizeof(path), "%.*s", (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), sun->sun_path); + py_lpath = psutil_PyUnicode_DecodeFSDefault(path); + if (! py_lpath) + goto error; - py_tuple = Py_BuildValue("(iiisOii)", -1, AF_UNIX, proto, path, - Py_None, PSUTIL_CONN_NONE, pid); + py_tuple = Py_BuildValue("(iiiOOii)", + -1, + AF_UNIX, + proto, + py_lpath, + Py_None, + PSUTIL_CONN_NONE, + pid); if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_lpath); Py_DECREF(py_tuple); Py_INCREF(Py_None); } @@ -434,8 +443,7 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { error: Py_XDECREF(py_tuple); - Py_XDECREF(py_laddr); - Py_XDECREF(py_raddr); + Py_XDECREF(py_lpath); free(buf); return 0; } From c2914a2bff7ec2ffff746b92873aff85ecd36b98 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 00:27:18 +0200 Subject: [PATCH 453/922] socket utils: make sure to close fd in case of err --- psutil/tests/__init__.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 4a97dba8c..c9fc7cdb6 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -803,11 +803,15 @@ def unix_socket_path(suffix=""): def bind_socket(addr, family, type): """Binds a generic socket.""" sock = socket.socket(family, type) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind(addr) - if type == socket.SOCK_STREAM: - sock.listen(10) - return sock + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(addr) + if type == socket.SOCK_STREAM: + sock.listen(10) + return sock + except Exception: + sock.close() + raise def bind_unix_socket(name, type=socket.SOCK_STREAM): @@ -817,11 +821,11 @@ def bind_unix_socket(name, type=socket.SOCK_STREAM): sock = socket.socket(socket.AF_UNIX, type) try: sock.bind(name) + if type == socket.SOCK_STREAM: + sock.listen(10) except Exception: sock.close() raise - if type == socket.SOCK_STREAM: - sock.listen(10) return sock @@ -854,12 +858,20 @@ def unix_socketpair(name): Return a (server, client) tuple. """ assert psutil.POSIX - server = bind_unix_socket(name, type=socket.SOCK_STREAM) - server.setblocking(0) - client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - client.setblocking(0) - client.connect(name) - # new = server.accept() + server = client = None + try: + server = bind_unix_socket(name, type=socket.SOCK_STREAM) + server.setblocking(0) + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.setblocking(0) + client.connect(name) + # new = server.accept() + except Exception: + if server is not None: + server.close() + if client is not None: + client.close() + raise return (server, client) From f90e6c6aa8efefb8a909d8d32a4b006d1f6c4623 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 00:39:00 +0200 Subject: [PATCH 454/922] bind_socket() change signature --- psutil/tests/__init__.py | 4 +++- psutil/tests/test_connections.py | 8 ++++---- psutil/tests/test_memory_leaks.py | 17 ++++++----------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index c9fc7cdb6..081cf28b8 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -800,8 +800,10 @@ def unix_socket_path(suffix=""): pass -def bind_socket(addr, family, type): +def bind_socket(family=AF_INET, type=SOCK_STREAM, addr=None): """Binds a generic socket.""" + if addr is None and family in (AF_INET, AF_INET6): + addr = ("", 0) sock = socket.socket(family, type) try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 8538fa511..2df26d10b 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -126,28 +126,28 @@ class TestUnconnectedSockets(Base, unittest.TestCase): def test_tcp_v4(self): addr = ("127.0.0.1", get_free_port()) - with closing(bind_socket(addr, AF_INET, SOCK_STREAM)) as sock: + with closing(bind_socket(AF_INET, SOCK_STREAM, addr=addr)) as sock: conn = self.check_socket(sock) assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_LISTEN) def test_tcp_v6(self): addr = ("::1", get_free_port()) - with closing(bind_socket(addr, AF_INET6, SOCK_STREAM)) as sock: + with closing(bind_socket(AF_INET6, SOCK_STREAM, addr=addr)) as sock: conn = self.check_socket(sock) assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_LISTEN) def test_udp_v4(self): addr = ("127.0.0.1", get_free_port()) - with closing(bind_socket(addr, AF_INET, SOCK_DGRAM)) as sock: + with closing(bind_socket(AF_INET, SOCK_DGRAM, addr=addr)) as sock: conn = self.check_socket(sock) assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_NONE) def test_udp_v6(self): addr = ("127.0.0.1", get_free_port()) - with closing(bind_socket(addr, AF_INET, SOCK_DGRAM)) as sock: + with closing(bind_socket(AF_INET, SOCK_DGRAM, addr=addr)) as sock: conn = self.check_socket(sock) assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_NONE) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 6e0992306..19b0923ba 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -31,6 +31,7 @@ from psutil import WINDOWS from psutil._common import supports_ipv6 from psutil._compat import xrange +from psutil.tests import bind_socket from psutil.tests import bind_unix_socket from psutil.tests import get_test_subprocess from psutil.tests import HAS_CPU_AFFINITY @@ -60,6 +61,7 @@ SKIP_PYTHON_IMPL = True if TRAVIS else False cext = psutil._psplatform.cext thisproc = psutil.Process() +SKIP_PYTHON_IMPL = True if TRAVIS else False # =================================================================== @@ -360,21 +362,14 @@ def test_rlimit_set(self): # function (tested later). @unittest.skipIf(WINDOWS, "worthless on WINDOWS") def test_connections(self): - def create_socket(family, type): - sock = socket.socket(family, type) - sock.bind(('', 0)) - if type == socket.SOCK_STREAM: - sock.listen(1) - return sock - # Open as many socket types as possible so that we excercise # as many C code sections as possible. socks = [] - socks.append(create_socket(socket.AF_INET, socket.SOCK_STREAM)) - socks.append(create_socket(socket.AF_INET, socket.SOCK_DGRAM)) + socks.append(bind_socket(socket.AF_INET, socket.SOCK_STREAM)) + socks.append(bind_socket(socket.AF_INET, socket.SOCK_DGRAM)) if supports_ipv6(): - socks.append(create_socket(socket.AF_INET6, socket.SOCK_STREAM)) - socks.append(create_socket(socket.AF_INET6, socket.SOCK_DGRAM)) + socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM)) + socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM)) if POSIX and not SUNOS: # TODO: SunOS name1 = unix_socket_path().__enter__() name2 = unix_socket_path().__enter__() From b37ddf7ca9e0e9650a2fee9bcfa1e6425207287f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 00:45:30 +0200 Subject: [PATCH 455/922] memleaks tests: create test sockets also for net_connections() so that we excercise as many C code sections as possible --- psutil/tests/test_memory_leaks.py | 47 +++++++++++++++++++------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 19b0923ba..a18f31428 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -181,6 +181,33 @@ def _get_mem(): def _call(fun, *args, **kwargs): fun(*args, **kwargs) + def create_sockets(self): + """Open as many socket families / types as possible. + This is needed to excercise as many C code sections as + possible for net_connections() functions. + The returned sockets are already scheduled for cleanup. + """ + socks = [] + try: + socks.append(bind_socket(socket.AF_INET, socket.SOCK_STREAM)) + socks.append(bind_socket(socket.AF_INET, socket.SOCK_DGRAM)) + if supports_ipv6(): + socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM)) + socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM)) + if POSIX and not SUNOS: # TODO: SunOS + name1 = unix_socket_path().__enter__() + name2 = unix_socket_path().__enter__() + s1, s2 = unix_socketpair(name1) + s3 = bind_unix_socket(name2, type=socket.SOCK_DGRAM) + self.addCleanup(safe_rmpath, name1) + self.addCleanup(safe_rmpath, name2) + for s in (s1, s2, s3): + socks.append(s) + return socks + finally: + for s in socks: + self.addCleanup(s.close) + # =================================================================== # Process class @@ -362,24 +389,7 @@ def test_rlimit_set(self): # function (tested later). @unittest.skipIf(WINDOWS, "worthless on WINDOWS") def test_connections(self): - # Open as many socket types as possible so that we excercise - # as many C code sections as possible. - socks = [] - socks.append(bind_socket(socket.AF_INET, socket.SOCK_STREAM)) - socks.append(bind_socket(socket.AF_INET, socket.SOCK_DGRAM)) - if supports_ipv6(): - socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM)) - socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM)) - if POSIX and not SUNOS: # TODO: SunOS - name1 = unix_socket_path().__enter__() - name2 = unix_socket_path().__enter__() - s1, s2 = unix_socketpair(name1) - s3 = bind_unix_socket(name2, type=socket.SOCK_DGRAM) - self.addCleanup(safe_rmpath, name1) - self.addCleanup(safe_rmpath, name2) - for s in (s1, s2, s3): - socks.append(s) - + socks = self.create_sockets() # TODO: UNIX sockets are temporarily implemented by parsing # 'pfiles' cmd output; we don't want that part of the code to # be executed. @@ -550,6 +560,7 @@ def test_net_io_counters(self): "worthless on Linux (pure python)") @unittest.skipIf(OSX and os.getuid() != 0, "need root access") def test_net_connections(self): + self.create_sockets() self.execute(psutil.net_connections) def test_net_if_addrs(self): From a85dfa78ad7d93cfb8bce31124c1e8ce96b8c082 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 01:56:09 +0200 Subject: [PATCH 456/922] #1041: add test for memory_maps() which loads a .so lib and makes sure it gets listed --- psutil/tests/__init__.py | 22 +++++++++++++++++++++- psutil/tests/test_process.py | 12 +++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 081cf28b8..2a45e8c8f 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -9,8 +9,10 @@ """ from __future__ import print_function + import atexit import contextlib +import ctypes import errno import functools import os @@ -74,6 +76,10 @@ 'PYPY', 'PYTHON', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFILE_PREFIX', 'TESTFN', 'TESTFN_UNICODE', 'TOX', 'TRAVIS', 'VALID_PROC_STATUSES', 'VERBOSITY', + "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", + "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", + "HAS_SENSORS_BATTERY", "HAS_BATTERY""HAS_SENSORS_FANS", + "HAS_SENSORS_TEMPERATURES", # classes 'ThreadTask' # test utils @@ -92,8 +98,10 @@ 'call_until', 'wait_for_pid', 'wait_for_file', # network 'check_connection_ntuple', 'check_net_address', + 'get_free_port', 'unix_socket_path', 'bind_socket', 'bind_unix_socket', + 'tcp_socketpair', 'unix_socketpair', # others - 'warn', + 'warn', 'copyload_shared_lib', 'is_namedtuple', ] @@ -995,3 +1003,15 @@ def is_namedtuple(x): if not isinstance(f, tuple): return False return all(type(n) == str for n in f) + + +def copyload_shared_lib(src, dst_prefix=TESTFILE_PREFIX): + """Given an existing shared so / DLL library copies it in + another location and loads it via ctypes. + Return the new path. + """ + newpath = tempfile.mktemp(prefix=dst_prefix, + suffix=os.path.splitext(src)[1]) + shutil.copyfile(src, newpath) + ctypes.CDLL(newpath) + return newpath diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 06b7de129..2133b0642 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -34,6 +34,7 @@ from psutil._compat import PY3 from psutil.tests import APPVEYOR from psutil.tests import call_until +from psutil.tests import copyload_shared_lib from psutil.tests import create_exe from psutil.tests import create_proc_children_pair from psutil.tests import enum @@ -43,6 +44,7 @@ from psutil.tests import HAS_CPU_AFFINITY from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_IONICE +from psutil.tests import HAS_MEMORY_MAPS from psutil.tests import HAS_PROC_CPU_NUM from psutil.tests import HAS_PROC_IO_COUNTERS from psutil.tests import HAS_RLIMIT @@ -611,7 +613,7 @@ def test_memory_full_info(self): self.assertGreaterEqual(mem.pss, 0) self.assertGreaterEqual(mem.swap, 0) - @unittest.skipIf(OPENBSD or NETBSD, "not supported") + @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") def test_memory_maps(self): p = psutil.Process() maps = p.memory_maps() @@ -652,6 +654,14 @@ def test_memory_maps(self): self.assertIsInstance(value, (int, long)) assert value >= 0, value + @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") + def test_memory_maps_lists_lib(self): + p = psutil.Process() + path = [x.path for x in p.memory_maps() if x.path.endswith(".so")][0] + newpath = copyload_shared_lib(path) + self.addCleanup(safe_rmpath, newpath) + assert any([x.path for x in p.memory_maps() if x.path == newpath]) + def test_memory_percent(self): p = psutil.Process() ret = p.memory_percent() From 17cb9c54cd7b8414634d38480fd61a10e62f4cde Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 02:13:42 +0200 Subject: [PATCH 457/922] #1014: make test work on Windows (use normcase) --- psutil/tests/test_process.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 2133b0642..351436146 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -657,10 +657,11 @@ def test_memory_maps(self): @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") def test_memory_maps_lists_lib(self): p = psutil.Process() - path = [x.path for x in p.memory_maps() if x.path.endswith(".so")][0] - newpath = copyload_shared_lib(path) - self.addCleanup(safe_rmpath, newpath) - assert any([x.path for x in p.memory_maps() if x.path == newpath]) + ext = ".so" if POSIX else ".dll" + old = [x.path for x in p.memory_maps() if x.path.endswith(ext)][0] + new = os.path.normcase(copyload_shared_lib(old)) + newpaths = [os.path.normcase(x.path) for x in p.memory_maps()] + self.assertIn(new, newpaths) def test_memory_percent(self): p = psutil.Process() From a3c1bc23c76359dd0093c0c8d0b1e6071b2fc0d7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 03:09:07 +0200 Subject: [PATCH 458/922] #1040: add funky unicode test for memory_maps() --- psutil/tests/test_process.py | 3 ++- psutil/tests/test_unicode.py | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 351436146..ec9c71754 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -658,7 +658,8 @@ def test_memory_maps(self): def test_memory_maps_lists_lib(self): p = psutil.Process() ext = ".so" if POSIX else ".dll" - old = [x.path for x in p.memory_maps() if x.path.endswith(ext)][0] + old = [x.path for x in p.memory_maps() + if os.path.normcase(x.path).endswith(ext)][0] new = os.path.normcase(copyload_shared_lib(old)) newpaths = [os.path.normcase(x.path) for x in p.memory_maps()] self.assertIn(new, newpaths) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 4a293fd6a..ea2367f50 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -17,7 +17,7 @@ - Process.cwd() - Process.environ() - Process.exe() -- Process.memory_maps() (not tested) +- Process.memory_maps() - Process.name() - Process.open_files() - Process.username() (not tested) @@ -72,9 +72,11 @@ from psutil.tests import ASCII_FS from psutil.tests import bind_unix_socket from psutil.tests import chdir +from psutil.tests import copyload_shared_lib from psutil.tests import create_exe from psutil.tests import get_test_subprocess from psutil.tests import HAS_ENVIRON +from psutil.tests import HAS_MEMORY_MAPS from psutil.tests import reap_children from psutil.tests import run_test_module_by_name from psutil.tests import safe_mkdir @@ -235,6 +237,17 @@ def test_disk_usage(self): safe_mkdir(self.funky_name) psutil.disk_usage(self.funky_name) + @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") + def test_memory_maps(self): + p = psutil.Process() + ext = ".so" if POSIX else ".dll" + old = [x.path for x in p.memory_maps() + if os.path.normcase(x.path).endswith(ext)][0] + new = os.path.normcase( + copyload_shared_lib(old, dst_prefix=self.funky_name)) + newpaths = [os.path.normcase(x.path) for x in p.memory_maps()] + self.assertIn(new, newpaths) + @unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # TODO @unittest.skipIf(ASCII_FS, "ASCII fs") From 2455af767dd571d11752ecb3a3b00a70136178d4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 03:19:02 +0200 Subject: [PATCH 459/922] skip funky unicode test if ctypes can't handle unicode --- psutil/tests/test_unicode.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index ea2367f50..fd0ab3c70 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -243,8 +243,14 @@ def test_memory_maps(self): ext = ".so" if POSIX else ".dll" old = [x.path for x in p.memory_maps() if os.path.normcase(x.path).endswith(ext)][0] - new = os.path.normcase( - copyload_shared_lib(old, dst_prefix=self.funky_name)) + try: + new = os.path.normcase( + copyload_shared_lib(old, dst_prefix=self.funky_name)) + except UnicodeEncodeError: + if PY3: + raise + else: + raise unittest.SkipTest("ctypes can't handle unicode") newpaths = [os.path.normcase(x.path) for x in p.memory_maps()] self.assertIn(new, newpaths) From 3685cfd03d478ce2fa4a4457760056b37794fb01 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 01:40:07 +0000 Subject: [PATCH 460/922] #1040 / unicode / freebsd: fix decode handling for memory_maps() and open_files() --- psutil/_psutil_bsd.c | 7 ++++++- psutil/arch/bsd/freebsd.c | 12 +++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index c158177f4..94a1770aa 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -463,6 +463,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { struct kinfo_file *kif; kinfo_proc kipp; PyObject *py_tuple = NULL; + PyObject *py_path = NULL; PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) @@ -498,12 +499,16 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { // XXX - it appears path is not exposed in the kinfo_file struct. path = ""; #endif + py_path = psutil_PyUnicode_DecodeFSDefault(path); + if (! py_path) + goto error; if (regular == 1) { - py_tuple = Py_BuildValue("(si)", path, fd); + py_tuple = Py_BuildValue("(Oi)", py_path, fd); if (py_tuple == NULL) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_path); Py_DECREF(py_tuple); } } diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index 4e8ebdfac..9a1f952c4 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -746,12 +746,13 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { int i, cnt; char addr[1000]; char perms[4]; - const char *path; + char *path; struct kinfo_proc kp; struct kinfo_vmentry *freep = NULL; struct kinfo_vmentry *kve; ptrwidth = 2 * sizeof(void *); PyObject *py_tuple = NULL; + PyObject *py_path = NULL; PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) @@ -820,10 +821,13 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { path = kve->kve_path; } - py_tuple = Py_BuildValue("sssiiii", + py_path = psutil_PyUnicode_DecodeFSDefault(path); + if (! py_path) + goto error; + py_tuple = Py_BuildValue("ssOiiii", addr, // "start-end" address perms, // "rwx" permissions - path, // path + py_path, // path kve->kve_resident, // rss kve->kve_private_resident, // private kve->kve_ref_count, // ref count @@ -832,6 +836,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_path); Py_DECREF(py_tuple); } free(freep); @@ -839,6 +844,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { error: Py_XDECREF(py_tuple); + Py_XDECREF(py_path); Py_DECREF(py_retlist); if (freep != NULL) free(freep); From 0fab2e5acc72bac522467a36aa7a2d83f659600d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 01:50:25 +0000 Subject: [PATCH 461/922] #1040 / unicode / bsd: fix unicode handling for disk_partitions() --- psutil/_psutil_bsd.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 94a1770aa..71a323873 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -542,6 +542,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { struct statfs *fs = NULL; #endif PyObject *py_retlist = PyList_New(0); + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; PyObject *py_tuple = NULL; if (py_retlist == NULL) @@ -654,15 +656,23 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { if (flags & MNT_NODEVMTIME) strlcat(opts, ",nodevmtime", sizeof(opts)); #endif - py_tuple = Py_BuildValue("(ssss)", - fs[i].f_mntfromname, // device - fs[i].f_mntonname, // mount point + py_dev = psutil_PyUnicode_DecodeFSDefault(fs[i].f_mntfromname); + if (! py_dev) + goto error; + py_mountp = psutil_PyUnicode_DecodeFSDefault(fs[i].f_mntonname); + if (! py_mountp) + goto error; + py_tuple = Py_BuildValue("(OOss)", + py_dev, // device + py_mountp, // mount point fs[i].f_fstypename, // fs type opts); // options if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_dev); + Py_DECREF(py_mountp); Py_DECREF(py_tuple); } @@ -670,6 +680,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { return py_retlist; error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); if (fs != NULL) From b19ddf8963399eb814e4d21564b71eacad716de2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 02:26:41 +0000 Subject: [PATCH 462/922] #1040 / unicode / bsd: fix unicode handling for users() --- psutil/_psutil_bsd.c | 49 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 71a323873..a3c3d56a8 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -791,6 +791,9 @@ psutil_net_io_counters(PyObject *self, PyObject *args) { static PyObject * psutil_users(PyObject *self, PyObject *args) { PyObject *py_retlist = PyList_New(0); + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; PyObject *py_tuple = NULL; if (py_retlist == NULL) @@ -809,12 +812,21 @@ psutil_users(PyObject *self, PyObject *args) { while (fread(&ut, sizeof(ut), 1, fp) == 1) { if (*ut.ut_name == '\0') continue; + py_username = psutil_PyUnicode_DecodeFSDefault(ut.ut_name); + if (! py_username) + goto error; + py_tty = psutil_PyUnicode_DecodeFSDefault(ut.ut_line); + if (! py_tty) + goto error; + py_hostname = psutil_PyUnicode_DecodeFSDefault(ut.ut_host); + if (! py_hostname) + goto error; py_tuple = Py_BuildValue( - "(sssfi)", - ut.ut_name, // username - ut.ut_line, // tty - ut.ut_host, // hostname - (float)ut.ut_time, // start time + "(OOOfi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname + (float)ut.ut_time, // start time ut.ut_pid // process id ); if (!py_tuple) { @@ -825,22 +837,33 @@ psutil_users(PyObject *self, PyObject *args) { fclose(fp); goto error; } + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); Py_DECREF(py_tuple); } fclose(fp); #else struct utmpx *utx; - setutxent(); while ((utx = getutxent()) != NULL) { if (utx->ut_type != USER_PROCESS) continue; + py_username = psutil_PyUnicode_DecodeFSDefault(utx->ut_user); + if (! py_username) + goto error; + py_tty = psutil_PyUnicode_DecodeFSDefault(utx->ut_line); + if (! py_tty) + goto error; + py_hostname = psutil_PyUnicode_DecodeFSDefault(utx->ut_host); + if (! py_hostname) + goto error; py_tuple = Py_BuildValue( - "(sssfi)", - utx->ut_user, // username - utx->ut_line, // tty - utx->ut_host, // hostname + "(OOOfi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname (float)utx->ut_tv.tv_sec, // start time utx->ut_pid // process id ); @@ -853,6 +876,9 @@ psutil_users(PyObject *self, PyObject *args) { endutxent(); goto error; } + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); Py_DECREF(py_tuple); } @@ -861,6 +887,9 @@ psutil_users(PyObject *self, PyObject *args) { return py_retlist; error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); return NULL; From 56f88f486a09d95e5152d9d2ed789d706d2f9a75 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 05:34:42 +0200 Subject: [PATCH 463/922] netbsd opens a UNIX socket to /var/log/run which breaks our connection tesrs; work around that --- psutil/tests/test_connections.py | 33 +++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 2df26d10b..05c39b36d 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -17,6 +17,7 @@ import psutil from psutil import FREEBSD +from psutil import NETBSD from psutil import OSX from psutil import POSIX from psutil import SUNOS @@ -71,26 +72,39 @@ def compare_procsys_connections(pid, proc_cons, kind='all'): class Base(object): def setUp(self): - cons = psutil.Process().connections(kind='all') - assert not cons, cons + if not NETBSD: + # NetBSD opens a UNIX socket to /var/log/run. + cons = psutil.Process().connections(kind='all') + assert not cons, cons def tearDown(self): safe_rmpath(TESTFN) reap_children() - # make sure we closed all resources + if not NETBSD: + # Make sure we closed all resources. + # NetBSD opens a UNIX socket to /var/log/run. + cons = psutil.Process().connections(kind='all') + assert not cons, cons + + def get_conn_from_socck(self, sock): cons = psutil.Process().connections(kind='all') - assert not cons, cons + smap = dict([(c.fd, c) for c in cons]) + if psutil.NETBSD: + # NetBSD opens a UNIX socket to /var/log/run + # so there may be more connections. + return smap[sock.fileno()] + else: + self.assertEqual(smap[sock.fileno()].fd, sock.fileno()) + self.assertEqual(len(cons), 1) + return cons[0] def check_socket(self, sock, conn=None): """Given a socket, makes sure it matches the one obtained via psutil. It assumes this process created one connection only (the one supposed to be checked). """ - cons = psutil.Process().connections(kind='all') - if not conn: - self.assertEqual(len(cons), 1) - conn = cons[0] - + if conn is None: + conn = self.get_conn_from_socck(sock) check_connection_ntuple(conn) # fd, family, type @@ -112,6 +126,7 @@ def check_socket(self, sock, conn=None): # XXX Solaris can't retrieve system-wide UNIX sockets if not (SUNOS and sock.family == AF_UNIX): + cons = psutil.Process().connections(kind='all') compare_procsys_connections(os.getpid(), cons) return conn From fccaa29d9efa9cd184ed4ab8b25830df1af0f696 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 05:35:50 +0200 Subject: [PATCH 464/922] refactoring --- psutil/tests/test_connections.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 05c39b36d..0462fb6ce 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -43,6 +43,9 @@ from psutil.tests import wait_for_file +thisproc = psutil.Process() + + def compare_procsys_connections(pid, proc_cons, kind='all'): """Given a process PID and its list of connections compare those against system-wide connections retrieved via @@ -74,7 +77,7 @@ class Base(object): def setUp(self): if not NETBSD: # NetBSD opens a UNIX socket to /var/log/run. - cons = psutil.Process().connections(kind='all') + cons = thisproc.connections(kind='all') assert not cons, cons def tearDown(self): @@ -83,11 +86,11 @@ def tearDown(self): if not NETBSD: # Make sure we closed all resources. # NetBSD opens a UNIX socket to /var/log/run. - cons = psutil.Process().connections(kind='all') + cons = thisproc.connections(kind='all') assert not cons, cons def get_conn_from_socck(self, sock): - cons = psutil.Process().connections(kind='all') + cons = thisproc.connections(kind='all') smap = dict([(c.fd, c) for c in cons]) if psutil.NETBSD: # NetBSD opens a UNIX socket to /var/log/run @@ -126,7 +129,7 @@ def check_socket(self, sock, conn=None): # XXX Solaris can't retrieve system-wide UNIX sockets if not (SUNOS and sock.family == AF_UNIX): - cons = psutil.Process().connections(kind='all') + cons = thisproc.connections(kind='all') compare_procsys_connections(os.getpid(), cons) return conn @@ -220,7 +223,7 @@ def test_tcp(self): addr = ("127.0.0.1", get_free_port()) server, client = tcp_socketpair(AF_INET, addr=addr) with nested(closing(server), closing(client)): - cons = psutil.Process().connections(kind='all') + cons = thisproc.connections(kind='all') server_conn, client_conn = self.distinguish_tcp_socks(cons, addr) self.check_socket(server, conn=server_conn) self.check_socket(client, conn=client_conn) @@ -229,7 +232,7 @@ def test_tcp(self): # May not be fast enough to change state so it stays # commenteed. # client.close() - # cons = psutil.Process().connections(kind='all') + # cons = thisproc.connections(kind='all') # self.assertEqual(len(cons), 1) # self.assertEqual(cons[0].status, psutil.CONN_CLOSE_WAIT) @@ -238,7 +241,7 @@ def test_unix(self): with unix_socket_path() as name: server, client = unix_socketpair(name) with nested(closing(server), closing(client)): - cons = psutil.Process().connections(kind='unix') + cons = thisproc.connections(kind='unix') self.assertEqual(len(cons), 2) server_conn, client_conn = self.distinguish_unix_socks(cons) self.check_socket(server, conn=server_conn) @@ -320,7 +323,7 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): tcp6_addr = None udp6_addr = None - for p in psutil.Process().children(): + for p in thisproc.children(): cons = p.connections() self.assertEqual(len(cons), 1) for conn in cons: From 0bcf3108f2fec248e5edeeeb0f19dc2f7281b756 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 05:40:47 +0200 Subject: [PATCH 465/922] refactoring --- psutil/tests/test_connections.py | 53 +++++++++++++++----------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 0462fb6ce..0d58a90f6 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -22,6 +22,7 @@ from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS +from psutil._common import pconn from psutil._common import supports_ipv6 from psutil._compat import nested from psutil._compat import PY3 @@ -46,32 +47,6 @@ thisproc = psutil.Process() -def compare_procsys_connections(pid, proc_cons, kind='all'): - """Given a process PID and its list of connections compare - those against system-wide connections retrieved via - psutil.net_connections. - """ - from psutil._common import pconn - try: - sys_cons = psutil.net_connections(kind=kind) - except psutil.AccessDenied: - # On OSX, system-wide connections are retrieved by iterating - # over all processes - if OSX: - return - else: - raise - # exclude PIDs from syscons - sys_cons = [c[:-1] for c in sys_cons if c.pid == pid] - if FREEBSD: - # On FreeBSD all fds are set to -1 so exclude them - # for comparison. - proc_cons = [pconn(*[-1] + list(x[1:])) for x in proc_cons] - proc_cons.sort() - sys_cons.sort() - assert proc_cons == sys_cons, (proc_cons, sys_cons) - - class Base(object): def setUp(self): @@ -130,9 +105,31 @@ def check_socket(self, sock, conn=None): # XXX Solaris can't retrieve system-wide UNIX sockets if not (SUNOS and sock.family == AF_UNIX): cons = thisproc.connections(kind='all') - compare_procsys_connections(os.getpid(), cons) + self.compare_procsys_connections(os.getpid(), cons) return conn + def compare_procsys_connections(self, pid, proc_cons, kind='all'): + """Given a process PID and its list of connections compare + those against system-wide connections retrieved via + psutil.net_connections. + """ + try: + sys_cons = psutil.net_connections(kind=kind) + except psutil.AccessDenied: + # On OSX, system-wide connections are retrieved by iterating + # over all processes + if OSX: + return + else: + raise + # exclude PIDs from syscons + sys_cons = [c[:-1] for c in sys_cons if c.pid == pid] + if FREEBSD: + # On FreeBSD all fds are set to -1 so exclude them + # for comparison. + proc_cons = [pconn(*[-1] + list(x[1:])) for x in proc_cons] + self.assertEqual(sorted(proc_cons), sorted(sys_cons)) + # ===================================================================== # --- Test unconnected sockets @@ -274,7 +271,7 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): # XXX Solaris can't retrieve system-wide UNIX # sockets. if not SUNOS: - compare_procsys_connections(proc.pid, [conn]) + self.compare_procsys_connections(proc.pid, [conn]) tcp_template = textwrap.dedent(""" import socket, time From f0666e5d0b4e8ba26398b98aeb441b6f565153a0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 05:53:32 +0200 Subject: [PATCH 466/922] refactoring --- psutil/tests/test_connections.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 0d58a90f6..202a0b170 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -122,13 +122,15 @@ def compare_procsys_connections(self, pid, proc_cons, kind='all'): return else: raise - # exclude PIDs from syscons + # Filter for this proc PID and exlucde PIDs from the tuple. sys_cons = [c[:-1] for c in sys_cons if c.pid == pid] if FREEBSD: # On FreeBSD all fds are set to -1 so exclude them - # for comparison. + # from comparison. proc_cons = [pconn(*[-1] + list(x[1:])) for x in proc_cons] - self.assertEqual(sorted(proc_cons), sorted(sys_cons)) + sys_cons.sort() + proc_cons.sort() + self.assertEqual(proc_cons, sys_cons) # ===================================================================== From b54f7b34b516c782348acde85a443ef9c696b530 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 06:19:23 +0200 Subject: [PATCH 467/922] refactoring --- psutil/tests/test_connections.py | 66 +++++++++++++------------------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 202a0b170..3555aed0d 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -17,6 +17,7 @@ import psutil from psutil import FREEBSD +from psutil import LINUX from psutil import NETBSD from psutil import OSX from psutil import POSIX @@ -67,7 +68,7 @@ def tearDown(self): def get_conn_from_socck(self, sock): cons = thisproc.connections(kind='all') smap = dict([(c.fd, c) for c in cons]) - if psutil.NETBSD: + if NETBSD: # NetBSD opens a UNIX socket to /var/log/run # so there may be more connections. return smap[sock.fileno()] @@ -196,38 +197,15 @@ class TestConnectedSocketPairs(Base, unittest.TestCase): each other. """ - @staticmethod - def distinguish_tcp_socks(cons, server_addr): - """Given a list of connections return a (server, client) - connection ntuple. - """ - if cons[0].laddr == server_addr: - return (cons[0], cons[1]) - else: - assert cons[1].laddr == server_addr, (cons, server_addr) - return (cons[1], cons[0]) - - @staticmethod - def distinguish_unix_socks(cons): - """Given a list of connections and 2 sockets return a - (server, client) connection ntuple. - """ - if cons[0].laddr: - return (cons[0], cons[1]) - else: - assert cons[1].laddr, cons - return (cons[1], cons[0]) - def test_tcp(self): addr = ("127.0.0.1", get_free_port()) + assert not thisproc.connections(kind='tcp4') server, client = tcp_socketpair(AF_INET, addr=addr) with nested(closing(server), closing(client)): - cons = thisproc.connections(kind='all') - server_conn, client_conn = self.distinguish_tcp_socks(cons, addr) - self.check_socket(server, conn=server_conn) - self.check_socket(client, conn=client_conn) - self.assertEqual(server_conn.status, psutil.CONN_ESTABLISHED) - self.assertEqual(client_conn.status, psutil.CONN_ESTABLISHED) + cons = thisproc.connections(kind='tcp4') + self.assertEqual(len(cons), 2) + self.assertEqual(cons[0].status, psutil.CONN_ESTABLISHED) + self.assertEqual(cons[1].status, psutil.CONN_ESTABLISHED) # May not be fast enough to change state so it stays # commenteed. # client.close() @@ -241,16 +219,24 @@ def test_unix(self): server, client = unix_socketpair(name) with nested(closing(server), closing(client)): cons = thisproc.connections(kind='unix') + if NETBSD: + # On NetBSD creating a UNIX socket will cause + # a UNIX connection to /var/run/log. + cons = [c for c in cons if c.raddr != '/var/run/log'] self.assertEqual(len(cons), 2) - server_conn, client_conn = self.distinguish_unix_socks(cons) - self.check_socket(server, conn=server_conn) - - self.check_socket(client, conn=client_conn) - self.assertEqual(server_conn.laddr, name) - # TODO: https://github.com/giampaolo/psutil/issues/1035 - self.assertIn(server_conn.raddr, ("", None)) - # TODO: https://github.com/giampaolo/psutil/issues/1035 - self.assertIn(client_conn.laddr, ("", None)) + if LINUX or FREEBSD: + # On linux the remote path is never set. Test + # that at least ONE address has our path. + one = (cons[0].laddr or cons[0].raddr or + cons[1].laddr or cons[1].raddr) + self.assertEqual(one, name) + else: + # On other systems either the laddr or raddr + # of both peers are set. + self.assertEqual(cons[0].laddr or cons[1].laddr, name) + self.assertEqual(cons[0].raddr or cons[1].raddr, name) + assert not (cons[0].laddr and cons[0].raddr) + assert not (cons[1].laddr and cons[1].raddr) @skip_on_access_denied(only_if=OSX) def test_combos(self): @@ -367,8 +353,8 @@ def test_connection_constants(self): num = getattr(psutil, name) str_ = str(num) assert str_.isupper(), str_ - assert str_ not in strs, str_ - assert num not in ints, num + self.assertNotIn(str, strs) + self.assertNotIn(num, ints) ints.append(num) strs.append(str_) if SUNOS: From 209980937428fc53cf26a3d22df138cc291d50c4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 06:56:52 +0200 Subject: [PATCH 468/922] move create_sockets() out of memory_leaks.py and make it available for all tests --- psutil/tests/__init__.py | 33 ++++++++++++++++++++- psutil/tests/test_memory_leaks.py | 49 ++++--------------------------- psutil/tests/test_misc.py | 17 +++++++++++ 3 files changed, 54 insertions(+), 45 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 2a45e8c8f..0dbb003bd 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -41,6 +41,7 @@ import psutil from psutil import POSIX from psutil import WINDOWS +from psutil._common import supports_ipv6 from psutil._compat import PY3 from psutil._compat import u from psutil._compat import unicode @@ -99,7 +100,7 @@ # network 'check_connection_ntuple', 'check_net_address', 'get_free_port', 'unix_socket_path', 'bind_socket', 'bind_unix_socket', - 'tcp_socketpair', 'unix_socketpair', + 'tcp_socketpair', 'unix_socketpair', 'create_sockets', # others 'warn', 'copyload_shared_lib', 'is_namedtuple', ] @@ -885,6 +886,36 @@ def unix_socketpair(name): return (server, client) +@contextlib.contextmanager +def create_sockets(): + """Open as many socket families / types as possible.""" + socks = [] + fname1 = fname2 = None + try: + socks.append(bind_socket(socket.AF_INET, socket.SOCK_STREAM)) + socks.append(bind_socket(socket.AF_INET, socket.SOCK_DGRAM)) + if supports_ipv6(): + socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM)) + socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM)) + if POSIX: + fname1 = unix_socket_path().__enter__() + fname2 = unix_socket_path().__enter__() + s1, s2 = unix_socketpair(fname1) + s3 = bind_unix_socket(fname2, type=socket.SOCK_DGRAM) + # self.addCleanup(safe_rmpath, fname1) + # self.addCleanup(safe_rmpath, fname2) + for s in (s1, s2, s3): + socks.append(s) + yield socks + finally: + for s in socks: + s.close() + if fname1 is not None: + safe_rmpath(fname1) + if fname2 is not None: + safe_rmpath(fname2) + + def check_net_address(addr, family): """Check a net address validity. Supported families are IPv4, IPv6 and MAC addresses. diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index a18f31428..3c562ea2d 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -17,7 +17,6 @@ import functools import gc import os -import socket import threading import time @@ -29,13 +28,11 @@ from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS -from psutil._common import supports_ipv6 from psutil._compat import xrange -from psutil.tests import bind_socket -from psutil.tests import bind_unix_socket from psutil.tests import get_test_subprocess from psutil.tests import HAS_CPU_AFFINITY from psutil.tests import HAS_CPU_FREQ +from psutil.tests import create_sockets from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_IONICE from psutil.tests import HAS_PROC_CPU_NUM @@ -50,8 +47,6 @@ from psutil.tests import TESTFN from psutil.tests import TRAVIS from psutil.tests import unittest -from psutil.tests import unix_socket_path -from psutil.tests import unix_socketpair LOOPS = 1000 @@ -181,33 +176,6 @@ def _get_mem(): def _call(fun, *args, **kwargs): fun(*args, **kwargs) - def create_sockets(self): - """Open as many socket families / types as possible. - This is needed to excercise as many C code sections as - possible for net_connections() functions. - The returned sockets are already scheduled for cleanup. - """ - socks = [] - try: - socks.append(bind_socket(socket.AF_INET, socket.SOCK_STREAM)) - socks.append(bind_socket(socket.AF_INET, socket.SOCK_DGRAM)) - if supports_ipv6(): - socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM)) - socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM)) - if POSIX and not SUNOS: # TODO: SunOS - name1 = unix_socket_path().__enter__() - name2 = unix_socket_path().__enter__() - s1, s2 = unix_socketpair(name1) - s3 = bind_unix_socket(name2, type=socket.SOCK_DGRAM) - self.addCleanup(safe_rmpath, name1) - self.addCleanup(safe_rmpath, name2) - for s in (s1, s2, s3): - socks.append(s) - return socks - finally: - for s in socks: - self.addCleanup(s.close) - # =================================================================== # Process class @@ -389,19 +357,12 @@ def test_rlimit_set(self): # function (tested later). @unittest.skipIf(WINDOWS, "worthless on WINDOWS") def test_connections(self): - socks = self.create_sockets() # TODO: UNIX sockets are temporarily implemented by parsing # 'pfiles' cmd output; we don't want that part of the code to # be executed. - kind = 'inet' if SUNOS else 'all' - # Make sure we did a proper setup. - self.assertEqual( - len(psutil.Process().connections(kind=kind)), len(socks)) - try: + with create_sockets(): + kind = 'inet' if SUNOS else 'all' self.execute(self.proc.connections, kind) - finally: - for s in socks: - s.close() @unittest.skipIf(not HAS_ENVIRON, "not supported") def test_environ(self): @@ -560,8 +521,8 @@ def test_net_io_counters(self): "worthless on Linux (pure python)") @unittest.skipIf(OSX and os.getuid() != 0, "need root access") def test_net_connections(self): - self.create_sockets() - self.execute(psutil.net_connections) + with create_sockets(): + self.execute(psutil.net_connections) def test_net_if_addrs(self): # Note: verified that on Windows this was a false positive. diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 4f653cea3..b24212a6c 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -10,6 +10,7 @@ """ import ast +import collections import contextlib import errno import imp @@ -32,6 +33,7 @@ from psutil.tests import bind_unix_socket from psutil.tests import chdir from psutil.tests import create_proc_children_pair +from psutil.tests import create_sockets from psutil.tests import get_free_port from psutil.tests import get_test_subprocess from psutil.tests import HAS_MEMORY_MAPS @@ -729,6 +731,21 @@ def test_unix_socketpair(self): client.close() server.close() + def test_create_sockets(self): + with create_sockets() as socks: + fams = collections.defaultdict(int) + types = collections.defaultdict(int) + for s in socks: + fams[s.family] += 1 + # work around http://bugs.python.org/issue30204 + types[s.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE)] += 1 + self.assertGreaterEqual(fams[socket.AF_INET], 2) + self.assertGreaterEqual(fams[socket.AF_INET6], 2) + if POSIX: + self.assertGreaterEqual(fams[socket.AF_UNIX], 2) + self.assertGreaterEqual(types[socket.SOCK_STREAM], 2) + self.assertGreaterEqual(types[socket.SOCK_DGRAM], 2) + if __name__ == '__main__': run_test_module_by_name(__file__) From 62097dae140d60e90e2e1088e8810d6a5c7e3a3c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 07:06:44 +0200 Subject: [PATCH 469/922] small refactoring (ignore me) --- psutil/tests/test_connections.py | 48 +++++++++++++++++++------------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 3555aed0d..ea63af3cb 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -342,29 +342,11 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): # ===================================================================== -class TestMisc(Base, unittest.TestCase): +class TestSystemWideConnections(unittest.TestCase): """Tests for net_connections().""" - def test_connection_constants(self): - ints = [] - strs = [] - for name in dir(psutil): - if name.startswith('CONN_'): - num = getattr(psutil, name) - str_ = str(num) - assert str_.isupper(), str_ - self.assertNotIn(str, strs) - self.assertNotIn(num, ints) - ints.append(num) - strs.append(str_) - if SUNOS: - psutil.CONN_IDLE - psutil.CONN_BOUND - if WINDOWS: - psutil.CONN_DELETE_TCB - @skip_on_access_denied() - def test_net_connections(self): + def test_it(self): def check(cons, families, types_): AF_UNIX = getattr(socket, 'AF_UNIX', object()) for conn in cons: @@ -385,5 +367,31 @@ def check(cons, families, types_): self.assertRaises(ValueError, psutil.net_connections, kind='???') +# ===================================================================== +# --- Miscellaneous tests +# ===================================================================== + + +class TestMisc(unittest.TestCase): + + def test_connection_constants(self): + ints = [] + strs = [] + for name in dir(psutil): + if name.startswith('CONN_'): + num = getattr(psutil, name) + str_ = str(num) + assert str_.isupper(), str_ + self.assertNotIn(str, strs) + self.assertNotIn(num, ints) + ints.append(num) + strs.append(str_) + if SUNOS: + psutil.CONN_IDLE + psutil.CONN_BOUND + if WINDOWS: + psutil.CONN_DELETE_TCB + + if __name__ == '__main__': run_test_module_by_name(__file__) From b6c2636c05057e610d68949d1832447953269aa8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 07:32:32 +0200 Subject: [PATCH 470/922] #1040 fix unicode for connections() on netbsd --- psutil/arch/bsd/netbsd.c | 6 +----- psutil/arch/bsd/netbsd_socks.c | 10 +++++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index 361ab1f9b..cc33d4910 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -386,11 +386,7 @@ psutil_get_cmdline(pid_t pid) { // separator if (argsize > 0) { while (pos < argsize) { -#if PY_MAJOR_VERSION >= 3 - py_arg = PyUnicode_DecodeFSDefault(&argstr[pos]); -#else - py_arg = Py_BuildValue("s", &argstr[pos]); -#endif + py_arg = psutil_PyUnicode_DecodeFSDefault(&argstr[pos]); if (!py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) diff --git a/psutil/arch/bsd/netbsd_socks.c b/psutil/arch/bsd/netbsd_socks.c index d05981d22..ae3ac3f1f 100644 --- a/psutil/arch/bsd/netbsd_socks.c +++ b/psutil/arch/bsd/netbsd_socks.c @@ -406,19 +406,17 @@ psutil_net_connections(PyObject *self, PyObject *args) { strcpy(laddr, sun_src->sun_path); strcpy(raddr, sun_dst->sun_path); status = PSUTIL_CONN_NONE; - // TODO: handle unicode - py_laddr = Py_BuildValue("s", laddr); + py_laddr = psutil_PyUnicode_DecodeFSDefault(laddr); if (! py_laddr) goto error; - // TODO: handle unicode - py_raddr = Py_BuildValue("s", raddr); + py_raddr = psutil_PyUnicode_DecodeFSDefault(raddr); if (! py_raddr) goto error; } // append tuple to list py_tuple = Py_BuildValue( - "(iiiNNii)", + "(iiiOOii)", k->kif->ki_fd, kp->kpcb->ki_family, kp->kpcb->ki_type, @@ -430,6 +428,8 @@ psutil_net_connections(PyObject *self, PyObject *args) { goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_laddr); + Py_DECREF(py_raddr); Py_DECREF(py_tuple); } } From 9c22b44dc27607e508036c1cb846b9008c7eb39e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 07:42:32 +0200 Subject: [PATCH 471/922] comment dead code --- psutil/arch/bsd/netbsd.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index cc33d4910..6a17b45e3 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -119,6 +119,7 @@ kinfo_getfile(pid_t pid, int* cnt) { // https://github.com/giampaolo/psutil/pull/557#issuecomment-171912820 // Current implementation uses /proc instead. // Left here just in case. +/* PyObject * psutil_proc_exe(PyObject *self, PyObject *args) { #if __NetBSD_Version__ >= 799000000 @@ -163,16 +164,12 @@ psutil_proc_exe(PyObject *self, PyObject *args) { strcpy(pathname, ""); } -#if PY_MAJOR_VERSION >= 3 return PyUnicode_DecodeFSDefault(pathname); -#else - return Py_BuildValue("s", pathname); -#endif - #else return Py_BuildValue("s", ""); #endif } +*/ PyObject * psutil_proc_num_threads(PyObject *self, PyObject *args) { From d93a437325d9028f717dfbc1344015b86311d24f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 07:51:47 +0200 Subject: [PATCH 472/922] #1040 fix unicode for memory_maps() on osx --- psutil/_psutil_osx.c | 10 ++++++++-- psutil/arch/osx/process_info.c | 8 ++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index db1f997ab..dbc690323 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -337,6 +337,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { vm_size_t size = 0; PyObject *py_tuple = NULL; + PyObject *py_path = NULL; PyObject *py_list = PyList_New(0); if (py_list == NULL) @@ -431,11 +432,14 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { } } + py_path = psutil_PyUnicode_DecodeFSDefault(buf); + if (! py_path) + goto error; py_tuple = Py_BuildValue( - "sssIIIIIH", + "ssOIIIIIH", addr_str, // "start-end"address perms, // "rwx" permissions - buf, // path + py_path, // path info.pages_resident * pagesize, // rss info.pages_shared_now_private * pagesize, // private info.pages_swapped_out * pagesize, // swapped @@ -448,6 +452,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { if (PyList_Append(py_list, py_tuple)) goto error; Py_DECREF(py_tuple); + Py_DECREF(py_path); } // increment address for the next map/file @@ -463,6 +468,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { if (task != MACH_PORT_NULL) mach_port_deallocate(mach_task_self(), task); Py_XDECREF(py_tuple); + Py_XDECREF(py_path); Py_DECREF(py_list); return NULL; } diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/osx/process_info.c index 1c97b69f5..28b02246d 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -177,12 +177,8 @@ psutil_get_cmdline(long pid) { goto error; while (arg_ptr < arg_end && nargs > 0) { if (*arg_ptr++ == '\0') { -#if PY_MAJOR_VERSION >= 3 - py_arg = PyUnicode_DecodeFSDefault(curr_arg); -#else - py_arg = Py_BuildValue("s", curr_arg); -#endif - if (!py_arg) + py_arg = psutil_PyUnicode_DecodeFSDefault(curr_arg); + if (! py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) goto error; From 40d73c49faf22b7fada7e644891d36fe3c8dd5c8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 08:01:34 +0200 Subject: [PATCH 473/922] #1040 use psutil_PyUnicode_DecodeFSDefault everywhere (osx) --- psutil/_psutil_osx.c | 60 ++++++++++---------------------------------- 1 file changed, 13 insertions(+), 47 deletions(-) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index dbc690323..fa04889ad 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -138,11 +138,7 @@ psutil_proc_kinfo_oneshot(PyObject *self, PyObject *args) { if (psutil_get_kinfo_proc(pid, &kp) == -1) return NULL; -#if PY_MAJOR_VERSION >= 3 - py_name = PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); -#else - py_name = Py_BuildValue("s", kp.kp_proc.p_comm); -#endif + py_name = psutil_PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); if (! py_name) { // Likely a decoding error. We don't want to fail the whole // operation. The python module may retry with proc_name(). @@ -224,11 +220,7 @@ psutil_proc_name(PyObject *self, PyObject *args) { return NULL; if (psutil_get_kinfo_proc(pid, &kp) == -1) return NULL; -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); -#else - return Py_BuildValue("s", kp.kp_proc.p_comm); -#endif + return psutil_PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); } @@ -248,12 +240,8 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { { return NULL; } - -#if PY_MAJOR_VERSION >= 3 + return PyUnicode_DecodeFSDefault(pathinfo.pvi_cdir.vip_path); -#else - return Py_BuildValue("s", pathinfo.pvi_cdir.vip_path); -#endif } @@ -277,11 +265,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { psutil_raise_for_pid(pid, "proc_pidpath() syscall failed"); return NULL; } -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_DecodeFSDefault(buf); -#else - return Py_BuildValue("s", buf); -#endif + return psutil_PyUnicode_DecodeFSDefault(buf); } @@ -1157,11 +1141,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { // --- /errors checking // --- construct python list -#if PY_MAJOR_VERSION >= 3 - py_path = PyUnicode_DecodeFSDefault(vi.pvip.vip_path); -#else - py_path = Py_BuildValue("s", vi.pvip.vip_path); -#endif + py_path = psutil_PyUnicode_DecodeFSDefault(vi.pvip.vip_path); if (! py_path) goto error; py_tuple = Py_BuildValue( @@ -1357,28 +1337,14 @@ psutil_proc_connections(PyObject *self, PyObject *args) { Py_DECREF(py_tuple); } else if (family == AF_UNIX) { - // decode laddr -#if PY_MAJOR_VERSION >= 3 - py_laddr = PyUnicode_DecodeFSDefault( -#else - py_laddr = Py_BuildValue("s", -#endif - si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path - ); - if (!py_laddr) - goto error; - - // decode raddr -#if PY_MAJOR_VERSION >= 3 - py_raddr = PyUnicode_DecodeFSDefault( -#else - py_raddr = Py_BuildValue("s", -#endif - si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path - ); - if (!py_raddr) - goto error; - + py_laddr = psutil_PyUnicode_DecodeFSDefault( + si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path); + if (!py_laddr) + goto error; + py_raddr = psutil_PyUnicode_DecodeFSDefault( + si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path); + if (!py_raddr) + goto error; // construct the python list py_tuple = Py_BuildValue( "(iiiOOi)", From 043d5737d6eeab96aedd97f3a2fef35797304677 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 08:09:49 +0200 Subject: [PATCH 474/922] #1040: add alias for psutil_PyUnicode_DecodeFSDefaultAndSize --- psutil/_psutil_common.c | 10 ++++++++++ psutil/_psutil_common.h | 1 + psutil/_psutil_osx.c | 2 +- psutil/arch/osx/process_info.c | 7 +------ 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 88b86202a..ab88b99b3 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -50,3 +50,13 @@ psutil_PyUnicode_DecodeFSDefault(char *s) { return Py_BuildValue("s", s); #endif } + + +PyObject* +psutil_PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size) { +#if PY_MAJOR_VERSION >= 3 + return PyUnicode_DecodeFSDefaultAndSize(s, size); +#else + return PyString_FromStringAndSize(s, size); +#endif +} diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 31d93fbde..84916d2d2 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -9,3 +9,4 @@ PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); PyObject* psutil_PyUnicode_DecodeFSDefault(char *s); +PyObject* psutil_PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index fa04889ad..59b7c8047 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -241,7 +241,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { return NULL; } - return PyUnicode_DecodeFSDefault(pathinfo.pvi_cdir.vip_path); + return psutil_PyUnicode_DecodeFSDefault(pathinfo.pvi_cdir.vip_path); } diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/osx/process_info.c index 28b02246d..ee3956296 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -289,13 +289,8 @@ psutil_get_environ(long pid) { arg_ptr = s + 1; } -#if PY_MAJOR_VERSION >= 3 - py_ret = PyUnicode_DecodeFSDefaultAndSize( + py_ret = psutil_PyUnicode_DecodeFSDefaultAndSize( procenv, arg_ptr - env_start + 1); -#else - py_ret = PyString_FromStringAndSize(procenv, arg_ptr - env_start + 1); -#endif - if (!py_ret) { // XXX: don't want to free() this as per: // https://github.com/giampaolo/psutil/issues/926 From 0cab2d1f98b4814e87b930193e5316897d70d047 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 08:21:00 +0200 Subject: [PATCH 475/922] #1040: add replacement for PyUnicode_DecodeFSDefaultAndSize on Python 2; also get rid of the pstuil_ prefix and use the original Python names --- psutil/_psutil_bsd.c | 22 +++++++++++----------- psutil/_psutil_common.c | 23 ++++++++--------------- psutil/_psutil_common.h | 6 ++++-- psutil/_psutil_osx.c | 16 ++++++++-------- psutil/arch/bsd/freebsd.c | 12 ++++++------ psutil/arch/bsd/freebsd_socks.c | 4 ++-- psutil/arch/bsd/netbsd.c | 2 +- psutil/arch/bsd/netbsd_socks.c | 4 ++-- psutil/arch/osx/process_info.c | 4 ++-- 9 files changed, 44 insertions(+), 49 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index a3c3d56a8..a8ff93fa5 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -215,7 +215,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { #elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) sprintf(str, "%s", kp.p_comm); #endif - py_name = psutil_PyUnicode_DecodeFSDefault(str); + py_name = PyUnicode_DecodeFSDefault(str); if (! py_name) { // Likely a decoding error. We don't want to fail the whole // operation. The python module may retry with proc_name(). @@ -368,7 +368,7 @@ psutil_proc_name(PyObject *self, PyObject *args) { #elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) sprintf(str, "%s", kp.p_comm); #endif - return psutil_PyUnicode_DecodeFSDefault(str); + return PyUnicode_DecodeFSDefault(str); } @@ -499,7 +499,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { // XXX - it appears path is not exposed in the kinfo_file struct. path = ""; #endif - py_path = psutil_PyUnicode_DecodeFSDefault(path); + py_path = PyUnicode_DecodeFSDefault(path); if (! py_path) goto error; if (regular == 1) { @@ -656,10 +656,10 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { if (flags & MNT_NODEVMTIME) strlcat(opts, ",nodevmtime", sizeof(opts)); #endif - py_dev = psutil_PyUnicode_DecodeFSDefault(fs[i].f_mntfromname); + py_dev = PyUnicode_DecodeFSDefault(fs[i].f_mntfromname); if (! py_dev) goto error; - py_mountp = psutil_PyUnicode_DecodeFSDefault(fs[i].f_mntonname); + py_mountp = PyUnicode_DecodeFSDefault(fs[i].f_mntonname); if (! py_mountp) goto error; py_tuple = Py_BuildValue("(OOss)", @@ -812,13 +812,13 @@ psutil_users(PyObject *self, PyObject *args) { while (fread(&ut, sizeof(ut), 1, fp) == 1) { if (*ut.ut_name == '\0') continue; - py_username = psutil_PyUnicode_DecodeFSDefault(ut.ut_name); + py_username = PyUnicode_DecodeFSDefault(ut.ut_name); if (! py_username) goto error; - py_tty = psutil_PyUnicode_DecodeFSDefault(ut.ut_line); + py_tty = PyUnicode_DecodeFSDefault(ut.ut_line); if (! py_tty) goto error; - py_hostname = psutil_PyUnicode_DecodeFSDefault(ut.ut_host); + py_hostname = PyUnicode_DecodeFSDefault(ut.ut_host); if (! py_hostname) goto error; py_tuple = Py_BuildValue( @@ -850,13 +850,13 @@ psutil_users(PyObject *self, PyObject *args) { while ((utx = getutxent()) != NULL) { if (utx->ut_type != USER_PROCESS) continue; - py_username = psutil_PyUnicode_DecodeFSDefault(utx->ut_user); + py_username = PyUnicode_DecodeFSDefault(utx->ut_user); if (! py_username) goto error; - py_tty = psutil_PyUnicode_DecodeFSDefault(utx->ut_line); + py_tty = PyUnicode_DecodeFSDefault(utx->ut_line); if (! py_tty) goto error; - py_hostname = psutil_PyUnicode_DecodeFSDefault(utx->ut_host); + py_hostname = PyUnicode_DecodeFSDefault(utx->ut_host); if (! py_hostname) goto error; py_tuple = Py_BuildValue( diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index ab88b99b3..bcbd623b6 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -37,26 +37,19 @@ AccessDenied(void) { /* - * Alias for PyUnicode_DecodeFSDefault which is not available - * on Python 2. On Python 2 we just return a plain byte string + * Backport of unicode FS APIs from Python 3. + * On Python 2 we just return a plain byte string * which is never supposed to raise decoding errors. * See: https://github.com/giampaolo/psutil/issues/1040 */ +#if PY_MAJOR_VERSION < 3 PyObject * -psutil_PyUnicode_DecodeFSDefault(char *s) { -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_DecodeFSDefault(s); -#else - return Py_BuildValue("s", s); -#endif +PyUnicode_DecodeFSDefault(char *s) { + return PyString_FromString(s); } - -PyObject* -psutil_PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size) { -#if PY_MAJOR_VERSION >= 3 - return PyUnicode_DecodeFSDefaultAndSize(s, size); -#else +PyObject * +PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size) { return PyString_FromStringAndSize(s, size); -#endif } +#endif \ No newline at end of file diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 84916d2d2..0507458aa 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -8,5 +8,7 @@ PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); -PyObject* psutil_PyUnicode_DecodeFSDefault(char *s); -PyObject* psutil_PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); +#if PY_MAJOR_VERSION < 3 +PyObject* PyUnicode_DecodeFSDefault(char *s); +PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); +#endif \ No newline at end of file diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 59b7c8047..536500f50 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -138,7 +138,7 @@ psutil_proc_kinfo_oneshot(PyObject *self, PyObject *args) { if (psutil_get_kinfo_proc(pid, &kp) == -1) return NULL; - py_name = psutil_PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); + py_name = PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); if (! py_name) { // Likely a decoding error. We don't want to fail the whole // operation. The python module may retry with proc_name(). @@ -220,7 +220,7 @@ psutil_proc_name(PyObject *self, PyObject *args) { return NULL; if (psutil_get_kinfo_proc(pid, &kp) == -1) return NULL; - return psutil_PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); + return PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); } @@ -241,7 +241,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { return NULL; } - return psutil_PyUnicode_DecodeFSDefault(pathinfo.pvi_cdir.vip_path); + return PyUnicode_DecodeFSDefault(pathinfo.pvi_cdir.vip_path); } @@ -265,7 +265,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { psutil_raise_for_pid(pid, "proc_pidpath() syscall failed"); return NULL; } - return psutil_PyUnicode_DecodeFSDefault(buf); + return PyUnicode_DecodeFSDefault(buf); } @@ -416,7 +416,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { } } - py_path = psutil_PyUnicode_DecodeFSDefault(buf); + py_path = PyUnicode_DecodeFSDefault(buf); if (! py_path) goto error; py_tuple = Py_BuildValue( @@ -1141,7 +1141,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { // --- /errors checking // --- construct python list - py_path = psutil_PyUnicode_DecodeFSDefault(vi.pvip.vip_path); + py_path = PyUnicode_DecodeFSDefault(vi.pvip.vip_path); if (! py_path) goto error; py_tuple = Py_BuildValue( @@ -1337,11 +1337,11 @@ psutil_proc_connections(PyObject *self, PyObject *args) { Py_DECREF(py_tuple); } else if (family == AF_UNIX) { - py_laddr = psutil_PyUnicode_DecodeFSDefault( + py_laddr = PyUnicode_DecodeFSDefault( si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path); if (!py_laddr) goto error; - py_raddr = psutil_PyUnicode_DecodeFSDefault( + py_raddr = PyUnicode_DecodeFSDefault( si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path); if (!py_raddr) goto error; diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index 9a1f952c4..11594b9d6 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -238,7 +238,7 @@ psutil_get_cmdline(long pid) { // separator if (argsize > 0) { while (pos < argsize) { - py_arg = psutil_PyUnicode_DecodeFSDefault(&argstr[pos]); + py_arg = PyUnicode_DecodeFSDefault(&argstr[pos]); if (!py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) @@ -288,7 +288,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (error == -1) { // see: https://github.com/giampaolo/psutil/issues/907 if (errno == ENOENT) - return psutil_PyUnicode_DecodeFSDefault(""); + return PyUnicode_DecodeFSDefault(""); else return PyErr_SetFromErrno(PyExc_OSError); } @@ -302,7 +302,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { strcpy(pathname, ""); } - return psutil_PyUnicode_DecodeFSDefault(pathname); + return PyUnicode_DecodeFSDefault(pathname); } @@ -555,7 +555,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { for (i = 0; i < cnt; i++) { kif = &freep[i]; if (kif->kf_fd == KF_FD_TYPE_CWD) { - py_path = psutil_PyUnicode_DecodeFSDefault(kif->kf_path); + py_path = PyUnicode_DecodeFSDefault(kif->kf_path); if (!py_path) goto error; break; @@ -567,7 +567,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { * as root we return an empty string instead of AccessDenied. */ if (py_path == NULL) - py_path = psutil_PyUnicode_DecodeFSDefault(""); + py_path = PyUnicode_DecodeFSDefault(""); free(freep); return py_path; @@ -821,7 +821,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { path = kve->kve_path; } - py_path = psutil_PyUnicode_DecodeFSDefault(path); + py_path = PyUnicode_DecodeFSDefault(path); if (! py_path) goto error; py_tuple = Py_BuildValue("ssOiiii", diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 708ff893f..28695658c 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -417,7 +417,7 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { snprintf(path, sizeof(path), "%.*s", (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), sun->sun_path); - py_lpath = psutil_PyUnicode_DecodeFSDefault(path); + py_lpath = PyUnicode_DecodeFSDefault(path); if (! py_lpath) goto error; @@ -604,7 +604,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), sun->sun_path); - py_laddr = psutil_PyUnicode_DecodeFSDefault(path); + py_laddr = PyUnicode_DecodeFSDefault(path); if (! py_laddr) goto error; diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index 6a17b45e3..5748000fa 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -383,7 +383,7 @@ psutil_get_cmdline(pid_t pid) { // separator if (argsize > 0) { while (pos < argsize) { - py_arg = psutil_PyUnicode_DecodeFSDefault(&argstr[pos]); + py_arg = PyUnicode_DecodeFSDefault(&argstr[pos]); if (!py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) diff --git a/psutil/arch/bsd/netbsd_socks.c b/psutil/arch/bsd/netbsd_socks.c index ae3ac3f1f..06b7c7556 100644 --- a/psutil/arch/bsd/netbsd_socks.c +++ b/psutil/arch/bsd/netbsd_socks.c @@ -406,10 +406,10 @@ psutil_net_connections(PyObject *self, PyObject *args) { strcpy(laddr, sun_src->sun_path); strcpy(raddr, sun_dst->sun_path); status = PSUTIL_CONN_NONE; - py_laddr = psutil_PyUnicode_DecodeFSDefault(laddr); + py_laddr = PyUnicode_DecodeFSDefault(laddr); if (! py_laddr) goto error; - py_raddr = psutil_PyUnicode_DecodeFSDefault(raddr); + py_raddr = PyUnicode_DecodeFSDefault(raddr); if (! py_raddr) goto error; } diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/osx/process_info.c index ee3956296..7d6861a52 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -177,7 +177,7 @@ psutil_get_cmdline(long pid) { goto error; while (arg_ptr < arg_end && nargs > 0) { if (*arg_ptr++ == '\0') { - py_arg = psutil_PyUnicode_DecodeFSDefault(curr_arg); + py_arg = PyUnicode_DecodeFSDefault(curr_arg); if (! py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) @@ -289,7 +289,7 @@ psutil_get_environ(long pid) { arg_ptr = s + 1; } - py_ret = psutil_PyUnicode_DecodeFSDefaultAndSize( + py_ret = PyUnicode_DecodeFSDefaultAndSize( procenv, arg_ptr - env_start + 1); if (!py_ret) { // XXX: don't want to free() this as per: From 552ee29eb1151ddec0b652db8d974abfa38440bd Mon Sep 17 00:00:00 2001 From: Himanshu Shekhar Date: Mon, 1 May 2017 20:09:26 +0530 Subject: [PATCH 476/922] update thread management to safer way --- scripts/internal/check_broken_links.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 9ae5e6440..b14e9f596 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -43,9 +43,8 @@ import os import re import sys -import time -from concurrent.futures import ThreadPoolExecutor +import concurrent.futures import requests @@ -103,23 +102,19 @@ def parallel_validator(urls): fails = [] # list of tuples (filename, url) completed = 0 total = len(urls) - threads = [] - with ThreadPoolExecutor() as executor: - for url in urls: - fut = executor.submit(validate_url, url[1]) - threads.append((url, fut)) - - # wait for threads to progress a little - time.sleep(2) - for thr in threads: - url = thr[0] - fut = thr[1] + with concurrent.futures.ThreadPoolExecutor() as executor: + fut_to_url = {executor.submit(validate_url, url[1]): url + for url in urls} + + for fut in concurrent.futures.as_completed(fut_to_url): if not fut.result(): - fails.append((url[0], url[1])) + url = fut_to_url[fut] + fails.append(url) # actually a tuple of url and filename completed += 1 sys.stdout.write("\r" + str(completed)+' / '+str(total)) sys.stdout.flush() + print() return fails @@ -141,10 +136,10 @@ def main(): if not fails: print("all links are valid. cheers!") else: - print("total :", len(fails), "fails!") for fail in fails: print(fail[1] + ' : ' + fail[0] + os.linesep) print('-' * 20) + print("total :", len(fails), "fails!") sys.exit(1) From 789773dfeae12b0b089887eb0e0b5360adf442d9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 18:16:51 +0200 Subject: [PATCH 477/922] #1036: add exception handling + some minor coding style adjustments --- CREDITS | 3 + DEVGUIDE.rst | 2 +- scripts/internal/check_broken_links.py | 78 +++++++++++++------------- 3 files changed, 42 insertions(+), 41 deletions(-) diff --git a/CREDITS b/CREDITS index 7f9da69d6..32d3d51c7 100644 --- a/CREDITS +++ b/CREDITS @@ -446,3 +446,6 @@ N: Alexander Hasselhuhn C: Germany W: https://github.com/alexanha +N: Himanshu Shekhar +W: https://github.com/himanshub16 +I: 1036 diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index df2113917..af34ee12a 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -132,7 +132,7 @@ Documentation - it uses `RsT syntax `_ and it's built with `sphinx `_. - doc can be built with ``make setup-dev-env; cd docs; make html``. -- public doc is hosted on http://pythonhosted.org/psutil/. +- public doc is hosted on http://pythonhosted.org/psutil/ ======================= Releasing a new version diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index b14e9f596..ec492f612 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -1,24 +1,24 @@ #!/usr/bin/env python -# Author : Himanshu Shekhar < https://github.com/himanshub16 > (2017) - -# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. +# Copyright (c) 2009, Giampaolo Rodola', Himanshu Shekhar. +# All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. """ -Checks for broken links in file names specified as command line parameters. - -There are a ton of a solutions available for validating URLs in string using -regex, but less for searching, of which very few are accurate. -This snippet is intended to just do the required work, and avoid complexities. -Django Validator has pretty good regex for validation, but we have to find -urls instead of validating them. (REFERENCES [7]) +Checks for broken links in file names specified as command line +parameters. + +There are a ton of a solutions available for validating URLs in string +using regex, but less for searching, of which very few are accurate. +This snippet is intended to just do the required work, and avoid +complexities. Django Validator has pretty good regex for validation, +but we have to find urls instead of validating them (REFERENCES [7]). There's always room for improvement. Method: * Match URLs using regex (REFERENCES [1]]) -* Some URLs need to be fixed, as they have < (or) > due to inefficient regex. +* Some URLs need to be fixed, as they have < (or) > due to inefficient + regex. * Remove duplicates (because regex is not 100% efficient as of now). * Check validity of URL, using HEAD request. (HEAD to save bandwidth) Uses requests module for others are painful to use. REFERENCES[9] @@ -36,6 +36,7 @@ [8] https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD [9] http://docs.python-requests.org/ +Author: Himanshu Shekhar (2017) """ from __future__ import print_function @@ -43,18 +44,16 @@ import os import re import sys - +import traceback import concurrent.futures + import requests HERE = os.path.abspath(os.path.dirname(__file__)) - REGEX = r'(?:http|ftp|https)?://' \ r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' - REQUEST_TIMEOUT = 30 - # There are some status codes sent by websites on HEAD request. # Like 503 by Microsoft, and 401 by Apple # They need to be sent GET request @@ -62,27 +61,21 @@ def get_urls(filename): - """Extracts all URLs available in specified filename - """ - # fname = os.path.abspath(os.path.join(HERE, filename)) - # expecting absolute path + """Extracts all URLs available in specified filename.""" with open(filename) as fs: text = fs.read() - urls = re.findall(REGEX, text) # remove duplicates, list for sets are not iterable urls = list(set(urls)) # correct urls which are between < and/or > for i, url in enumerate(urls): urls[i] = re.sub("[\*<>\(\)\)]", '', url) - return urls def validate_url(url): """Validate the URL by attempting an HTTP connection. Makes an HTTP-HEAD request for each URL. - Uses requests module. """ try: res = requests.head(url, timeout=REQUEST_TIMEOUT) @@ -100,32 +93,36 @@ def parallel_validator(urls): urls: tuple(filename, url) """ fails = [] # list of tuples (filename, url) - completed = 0 + current = 0 total = len(urls) - with concurrent.futures.ThreadPoolExecutor() as executor: fut_to_url = {executor.submit(validate_url, url[1]): url for url in urls} - for fut in concurrent.futures.as_completed(fut_to_url): - if not fut.result(): - url = fut_to_url[fut] - fails.append(url) # actually a tuple of url and filename - completed += 1 - sys.stdout.write("\r" + str(completed)+' / '+str(total)) + current += 1 + sys.stdout.write("\r%s / %s" % (current, total)) sys.stdout.flush() - - print() + fname, url = fut_to_url[fut] + try: + ok = fut.result() + except Exception: + fails.append((fname, url)) + print() + print("warn: error while validating %s" % url, file=sys.stderr) + traceback.print_exc() + else: + if not ok: + fails.append((fname, url)) + if fails: + print() return fails def main(): - """Main function - """ files = sys.argv[1:] - if not files: return sys.exit("usage: %s " % __name__) + all_urls = [] for fname in files: urls = get_urls(fname) @@ -134,12 +131,13 @@ def main(): fails = parallel_validator(all_urls) if not fails: - print("all links are valid. cheers!") + print("all links are valid; cheers!") else: for fail in fails: - print(fail[1] + ' : ' + fail[0] + os.linesep) + fname, url = fail + print("%s : %s " % (url, fname)) print('-' * 20) - print("total :", len(fails), "fails!") + print("total: %s fails!" % len(fails)) sys.exit(1) From 98c792720b389c6ceb7f2dc5c6924684daf74aa9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 21:40:35 +0200 Subject: [PATCH 478/922] fix mem maps test with regard for realpaths --- psutil/tests/test_process.py | 8 +++++--- psutil/tests/test_unicode.py | 12 +++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index ec9c71754..58dc6829b 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -656,12 +656,14 @@ def test_memory_maps(self): @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") def test_memory_maps_lists_lib(self): + normcase = os.path.normcase + realpath = os.path.realpath p = psutil.Process() ext = ".so" if POSIX else ".dll" old = [x.path for x in p.memory_maps() - if os.path.normcase(x.path).endswith(ext)][0] - new = os.path.normcase(copyload_shared_lib(old)) - newpaths = [os.path.normcase(x.path) for x in p.memory_maps()] + if normcase(x.path).endswith(ext)][0] + new = realpath(normcase(copyload_shared_lib(old))) + newpaths = [realpath(normcase(x.path)) for x in p.memory_maps()] self.assertIn(new, newpaths) def test_memory_percent(self): diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index fd0ab3c70..7b492447a 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -239,19 +239,21 @@ def test_disk_usage(self): @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") def test_memory_maps(self): + normcase = os.path.normcase + realpath = os.path.realpath p = psutil.Process() ext = ".so" if POSIX else ".dll" - old = [x.path for x in p.memory_maps() - if os.path.normcase(x.path).endswith(ext)][0] + old = [realpath(x.path) for x in p.memory_maps() + if normcase(x.path).endswith(ext)][0] try: - new = os.path.normcase( - copyload_shared_lib(old, dst_prefix=self.funky_name)) + new = realpath(normcase( + copyload_shared_lib(old, dst_prefix=self.funky_name))) except UnicodeEncodeError: if PY3: raise else: raise unittest.SkipTest("ctypes can't handle unicode") - newpaths = [os.path.normcase(x.path) for x in p.memory_maps()] + newpaths = [realpath(normcase(x.path)) for x in p.memory_maps()] self.assertIn(new, newpaths) From ed98d3ca2bdb2e4989de4fc9fdbea08274734d7b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 22:03:51 +0200 Subject: [PATCH 479/922] #1022 / users(): PID cannot be determined on OpenBSD so we set it to None --- docs/index.rst | 2 +- psutil/_psbsd.py | 3 +++ psutil/_psutil_bsd.c | 10 +++++++++- psutil/tests/test_contracts.py | 1 + psutil/tests/test_system.py | 6 +++--- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 1ce8bee89..e6f40058f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -755,7 +755,7 @@ Other system info - **started**: the creation time as a floating point number expressed in seconds since the epoch. - **pid**: the PID of the login process (like sshd, tmux, gdm-session-worker, - ...). On Windows this is always set to ``None``. + ...). On Windows and OpenBSD this is always set to ``None``. Example:: diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 86c0bdd57..8b44deeb7 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -435,6 +435,9 @@ def users(): rawlist = cext.users() for item in rawlist: user, tty, hostname, tstamp, pid = item + if pid == -1: + assert OPENBSD + pid = None if tty == '~': continue # reboot or shutdown nt = _common.suser(user, tty or None, hostname, tstamp, pid) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 75de731de..537df1511 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -807,7 +807,11 @@ psutil_users(PyObject *self, PyObject *args) { ut.ut_line, // tty ut.ut_host, // hostname (float)ut.ut_time, // start time +#ifdef PSUTIL_OPENBSD + -1 // process id (set to None later) +#else ut.ut_pid // process id +#endif ); if (!py_tuple) { fclose(fp); @@ -834,7 +838,11 @@ psutil_users(PyObject *self, PyObject *args) { utx->ut_line, // tty utx->ut_host, // hostname (float)utx->ut_tv.tv_sec, // start time - utx->ut_pid // process id +#ifdef PSUTIL_OPENBSD + -1 // process id (set to None later) +#else + utx->ut_pid // process id +#endif ); if (!py_tuple) { diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 34015b9e2..a3dcaa17e 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -251,6 +251,7 @@ def test_users(self): self.assertIsInstance(user.name, str) self.assertIsInstance(user.terminal, (str, type(None))) self.assertIsInstance(user.host, (str, type(None))) + self.assertIsInstance(user.pid, (int, type(None))) # =================================================================== diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 4f1781b88..19f997a85 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -704,10 +704,10 @@ def test_users(self): user.host assert user.started > 0.0, user datetime.datetime.fromtimestamp(user.started) - if POSIX: - psutil.Process(user.pid) - else: + if WINDOWS or OPENBSD: self.assertIsNone(user.pid) + else: + psutil.Process(user.pid) def test_cpu_stats(self): # Tested more extensively in per-platform test modules. From e2ab08d50aab5c2cc4a92cdbee0e1325da470ddf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 22:44:28 +0200 Subject: [PATCH 480/922] fix different tests on openbsd --- psutil/tests/__init__.py | 5 ++++- psutil/tests/test_bsd.py | 5 +++-- psutil/tests/test_misc.py | 29 +++++++++++------------------ scripts/fans.py | 3 ++- scripts/sensors.py | 4 ++-- 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 0dbb003bd..7023ab376 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -41,6 +41,8 @@ import psutil from psutil import POSIX from psutil import WINDOWS +from psutil import LINUX +from psutil import OSX from psutil._common import supports_ipv6 from psutil._compat import PY3 from psutil._compat import u @@ -80,7 +82,7 @@ "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", "HAS_SENSORS_BATTERY", "HAS_BATTERY""HAS_SENSORS_FANS", - "HAS_SENSORS_TEMPERATURES", + "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", # classes 'ThreadTask' # test utils @@ -158,6 +160,7 @@ HAS_ENVIRON = hasattr(psutil.Process, "environ") HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters") HAS_IONICE = hasattr(psutil.Process, "ionice") +HAS_MEMORY_FULL_INFO = LINUX or OSX or WINDOWS HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps") HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") HAS_RLIMIT = hasattr(psutil.Process, "rlimit") diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 0f98a3c6c..8fd7fe1d5 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -136,8 +136,9 @@ def test_net_if_stats(self): pass else: self.assertEqual(stats.isup, 'RUNNING' in out, msg=out) - self.assertEqual(stats.mtu, - int(re.findall('mtu (\d+)', out)[0])) + if "mtu" in out: + self.assertEqual(stats.mtu, + int(re.findall('mtu (\d+)', out)[0])) # ===================================================================== diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index b24212a6c..cb22c5fcb 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -22,7 +22,6 @@ import sys from psutil import LINUX -from psutil import OSX from psutil import POSIX from psutil import WINDOWS from psutil._common import memoize @@ -36,7 +35,11 @@ from psutil.tests import create_sockets from psutil.tests import get_free_port from psutil.tests import get_test_subprocess +from psutil.tests import HAS_MEMORY_FULL_INFO from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_SENSORS_BATTERY +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import importlib from psutil.tests import mock from psutil.tests import reap_children @@ -459,7 +462,7 @@ def test_ifconfig(self): def test_pmap(self): self.assert_stdout('pmap.py', args=str(os.getpid())) - @unittest.skipIf(not OSX or WINDOWS or LINUX, "platform not supported") + @unittest.skipIf(not HAS_MEMORY_FULL_INFO, "not supported") def test_procsmem(self): self.assert_stdout('procsmem.py') @@ -486,30 +489,20 @@ def test_winservices(self): def test_cpu_distribution(self): self.assert_syntax('cpu_distribution.py') + @unittest.skipIf(not HAS_SENSORS_TEMPERATURES, "not supported") @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_temperatures(self): - if hasattr(psutil, "sensors_temperatures") and \ - psutil.sensors_temperatures(): - self.assert_stdout('temperatures.py') - else: - self.assert_syntax('temperatures.py') + self.assert_stdout('temperatures.py') + @unittest.skipIf(not HAS_SENSORS_FANS, "not supported") @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_fans(self): - if hasattr(psutil, "sensors_fans") and psutil.sensors_fans(): - self.assert_stdout('fans.py') - else: - self.assert_syntax('fans.py') + self.assert_stdout('fans.py') + @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") def test_battery(self): - if hasattr(psutil, "sensors_battery") and \ - psutil.sensors_battery() is not None: - self.assert_stdout('battery.py') - else: - self.assert_syntax('battery.py') + self.assert_stdout('battery.py') - @unittest.skipIf(APPVEYOR or TRAVIS, "unreliable on CI") - @unittest.skipIf(OSX, "platform not supported") def test_sensors(self): self.assert_stdout('sensors.py') diff --git a/scripts/fans.py b/scripts/fans.py index e302aec5d..7a0ccf91d 100755 --- a/scripts/fans.py +++ b/scripts/fans.py @@ -23,7 +23,8 @@ def main(): return sys.exit("platform not supported") fans = psutil.sensors_fans() if not fans: - return sys.exit("no fans detected") + print("no fans detected") + return for name, entries in fans.items(): print(name) for entry in entries: diff --git a/scripts/sensors.py b/scripts/sensors.py index e3301ebfe..bbf3ac908 100755 --- a/scripts/sensors.py +++ b/scripts/sensors.py @@ -30,7 +30,6 @@ """ from __future__ import print_function -import sys import psutil @@ -56,7 +55,8 @@ def main(): battery = None if not any((temps, fans, battery)): - return sys.exit("can't read any temperature, fans or battery info") + print("can't read any temperature, fans or battery info") + return names = set(list(temps.keys()) + list(fans.keys())) for name in names: From 8a053328e2238d3b7a0bb632798acf4463ed2bed Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 23:07:48 +0200 Subject: [PATCH 481/922] add more test utils tests --- psutil/tests/__init__.py | 22 +--------------------- psutil/tests/test_misc.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 7023ab376..16d4f5a9b 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -41,8 +41,6 @@ import psutil from psutil import POSIX from psutil import WINDOWS -from psutil import LINUX -from psutil import OSX from psutil._common import supports_ipv6 from psutil._compat import PY3 from psutil._compat import u @@ -160,7 +158,7 @@ HAS_ENVIRON = hasattr(psutil.Process, "environ") HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters") HAS_IONICE = hasattr(psutil.Process, "ionice") -HAS_MEMORY_FULL_INFO = LINUX or OSX or WINDOWS +HAS_MEMORY_FULL_INFO = 'uss' in psutil.Process().memory_full_info()._fields HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps") HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") HAS_RLIMIT = hasattr(psutil.Process, "rlimit") @@ -611,24 +609,6 @@ def create_exe(outpath, c_code=None): os.chmod(outpath, st.st_mode | stat.S_IEXEC) -# In Python 3 paths are unicode objects by default. Surrogate escapes -# are used to handle non-character data. -def encode_path(path): - if PY3: - return path.encode(sys.getfilesystemencoding(), - errors="surrogateescape") - else: - return path - - -def decode_path(path): - if PY3: - return path.decode(sys.getfilesystemencoding(), - errors="surrogateescape") - else: - return path - - # =================================================================== # --- testing # =================================================================== diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index cb22c5fcb..29012cd49 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -29,7 +29,9 @@ from psutil._common import supports_ipv6 from psutil._compat import PY3 from psutil.tests import APPVEYOR +from psutil.tests import bind_socket from psutil.tests import bind_unix_socket +from psutil.tests import call_until from psutil.tests import chdir from psutil.tests import create_proc_children_pair from psutil.tests import create_sockets @@ -41,6 +43,7 @@ from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES from psutil.tests import importlib +from psutil.tests import is_namedtuple from psutil.tests import mock from psutil.tests import reap_children from psutil.tests import retry @@ -612,6 +615,10 @@ def test_wait_for_file_no_delete(self): wait_for_file(TESTFN, delete=False) assert os.path.exists(TESTFN) + def test_call_until(self): + ret = call_until(lambda: 1, "ret == 1") + self.assertEqual(ret, 1) + class TestFSTestUtils(unittest.TestCase): @@ -679,6 +686,11 @@ def test_create_proc_children_pair(self): class TestNetUtils(unittest.TestCase): + def bind_socket(self): + port = get_free_port() + with contextlib.closing(bind_socket(addr=('', port))) as s: + self.assertEqual(s.getsockname()[1], port) + @unittest.skipIf(not POSIX, "POSIX only") def test_bind_unix_socket(self): with unix_socket_path() as name: @@ -740,5 +752,12 @@ def test_create_sockets(self): self.assertGreaterEqual(types[socket.SOCK_DGRAM], 2) +class TestOtherUtils(unittest.TestCase): + + def test_is_namedtuple(self): + assert is_namedtuple(collections.namedtuple('foo', 'a b c')(1, 2, 3)) + assert not is_namedtuple(tuple()) + + if __name__ == '__main__': run_test_module_by_name(__file__) From d1ba75b57d08300c7a2f190e69eb7af902c25125 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 23:12:59 +0200 Subject: [PATCH 482/922] skip test in case 'df' gives 'permission denied' --- psutil/tests/test_posix.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 1b2dc5063..819da0d20 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -17,6 +17,7 @@ import psutil from psutil import BSD from psutil import LINUX +from psutil import OPENBSD from psutil import OSX from psutil import POSIX from psutil import SUNOS @@ -273,8 +274,8 @@ def test_pids(self): pids_ps.sort() pids_psutil.sort() - # on OSX ps doesn't show pid 0 - if OSX and 0 not in pids_ps: + # on OSX and OPENBSD ps doesn't show pid 0 + if OSX or OPENBSD and 0 not in pids_ps: pids_ps.insert(0, 0) if pids_ps != pids_psutil: @@ -365,8 +366,10 @@ def df(device): # see: # https://travis-ci.org/giampaolo/psutil/jobs/138338464 # https://travis-ci.org/giampaolo/psutil/jobs/138343361 - if "no such file or directory" in str(err).lower() or \ - "raw devices not supported" in str(err).lower(): + err = str(err).lower() + if "no such file or directory" in err or \ + "raw devices not supported" in err or \ + "permission denied" in err: continue else: raise From ece4be8ff929ed405160700e976c5f54b20e8e47 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 May 2017 23:17:36 +0200 Subject: [PATCH 483/922] fix contract test --- psutil/tests/test_contracts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index a3dcaa17e..f44b7a41e 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -56,7 +56,7 @@ class TestAvailability(unittest.TestCase): """ def test_cpu_affinity(self): - hasit = LINUX or WINDOWS or BSD + hasit = LINUX or WINDOWS or FREEBSD self.assertEqual(hasattr(psutil.Process, "cpu_affinity"), hasit) def test_win_service(self): From 3cb16ddf6b50037310a99d5c5eed555db7610732 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 00:56:21 +0200 Subject: [PATCH 484/922] #1039 / connections('unix') / linux: set laddr and raddr to an empty string instead of None --- HISTORY.rst | 3 ++- docs/index.rst | 21 +++++++++++++++++---- psutil/_pslinux.py | 5 ++++- psutil/arch/bsd/openbsd.c | 18 +++++++++++++----- psutil/tests/__init__.py | 2 +- 5 files changed, 37 insertions(+), 12 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 54e2d99da..8de15018e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -30,7 +30,8 @@ - 1033_: [OSX, FreeBSD] memory leak for net_connections() and Process.connections() when retrieving UNIX sockets (kind='unix'). - 1039_: returned types consolidation: - - Windows: Process.cpu_times()'s fields #3 and #4 were int instead of float + - Windows / Process.cpu_times(): fields #3 and #4 were int instead of float + - Linux / connections('unix'): raddr is now set to "" instead of None *2017-04-10* diff --git a/docs/index.rst b/docs/index.rst index e6f40058f..84c8ae403 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -543,6 +543,13 @@ Network .. note:: (Solaris) UNIX sockets are not supported. + .. note:: + (Linux) "raddr" field for UNIX sockets is always set to "". + + .. note:: + (OpenBSD) "laddr" and "raddr" fields for UNIX sockets are always set to + "". + .. versionadded:: 2.1.0 .. function:: net_if_addrs() @@ -1753,10 +1760,9 @@ Process class `__, `AF_INET6 `__ or `AF_UNIX `__. - - **type**: the address type, either `SOCK_STREAM - `__ or - `SOCK_DGRAM - `__. + - **type**: the address type, either + `SOCK_STREAM `__ or + `SOCK_DGRAM `__. - **laddr**: the local address as a ``(ip, port)`` tuple or a ``path`` in case of AF_UNIX sockets. - **raddr**: the remote address as a ``(ip, port)`` tuple or an absolute @@ -1810,6 +1816,13 @@ Process class pconn(fd=119, family=, type=, laddr=('10.0.0.1', 60759), raddr=('72.14.234.104', 80), status='ESTABLISHED'), pconn(fd=123, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT')] + .. note:: + (Linux) "raddr" field for UNIX sockets is always set to "". + + .. note:: + (OpenBSD) "laddr" and "raddr" fields for UNIX sockets are always set to + "". + .. method:: is_running() Return whether the current process is running in the current process list. diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index c0ccb4669..1a890a874 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -898,7 +898,10 @@ def process_unix(file, family, inodes, filter_pid=None): else: path = "" type_ = int(type_) - raddr = None + # XXX: determining the remote endpoint of a + # UNIX socket on Linux is not possible, see: + # https://serverfault.com/questions/252723/ + raddr = "" status = _common.CONN_NONE yield (fd, family, type_, path, raddr, status, pid) diff --git a/psutil/arch/bsd/openbsd.c b/psutil/arch/bsd/openbsd.c index c86b003c1..667abbfc9 100644 --- a/psutil/arch/bsd/openbsd.c +++ b/psutil/arch/bsd/openbsd.c @@ -479,15 +479,21 @@ psutil_inet6_addrstr(struct in6_addr *p) } +/* + * List process connections. + * Note: there is no net_connections() on OpenBSD. The Python + * implementation will iterate over all processes and use this + * function. + * Note: path cannot be determined for UNIX sockets. + */ PyObject * psutil_proc_connections(PyObject *self, PyObject *args) { long pid; - int i, cnt; - + int i; + int cnt; struct kinfo_file *freep = NULL; struct kinfo_file *kif; char *tcplist = NULL; - PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; PyObject *py_laddr = NULL; @@ -608,14 +614,16 @@ psutil_proc_connections(PyObject *self, PyObject *args) { goto error; Py_DECREF(py_tuple); } - // UNIX socket + // UNIX socket. + // XXX: "unp_path" is always an empty string; also "fstat" + // command is not able to show UNIX socket paths. else if (kif->so_family == AF_UNIX) { py_tuple = Py_BuildValue( "(iiisOi)", kif->fd_fd, kif->so_family, kif->so_type, - kif->unp_path, + "", // kif->unp_path Py_None, PSUTIL_CONN_NONE); if (!py_tuple) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 16d4f5a9b..dd95755f1 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -989,7 +989,7 @@ def check_connection_ntuple(conn): assert 0 <= port <= 65535, port check_net_address(ip, conn.family) elif conn.family == AF_UNIX: - assert isinstance(addr, (str, type(None))), addr + assert isinstance(addr, str), addr # check status assert isinstance(conn.status, str), conn From f3b2ab32da30c659a6806d7aead1c19f12980751 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 01:23:51 +0200 Subject: [PATCH 485/922] #1039 / freebsd / openbsd / connections('unix'): set laddr|raddr to '' instead of None --- HISTORY.rst | 5 ++++- docs/index.rst | 14 ++++++-------- psutil/arch/bsd/freebsd_socks.c | 8 ++++---- psutil/arch/bsd/openbsd.c | 13 +++++++------ psutil/tests/test_connections.py | 26 ++++++++++++++++++-------- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8de15018e..85092fa68 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -31,7 +31,10 @@ Process.connections() when retrieving UNIX sockets (kind='unix'). - 1039_: returned types consolidation: - Windows / Process.cpu_times(): fields #3 and #4 were int instead of float - - Linux / connections('unix'): raddr is now set to "" instead of None + - Linux / FreeBSD: connections('unix'): raddr is now set to "" instead of + None + - OpenBSD: connections('unix'): laddr and raddr are now set to "" instead of + None *2017-04-10* diff --git a/docs/index.rst b/docs/index.rst index 84c8ae403..394aeae86 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -476,12 +476,11 @@ Network `SOCK_DGRAM `__. - **laddr**: the local address as a ``(ip, port)`` tuple or a ``path`` - in case of AF_UNIX sockets. + in case of AF_UNIX sockets. For UNIX sockets see notes below. - **raddr**: the remote address as a ``(ip, port)`` tuple or an absolute ``path`` in case of UNIX sockets. When the remote endpoint is not connected you'll get an empty tuple - (AF_INET*) or ``None`` (AF_UNIX). - On Linux AF_UNIX sockets will always have this set to ``None``. + (AF_INET*) or ``""`` (AF_UNIX). For UNIX sockets see notes below. - **status**: represents the status of a TCP connection. The return value is one of the :data:`psutil.CONN_* ` constants (a string). @@ -544,7 +543,7 @@ Network (Solaris) UNIX sockets are not supported. .. note:: - (Linux) "raddr" field for UNIX sockets is always set to "". + (Linux, FreeBSD) "raddr" field for UNIX sockets is always set to "". .. note:: (OpenBSD) "laddr" and "raddr" fields for UNIX sockets are always set to @@ -1764,12 +1763,11 @@ Process class `SOCK_STREAM `__ or `SOCK_DGRAM `__. - **laddr**: the local address as a ``(ip, port)`` tuple or a ``path`` - in case of AF_UNIX sockets. + in case of AF_UNIX sockets. For UNIX sockets see notes below. - **raddr**: the remote address as a ``(ip, port)`` tuple or an absolute ``path`` in case of UNIX sockets. When the remote endpoint is not connected you'll get an empty tuple - (AF_INET) or ``None`` (AF_UNIX). - On Linux AF_UNIX sockets will always have this set to ``None``. + (AF_INET*) or ``""`` (AF_UNIX). For UNIX sockets see notes below. - **status**: represents the status of a TCP connection. The return value is one of the :data:`psutil.CONN_* ` constants. For UDP and UNIX sockets this is always going to be @@ -1817,7 +1815,7 @@ Process class pconn(fd=123, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT')] .. note:: - (Linux) "raddr" field for UNIX sockets is always set to "". + (Linux, FreeBSD) "raddr" field for UNIX sockets is always set to "". .. note:: (OpenBSD) "laddr" and "raddr" fields for UNIX sockets are always set to diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 7d216280d..521868ba9 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -586,7 +586,8 @@ psutil_proc_connections(PyObject *self, PyObject *args) { goto error; Py_DECREF(py_tuple); } - // UNIX socket + // UNIX socket. + // Note: remote path cannot be determined. else if (kif->kf_sock_domain == AF_UNIX) { struct sockaddr_un *sun; @@ -605,12 +606,12 @@ psutil_proc_connections(PyObject *self, PyObject *args) { goto error; py_tuple = Py_BuildValue( - "(iiiOOi)", + "(iiiOsi)", kif->kf_fd, kif->kf_sock_domain, kif->kf_sock_type, py_laddr, - Py_None, + "", // raddr can't be determined PSUTIL_CONN_NONE ); if (!py_tuple) @@ -619,7 +620,6 @@ psutil_proc_connections(PyObject *self, PyObject *args) { goto error; Py_DECREF(py_tuple); Py_DECREF(py_laddr); - Py_INCREF(Py_None); } } } diff --git a/psutil/arch/bsd/openbsd.c b/psutil/arch/bsd/openbsd.c index 667abbfc9..fa6edc02d 100644 --- a/psutil/arch/bsd/openbsd.c +++ b/psutil/arch/bsd/openbsd.c @@ -484,7 +484,7 @@ psutil_inet6_addrstr(struct in6_addr *p) * Note: there is no net_connections() on OpenBSD. The Python * implementation will iterate over all processes and use this * function. - * Note: path cannot be determined for UNIX sockets. + * Note: local and remote paths cannot be determined for UNIX sockets. */ PyObject * psutil_proc_connections(PyObject *self, PyObject *args) { @@ -615,16 +615,17 @@ psutil_proc_connections(PyObject *self, PyObject *args) { Py_DECREF(py_tuple); } // UNIX socket. - // XXX: "unp_path" is always an empty string; also "fstat" - // command is not able to show UNIX socket paths. + // XXX: local addr is supposed to be in "unp_path" but it + // always empty; also "fstat" command is not able to show + // UNIX socket paths. else if (kif->so_family == AF_UNIX) { py_tuple = Py_BuildValue( - "(iiisOi)", + "(iiissi)", kif->fd_fd, kif->so_family, kif->so_type, - "", // kif->unp_path - Py_None, + "", // laddr (kif->unp_path is empty) + "", // raddr PSUTIL_CONN_NONE); if (!py_tuple) goto error; diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index ea63af3cb..c47160c7f 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -19,6 +19,7 @@ from psutil import FREEBSD from psutil import LINUX from psutil import NETBSD +from psutil import OPENBSD from psutil import OSX from psutil import POSIX from psutil import SUNOS @@ -101,7 +102,11 @@ def check_socket(self, sock, conn=None): laddr = laddr.decode() if sock.family == AF_INET6: laddr = laddr[:2] - self.assertEqual(conn.laddr, laddr) + if sock.family == AF_UNIX and OPENBSD: + # No addresses are set for UNIX sockets on OpenBSD. + pass + else: + self.assertEqual(conn.laddr, laddr) # XXX Solaris can't retrieve system-wide UNIX sockets if not (SUNOS and sock.family == AF_UNIX): @@ -219,24 +224,29 @@ def test_unix(self): server, client = unix_socketpair(name) with nested(closing(server), closing(client)): cons = thisproc.connections(kind='unix') + assert not (cons[0].laddr and cons[0].raddr) + assert not (cons[1].laddr and cons[1].raddr) if NETBSD: # On NetBSD creating a UNIX socket will cause # a UNIX connection to /var/run/log. cons = [c for c in cons if c.raddr != '/var/run/log'] self.assertEqual(len(cons), 2) if LINUX or FREEBSD: - # On linux the remote path is never set. Test - # that at least ONE address has our path. - one = (cons[0].laddr or cons[0].raddr or - cons[1].laddr or cons[1].raddr) - self.assertEqual(one, name) + # remote path is never set + self.assertEqual(cons[0].raddr, "") + self.assertEqual(cons[1].raddr, "") + # one local address should though + self.assertEqual(name, cons[0].laddr or cons[1].laddr) + elif OPENBSD: + # No addresses whatsoever here. + for addr in (cons[0].laddr, cons[0].raddr, + cons[1].laddr, cons[1].raddr): + self.assertEqual(addr, "") else: # On other systems either the laddr or raddr # of both peers are set. self.assertEqual(cons[0].laddr or cons[1].laddr, name) self.assertEqual(cons[0].raddr or cons[1].raddr, name) - assert not (cons[0].laddr and cons[0].raddr) - assert not (cons[1].laddr and cons[1].raddr) @skip_on_access_denied(only_if=OSX) def test_combos(self): From 8f49ae37083db80b07081731dd5133bd4e93b8ea Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 01:34:20 +0200 Subject: [PATCH 486/922] skip failing tests on openbsd --- psutil/tests/test_unicode.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 7b492447a..6dc1e6032 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -65,6 +65,7 @@ from contextlib import closing from psutil import BSD +from psutil import OPENBSD from psutil import OSX from psutil import POSIX from psutil import WINDOWS @@ -207,7 +208,9 @@ def test_proc_connections(self): with closing(sock): conn = psutil.Process().connections('unix')[0] self.assertIsInstance(conn.laddr, str) - self.assertEqual(conn.laddr, name) + # AF_UNIX addr not set on OpenBSD + if not OPENBSD: + self.assertEqual(conn.laddr, name) @unittest.skipIf(not POSIX, "POSIX only") @skip_on_access_denied() @@ -229,9 +232,11 @@ def find_sock(cons): raise unittest.SkipTest("not supported") with closing(sock): cons = psutil.net_connections(kind='unix') - conn = find_sock(cons) - self.assertIsInstance(conn.laddr, str) - self.assertEqual(conn.laddr, name) + # AF_UNIX addr not set on OpenBSD + if not OPENBSD: + conn = find_sock(cons) + self.assertIsInstance(conn.laddr, str) + self.assertEqual(conn.laddr, name) def test_disk_usage(self): safe_mkdir(self.funky_name) From ec26725472a4c967322e912c529ad43cb863285d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 01:41:18 +0200 Subject: [PATCH 487/922] remove dead C unicode code on openbsd --- psutil/arch/bsd/openbsd.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/psutil/arch/bsd/openbsd.c b/psutil/arch/bsd/openbsd.c index fa6edc02d..8891c4611 100644 --- a/psutil/arch/bsd/openbsd.c +++ b/psutil/arch/bsd/openbsd.c @@ -203,11 +203,7 @@ psutil_get_cmdline(long pid) { goto error; for (p = argv; *p != NULL; p++) { -#if PY_MAJOR_VERSION >= 3 py_arg = PyUnicode_DecodeFSDefault(*p); -#else - py_arg = Py_BuildValue("s", *p); -#endif if (!py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) @@ -435,11 +431,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } -#if PY_MAJOR_VERSION >= 3 return PyUnicode_DecodeFSDefault(path); -#else - return Py_BuildValue("s", path); -#endif } From 861bb930f4fff1387bde28dd2222ea9b4bd01010 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 02:44:13 +0200 Subject: [PATCH 488/922] Add an 'expert' section in the CREDITS CC @0-wiz-0 @0-wiz-0 @landryb @whitlockjc @mrjefftang @wj32 @fbenkstein @wiggin15 --- CREDITS | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CREDITS b/CREDITS index 32d3d51c7..148c7bbc3 100644 --- a/CREDITS +++ b/CREDITS @@ -20,6 +20,25 @@ C: Italy E: g.rodola@gmail.com W: http://grodola.blogspot.com/ +Experts +======= + +Github usernames of people to CC on github when in need of help. + +- NetBSD: + - 0-wiz-0, Thomas Klausner + - ryoqun, Ryo Onodera +- OpenBSD: + - landryb, Landry Breuil +- OSX: + - whitlockjc, Jeremy Whitlock +- Windows: + - mrjefftang, Jeff Tang + - wj32, Wen Jia Liu + - fbenkstein, Frank Benkstein +- SunOS: + - wiggin15, Arnon Yaari + Contributors ============ From 9c6e126e57ae87eddc43cfcb96e655e0b88f5308 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 05:32:12 +0200 Subject: [PATCH 489/922] #1040: remove dead unicode C code --- psutil/_psutil_sunos.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 3e2262ffb..6a0471d8e 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -130,17 +130,19 @@ psutil_proc_name_and_args(PyObject *self, PyObject *args) { if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) return NULL; -#if PY_MAJOR_VERSION >= 3 + // TODO: probably have to Py_INCREF here. py_name = PyUnicode_DecodeFSDefault(info.pr_fname); if (!py_name) - return NULL; + goto error; py_args = PyUnicode_DecodeFSDefault(info.pr_psargs); if (!py_args) - return NULL; + goto error; return Py_BuildValue("OO", py_name, py_args); -#else - return Py_BuildValue("ss", info.pr_fname, info.pr_psargs); -#endif + +error: + Py_XDECREF(py_name); + Py_XDECREF(py_args); + return NULL; } From d1ddeb5b4ec34afa86f168243e25699234ac22b4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 05:38:03 +0200 Subject: [PATCH 490/922] #1040 / users() / sunos: fix unicode --- psutil/_psutil_sunos.c | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 6a0471d8e..ceb32cf5c 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -454,6 +454,9 @@ psutil_users(PyObject *self, PyObject *args) { struct utmpx *ut; PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; PyObject *py_user_proc = NULL; if (py_retlist == NULL) @@ -464,11 +467,20 @@ psutil_users(PyObject *self, PyObject *args) { py_user_proc = Py_True; else py_user_proc = Py_False; + py_username = PyUnicode_DecodeFSDefault(ut->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(ut->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(ut->ut_host); + if (! py_hostname) + goto error; py_tuple = Py_BuildValue( "(sssfOi)", - ut->ut_user, // username - ut->ut_line, // tty - ut->ut_host, // hostname + py_username, // username + py_tty, // tty + py_hostname, // hostname (float)ut->ut_tv.tv_sec, // tstamp py_user_proc, // (bool) user process ut->ut_pid // process id @@ -477,6 +489,9 @@ psutil_users(PyObject *self, PyObject *args) { goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); Py_DECREF(py_tuple); } endutent(); @@ -484,6 +499,9 @@ psutil_users(PyObject *self, PyObject *args) { return py_retlist; error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); if (ut != NULL) From f23d7b2a8cb2b91bebbad318d7bdb45357636e62 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 05:43:54 +0200 Subject: [PATCH 491/922] #1040 / disk_partitions() / sunos: fix unicode --- psutil/_psutil_sunos.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index ceb32cf5c..aaca1ffb8 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -518,8 +518,10 @@ static PyObject * psutil_disk_partitions(PyObject *self, PyObject *args) { FILE *file; struct mnttab mt; - PyObject *py_retlist = PyList_New(0); + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; @@ -531,23 +533,32 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { } while (getmntent(file, &mt) == 0) { + py_dev = PyUnicode_DecodeFSDefault(mt.mnt_special); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(mt.mnt_mountp); + if (! py_mountp) + goto error; py_tuple = Py_BuildValue( - "(ssss)", - mt.mnt_special, // device - mt.mnt_mountp, // mount point + "(OOss)", + py_dev, // device + py_mountp, // mount point mt.mnt_fstype, // fs type mt.mnt_mntopts); // options if (py_tuple == NULL) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_dev); + Py_DECREF(py_mountp); Py_DECREF(py_tuple); - } fclose(file); return py_retlist; error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); if (file != NULL) From b6cb551bb68d7985b8796370fd8ba856dbc2859e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 05:46:13 +0200 Subject: [PATCH 492/922] fix type --- psutil/_psutil_sunos.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index aaca1ffb8..dc61ce1a2 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -477,7 +477,7 @@ psutil_users(PyObject *self, PyObject *args) { if (! py_hostname) goto error; py_tuple = Py_BuildValue( - "(sssfOi)", + "(OOOfOi)", py_username, // username py_tty, // tty py_hostname, // hostname From 986fb8aeb82013bf9e18a0c006138f7d3032dfb7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 05:53:11 +0200 Subject: [PATCH 493/922] #1040 / memory_maps() / sunos: fix unicode --- psutil/_psutil_sunos.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index dc61ce1a2..8e783d59c 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -452,12 +452,12 @@ psutil_swap_mem(PyObject *self, PyObject *args) { static PyObject * psutil_users(PyObject *self, PyObject *args) { struct utmpx *ut; - PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; PyObject *py_username = NULL; PyObject *py_tty = NULL; PyObject *py_hostname = NULL; PyObject *py_user_proc = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; @@ -699,6 +699,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { const char *procfs_path; PyObject *py_tuple = NULL; + PyObject *py_path = NULL; PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) @@ -777,12 +778,15 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { } } + py_path = PyUnicode_DecodeFSDefault(name); + if (! py_path) + goto error; py_tuple = Py_BuildValue( - "iisslll", + "iisOlll", p->pr_vaddr, pr_addr_sz, perms, - name, + py_path, (long)p->pr_rss * p->pr_pagesize, (long)p->pr_anon * p->pr_pagesize, (long)p->pr_locked * p->pr_pagesize); @@ -790,6 +794,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_path); Py_DECREF(py_tuple); // increment pointer @@ -804,6 +809,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { if (fd != -1) close(fd); Py_XDECREF(py_tuple); + Py_XDECREF(py_path); Py_DECREF(py_retlist); if (xmap != NULL) free(xmap); From 85744f6db64f388ad2d2ceffcf3004d3372ca581 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 06:03:35 +0200 Subject: [PATCH 494/922] fix memleak --- psutil/_psutil_sunos.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 8e783d59c..c205a3ac5 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -123,6 +123,7 @@ psutil_proc_name_and_args(PyObject *self, PyObject *args) { const char *procfs_path; PyObject *py_name; PyObject *py_args; + PyObject *py_retlist; if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) return NULL; @@ -130,18 +131,23 @@ psutil_proc_name_and_args(PyObject *self, PyObject *args) { if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) return NULL; - // TODO: probably have to Py_INCREF here. py_name = PyUnicode_DecodeFSDefault(info.pr_fname); if (!py_name) goto error; py_args = PyUnicode_DecodeFSDefault(info.pr_psargs); if (!py_args) goto error; - return Py_BuildValue("OO", py_name, py_args); + py_retlist = Py_BuildValue("OO", py_name, py_args); + if (!py_retlist) + goto error; + Py_DECREF(py_name); + Py_DECREF(py_args); + return py_retlist; error: Py_XDECREF(py_name); Py_XDECREF(py_args); + Py_XDECREF(py_retlist); return NULL; } From bb60602631a0e1c7569455f97e2735a5a9e8fcfa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 04:08:51 +0200 Subject: [PATCH 495/922] #1040 / users() / osx: fix unicode --- psutil/_psutil_osx.c | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 536500f50..eac8d7fbb 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -240,7 +240,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { { return NULL; } - + return PyUnicode_DecodeFSDefault(pathinfo.pvi_cdir.vip_path); } @@ -1677,19 +1677,31 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { static PyObject * psutil_users(PyObject *self, PyObject *args) { struct utmpx *utx; - PyObject *py_retlist = PyList_New(0); + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; while ((utx = getutxent()) != NULL) { if (utx->ut_type != USER_PROCESS) continue; + py_username = PyUnicode_DecodeFSDefault(utx->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(utx->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(utx->ut_host); + if (! py_hostname) + goto error; py_tuple = Py_BuildValue( - "(sssfi)", - utx->ut_user, // username - utx->ut_line, // tty - utx->ut_host, // hostname + "(OOOfi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname (float)utx->ut_tv.tv_sec, // start time utx->ut_pid // process id ); @@ -1701,6 +1713,9 @@ psutil_users(PyObject *self, PyObject *args) { endutxent(); goto error; } + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); Py_DECREF(py_tuple); } @@ -1708,6 +1723,9 @@ psutil_users(PyObject *self, PyObject *args) { return py_retlist; error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); return NULL; From 3681cd7b4a404ad15c5117c78123bd0cdbb462e1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 04:14:57 +0200 Subject: [PATCH 496/922] =?UTF-8?q?#1040=20/=20disk=5F=C3=A8artitions()=20?= =?UTF-8?q?/=20osx:=20fix=20unicode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- psutil/_psutil_osx.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index eac8d7fbb..a831441a4 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -853,8 +853,10 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { uint64_t flags; char opts[400]; struct statfs *fs = NULL; - PyObject *py_retlist = PyList_New(0); + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; @@ -939,15 +941,24 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { if (flags & MNT_CMDFLAGS) strlcat(opts, ",cmdflags", sizeof(opts)); + py_dev = PyUnicode_DecodeFSDefault(fs[i].f_mntfromname); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(fs[i].f_mntonname); + if (! py_mountp) + goto error; py_tuple = Py_BuildValue( - "(ssss)", fs[i].f_mntfromname, // device - fs[i].f_mntonname, // mount point + "(OOss)", + py_dev, // device + py_mountp, // mount point fs[i].f_fstypename, // fs type opts); // options if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_dev); + Py_DECREF(py_mountp); Py_DECREF(py_tuple); } @@ -955,6 +966,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { return py_retlist; error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); if (fs != NULL) From ad579702c39bbe5d3c1f6d1c0b3f3091399e8e49 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 04:40:42 +0200 Subject: [PATCH 497/922] windows: fix battery tests --- psutil/tests/test_connections.py | 3 ++- psutil/tests/test_windows.py | 36 +++++++++++++++++--------------- scripts/internal/winmake.py | 7 +++++++ 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index c47160c7f..d0c5445a6 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -74,8 +74,9 @@ def get_conn_from_socck(self, sock): # so there may be more connections. return smap[sock.fileno()] else: - self.assertEqual(smap[sock.fileno()].fd, sock.fileno()) self.assertEqual(len(cons), 1) + if cons[0].fd != -1: + self.assertEqual(smap[sock.fileno()].fd, sock.fileno()) return cons[0] def check_socket(self, sock, conn=None): diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 776476526..677df34d4 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -33,7 +33,6 @@ from psutil.tests import APPVEYOR from psutil.tests import get_test_subprocess from psutil.tests import HAS_BATTERY -from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import mock from psutil.tests import reap_children from psutil.tests import retry_before_failing @@ -190,28 +189,31 @@ def test_net_if_stats(self): @unittest.skipIf(not WINDOWS, "WINDOWS only") class TestSensorsBattery(unittest.TestCase): - @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") + def test_has_battery(self): + if psutil.sensors_battery() is None: + w = wmi.WMI() + with self.assertRaises(IndexError): + w.query('select * from Win32_Battery')[0] + @unittest.skipIf(not HAS_BATTERY, "no battery") def test_percent(self): w = wmi.WMI() + battery_wmi = w.query('select * from Win32_Battery')[0] battery_psutil = psutil.sensors_battery() - if battery_psutil is None: - with self.assertRaises(IndexError): - w.query('select * from Win32_Battery')[0] - else: - battery_wmi = w.query('select * from Win32_Battery')[0] - if battery_psutil is None: - self.assertNot(battery_wmi.EstimatedChargeRemaining) - return + self.assertAlmostEqual( + battery_psutil.percent, battery_wmi.EstimatedChargeRemaining, + delta=1) - self.assertAlmostEqual( - battery_psutil.percent, battery_wmi.EstimatedChargeRemaining, - delta=1) - self.assertEqual( - battery_psutil.power_plugged, battery_wmi.BatteryStatus == 1) - - @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") @unittest.skipIf(not HAS_BATTERY, "no battery") + def test_power_plugged(self): + w = wmi.WMI() + battery_wmi = w.query('select * from Win32_Battery')[0] + battery_psutil = psutil.sensors_battery() + # Status codes: + # https://msdn.microsoft.com/en-us/library/aa394074(v=vs.85).aspx + self.assertEqual(battery_psutil.power_plugged, + battery_wmi.BatteryStatus == 2) + def test_battery_present(self): if win32api.GetPwrCapabilities()['SystemBatteriesPresent']: self.assertIsNotNone(psutil.sensors_battery()) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index c91399775..7b70404ad 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -353,6 +353,13 @@ def test_unicode(): sh("%s -m unittest -v psutil.tests.test_unicode" % PYTHON) +@cmd +def test_connections(): + """Run connections tests""" + install() + sh("%s -m unittest -v psutil.tests.test_connections" % PYTHON) + + @cmd def test_contracts(): """Run contracts tests""" From b47f72620fe547e53ff7ab1247509d07b586e39f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 04:43:00 +0200 Subject: [PATCH 498/922] windows: remove dupe test to check if there's a battery --- psutil/tests/test_windows.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 677df34d4..2a883132d 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -190,10 +190,10 @@ def test_net_if_stats(self): class TestSensorsBattery(unittest.TestCase): def test_has_battery(self): - if psutil.sensors_battery() is None: - w = wmi.WMI() - with self.assertRaises(IndexError): - w.query('select * from Win32_Battery')[0] + if win32api.GetPwrCapabilities()['SystemBatteriesPresent']: + self.assertIsNotNone(psutil.sensors_battery()) + else: + self.assertIsNone(psutil.sensors_battery()) @unittest.skipIf(not HAS_BATTERY, "no battery") def test_percent(self): @@ -214,12 +214,6 @@ def test_power_plugged(self): self.assertEqual(battery_psutil.power_plugged, battery_wmi.BatteryStatus == 2) - def test_battery_present(self): - if win32api.GetPwrCapabilities()['SystemBatteriesPresent']: - self.assertIsNotNone(psutil.sensors_battery()) - else: - self.assertIsNone(psutil.sensors_battery()) - def test_emulate_no_battery(self): with mock.patch("psutil._pswindows.cext.sensors_battery", return_value=(0, 128, 0, 0)) as m: From 79e30c23053a2eb9ecdea6a78cbd43e4cb2dfe91 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 04:54:27 +0200 Subject: [PATCH 499/922] #1040 disk_partitions() / linux: fix unicode --- psutil/_psutil_linux.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 9abe44e09..5c0907127 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -195,8 +195,10 @@ static PyObject * psutil_disk_partitions(PyObject *self, PyObject *args) { FILE *file = NULL; struct mntent *entry; - PyObject *py_retlist = PyList_New(0); + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; @@ -215,15 +217,23 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { PyErr_Format(PyExc_RuntimeError, "getmntent() syscall failed"); goto error; } - py_tuple = Py_BuildValue("(ssss)", - entry->mnt_fsname, // device - entry->mnt_dir, // mount point + py_dev = PyUnicode_DecodeFSDefault(entry->mnt_fsname); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(entry->mnt_dir); + if (! py_mountp) + goto error; + py_tuple = Py_BuildValue("(OOss)", + py_dev, // device + py_mountp, // mount point entry->mnt_type, // fs type entry->mnt_opts); // options if (! py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_dev); + Py_DECREF(py_mountp); Py_DECREF(py_tuple); } endmntent(file); @@ -232,6 +242,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { error: if (file != NULL) endmntent(file); + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); return NULL; From 753faf8488796ad5ed838c901788ea0fc8a4f57e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 04:57:08 +0200 Subject: [PATCH 500/922] #1040 users() / linux: fix unicode --- psutil/_psutil_linux.c | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 5c0907127..1a96fea08 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -451,6 +451,9 @@ psutil_users(PyObject *self, PyObject *args) { struct utmp *ut; PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; PyObject *py_user_proc = NULL; if (py_retlist == NULL) @@ -463,11 +466,20 @@ psutil_users(PyObject *self, PyObject *args) { py_user_proc = Py_True; else py_user_proc = Py_False; + py_username = PyUnicode_DecodeFSDefault(ut->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(ut->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(ut->ut_host); + if (! py_hostname) + goto error; py_tuple = Py_BuildValue( - "(sssfOi)", - ut->ut_user, // username - ut->ut_line, // tty - ut->ut_host, // hostname + "(OOOfOi)", + py_username, // username + py_tty, // tty + py_username, // hostname (float)ut->ut_tv.tv_sec, // tstamp py_user_proc, // (bool) user process ut->ut_pid // process id @@ -476,14 +488,19 @@ psutil_users(PyObject *self, PyObject *args) { goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); Py_DECREF(py_tuple); } endutent(); return py_retlist; error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); Py_XDECREF(py_tuple); - Py_XDECREF(py_user_proc); Py_DECREF(py_retlist); endutent(); return NULL; From e16005089ce4af80b92d66ef017bcfdfba81fc4e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 05:18:07 +0200 Subject: [PATCH 501/922] make.bad: add test_script cmd to quickly run a script on the fly --- scripts/internal/winmake.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 7b70404ad..82e99d961 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -379,6 +379,18 @@ def test_by_name(): sh("%s -m unittest -v %s" % (PYTHON, name)) +@cmd +def test_script(): + """Quick way to test a script""" + try: + print(sys.argv) + name = sys.argv[2] + except IndexError: + sys.exit('second arg missing') + install() + sh("%s %s" % (PYTHON, name)) + + @cmd def test_memleaks(): """Run memory leaks tests""" From a8a535f7c5bd41f7513acfc4843b78c2b48be333 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 06:12:06 +0200 Subject: [PATCH 502/922] tunr copyload_shared_lib into a ctx manager and refactor tests --- psutil/tests/__init__.py | 27 ++++++++++++++------- psutil/tests/test_process.py | 14 ++++------- psutil/tests/test_unicode.py | 47 +++++++++++++++++++----------------- 3 files changed, 48 insertions(+), 40 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index dd95755f1..3ccb30445 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -16,6 +16,7 @@ import errno import functools import os +import random import re import shutil import socket @@ -1019,13 +1020,21 @@ def is_namedtuple(x): return all(type(n) == str for n in f) -def copyload_shared_lib(src, dst_prefix=TESTFILE_PREFIX): - """Given an existing shared so / DLL library copies it in - another location and loads it via ctypes. - Return the new path. +@contextlib.contextmanager +def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): + """Ctx manager which picks up a random shared so/dll lib used + by this process, copies it in another location and loads it + in memory via ctypes. + Return the new absolutized path. """ - newpath = tempfile.mktemp(prefix=dst_prefix, - suffix=os.path.splitext(src)[1]) - shutil.copyfile(src, newpath) - ctypes.CDLL(newpath) - return newpath + ext = ".so" if POSIX else ".dll" + dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) + libs = [x.path for x in psutil.Process().memory_maps() + if os.path.normcase(x.path).endswith(ext)] + src = random.choice(libs) + try: + shutil.copyfile(src, dst) + ctypes.CDLL(dst) + yield os.path.realpath(dst) + finally: + safe_rmpath(dst) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 58dc6829b..86a784d2c 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -656,15 +656,11 @@ def test_memory_maps(self): @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") def test_memory_maps_lists_lib(self): - normcase = os.path.normcase - realpath = os.path.realpath - p = psutil.Process() - ext = ".so" if POSIX else ".dll" - old = [x.path for x in p.memory_maps() - if normcase(x.path).endswith(ext)][0] - new = realpath(normcase(copyload_shared_lib(old))) - newpaths = [realpath(normcase(x.path)) for x in p.memory_maps()] - self.assertIn(new, newpaths) + with copyload_shared_lib() as path: + # Make sure a newly loaded shared lib is listed. + libpaths = [os.path.realpath(os.path.normcase(x.path)) + for x in psutil.Process().memory_maps()] + self.assertIn(path, libpaths) def test_memory_percent(self): p = psutil.Process() diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 6dc1e6032..2599b2f8d 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -93,9 +93,9 @@ import psutil.tests -def can_deal_with_funky_name(name): +def subprocess_supports_unicode(name): """Return True if both the fs and the subprocess module can - deal with a funky file name. + deal with a unicode file name. """ if PY3: return True @@ -110,13 +110,22 @@ def can_deal_with_funky_name(name): return True +def ctypes_supports_unicode(name): + if PY3: + return True + try: + with copyload_shared_lib(): + pass + except UnicodeEncodeError: + return False + + +# An invalid unicode string. if PY3: INVALID_NAME = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( 'utf8', 'surrogateescape') else: INVALID_NAME = TESTFN + "f\xc0\x80" -UNICODE_OK = can_deal_with_funky_name(TESTFN_UNICODE) -INVALID_UNICODE_OK = can_deal_with_funky_name(INVALID_NAME) # =================================================================== @@ -244,27 +253,21 @@ def test_disk_usage(self): @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") def test_memory_maps(self): - normcase = os.path.normcase - realpath = os.path.realpath - p = psutil.Process() - ext = ".so" if POSIX else ".dll" - old = [realpath(x.path) for x in p.memory_maps() - if normcase(x.path).endswith(ext)][0] - try: - new = realpath(normcase( - copyload_shared_lib(old, dst_prefix=self.funky_name))) - except UnicodeEncodeError: - if PY3: - raise - else: - raise unittest.SkipTest("ctypes can't handle unicode") - newpaths = [realpath(normcase(x.path)) for x in p.memory_maps()] - self.assertIn(new, newpaths) + if not ctypes_supports_unicode(self.funky_name): + raise unittest.SkipTest("ctypes can't handle unicode") + + with copyload_shared_lib(dst_prefix=self.funky_name) as funky_path: + libpaths = [os.path.realpath(os.path.normcase(x.path)) + for x in psutil.Process().memory_maps()] + self.assertIn(funky_path, libpaths) + for path in libpaths: + self.assertIsInstance(path, str) @unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # TODO @unittest.skipIf(ASCII_FS, "ASCII fs") -@unittest.skipIf(not UNICODE_OK, "subprocess can't deal with unicode") +@unittest.skipIf(not subprocess_supports_unicode(TESTFN_UNICODE), + "subprocess can't deal with unicode") class TestFSAPIs(_BaseFSAPIsTests, unittest.TestCase): """Test FS APIs with a funky, valid, UTF8 path name.""" funky_name = TESTFN_UNICODE @@ -277,7 +280,7 @@ def expect_exact_path_match(cls): @unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # TODO -@unittest.skipIf(not INVALID_UNICODE_OK, +@unittest.skipIf(not subprocess_supports_unicode(INVALID_NAME), "subprocess can't deal with invalid unicode") class TestFSAPIsWithInvalidPath(_BaseFSAPIsTests, unittest.TestCase): """Test FS APIs with a funky, invalid path name.""" From 56b5aad8ae01d5f7f50d9f45d2faf57785293326 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 06:46:23 +0200 Subject: [PATCH 503/922] make.bat: add utility function which prints to console without producing encoding errors --- scripts/internal/winmake.py | 49 +++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 82e99d961..b88149a18 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -11,6 +11,7 @@ that they should be deemed illegal! """ +from __future__ import print_function import errno import fnmatch import functools @@ -44,15 +45,36 @@ "requests" ] _cmds = {} - +if PY3: + basestring = str # =================================================================== # utils # =================================================================== +def safe_print(text, file=sys.stdout, flush=False): + """Prints a (unicode) string to the console, encoded depending on + the stdout/file encoding (eg. cp437 on Windows). This is to avoid + encoding errors in case of funky path names. + Works with Python 2 and 3. + """ + if not isinstance(text, basestring): + return print(text, file=file, flush=flush) + try: + file.write(text) + except UnicodeEncodeError: + bytes_string = text.encode(file.encoding, 'backslashreplace') + if hasattr(file, 'buffer'): + file.buffer.write(bytes_string) + else: + text = bytes_string.decode(file.encoding, 'strict') + file.write(text) + file.write("\n") + + def sh(cmd): - print("cmd: " + cmd) + safe_print("cmd: " + cmd) code = os.system(cmd) if code: raise SystemExit @@ -76,7 +98,7 @@ def safe_remove(path): if err.errno != errno.ENOENT: raise else: - print("rm %s" % path) + safe_print("rm %s" % path) def safe_rmtree(path): def onerror(fun, path, excinfo): @@ -87,7 +109,7 @@ def onerror(fun, path, excinfo): existed = os.path.isdir(path) shutil.rmtree(path, onerror=onerror) if existed: - print("rmdir -f %s" % path) + safe_print("rmdir -f %s" % path) if "*" not in pattern: if directory: @@ -104,10 +126,10 @@ def onerror(fun, path, excinfo): for name in found: path = os.path.join(root, name) if directory: - print("rmdir -f %s" % path) + safe_print("rmdir -f %s" % path) safe_rmtree(path) else: - print("rm %s" % path) + safe_print("rm %s" % path) safe_remove(path) @@ -118,7 +140,7 @@ def safe_remove(path): if err.errno != errno.ENOENT: raise else: - print("rm %s" % path) + safe_print("rm %s" % path) def safe_rmtree(path): @@ -130,7 +152,7 @@ def onerror(fun, path, excinfo): existed = os.path.isdir(path) shutil.rmtree(path, onerror=onerror) if existed: - print("rmdir -f %s" % path) + safe_print("rmdir -f %s" % path) def recursive_rm(*patterns): @@ -157,9 +179,10 @@ def recursive_rm(*patterns): @cmd def help(): """Print this help""" - print('Run "make " where is one of:') + safe_print('Run "make " where is one of:') for name in sorted(_cmds): - print(" %-20s %s" % (name.replace('_', '-'), _cmds[name] or '')) + safe_print( + " %-20s %s" % (name.replace('_', '-'), _cmds[name] or '')) @cmd @@ -202,7 +225,7 @@ def install_pip(): else: ctx = None kw = dict(context=ctx) if ctx else {} - print("downloading %s" % GET_PIP_URL) + safe_print("downloading %s" % GET_PIP_URL) req = urlopen(GET_PIP_URL, **kw) data = req.read() @@ -371,7 +394,7 @@ def test_contracts(): def test_by_name(): """Run test by name""" try: - print(sys.argv) + safe_print(sys.argv) name = sys.argv[2] except IndexError: sys.exit('second arg missing') @@ -383,7 +406,7 @@ def test_by_name(): def test_script(): """Quick way to test a script""" try: - print(sys.argv) + safe_print(sys.argv) name = sys.argv[2] except IndexError: sys.exit('second arg missing') From 4c023838c0459f77e8d528ce31b2f99ff96944fc Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 07:10:46 +0200 Subject: [PATCH 504/922] refactor copyload_shared_lib --- psutil/tests/__init__.py | 11 +++++++---- psutil/tests/test_process.py | 8 +++++--- psutil/tests/test_unicode.py | 6 ++++-- scripts/internal/winmake.py | 2 +- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 3ccb30445..7c14680bb 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -1025,16 +1025,19 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): """Ctx manager which picks up a random shared so/dll lib used by this process, copies it in another location and loads it in memory via ctypes. - Return the new absolutized path. + Return the new absolutized, normcased path. """ ext = ".so" if POSIX else ".dll" dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() - if os.path.normcase(x.path).endswith(ext)] + if os.path.normcase(os.path.splitext(x.path)[1]) == ext] src = random.choice(libs) + cfile = None try: shutil.copyfile(src, dst) - ctypes.CDLL(dst) - yield os.path.realpath(dst) + cfile = ctypes.CDLL(dst) + yield dst finally: + if WINDOWS and cfile is not None: + ctypes.windll.kernel32.FreeLibrary(cfile._handle) safe_rmpath(dst) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 86a784d2c..7bb83b63e 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -656,11 +656,13 @@ def test_memory_maps(self): @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") def test_memory_maps_lists_lib(self): + # Make sure a newly loaded shared lib is listed. with copyload_shared_lib() as path: - # Make sure a newly loaded shared lib is listed. - libpaths = [os.path.realpath(os.path.normcase(x.path)) + def normpath(p): + return os.path.realpath(os.path.normcase(p)) + libpaths = [normpath(x.path) for x in psutil.Process().memory_maps()] - self.assertIn(path, libpaths) + self.assertIn(normpath(path), libpaths) def test_memory_percent(self): p = psutil.Process() diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 2599b2f8d..f67e14c20 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -257,9 +257,11 @@ def test_memory_maps(self): raise unittest.SkipTest("ctypes can't handle unicode") with copyload_shared_lib(dst_prefix=self.funky_name) as funky_path: - libpaths = [os.path.realpath(os.path.normcase(x.path)) + def normpath(p): + return os.path.realpath(os.path.normcase(p)) + libpaths = [normpath(x.path) for x in psutil.Process().memory_maps()] - self.assertIn(funky_path, libpaths) + self.assertIn(normpath(funky_path), libpaths) for path in libpaths: self.assertIsInstance(path, str) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index b88149a18..69d4d9724 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -60,7 +60,7 @@ def safe_print(text, file=sys.stdout, flush=False): Works with Python 2 and 3. """ if not isinstance(text, basestring): - return print(text, file=file, flush=flush) + return print(text, file=file) try: file.write(text) except UnicodeEncodeError: From 881c423129262cd20b4de803701e2bee46fcbe81 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 08:10:22 +0200 Subject: [PATCH 505/922] finally! fix the test bug which was causing wait_for_file() to hang sometimes --- psutil/tests/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 7c14680bb..47c09a87e 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -91,6 +91,7 @@ 'install_pip', 'install_test_deps', # fs utils 'chdir', 'safe_rmpath', 'create_exe', 'decode_path', 'encode_path', + 'unique_filename', # subprocesses 'pyrun', 'reap_children', 'get_test_subprocess', 'create_proc_children_pair', @@ -269,6 +270,7 @@ def create_proc_children_pair(): Return a (child, grandchild) tuple. The 2 processes are fully initialized and will live for 60 secs. """ + _TESTFN2 = os.path.basename(_TESTFN) + '2' # need to be relative s = textwrap.dedent("""\ import subprocess, os, sys, time PYTHON = os.path.realpath(sys.executable) @@ -279,10 +281,10 @@ def create_proc_children_pair(): s += "time.sleep(60);" subprocess.Popen([PYTHON, '-c', s]) time.sleep(60) - """ % _TESTFN) + """ % _TESTFN2) child1 = psutil.Process(pyrun(s).pid) - data = wait_for_file(_TESTFN, delete=False, empty=False) - os.remove(_TESTFN) + data = wait_for_file(_TESTFN2, delete=False, empty=False) + os.remove(_TESTFN2) child2_pid = int(data) _pids_started.add(child2_pid) child2 = psutil.Process(child2_pid) @@ -610,6 +612,10 @@ def create_exe(outpath, c_code=None): os.chmod(outpath, st.st_mode | stat.S_IEXEC) +def unique_filename(prefix=TESTFILE_PREFIX, suffix=""): + return tempfile.mktemp(prefix=prefix, suffix=suffix) + + # =================================================================== # --- testing # =================================================================== @@ -783,7 +789,7 @@ def unix_socket_path(suffix=""): and tries to delete it on exit. """ assert psutil.POSIX - path = tempfile.mktemp(prefix=TESTFILE_PREFIX, suffix=suffix) + path = unique_filename(suffix=suffix) try: yield path finally: From c3f4f8999158cf8a0ede96ff886c5b69adf46522 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 08:22:20 +0200 Subject: [PATCH 506/922] test subprocesses: clean them up in case of exception --- psutil/tests/__init__.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 47c09a87e..3cd3398c9 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -189,7 +189,7 @@ _subprocesses_started = set() _pids_started = set() -_testfiles = set() +_testfiles_created = set() # =================================================================== @@ -256,7 +256,11 @@ def get_test_subprocess(cmd=None, **kwds): cmd = [PYTHON, "-c", pyline] sproc = subprocess.Popen(cmd, **kwds) _subprocesses_started.add(sproc) - wait_for_file(_TESTFN, delete=True, empty=True) + try: + wait_for_file(_TESTFN, delete=True, empty=True) + except Exception: + reap_children() + raise else: sproc = subprocess.Popen(cmd, **kwds) _subprocesses_started.add(sproc) @@ -282,24 +286,27 @@ def create_proc_children_pair(): subprocess.Popen([PYTHON, '-c', s]) time.sleep(60) """ % _TESTFN2) - child1 = psutil.Process(pyrun(s).pid) - data = wait_for_file(_TESTFN2, delete=False, empty=False) - os.remove(_TESTFN2) - child2_pid = int(data) - _pids_started.add(child2_pid) - child2 = psutil.Process(child2_pid) - return (child1, child2) + subp = pyrun(s) + try: + child1 = psutil.Process(subp.pid) + data = wait_for_file(_TESTFN2, delete=False, empty=False) + os.remove(_TESTFN2) + child2_pid = int(data) + _pids_started.add(child2_pid) + child2 = psutil.Process(child2_pid) + return (child1, child2) + except Exception: + reap_children() + raise def pyrun(src): """Run python 'src' code in a separate interpreter. Returns a subprocess.Popen instance. """ - if PY3: - src = bytes(src, 'ascii') with tempfile.NamedTemporaryFile( - prefix=TESTFILE_PREFIX, delete=False) as f: - _testfiles.add(f.name) + prefix=TESTFILE_PREFIX, mode="wt", delete=False) as f: + _testfiles_created.add(f.name) f.write(src) f.flush() subp = get_test_subprocess([PYTHON, f.name], stdout=None, @@ -720,7 +727,7 @@ def cleanup(): safe_rmpath(name) except UnicodeEncodeError as exc: warn(exc) - for path in _testfiles: + for path in _testfiles_created: safe_rmpath(path) From a40c3187a491d22826a1cbcf576f94e00c9a7cc2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 08:38:39 +0200 Subject: [PATCH 507/922] move some functions out of the main test util module as they don't belong in there --- psutil/tests/__init__.py | 87 +++------------------------------------- psutil/tests/__main__.py | 76 +++++++++++++++++++++++++++++++++-- 2 files changed, 77 insertions(+), 86 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 3cd3398c9..771a3cc30 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -20,7 +20,6 @@ import re import shutil import socket -import ssl import stat import subprocess import sys @@ -34,11 +33,6 @@ from socket import SOCK_DGRAM from socket import SOCK_STREAM -try: - from urllib.request import urlopen # py3 -except ImportError: - from urllib2 import urlopen - import psutil from psutil import POSIX from psutil import WINDOWS @@ -82,7 +76,10 @@ "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", "HAS_SENSORS_BATTERY", "HAS_BATTERY""HAS_SENSORS_FANS", "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", - # classes + # subprocesses + 'pyrun', 'reap_children', 'get_test_subprocess', + 'create_proc_children_pair', + # threads 'ThreadTask' # test utils 'unittest', 'cleanup', 'skip_on_access_denied', 'skip_on_not_implemented', @@ -92,9 +89,6 @@ # fs utils 'chdir', 'safe_rmpath', 'create_exe', 'decode_path', 'encode_path', 'unique_filename', - # subprocesses - 'pyrun', 'reap_children', 'get_test_subprocess', - 'create_proc_children_pair', # os 'get_winver', 'get_kernel_version', # sync primitives @@ -149,7 +143,6 @@ # --- paths -HERE = os.path.abspath(os.path.dirname(__file__)) ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') @@ -175,25 +168,16 @@ DEVNULL = open(os.devnull, 'r+') VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) if x.startswith('STATUS_')] -GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" AF_UNIX = getattr(socket, "AF_UNIX", object()) SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) -TEST_DEPS = [] -if sys.version_info[:2] == (2, 6): - TEST_DEPS.extend(["ipaddress", "unittest2", "argparse", "mock==1.0.1"]) -elif sys.version_info[:2] == (2, 7) or sys.version_info[:2] <= (3, 2): - TEST_DEPS.extend(["ipaddress", "mock"]) -elif sys.version_info[:2] == (3, 3): - TEST_DEPS.extend(["ipaddress"]) - _subprocesses_started = set() _pids_started = set() _testfiles_created = set() # =================================================================== -# --- classes +# --- threads # =================================================================== @@ -647,25 +631,6 @@ def __str__(self): unittest.TestCase = TestCase -def get_suite(): - testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) - if x.endswith('.py') and x.startswith('test_') and not - x.startswith('test_memory_leaks')] - suite = unittest.TestSuite() - for tm in testmodules: - # ...so that the full test paths are printed on screen - tm = "psutil.tests.%s" % tm - suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) - return suite - - -def run_suite(): - """Run unit tests.""" - result = unittest.TextTestRunner(verbosity=VERBOSITY).run(get_suite()) - success = result.wasSuccessful() - sys.exit(0 if success else 1) - - def run_test_module_by_name(name): # testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) # if x.endswith('.py') and x.startswith('test_')] @@ -735,48 +700,6 @@ def cleanup(): atexit.register(lambda: DEVNULL.close()) -# =================================================================== -# --- install -# =================================================================== - - -def install_pip(): - """Install pip. Returns the exit code of the subprocess.""" - try: - import pip # NOQA - except ImportError: - f = tempfile.NamedTemporaryFile(suffix='.py') - with contextlib.closing(f): - print("downloading %s to %s" % (GET_PIP_URL, f.name)) - if hasattr(ssl, '_create_unverified_context'): - ctx = ssl._create_unverified_context() - else: - ctx = None - kwargs = dict(context=ctx) if ctx else {} - req = urlopen(GET_PIP_URL, **kwargs) - data = req.read() - f.write(data) - f.flush() - - print("installing pip") - code = os.system('%s %s --user' % (sys.executable, f.name)) - return code - - -def install_test_deps(deps=None): - """Install test dependencies via pip.""" - if deps is None: - deps = TEST_DEPS - deps = set(deps) - if deps: - is_venv = hasattr(sys, 'real_prefix') - opts = "--user" if not is_venv else "" - install_pip() - code = os.system('%s -m pip install %s --upgrade %s' % ( - sys.executable, opts, " ".join(deps))) - return code - - # =================================================================== # --- network # =================================================================== diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py index b57914d48..896b00cc7 100755 --- a/psutil/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -10,17 +10,85 @@ $ python -m psutil.tests """ +import contextlib import optparse import os +import ssl import sys +import tempfile +try: + from urllib.request import urlopen # py3 +except ImportError: + from urllib2 import urlopen -from psutil.tests import install_pip -from psutil.tests import install_test_deps -from psutil.tests import run_suite -from psutil.tests import TEST_DEPS +from psutil.tests import unittest +from psutil.tests import VERBOSITY +HERE = os.path.abspath(os.path.dirname(__file__)) PYTHON = os.path.basename(sys.executable) +GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" +TEST_DEPS = [] +if sys.version_info[:2] == (2, 6): + TEST_DEPS.extend(["ipaddress", "unittest2", "argparse", "mock==1.0.1"]) +elif sys.version_info[:2] == (2, 7) or sys.version_info[:2] <= (3, 2): + TEST_DEPS.extend(["ipaddress", "mock"]) +elif sys.version_info[:2] == (3, 3): + TEST_DEPS.extend(["ipaddress"]) + + +def install_pip(): + try: + import pip # NOQA + except ImportError: + f = tempfile.NamedTemporaryFile(suffix='.py') + with contextlib.closing(f): + print("downloading %s to %s" % (GET_PIP_URL, f.name)) + if hasattr(ssl, '_create_unverified_context'): + ctx = ssl._create_unverified_context() + else: + ctx = None + kwargs = dict(context=ctx) if ctx else {} + req = urlopen(GET_PIP_URL, **kwargs) + data = req.read() + f.write(data) + f.flush() + + print("installing pip") + code = os.system('%s %s --user' % (sys.executable, f.name)) + return code + + +def install_test_deps(deps=None): + """Install test dependencies via pip.""" + if deps is None: + deps = TEST_DEPS + deps = set(deps) + if deps: + is_venv = hasattr(sys, 'real_prefix') + opts = "--user" if not is_venv else "" + install_pip() + code = os.system('%s -m pip install %s --upgrade %s' % ( + sys.executable, opts, " ".join(deps))) + return code + + +def get_suite(): + testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) + if x.endswith('.py') and x.startswith('test_') and not + x.startswith('test_memory_leaks')] + suite = unittest.TestSuite() + for tm in testmodules: + # ...so that the full test paths are printed on screen + tm = "psutil.tests.%s" % tm + suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) + return suite + + +def run_suite(): + result = unittest.TextTestRunner(verbosity=VERBOSITY).run(get_suite()) + success = result.wasSuccessful() + sys.exit(0 if success else 1) def main(): From 275dc089ab04cb86647eb56d405384bc00ebea86 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 08:49:00 +0200 Subject: [PATCH 508/922] clever atexit logic --- psutil/tests/__init__.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 771a3cc30..f1e0903e6 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -145,6 +145,7 @@ ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') +_HERE = os.path.abspath(os.path.dirname(__file__)) # --- support @@ -176,6 +177,19 @@ _testfiles_created = set() +@atexit.register +def _cleanup(): + DEVNULL.close() + for name in os.listdir(_HERE): + if name.startswith(TESTFILE_PREFIX): + _testfiles_created.add(name) + for name in _testfiles_created: + try: + safe_rmpath(name) + except UnicodeEncodeError as exc: + warn(exc) + + # =================================================================== # --- threads # =================================================================== @@ -226,7 +240,7 @@ def get_test_subprocess(cmd=None, **kwds): """Creates a python subprocess which does nothing for 60 secs and return it as subprocess.Popen instance. If "cmd" is specified that is used instead of python. - By default stdout and stderr are redirected to /dev/null. + By default sdting and stderr are redirected to /dev/null. It also attemps to make sure the process is in a reasonably initialized state. """ @@ -685,21 +699,6 @@ def wrapper(*args, **kwargs): return decorator -def cleanup(): - for name in os.listdir('.'): - if name.startswith(TESTFILE_PREFIX): - try: - safe_rmpath(name) - except UnicodeEncodeError as exc: - warn(exc) - for path in _testfiles_created: - safe_rmpath(path) - - -atexit.register(cleanup) -atexit.register(lambda: DEVNULL.close()) - - # =================================================================== # --- network # =================================================================== From 1eba9d0515e2528f504d0d7e277f6bf39cdc8e2e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 09:29:52 +0200 Subject: [PATCH 509/922] test utils refaactoring --- psutil/tests/__init__.py | 65 ++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index f1e0903e6..d1f1fdef3 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -178,7 +178,7 @@ @atexit.register -def _cleanup(): +def _cleanup_files(): DEVNULL.close() for name in os.listdir(_HERE): if name.startswith(TESTFILE_PREFIX): @@ -190,6 +190,12 @@ def _cleanup(): warn(exc) +# this is executed first +@atexit.register +def _cleanup_procs(): + reap_children(recursive=True) + + # =================================================================== # --- threads # =================================================================== @@ -237,12 +243,15 @@ def stop(self): def get_test_subprocess(cmd=None, **kwds): - """Creates a python subprocess which does nothing for 60 secs and + """Create a python subprocess which does nothing for 60 secs and return it as subprocess.Popen instance. + If "cmd" is specified that is used instead of python. - By default sdting and stderr are redirected to /dev/null. + By default sdtin and stdout are redirected to /dev/null. It also attemps to make sure the process is in a reasonably initialized state. + + The caller is supposed to clean this up with reap_children(). """ kwds.setdefault("stdin", DEVNULL) kwds.setdefault("stdout", DEVNULL) @@ -271,6 +280,8 @@ def create_proc_children_pair(): A (us) -> B (child) -> C (grandchild). Return a (child, grandchild) tuple. The 2 processes are fully initialized and will live for 60 secs. + + The caller is supposed to clean them up with reap_children(). """ _TESTFN2 = os.path.basename(_TESTFN) + '2' # need to be relative s = textwrap.dedent("""\ @@ -299,8 +310,10 @@ def create_proc_children_pair(): def pyrun(src): - """Run python 'src' code in a separate interpreter. - Returns a subprocess.Popen instance. + """Run python 'src' code (a string) in a separate interpreter + and return it as a subprocess.Popen instance. + + The caller is supposed to clean this up with reap_children(). """ with tempfile.NamedTemporaryFile( prefix=TESTFILE_PREFIX, mode="wt", delete=False) as f: @@ -313,29 +326,37 @@ def pyrun(src): return subp -def sh(cmd): - """run cmd in a subprocess and return its output. +def sh(cmd, **kwargs): + """run cmd as subprocess.Popen and return its output. raises RuntimeError on error. """ - shell = True if isinstance(cmd, (str, unicode)) else False - p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True) - stdout, stderr = p.communicate() - if p.returncode != 0: - raise RuntimeError(stderr) - if stderr: - warn(stderr) - if stdout.endswith('\n'): - stdout = stdout[:-1] - return stdout + kwargs.setdefault("stdout", subprocess.PIPE) + kwargs.setdefault("stderr", subprocess.PIPE) + kwargs.setdefault( + "shell", True if isinstance(cmd, (str, unicode)) else False) + kwargs.setdefault("universal_newlines", True) + p = subprocess.Popen(cmd, **kwargs) + _subprocesses_started.add(p) + try: + stdout, stderr = p.communicate() + if p.returncode != 0: + raise RuntimeError(stderr) + if stderr: + warn(stderr) + if stdout.endswith('\n'): + stdout = stdout[:-1] + return stdout + except Exception: + reap_children() + raise def reap_children(recursive=False): - """Terminate and wait() any subprocess started by this test suite - and ensure that no zombies stick around to hog resources and - create problems when looking for refleaks. + """Terminate and wait() ANY subprocess which was started by this + test suite and ensure that no zombies stick around to hog resources + and create problems when looking for refleaks. - If resursive is True it also tries to terminate and wait() + If "esursive" is True it also tries hard to terminate and wait() all grandchildren started by this process. """ # Get the children here, before terminating the children sub From 51f37a8aab7a0d34231e945b9eeea12de5761fbb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 18:55:21 +0200 Subject: [PATCH 510/922] register testfiles which are created during tests so that they can be cleanup up more easily --- psutil/tests/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index d1f1fdef3..2db667edf 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -27,6 +27,7 @@ import textwrap import threading import time +import traceback import warnings from socket import AF_INET from socket import AF_INET6 @@ -174,7 +175,7 @@ _subprocesses_started = set() _pids_started = set() -_testfiles_created = set() +_testfiles = set() @atexit.register @@ -182,12 +183,12 @@ def _cleanup_files(): DEVNULL.close() for name in os.listdir(_HERE): if name.startswith(TESTFILE_PREFIX): - _testfiles_created.add(name) - for name in _testfiles_created: + _testfiles.add(name) + for name in _testfiles: try: safe_rmpath(name) - except UnicodeEncodeError as exc: - warn(exc) + except Exception: + traceback.print_exc() # this is executed first @@ -284,6 +285,7 @@ def create_proc_children_pair(): The caller is supposed to clean them up with reap_children(). """ _TESTFN2 = os.path.basename(_TESTFN) + '2' # need to be relative + _testfiles.add(_TESTFN2) s = textwrap.dedent("""\ import subprocess, os, sys, time PYTHON = os.path.realpath(sys.executable) @@ -306,6 +308,7 @@ def create_proc_children_pair(): return (child1, child2) except Exception: reap_children() + safe_rmpath(_TESTFN2) raise @@ -317,7 +320,7 @@ def pyrun(src): """ with tempfile.NamedTemporaryFile( prefix=TESTFILE_PREFIX, mode="wt", delete=False) as f: - _testfiles_created.add(f.name) + _testfiles.add(f.name) f.write(src) f.flush() subp = get_test_subprocess([PYTHON, f.name], stdout=None, @@ -639,8 +642,9 @@ def create_exe(outpath, c_code=None): def unique_filename(prefix=TESTFILE_PREFIX, suffix=""): - return tempfile.mktemp(prefix=prefix, suffix=suffix) - + ret = tempfile.mktemp(prefix=prefix, suffix=suffix) + _testfiles.add(ret) + return ret # =================================================================== # --- testing From 743fc935a325370d5cb86e64c18317800276f4ae Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 2 May 2017 20:56:29 +0200 Subject: [PATCH 511/922] create_exe(): change signature --- psutil/tests/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 2db667edf..e7e6f5432 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -611,13 +611,13 @@ def chdir(dirname): os.chdir(curdir) -def create_exe(outpath, c_code=None): +def create_exe(outpath, use_gcc=False, c_code=""): """Creates an executable file in the given location.""" assert not os.path.exists(outpath), outpath - if c_code: + if use_gcc or c_code: if not which("gcc"): raise ValueError("gcc is not installed") - if c_code is None: + if not c_code: c_code = textwrap.dedent( """ #include From d3e41552788cd393170769f7a3887d7aeebb5f2a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 00:08:40 +0200 Subject: [PATCH 512/922] clearer msg in case of failure --- psutil/tests/test_unicode.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index f67e14c20..cf1c02af6 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -261,6 +261,8 @@ def normpath(p): return os.path.realpath(os.path.normcase(p)) libpaths = [normpath(x.path) for x in psutil.Process().memory_maps()] + # ...just to have a clearer msg in case of failure + libpaths = [x for x in libpaths if TESTFILE_PREFIX in x] self.assertIn(normpath(funky_path), libpaths) for path in libpaths: self.assertIsInstance(path, str) From 721dfe2598c8c4d0f2c5bbfdbe5d88d68d6fd0ec Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 00:38:16 +0200 Subject: [PATCH 513/922] avoid to load wow64 dlls as they raise WindowsError --- psutil/tests/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 771a3cc30..c813ce463 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -967,6 +967,8 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() if os.path.normcase(os.path.splitext(x.path)[1]) == ext] + if WINDOWS: + libs = [x for x in libs if 'wow64' not in x.lower()] src = random.choice(libs) cfile = None try: From 4c07213c20840325eec14c24182d64b5abaae2d0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 00:52:48 +0200 Subject: [PATCH 514/922] #fix 1046: reset user SetErrorMode instead of resetting it to sysstem default --- HISTORY.rst | 1 + psutil/_psutil_windows.c | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 85092fa68..94126cd32 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -35,6 +35,7 @@ None - OpenBSD: connections('unix'): laddr and raddr are now set to "" instead of None +- 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. *2017-04-10* diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index b1a629774..1c742afb2 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2447,6 +2447,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { int all; int type; int ret; + unsigned int old_mode = 0; char opts[20]; LPTSTR fs_type[MAX_PATH + 1] = { 0 }; DWORD pflags = 0; @@ -2460,7 +2461,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { // avoid to visualize a message box in case something goes wrong // see https://github.com/giampaolo/psutil/issues/264 - SetErrorMode(SEM_FAILCRITICALERRORS); + old_mode = SetErrorMode(SEM_FAILCRITICALERRORS); if (! PyArg_ParseTuple(args, "O", &py_all)) goto error; @@ -2541,11 +2542,11 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { drive_letter = strchr(drive_letter, 0) + 1; } - SetErrorMode(0); + SetErrorMode(old_mode); return py_retlist; error: - SetErrorMode(0); + SetErrorMode(old_mode); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); return NULL; From 68588c3f97d7609c1e8feb6da24d97d06dfbaed0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 00:57:42 +0200 Subject: [PATCH 515/922] skip memory_maps unicode test: ctypes opens a message box which blocks the test suite --- psutil/tests/test_unicode.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index cf1c02af6..ce881eac6 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -110,16 +110,6 @@ def subprocess_supports_unicode(name): return True -def ctypes_supports_unicode(name): - if PY3: - return True - try: - with copyload_shared_lib(): - pass - except UnicodeEncodeError: - return False - - # An invalid unicode string. if PY3: INVALID_NAME = (TESTFN.encode('utf8') + b"f\xc0\x80").decode( @@ -252,10 +242,10 @@ def test_disk_usage(self): psutil.disk_usage(self.funky_name) @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") + @unittest.skipIf(not PY3, "ctypes opens err msg box on Python 2") def test_memory_maps(self): - if not ctypes_supports_unicode(self.funky_name): - raise unittest.SkipTest("ctypes can't handle unicode") - + # XXX: on Python 2, using ctypes.CDLL with a unicode path + # opens a message box which blocks the test run. with copyload_shared_lib(dst_prefix=self.funky_name) as funky_path: def normpath(p): return os.path.realpath(os.path.normcase(p)) From b4b3892f42bfaed40d99a729456a560bb69f2600 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 01:18:20 +0200 Subject: [PATCH 516/922] windows: use trick to avoid creating error boxes on subprocess.Popen --- psutil/tests/__init__.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index c813ce463..987907e53 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -232,6 +232,9 @@ def get_test_subprocess(cmd=None, **kwds): """ kwds.setdefault("stdin", DEVNULL) kwds.setdefault("stdout", DEVNULL) + if WINDOWS: + # avoid creating error boxes + kwds.setdefault("creationflags", 0x8000000) # CREATE_NO_WINDOW if cmd is None: safe_rmpath(_TESTFN) pyline = "from time import sleep;" @@ -304,8 +307,11 @@ def sh(cmd): raises RuntimeError on error. """ shell = True if isinstance(cmd, (str, unicode)) else False + # avoid creating error boxes on windows + creationflags = 0x8000000 if WINDOWS else 0 p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True) + stderr=subprocess.PIPE, universal_newlines=True, + creationflags=creationflags) stdout, stderr = p.communicate() if p.returncode != 0: raise RuntimeError(stderr) From 36fe8dacf24098502fb782cfc4b05ccc24734d13 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 01:45:29 +0200 Subject: [PATCH 517/922] actualy passing CREATE_NO_WINDOWS to Popen makes test_children fail because it creates an extra subprocess for some reason --- psutil/tests/__init__.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 987907e53..c813ce463 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -232,9 +232,6 @@ def get_test_subprocess(cmd=None, **kwds): """ kwds.setdefault("stdin", DEVNULL) kwds.setdefault("stdout", DEVNULL) - if WINDOWS: - # avoid creating error boxes - kwds.setdefault("creationflags", 0x8000000) # CREATE_NO_WINDOW if cmd is None: safe_rmpath(_TESTFN) pyline = "from time import sleep;" @@ -307,11 +304,8 @@ def sh(cmd): raises RuntimeError on error. """ shell = True if isinstance(cmd, (str, unicode)) else False - # avoid creating error boxes on windows - creationflags = 0x8000000 if WINDOWS else 0 p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True, - creationflags=creationflags) + stderr=subprocess.PIPE, universal_newlines=True) stdout, stderr = p.communicate() if p.returncode != 0: raise RuntimeError(stderr) From 530ba0677c5feef27eb5af054af54e80d20392f3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 01:53:44 +0200 Subject: [PATCH 518/922] fix ionice test on windows + update ionice doc --- docs/index.rst | 3 ++- psutil/tests/test_contracts.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 394aeae86..16301958f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1245,7 +1245,8 @@ Process class >>> On Windows only *ioclass* is used and it can be set to ``2`` (normal), - ``1`` (low) or ``0`` (very low). + ``1`` (low) or ``0`` (very low). Also it returns an integer instead of a + named tuple. Availability: Linux and Windows > Vista diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index f44b7a41e..d3c0377a5 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -430,9 +430,10 @@ def io_counters(self, ret, proc): self.assertGreaterEqual(field, 0) def ionice(self, ret, proc): - assert is_namedtuple(ret) - for field in ret: - self.assertIsInstance(field, int) + if POSIX: + assert is_namedtuple(ret) + for field in ret: + self.assertIsInstance(field, int) if LINUX: self.assertGreaterEqual(ret.ioclass, 0) self.assertGreaterEqual(ret.value, 0) From ace8d289f77febf5824a7c3bff26f3c9e10f0dae Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 04:03:45 +0200 Subject: [PATCH 519/922] re #1040: bif one win/ unicode / memory_maps() Fix memory_maps() which was returning an invalid encoded path in case of non ASCII path on both Python 2 and 3. Use GetMappedFileNameW instead of GetMappedFilenameA in order to ask the system an actual unicode path. Also, on Windows encode unicode back to str/bytes by using default fs-encoding + "replace" error handler. This paves the way for fixing other APIs in an identical manner. --- psutil/_psutil_windows.c | 15 ++++----------- psutil/_pswindows.py | 8 +++++++- psutil/tests/test_misc.py | 1 + 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 1c742afb2..c55aedd73 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2855,7 +2855,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { HANDLE hProcess = NULL; PVOID baseAddress; PVOID previousAllocationBase; - CHAR mappedFileName[MAX_PATH]; + LPWSTR mappedFileName[MAX_PATH]; SYSTEM_INFO system_info; LPVOID maxAddr; PyObject *py_retlist = PyList_New(0); @@ -2881,20 +2881,13 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { py_tuple = NULL; if (baseAddress > maxAddr) break; - if (GetMappedFileNameA(hProcess, baseAddress, mappedFileName, + if (GetMappedFileNameW(hProcess, baseAddress, mappedFileName, sizeof(mappedFileName))) { - -#if PY_MAJOR_VERSION >= 3 - py_str = PyUnicode_Decode( - mappedFileName, _tcslen(mappedFileName), - Py_FileSystemDefaultEncoding, "surrogateescape"); -#else - py_str = Py_BuildValue("s", mappedFileName); -#endif + py_str = PyUnicode_FromWideChar(mappedFileName, + wcslen(mappedFileName)); if (py_str == NULL) goto error; - #ifdef _WIN64 py_tuple = Py_BuildValue( "(KsOI)", diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 54b094b39..67fdcc877 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -36,11 +36,12 @@ from ._common import parse_environ_block from ._common import sockfam_to_enum from ._common import socktype_to_enum -from ._common import usage_percent from ._common import memoize_when_activated +from ._common import usage_percent from ._compat import long from ._compat import lru_cache from ._compat import PY3 +from ._compat import unicode from ._compat import xrange from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS @@ -754,6 +755,11 @@ def memory_maps(self): raise else: for addr, perm, path, rss in raw: + # TODO: refactor + assert isinstance(path, unicode), path + if not PY3: + path = path.encode( + sys.getfilesystemencoding(), errors='replace') path = convert_dos_path(path) addr = hex(addr) yield (addr, perm, path, rss) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 29012cd49..272253b61 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -503,6 +503,7 @@ def test_fans(self): self.assert_stdout('fans.py') @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") + @unittest.skipIf(TRAVIS, "not battery on TRAVIS") def test_battery(self): self.assert_stdout('battery.py') From 2d1028c7d4770a83937e9b80e9d25ef2774961b2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 04:59:05 +0200 Subject: [PATCH 520/922] #1040: have net_if_stats() and proc environ() return str instead of unicode on python 2 --- psutil/_pswindows.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 67fdcc877..3d54b86c2 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -71,7 +71,8 @@ # --- globals # ===================================================================== - +FS_ENCODING = sys.getfilesystemencoding() +PY2_ENCODING_ERRS = "replace" CONN_DELETE_TCB = "DELETE_TCB" WAIT_TIMEOUT = 0x00000102 # 258 in decimal ACCESS_DENIED_SET = frozenset([errno.EPERM, errno.EACCES, @@ -189,19 +190,17 @@ def convert_dos_path(s): return os.path.join(driveletter, s[len(rawdrive):]) -def py2_strencode(s, encoding=sys.getfilesystemencoding()): +def py2_strencode(s): """Encode a string in the given encoding. Falls back on returning the string as is if it can't be encoded. """ - if PY3 or isinstance(s, str): + if PY3: return s else: - try: - return s.encode(encoding) - except UnicodeEncodeError: - # Filesystem codec failed, return the plain unicode - # string (this should never happen). + if isinstance(s, str): return s + else: + return s.encode(FS_ENCODING, errors=PY2_ENCODING_ERRS) # ===================================================================== @@ -342,9 +341,12 @@ def net_connections(kind, _pid=-1): def net_if_stats(): """Get NIC stats (isup, duplex, speed, mtu).""" - ret = cext.net_if_stats() - for name, items in ret.items(): - name = py2_strencode(name) + ret = {} + rawdict = cext.net_if_stats() + for name, items in rawdict.items(): + assert isinstance(name, unicode), name + if not PY3: + name = py2_strencode(name) isup, duplex, speed, mtu = items if hasattr(_common, 'NicDuplex'): duplex = _common.NicDuplex(duplex) @@ -696,7 +698,10 @@ def cmdline(self): @wrap_exceptions def environ(self): - return parse_environ_block(cext.proc_environ(self.pid)) + ustr = cext.proc_environ(self.pid) + if ustr: + assert isinstance(ustr, unicode), ustr + return parse_environ_block(py2_strencode(ustr)) def ppid(self): try: @@ -755,11 +760,9 @@ def memory_maps(self): raise else: for addr, perm, path, rss in raw: - # TODO: refactor assert isinstance(path, unicode), path if not PY3: - path = path.encode( - sys.getfilesystemencoding(), errors='replace') + path = py2_strencode(path) path = convert_dos_path(path) addr = hex(addr) yield (addr, perm, path, rss) From a655055d198dd2f3cfb160a23173f1767b36d7aa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 05:20:21 +0200 Subject: [PATCH 521/922] #1040: use default fs encoding instead of utf8 when decoding from ubytes to unicode --- psutil/_pswindows.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 3d54b86c2..266c12200 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -184,7 +184,9 @@ def convert_dos_path(s): "C:\Windows\systemew\file.txt" """ if PY3 and not isinstance(s, str): - s = s.decode('utf8') + # TODO: probably getting here means there's something wrong; + # probably needs to be removed. + s = s.decode(FS_ENCODING, errors=PY2_ENCODING_ERRS) rawdrive = '\\'.join(s.split('\\')[:3]) driveletter = cext.win32_QueryDosDevice(rawdrive) return os.path.join(driveletter, s[len(rawdrive):]) @@ -761,9 +763,9 @@ def memory_maps(self): else: for addr, perm, path, rss in raw: assert isinstance(path, unicode), path + path = convert_dos_path(path) if not PY3: path = py2_strencode(path) - path = convert_dos_path(path) addr = hex(addr) yield (addr, perm, path, rss) From f0c0373ea67d5e42a7553fb27fac3df823e62655 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 06:05:32 +0200 Subject: [PATCH 522/922] fix #1047: potential memory leak of process username() on windows --- HISTORY.rst | 1 + psutil/_psutil_windows.c | 86 +++++++++++++++++++++++---------------- psutil/tests/test_misc.py | 1 + 3 files changed, 54 insertions(+), 34 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 94126cd32..67213b2ca 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -36,6 +36,7 @@ - OpenBSD: connections('unix'): laddr and raddr are now set to "" instead of None - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. +- 1047_: [Windows] Process username(): memory leak in case exception is thrown. *2017-04-10* diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 1c742afb2..47376499a 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1305,16 +1305,16 @@ psutil_win32_QueryDosDevice(PyObject *self, PyObject *args) { static PyObject * psutil_proc_username(PyObject *self, PyObject *args) { long pid; - HANDLE processHandle; - HANDLE tokenHandle; - PTOKEN_USER user; + HANDLE processHandle = NULL; + HANDLE tokenHandle = NULL; + PTOKEN_USER user = NULL; ULONG bufferSize; - PTSTR name; + PTSTR name = NULL; ULONG nameSize; - PTSTR domainName; + PTSTR domainName = NULL; ULONG domainNameSize; SID_NAME_USE nameUse; - PTSTR fullName; + PTSTR fullName = NULL; PyObject *py_unicode; if (! PyArg_ParseTuple(args, "l", &pid)) @@ -1326,8 +1326,8 @@ psutil_proc_username(PyObject *self, PyObject *args) { return NULL; if (!OpenProcessToken(processHandle, TOKEN_QUERY, &tokenHandle)) { - CloseHandle(processHandle); - return PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(0); + goto error; } CloseHandle(processHandle); @@ -1336,8 +1336,10 @@ psutil_proc_username(PyObject *self, PyObject *args) { bufferSize = 0x100; user = malloc(bufferSize); - if (user == NULL) - return PyErr_NoMemory(); + if (user == NULL) { + PyErr_NoMemory(); + goto error; + } if (!GetTokenInformation(tokenHandle, TokenUser, user, bufferSize, &bufferSize)) @@ -1345,15 +1347,14 @@ psutil_proc_username(PyObject *self, PyObject *args) { free(user); user = malloc(bufferSize); if (user == NULL) { - CloseHandle(tokenHandle); - return PyErr_NoMemory(); + PyErr_NoMemory(); + goto error; } if (!GetTokenInformation(tokenHandle, TokenUser, user, bufferSize, &bufferSize)) { - free(user); - CloseHandle(tokenHandle); - return PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(0); + goto error; } } @@ -1364,11 +1365,16 @@ psutil_proc_username(PyObject *self, PyObject *args) { domainNameSize = 0x100; name = malloc(nameSize * sizeof(TCHAR)); - if (name == NULL) - return PyErr_NoMemory(); + if (name == NULL) { + PyErr_NoMemory(); + goto error; + } + domainName = malloc(domainNameSize * sizeof(TCHAR)); - if (domainName == NULL) - return PyErr_NoMemory(); + if (domainName == NULL) { + PyErr_NoMemory(); + goto error; + } if (!LookupAccountSid(NULL, user->User.Sid, name, &nameSize, domainName, &domainNameSize, &nameUse)) @@ -1376,19 +1382,20 @@ psutil_proc_username(PyObject *self, PyObject *args) { free(name); free(domainName); name = malloc(nameSize * sizeof(TCHAR)); - if (name == NULL) - return PyErr_NoMemory(); + if (name == NULL) { + PyErr_NoMemory(); + goto error; + } domainName = malloc(domainNameSize * sizeof(TCHAR)); - if (domainName == NULL) - return PyErr_NoMemory(); + if (domainName == NULL) { + PyErr_NoMemory(); + goto error; + } if (!LookupAccountSid(NULL, user->User.Sid, name, &nameSize, domainName, &domainNameSize, &nameUse)) { - free(name); - free(domainName); - free(user); - - return PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(0); + goto error; } } @@ -1398,10 +1405,8 @@ psutil_proc_username(PyObject *self, PyObject *args) { // build the full username string fullName = malloc((domainNameSize + 1 + nameSize + 1) * sizeof(TCHAR)); if (fullName == NULL) { - free(name); - free(domainName); - free(user); - return PyErr_NoMemory(); + PyErr_NoMemory(); + goto error; } memcpy(fullName, domainName, domainNameSize); fullName[domainNameSize] = '\\'; @@ -1415,13 +1420,26 @@ psutil_proc_username(PyObject *self, PyObject *args) { py_unicode = PyUnicode_Decode( fullName, _tcslen(fullName), Py_FileSystemDefaultEncoding, "replace"); #endif - free(fullName); free(name); free(domainName); free(user); - return py_unicode; + +error: + if (processHandle != NULL) + CloseHandle(processHandle); + if (tokenHandle != NULL) + CloseHandle(tokenHandle); + if (fullName != NULL) + free(fullName); + if (name != NULL) + free(name); + if (domainName != NULL) + free(domainName); + if (user != NULL) + free(user); + return NULL; } diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 29012cd49..272253b61 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -503,6 +503,7 @@ def test_fans(self): self.assert_stdout('fans.py') @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") + @unittest.skipIf(TRAVIS, "not battery on TRAVIS") def test_battery(self): self.assert_stdout('battery.py') From b06bb03ea543e6e53dc5eda601111855c15e3f7d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 14:34:54 +0200 Subject: [PATCH 523/922] re #1040 have users() use WTSQuerySessionInformationW in order to return unicode. Also fixes #1048 (host IP address was invalid). --- HISTORY.rst | 1 + psutil/_psutil_windows.c | 30 ++++++++++++------------------ 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 67213b2ca..30bdecc3b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -37,6 +37,7 @@ None - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. +- 1048_: [Windows] users()'s host field report an invalid IP address. *2017-04-10* diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 5758c3e33..2ce98e175 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2576,7 +2576,7 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { static PyObject * psutil_users(PyObject *self, PyObject *args) { HANDLE hServer = WTS_CURRENT_SERVER_HANDLE; - LPTSTR buffer_user = NULL; + WCHAR *buffer_user = NULL; LPTSTR buffer_addr = NULL; PWTS_SESSION_INFO sessions = NULL; DWORD count; @@ -2595,7 +2595,7 @@ psutil_users(PyObject *self, PyObject *args) { PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; PyObject *py_address = NULL; - PyObject *py_buffer_user_encoded = NULL; + PyObject *py_username = NULL; if (py_retlist == NULL) return NULL; @@ -2623,12 +2623,12 @@ psutil_users(PyObject *self, PyObject *args) { // username bytes = 0; - if (WTSQuerySessionInformation(hServer, sessionId, WTSUserName, - &buffer_user, &bytes) == 0) { + if (WTSQuerySessionInformationW(hServer, sessionId, WTSUserName, + &buffer_user, &bytes) == 0) { PyErr_SetFromWindowsErr(0); goto error; } - if (bytes == 1) + if (bytes <= 2) continue; // address @@ -2672,24 +2672,18 @@ psutil_users(PyObject *self, PyObject *args) { station_info.ConnectTime.dwLowDateTime - 116444736000000000LL; unix_time /= 10000000; -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 - py_buffer_user_encoded = PyUnicode_DecodeLocaleAndSize( - buffer_user, _tcslen(buffer_user), "surrogateescape"); -#else - py_buffer_user_encoded = PyUnicode_Decode( - buffer_user, _tcslen(buffer_user), Py_FileSystemDefaultEncoding, - "replace"); -#endif - - if (py_buffer_user_encoded == NULL) + py_username = PyUnicode_FromWideChar(buffer_user, wcslen(buffer_user)); + if (py_username == NULL) goto error; - py_tuple = Py_BuildValue("OOd", py_buffer_user_encoded, py_address, + py_tuple = Py_BuildValue("OOd", + py_username, + py_address, (double)unix_time); if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; - Py_XDECREF(py_buffer_user_encoded); + Py_XDECREF(py_username); Py_XDECREF(py_address); Py_XDECREF(py_tuple); } @@ -2701,7 +2695,7 @@ psutil_users(PyObject *self, PyObject *args) { return py_retlist; error: - Py_XDECREF(py_buffer_user_encoded); + Py_XDECREF(py_username); Py_XDECREF(py_tuple); Py_XDECREF(py_address); Py_DECREF(py_retlist); From 061c15513415b932218b35cbf8ddbe4ce6cece34 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 16:43:29 +0200 Subject: [PATCH 524/922] LookupAccountSid() retry only in case of INSUFFICIENT_BUFFER --- psutil/_psutil_windows.c | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 47376499a..3836f9e4d 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1331,6 +1331,7 @@ psutil_proc_username(PyObject *self, PyObject *args) { } CloseHandle(processHandle); + processHandle = NULL; // Get the user SID. @@ -1359,28 +1360,12 @@ psutil_proc_username(PyObject *self, PyObject *args) { } CloseHandle(tokenHandle); + tokenHandle = NULL; // resolve the SID to a name nameSize = 0x100; domainNameSize = 0x100; - - name = malloc(nameSize * sizeof(TCHAR)); - if (name == NULL) { - PyErr_NoMemory(); - goto error; - } - - domainName = malloc(domainNameSize * sizeof(TCHAR)); - if (domainName == NULL) { - PyErr_NoMemory(); - goto error; - } - - if (!LookupAccountSid(NULL, user->User.Sid, name, &nameSize, domainName, - &domainNameSize, &nameUse)) - { - free(name); - free(domainName); + while (1) { name = malloc(nameSize * sizeof(TCHAR)); if (name == NULL) { PyErr_NoMemory(); @@ -1394,15 +1379,22 @@ psutil_proc_username(PyObject *self, PyObject *args) { if (!LookupAccountSid(NULL, user->User.Sid, name, &nameSize, domainName, &domainNameSize, &nameUse)) { - PyErr_SetFromWindowsErr(0); - goto error; + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + free(name); + free(domainName); + continue; + } + else { + PyErr_SetFromWindowsErr(0); + goto error; + } } + break; } + // build the "domain\\username" username string nameSize = _tcslen(name); domainNameSize = _tcslen(domainName); - - // build the full username string fullName = malloc((domainNameSize + 1 + nameSize + 1) * sizeof(TCHAR)); if (fullName == NULL) { PyErr_NoMemory(); From 5580428550896c66a9972682be0997035382d468 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 17:02:12 +0200 Subject: [PATCH 525/922] have GetTokenInformation() retry only in case of INSUFFICIENT_BUFFER --- psutil/_psutil_windows.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 3836f9e4d..69d1fdfda 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1334,18 +1334,8 @@ psutil_proc_username(PyObject *self, PyObject *args) { processHandle = NULL; // Get the user SID. - bufferSize = 0x100; - user = malloc(bufferSize); - if (user == NULL) { - PyErr_NoMemory(); - goto error; - } - - if (!GetTokenInformation(tokenHandle, TokenUser, user, bufferSize, - &bufferSize)) - { - free(user); + while (1) { user = malloc(bufferSize); if (user == NULL) { PyErr_NoMemory(); @@ -1354,9 +1344,16 @@ psutil_proc_username(PyObject *self, PyObject *args) { if (!GetTokenInformation(tokenHandle, TokenUser, user, bufferSize, &bufferSize)) { - PyErr_SetFromWindowsErr(0); - goto error; + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + free(user); + continue; + } + else { + PyErr_SetFromWindowsErr(0); + goto error; + } } + break; } CloseHandle(tokenHandle); From 734d603f58e65ed7353e95f2a0d9afaedb989e56 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 18:54:13 +0200 Subject: [PATCH 526/922] #1040: for proc username() on Windows use LookupAccountSidW in order to return proper unicode; also return a (domain, user) tuple instead of concatenating the string in C (I feel safer) --- psutil/_psutil_windows.c | 54 ++++++++++++++++++---------------------- psutil/_pswindows.py | 6 ++++- 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 3b189b1f4..83922a529 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1309,13 +1309,14 @@ psutil_proc_username(PyObject *self, PyObject *args) { HANDLE tokenHandle = NULL; PTOKEN_USER user = NULL; ULONG bufferSize; - PTSTR name = NULL; + WCHAR *name = NULL; + WCHAR *domainName = NULL; ULONG nameSize; - PTSTR domainName = NULL; ULONG domainNameSize; SID_NAME_USE nameUse; - PTSTR fullName = NULL; - PyObject *py_unicode; + PyObject *py_username; + PyObject *py_domain; + PyObject *py_tuple; if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; @@ -1363,18 +1364,18 @@ psutil_proc_username(PyObject *self, PyObject *args) { nameSize = 0x100; domainNameSize = 0x100; while (1) { - name = malloc(nameSize * sizeof(TCHAR)); + name = malloc(nameSize * sizeof(WCHAR)); if (name == NULL) { PyErr_NoMemory(); goto error; } - domainName = malloc(domainNameSize * sizeof(TCHAR)); + domainName = malloc(domainNameSize * sizeof(WCHAR)); if (domainName == NULL) { PyErr_NoMemory(); goto error; } - if (!LookupAccountSid(NULL, user->User.Sid, name, &nameSize, - domainName, &domainNameSize, &nameUse)) + if (!LookupAccountSidW(NULL, user->User.Sid, name, &nameSize, + domainName, &domainNameSize, &nameUse)) { if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { free(name); @@ -1389,45 +1390,38 @@ psutil_proc_username(PyObject *self, PyObject *args) { break; } - // build the "domain\\username" username string - nameSize = _tcslen(name); - domainNameSize = _tcslen(domainName); - fullName = malloc((domainNameSize + 1 + nameSize + 1) * sizeof(TCHAR)); - if (fullName == NULL) { - PyErr_NoMemory(); + py_domain = PyUnicode_FromWideChar(domainName, wcslen(domainName)); + if (! py_domain) goto error; - } - memcpy(fullName, domainName, domainNameSize); - fullName[domainNameSize] = '\\'; - memcpy(&fullName[domainNameSize + 1], name, nameSize); - fullName[domainNameSize + 1 + nameSize] = '\0'; + py_username = PyUnicode_FromWideChar(name, wcslen(name)); + if (! py_username) + goto error; + py_tuple = Py_BuildValue("OO", py_domain, py_username); + if (! py_tuple) + goto error; + Py_DECREF(py_domain); + Py_DECREF(py_username); -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 - py_unicode = PyUnicode_DecodeLocaleAndSize( - fullName, _tcslen(fullName), "surrogateescape"); -#else - py_unicode = PyUnicode_Decode( - fullName, _tcslen(fullName), Py_FileSystemDefaultEncoding, "replace"); -#endif - free(fullName); free(name); free(domainName); free(user); - return py_unicode; + + return py_tuple; error: if (processHandle != NULL) CloseHandle(processHandle); if (tokenHandle != NULL) CloseHandle(tokenHandle); - if (fullName != NULL) - free(fullName); if (name != NULL) free(name); if (domainName != NULL) free(domainName); if (user != NULL) free(user); + Py_XDECREF(py_domain); + Py_XDECREF(py_username); + Py_XDECREF(py_tuple); return NULL; } diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 266c12200..677e3426a 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -793,7 +793,11 @@ def wait(self, timeout=None): def username(self): if self.pid in (0, 4): return 'NT AUTHORITY\\SYSTEM' - return cext.proc_username(self.pid) + domain, user = cext.proc_username(self.pid) + if not PY3: + domain = py2_strencode(domain) + user = py2_strencode(user) + return domain + '\\' + user @wrap_exceptions def create_time(self): From d1500241a4ebb860d59c56e091200e4777b36b00 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 19:17:51 +0200 Subject: [PATCH 527/922] refactor tests --- psutil/tests/test_misc.py | 3 ++- psutil/tests/test_posix.py | 9 +++++++++ psutil/tests/test_process.py | 23 ++++++++--------------- psutil/tests/test_windows.py | 5 ++--- scripts/internal/winmake.py | 7 ++++--- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 272253b61..6bc2e28c9 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -37,6 +37,7 @@ from psutil.tests import create_sockets from psutil.tests import get_free_port from psutil.tests import get_test_subprocess +from psutil.tests import HAS_BATTERY from psutil.tests import HAS_MEMORY_FULL_INFO from psutil.tests import HAS_MEMORY_MAPS from psutil.tests import HAS_SENSORS_BATTERY @@ -503,7 +504,7 @@ def test_fans(self): self.assert_stdout('fans.py') @unittest.skipIf(not HAS_SENSORS_BATTERY, "not supported") - @unittest.skipIf(TRAVIS, "not battery on TRAVIS") + @unittest.skipIf(not HAS_BATTERY, "no battery") def test_battery(self): self.assert_stdout('battery.py') diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 819da0d20..3274c02ca 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -92,6 +92,15 @@ def test_username(self): username_psutil = psutil.Process(self.pid).username() self.assertEqual(username_ps, username_psutil) + def test_username_no_resolution(self): + # Emulate a case where the system can't resolve the uid to + # a username in which case psutil is supposed to return + # the stringified uid. + p = psutil.Process() + with mock.patch("psutil.pwd.getpwuid", side_effect=KeyError) as fun: + self.assertEqual(p.username(), str(p.uids().real)) + assert fun.called + @skip_on_access_denied() @retry_before_failing() def test_rss_memory(self): diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 7bb83b63e..d1cb96573 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -9,6 +9,7 @@ import collections import contextlib import errno +import getpass import os import select import signal @@ -836,22 +837,14 @@ def test_status(self): def test_username(self): sproc = get_test_subprocess() p = psutil.Process(sproc.pid) - if POSIX: - import pwd - self.assertEqual(p.username(), pwd.getpwuid(os.getuid()).pw_name) - with mock.patch("psutil.pwd.getpwuid", - side_effect=KeyError) as fun: - self.assertEqual(p.username(), str(p.uids().real)) - assert fun.called - - elif WINDOWS and 'USERNAME' in os.environ: - expected_username = os.environ['USERNAME'] - expected_domain = os.environ['USERDOMAIN'] - domain, username = p.username().split('\\') - self.assertEqual(domain, expected_domain) - self.assertEqual(username, expected_username) + username = p.username() + if WINDOWS: + domain, username = username.split('\\') + self.assertEqual(username, getpass.getuser()) + if 'USERDOMAIN' in os.environ: + self.assertEqual(domain, os.environ['USERDOMAIN']) else: - p.username() + self.assertEqual(username, getpass.getuser()) def test_cwd(self): sproc = get_test_subprocess() diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 2a883132d..2433849f5 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -369,9 +369,8 @@ def test_compare_name_exe(self): self.assertEqual(a, b) def test_username(self): - sys_value = win32api.GetUserName() - psutil_value = psutil.Process().username() - self.assertEqual(sys_value, psutil_value.split('\\')[1]) + self.assertEqual(psutil.Process().username(), + win32api.GetUserNameEx(win32con.NameSamCompatible)) def test_cmdline(self): sys_value = re.sub(' +', ' ', win32api.GetCommandLine()).strip() diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 69d4d9724..b8d111a4b 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -73,8 +73,9 @@ def safe_print(text, file=sys.stdout, flush=False): file.write("\n") -def sh(cmd): - safe_print("cmd: " + cmd) +def sh(cmd, nolog=False): + if not nolog: + safe_print("cmd: " + cmd) code = os.system(cmd) if code: raise SystemExit @@ -320,7 +321,7 @@ def flake8(): py_files = py_files.decode() py_files = [x for x in py_files.split() if x.endswith('.py')] py_files = ' '.join(py_files) - sh("%s -m flake8 %s" % (PYTHON, py_files)) + sh("%s -m flake8 %s" % (PYTHON, py_files), nolog=True) @cmd From b1a2bcaff2ad130386ea1ffe75b06004dc3aca28 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 19:24:24 +0200 Subject: [PATCH 528/922] fix test --- psutil/tests/test_memory_leaks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 3c562ea2d..28a083f26 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -568,6 +568,9 @@ def test_users(self): def test_win_service_iter(self): self.execute(cext.winservice_enumerate) + def test_win_service_get(self): + pass + def test_win_service_get_config(self): name = next(psutil.win_service_iter()).name() self.execute(cext.winservice_query_config, name) From e6f5f498d508335f359cd2d01a1098afcbcb1b6c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 19:32:17 +0200 Subject: [PATCH 529/922] fix 1050: memory leak in memory_maps() on Windows because we forgot to Py_DECREF --- HISTORY.rst | 1 + psutil/_psutil_windows.c | 1 + 2 files changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 67213b2ca..c193dfc8c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -37,6 +37,7 @@ None - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. +- 1050_: [Windows] Process.memory_maps memory() leaks memory. *2017-04-10* diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 69d1fdfda..20d20b824 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2920,6 +2920,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { if (PyList_Append(py_retlist, py_tuple)) goto error; Py_DECREF(py_tuple); + Py_DECREF(py_str); } previousAllocationBase = basicInfo.AllocationBase; baseAddress = (PCHAR)baseAddress + basicInfo.RegionSize; From 443763ff32d3bd23fee2a25e1812585cb5f20496 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 20:03:28 +0200 Subject: [PATCH 530/922] be explicit in using Py_UnicodeFromWideChar --- psutil/_psutil_windows.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 83922a529..2366d0305 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -669,7 +669,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { return PyErr_SetFromWindowsErr(0); } CloseHandle(hProcess); - return Py_BuildValue("u", exe); + return PyUnicode_FromWideChar(exe, wcslen(exe)); } From e47981f672ac9336ab0d61ea766c807f531e80a8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 21:19:36 +0200 Subject: [PATCH 531/922] #1040: just use Py_BuildValue on both py 2 and 3 --- psutil/_psutil_windows.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 2366d0305..31a0e2ea7 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -3035,11 +3035,7 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { } *--ptr = '\0'; -#if PY_MAJOR_VERSION >= 3 - py_mac_address = PyUnicode_FromString(buff); -#else - py_mac_address = PyString_FromString(buff); -#endif + py_mac_address = Py_BuildValue("s", buff); if (py_mac_address == NULL) goto error; From 7e86fc4d18ef6715b38c9e1122329e16c4d33e89 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 22:52:50 +0200 Subject: [PATCH 532/922] #1040: use QueryServiceConfig2W() and return unicode for windows service description() --- psutil/arch/windows/services.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/psutil/arch/windows/services.c b/psutil/arch/windows/services.c index 85ea6ff42..c65b50e5f 100644 --- a/psutil/arch/windows/services.c +++ b/psutil/arch/windows/services.c @@ -21,7 +21,7 @@ psutil_get_service_handler(char *service_name, DWORD scm_access, DWORD access) ENUM_SERVICE_STATUS_PROCESS *lpService = NULL; SC_HANDLE sc = NULL; SC_HANDLE hService = NULL; - SERVICE_DESCRIPTION *scd = NULL; + SERVICE_DESCRIPTIONW *scd = NULL; sc = OpenSCManager(NULL, NULL, scm_access); if (sc == NULL) { @@ -366,7 +366,7 @@ psutil_winservice_query_descr(PyObject *self, PyObject *args) { DWORD resumeHandle = 0; DWORD dwBytes = 0; SC_HANDLE hService = NULL; - SERVICE_DESCRIPTION *scd = NULL; + SERVICE_DESCRIPTIONW *scd = NULL; char *service_name; PyObject *py_retstr = NULL; @@ -380,22 +380,22 @@ psutil_winservice_query_descr(PyObject *self, PyObject *args) { // This first call to QueryServiceConfig2() is necessary in order // to get the right size. bytesNeeded = 0; - QueryServiceConfig2(hService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, - &bytesNeeded); + QueryServiceConfig2W(hService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, + &bytesNeeded); if (GetLastError() == ERROR_MUI_FILE_NOT_FOUND) { // Also services.msc fails in the same manner, so we return an // empty string. CloseServiceHandle(hService); - return Py_BuildValue("s", ""); + return Py_BuildValue("u", ""); } if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { PyErr_SetFromWindowsErr(0); goto error; } - scd = (SERVICE_DESCRIPTION *)malloc(bytesNeeded); - ok = QueryServiceConfig2(hService, SERVICE_CONFIG_DESCRIPTION, - (LPBYTE)scd, bytesNeeded, &bytesNeeded); + scd = (SERVICE_DESCRIPTIONW *)malloc(bytesNeeded); + ok = QueryServiceConfig2W(hService, SERVICE_CONFIG_DESCRIPTION, + (LPBYTE)scd, bytesNeeded, &bytesNeeded); if (ok == 0) { PyErr_SetFromWindowsErr(0); goto error; @@ -405,11 +405,8 @@ psutil_winservice_query_descr(PyObject *self, PyObject *args) { py_retstr = Py_BuildValue("s", ""); } else { - py_retstr = PyUnicode_Decode( - scd->lpDescription, - _tcslen(scd->lpDescription), - Py_FileSystemDefaultEncoding, - "replace"); + py_retstr = PyUnicode_FromWideChar( + scd->lpDescription, wcslen(scd->lpDescription)); } if (!py_retstr) goto error; From 3fccfda70bec96223b3447594b23789e8755854c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 23:09:34 +0200 Subject: [PATCH 533/922] #1040: use EnumServiceStatusExW and return proper unicode for service enumerate() and display_name() --- psutil/arch/windows/services.c | 47 +++++++++++++++++----------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/psutil/arch/windows/services.c b/psutil/arch/windows/services.c index c65b50e5f..52b3d99c0 100644 --- a/psutil/arch/windows/services.c +++ b/psutil/arch/windows/services.c @@ -18,10 +18,9 @@ SC_HANDLE psutil_get_service_handler(char *service_name, DWORD scm_access, DWORD access) { - ENUM_SERVICE_STATUS_PROCESS *lpService = NULL; + ENUM_SERVICE_STATUS_PROCESSW *lpService = NULL; SC_HANDLE sc = NULL; SC_HANDLE hService = NULL; - SERVICE_DESCRIPTIONW *scd = NULL; sc = OpenSCManager(NULL, NULL, scm_access); if (sc == NULL) { @@ -96,7 +95,7 @@ get_state_string(DWORD state) { */ PyObject * psutil_winservice_enumerate(PyObject *self, PyObject *args) { - ENUM_SERVICE_STATUS_PROCESS *lpService = NULL; + ENUM_SERVICE_STATUS_PROCESSW *lpService = NULL; BOOL ok; SC_HANDLE sc = NULL; DWORD bytesNeeded = 0; @@ -106,7 +105,8 @@ psutil_winservice_enumerate(PyObject *self, PyObject *args) { DWORD i; PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; - PyObject *py_unicode_display_name = NULL; + PyObject *py_name = NULL; + PyObject *py_display_name = NULL; if (py_retlist == NULL) return NULL; @@ -118,7 +118,7 @@ psutil_winservice_enumerate(PyObject *self, PyObject *args) { } for (;;) { - ok = EnumServicesStatusEx( + ok = EnumServicesStatusExW( sc, SC_ENUM_PROCESS_INFO, SERVICE_WIN32, // XXX - extend this to include drivers etc.? @@ -134,31 +134,31 @@ psutil_winservice_enumerate(PyObject *self, PyObject *args) { if (lpService) free(lpService); dwBytes = bytesNeeded; - lpService = (ENUM_SERVICE_STATUS_PROCESS*)malloc(dwBytes); + lpService = (ENUM_SERVICE_STATUS_PROCESSW*)malloc(dwBytes); } for (i = 0; i < srvCount; i++) { - // Get unicode display name. - py_unicode_display_name = NULL; - py_unicode_display_name = PyUnicode_Decode( - lpService[i].lpDisplayName, - _tcslen(lpService[i].lpDisplayName), - Py_FileSystemDefaultEncoding, - "replace"); - if (py_unicode_display_name == NULL) + // Get unicode name / display name. + py_name = NULL; + py_name = PyUnicode_FromWideChar( + lpService[i].lpServiceName, wcslen(lpService[i].lpServiceName)); + if (py_name == NULL) + goto error; + + py_display_name = NULL; + py_display_name = PyUnicode_FromWideChar( + lpService[i].lpDisplayName, wcslen(lpService[i].lpDisplayName)); + if (py_display_name == NULL) goto error; // Construct the result. - py_tuple = Py_BuildValue( - "(sO)", - lpService[i].lpServiceName, // name - py_unicode_display_name // display_name - ); + py_tuple = Py_BuildValue("(OO)", py_name, py_display_name); if (py_tuple == NULL) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; - Py_DECREF(py_unicode_display_name); + Py_DECREF(py_display_name); + Py_DECREF(py_name); Py_DECREF(py_tuple); } @@ -168,7 +168,8 @@ psutil_winservice_enumerate(PyObject *self, PyObject *args) { return py_retlist; error: - Py_XDECREF(py_unicode_display_name); + Py_DECREF(py_name); + Py_XDECREF(py_display_name); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); if (sc != NULL) @@ -360,7 +361,7 @@ psutil_winservice_query_status(PyObject *self, PyObject *args) { */ PyObject * psutil_winservice_query_descr(PyObject *self, PyObject *args) { - ENUM_SERVICE_STATUS_PROCESS *lpService = NULL; + ENUM_SERVICE_STATUS_PROCESSW *lpService = NULL; BOOL ok; DWORD bytesNeeded = 0; DWORD resumeHandle = 0; @@ -386,7 +387,7 @@ psutil_winservice_query_descr(PyObject *self, PyObject *args) { // Also services.msc fails in the same manner, so we return an // empty string. CloseServiceHandle(hService); - return Py_BuildValue("u", ""); + return Py_BuildValue("s", ""); } if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { PyErr_SetFromWindowsErr(0); From 4fe26b06dda7357a1928a5e6e7b8b4ae80dae427 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 23:23:51 +0200 Subject: [PATCH 534/922] #1040 use QueryServiceConfigW() and return proper unicode strings on service display_name() and username() --- psutil/arch/windows/services.c | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/psutil/arch/windows/services.c b/psutil/arch/windows/services.c index 52b3d99c0..e82f2887a 100644 --- a/psutil/arch/windows/services.c +++ b/psutil/arch/windows/services.c @@ -195,7 +195,7 @@ psutil_winservice_query_config(PyObject *self, PyObject *args) { DWORD bytesNeeded = 0; DWORD resumeHandle = 0; DWORD dwBytes = 0; - QUERY_SERVICE_CONFIG *qsc = NULL; + QUERY_SERVICE_CONFIGW *qsc = NULL; PyObject *py_tuple = NULL; PyObject *py_unicode_display_name = NULL; PyObject *py_unicode_binpath = NULL; @@ -208,45 +208,36 @@ psutil_winservice_query_config(PyObject *self, PyObject *args) { if (hService == NULL) goto error; - // First call to QueryServiceConfig() is necessary to get the + // First call to QueryServiceConfigW() is necessary to get the // right size. bytesNeeded = 0; - QueryServiceConfig(hService, NULL, 0, &bytesNeeded); + QueryServiceConfigW(hService, NULL, 0, &bytesNeeded); if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { PyErr_SetFromWindowsErr(0); goto error; } qsc = (QUERY_SERVICE_CONFIG *)malloc(bytesNeeded); - ok = QueryServiceConfig(hService, qsc, bytesNeeded, &bytesNeeded); + ok = QueryServiceConfigW(hService, qsc, bytesNeeded, &bytesNeeded); if (ok == 0) { PyErr_SetFromWindowsErr(0); goto error; } // Get unicode display name. - py_unicode_display_name = PyUnicode_Decode( - qsc->lpDisplayName, - _tcslen(qsc->lpDisplayName), - Py_FileSystemDefaultEncoding, - "replace"); + py_unicode_display_name = PyUnicode_FromWideChar( + qsc->lpDisplayName, wcslen(qsc->lpDisplayName)); if (py_unicode_display_name == NULL) goto error; // Get unicode bin path. - py_unicode_binpath = PyUnicode_Decode( - qsc->lpBinaryPathName, - _tcslen(qsc->lpBinaryPathName), - Py_FileSystemDefaultEncoding, - "replace"); + py_unicode_binpath = PyUnicode_FromWideChar( + qsc->lpBinaryPathName, wcslen(qsc->lpBinaryPathName)); if (py_unicode_binpath == NULL) goto error; // Get unicode username. - py_unicode_username = PyUnicode_Decode( - qsc->lpServiceStartName, - _tcslen(qsc->lpServiceStartName), - Py_FileSystemDefaultEncoding, - "replace"); + py_unicode_username = PyUnicode_FromWideChar( + qsc->lpServiceStartName, wcslen(qsc->lpServiceStartName)); if (py_unicode_username == NULL) goto error; @@ -378,7 +369,7 @@ psutil_winservice_query_descr(PyObject *self, PyObject *args) { if (hService == NULL) goto error; - // This first call to QueryServiceConfig2() is necessary in order + // This first call to QueryServiceConfig2W() is necessary in order // to get the right size. bytesNeeded = 0; QueryServiceConfig2W(hService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, From 164861df38ac1f597c4139ee20bb08144fd9591e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 23:27:28 +0200 Subject: [PATCH 535/922] 1040: on python 2 convert all service strings from unicode to bytes --- psutil/_pswindows.py | 14 +++++++------- psutil/tests/test_windows.py | 15 +++++++-------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 677e3426a..cb9cd8bed 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -427,9 +427,9 @@ def users(): def win_service_iter(): - """Return a list of WindowsService instances.""" + """Yields a list of WindowsService instances.""" for name, display_name in cext.winservice_enumerate(): - yield WindowsService(name, display_name) + yield WindowsService(py2_strencode(name), py2_strencode(display_name)) def win_service_get(name): @@ -470,10 +470,10 @@ def _query_config(self): cext.winservice_query_config(self._name) # XXX - update _self.display_name? return dict( - display_name=display_name, - binpath=binpath, - username=username, - start_type=start_type) + display_name=py2_strencode(display_name), + binpath=py2_strencode(binpath), + username=py2_strencode(username), + start_type=py2_strencode(start_type)) def _query_status(self): with self._wrap_exceptions(): @@ -550,7 +550,7 @@ def status(self): def description(self): """Service long description.""" - return cext.winservice_query_descr(self.name()) + return py2_strencode(cext.winservice_query_descr(self.name())) # utils diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 2a883132d..7acec9e8e 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -28,7 +28,6 @@ import psutil from psutil import WINDOWS -from psutil._compat import basestring from psutil._compat import callable from psutil.tests import APPVEYOR from psutil.tests import get_test_subprocess @@ -754,19 +753,19 @@ def test_win_service_iter(self): ]) for serv in psutil.win_service_iter(): data = serv.as_dict() - self.assertIsInstance(data['name'], basestring) + self.assertIsInstance(data['name'], str) self.assertNotEqual(data['name'].strip(), "") - self.assertIsInstance(data['display_name'], basestring) - self.assertIsInstance(data['username'], basestring) + self.assertIsInstance(data['display_name'], str) + self.assertIsInstance(data['username'], str) self.assertIn(data['status'], valid_statuses) if data['pid'] is not None: psutil.Process(data['pid']) - self.assertIsInstance(data['binpath'], basestring) - self.assertIsInstance(data['username'], basestring) - self.assertIsInstance(data['start_type'], basestring) + self.assertIsInstance(data['binpath'], str) + self.assertIsInstance(data['username'], str) + self.assertIsInstance(data['start_type'], str) self.assertIn(data['start_type'], valid_start_types) self.assertIn(data['status'], valid_statuses) - self.assertIsInstance(data['description'], basestring) + self.assertIsInstance(data['description'], str) pid = serv.pid() if pid is not None: p = psutil.Process(pid) From 8393e9c78068d0cd168251927d7952019af575db Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 3 May 2017 23:58:43 +0200 Subject: [PATCH 536/922] assume that internally python 3 never deals with bytes; also update unicode notes --- psutil/_pswindows.py | 8 +-- psutil/tests/test_unicode.py | 95 +++++++++++++++++++----------------- 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index cb9cd8bed..e284bb699 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -183,18 +183,14 @@ def convert_dos_path(s): into: "C:\Windows\systemew\file.txt" """ - if PY3 and not isinstance(s, str): - # TODO: probably getting here means there's something wrong; - # probably needs to be removed. - s = s.decode(FS_ENCODING, errors=PY2_ENCODING_ERRS) rawdrive = '\\'.join(s.split('\\')[:3]) driveletter = cext.win32_QueryDosDevice(rawdrive) return os.path.join(driveletter, s[len(rawdrive):]) def py2_strencode(s): - """Encode a string in the given encoding. Falls back on returning - the string as is if it can't be encoded. + """Encode a unicode string to a byte string by using the default fs + encoding + "replace" error handler. """ if PY3: return s diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index ce881eac6..429254919 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -12,53 +12,58 @@ In psutil these are the APIs returning or dealing with a string ('not tested' means they are not tested to deal with non-ASCII strings): -- Process.cmdline() -- Process.connections('unix') -- Process.cwd() -- Process.environ() -- Process.exe() -- Process.memory_maps() -- Process.name() -- Process.open_files() -- Process.username() (not tested) - -- disk_io_counters() (not tested) -- disk_partitions() (not tested) -- disk_usage(str) -- net_connections('unix') -- net_if_addrs() (not tested) -- net_if_stats() (not tested) -- net_io_counters() (not tested) -- sensors_fans() (not tested) -- sensors_temperatures() (not tested) -- users() (not tested) - -- WindowsService.binpath() (not tested) -- WindowsService.description() (not tested) -- WindowsService.display_name() (not tested) -- WindowsService.name() (not tested) -- WindowsService.status() (not tested) -- WindowsService.username() (not tested) +* Process.cmdline() +* Process.connections('unix') +* Process.cwd() +* Process.environ() +* Process.exe() +* Process.memory_maps() +* Process.name() +* Process.open_files() +* Process.username() (not tested) + +* disk_io_counters() (not tested) +* disk_partitions() (not tested) +* disk_usage(str) +* net_connections('unix') +* net_if_addrs() (not tested) +* net_if_stats() (not tested) +* net_io_counters() (not tested) +* sensors_fans() (not tested) +* sensors_temperatures() (not tested) +* users() (not tested) + +* WindowsService.binpath() (not tested) +* WindowsService.description() (not tested) +* WindowsService.display_name() (not tested) +* WindowsService.name() (not tested) +* WindowsService.status() (not tested) +* WindowsService.username() (not tested) In here we create a unicode path with a funky non-ASCII name and (where -possible) make psutil return it back (e.g. on name(), exe(), -open_files(), etc.) and make sure psutil never crashes with -UnicodeDecodeError. - -On Python 3 the returned path is supposed to match 100% (and this -is tested). -Not on Python 2 though, where we assume correct unicode path handling -is broken. In fact it is broken for most os.* functions, see: -http://bugs.python.org/issue18695 -There really is no way for psutil to handle unicode correctly on -Python 2 unless we make such APIs return a unicode type in certain -circumstances. -I'd rather have unicode support broken on Python 2 than having APIs -returning variable str/unicode types, see: -https://github.com/giampaolo/psutil/issues/655#issuecomment-136131180 - -As such we also test that all APIs on Python 2 always return str and -never unicode (in test_contracts.py). +possible) make psutil return it back (e.g. on name(), exe(), open_files(), +etc.) and make sure that: + +* psutil never crashes with UnicodeDecodeError +* the returned path matches + +Notes about unicode handling in psutil: + +* all strings are encoded by using the default filesystem encoding which + varies depending on the platform (e.g. UTF-8 on Linux, mbcs on Win) +* no API is supposed to crash with UnicodeDecodeError +* in case of badly encoded data returned by the OS the following error + handlers are used to replace the bad chars in the string: + * Python 2: "replace" + * Python 3 on POSIX: "surrogateescape" + * Python 3 on Windows: "surrogatepass" (3.6+) or "replace" (<= 3.5) +* on Python 2 all APIs return bytes (str type), never unicode +* on Python 2 you can go back to unicode by doing: + >>> unicode(p.exe(), sys.getdefaultencoding(), errors="replace") + ...and make proper comparisons. +* there is no API on Python 2 to tell psutil to return unicode + +See: https://github.com/giampaolo/psutil/issues/1040 """ import os From 9ea0636f8c50f532661a9a894bd4f10b4c6d7c43 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 00:57:28 +0200 Subject: [PATCH 537/922] #1040: add notes about unicode in the doc --- docs/index.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 16301958f..509d5f9c9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2239,6 +2239,44 @@ Constants >>> if psutil.version_info >= (4, 5): ... pass +---- + +Unicode +======= + +Starting from version 5.3.0 psutil +`fully supports unicode `__. +The notes below apply to *any* method returning a string such as +:meth:`Process.exe` or :meth:`Process.cwd`, including non-filesystem related +methods such as :meth:`Process.username`: + +* all strings are encoded by using the OS filesystem encoding which varies + depending on the platform you're on (e.g. UTF-8 on Linux, mbcs on Win) +* no API call is supposed to crash with ``UnicodeDecodeError`` +* instead, in case of badly encoded data returned by the OS, the following error handlers are used to replace the bad characters in the string: + * Python 2: ``"replace"`` + * Python 3: ``"surrogatescape"`` on POSIX and ``"replace"`` on Windows +* on Python 2 all APIs return bytes (``str`` type), never ``unicode`` +* on Python 2 you can go back to unicode by doing: + +.. code-block:: python + + >>> unicode(p.exe(), sys.getdefaultencoding(), errors="replace") + +Example which filters processes with a funky name working with Python 2 and 3:: + + # -*- coding: utf-8 -*- + import psutil, sys + + PY3 = sys.version_info[0] == 2 + LOOKFOR = u"ƒőő" + for proc in psutil.process_iter(attrs=['name']): + name = proc.info['name'] + if not PY3: + name = unicode(name, sys.getdefaultencoding(), errors="replace") + if LOOKFOR == name: + print("process %s found" % p) + Recipes ======= From de2b223686d5f8b8fd8cf28a4eef4c91468bf9c2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 01:15:52 +0200 Subject: [PATCH 538/922] #1040 update doc/notes --- HISTORY.rst | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 30bdecc3b..c25491fb3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ Process.as_dict(): "attrs" and "ad_value". With this you can iterate over all processes in one shot without needing to catch NoSuchProcess and do list/dict comprehensions. +- 1040_: implemented full unicode support. **Bug fixes** @@ -29,15 +30,27 @@ properly handle unicode paths and may raise UnicodeDecodeError. - 1033_: [OSX, FreeBSD] memory leak for net_connections() and Process.connections() when retrieving UNIX sockets (kind='unix'). +- 1040_: fixed many unicode related issues such as UnicodeDecodeError on + Python 3 + UNIX and invalid encoded data on Windows. +- 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. +- 1047_: [Windows] Process username(): memory leak in case exception is thrown. +- 1048_: [Windows] users()'s host field report an invalid IP address. + +**Porting notes** + - 1039_: returned types consolidation: - Windows / Process.cpu_times(): fields #3 and #4 were int instead of float - Linux / FreeBSD: connections('unix'): raddr is now set to "" instead of None - OpenBSD: connections('unix'): laddr and raddr are now set to "" instead of None -- 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. -- 1047_: [Windows] Process username(): memory leak in case exception is thrown. -- 1048_: [Windows] users()'s host field report an invalid IP address. +- 1040_: the following Windows APIs returned unicode and now they return str: + - Process.memory_maps().path + - WindosService.bin_path() + - WindosService.description() + - WindosService.display_name() + - WindosService.username() +- 1040_: all strings are encoded by using OS fs encoding. *2017-04-10* From 7ec4a8e75cf831a61d0e225c3e63fe83c9a0c5df Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 01:20:25 +0200 Subject: [PATCH 539/922] update notes --- psutil/tests/test_unicode.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 429254919..47e8d15c3 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -47,23 +47,9 @@ * psutil never crashes with UnicodeDecodeError * the returned path matches -Notes about unicode handling in psutil: - -* all strings are encoded by using the default filesystem encoding which - varies depending on the platform (e.g. UTF-8 on Linux, mbcs on Win) -* no API is supposed to crash with UnicodeDecodeError -* in case of badly encoded data returned by the OS the following error - handlers are used to replace the bad chars in the string: - * Python 2: "replace" - * Python 3 on POSIX: "surrogateescape" - * Python 3 on Windows: "surrogatepass" (3.6+) or "replace" (<= 3.5) -* on Python 2 all APIs return bytes (str type), never unicode -* on Python 2 you can go back to unicode by doing: - >>> unicode(p.exe(), sys.getdefaultencoding(), errors="replace") - ...and make proper comparisons. -* there is no API on Python 2 to tell psutil to return unicode - -See: https://github.com/giampaolo/psutil/issues/1040 +For a detailed explanation of how psutil handles unicode see: +- https://github.com/giampaolo/psutil/issues/1040 +- https://pythonhosted.org/psutil/#unicode """ import os From 04111d9a048006ecb24ca7a7a92e84bb138e4618 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 01:51:37 +0200 Subject: [PATCH 540/922] minor refactoring --- psutil/_pswindows.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index e284bb699..f2cf49e13 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -342,8 +342,8 @@ def net_if_stats(): ret = {} rawdict = cext.net_if_stats() for name, items in rawdict.items(): - assert isinstance(name, unicode), name if not PY3: + assert isinstance(name, unicode), type(name) name = py2_strencode(name) isup, duplex, speed, mtu = items if hasattr(_common, 'NicDuplex'): @@ -697,8 +697,8 @@ def cmdline(self): @wrap_exceptions def environ(self): ustr = cext.proc_environ(self.pid) - if ustr: - assert isinstance(ustr, unicode), ustr + if ustr and not PY3: + assert isinstance(ustr, unicode), type(ustr) return parse_environ_block(py2_strencode(ustr)) def ppid(self): @@ -758,9 +758,9 @@ def memory_maps(self): raise else: for addr, perm, path, rss in raw: - assert isinstance(path, unicode), path path = convert_dos_path(path) if not PY3: + assert isinstance(path, unicode), type(path) path = py2_strencode(path) addr = hex(addr) yield (addr, perm, path, rss) @@ -790,10 +790,7 @@ def username(self): if self.pid in (0, 4): return 'NT AUTHORITY\\SYSTEM' domain, user = cext.proc_username(self.pid) - if not PY3: - domain = py2_strencode(domain) - user = py2_strencode(user) - return domain + '\\' + user + return py2_strencode(domain) + '\\' + py2_strencode(user) @wrap_exceptions def create_time(self): From ec42f77f5ab929e6557b44810c92313e9d0d89d6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 06:29:36 +0200 Subject: [PATCH 541/922] fix #1051: make disk_usage() on python 3 able to deal with bytes --- HISTORY.rst | 1 + psutil/_pswindows.py | 10 +++------- psutil/tests/test_system.py | 8 +++----- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 30bdecc3b..52c48c73e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ Process.as_dict(): "attrs" and "ad_value". With this you can iterate over all processes in one shot without needing to catch NoSuchProcess and do list/dict comprehensions. +- 1051_: disk_usage() on Python 3 is now able to accept bytes. **Bug fixes** diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index e284bb699..c0d6d001a 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -239,13 +239,9 @@ def swap_memory(): def disk_usage(path): """Return disk usage associated with path.""" - try: - total, free = cext.disk_usage(path) - except WindowsError: - if not os.path.exists(path): - msg = "No such file or directory: '%s'" % path - raise OSError(errno.ENOENT, msg) - raise + if PY3 and isinstance(path, bytes): + path = path.decode(FS_ENCODING) + total, free = cext.disk_usage(path) used = total - free percent = usage_percent(used, total, _round=1) return _common.sdiskusage(total, used, free, percent) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 19f997a85..bb485296c 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -463,11 +463,9 @@ def test_disk_usage_unicode(self): if ASCII_FS: with self.assertRaises(UnicodeEncodeError): psutil.disk_usage(TESTFN_UNICODE) - else: - safe_rmpath(TESTFN_UNICODE) - self.addCleanup(safe_rmpath, TESTFN_UNICODE) - os.mkdir(TESTFN_UNICODE) - psutil.disk_usage(TESTFN_UNICODE) + + def test_disk_usage_bytes(self): + psutil.disk_usage(b'.') def test_disk_partitions(self): # all = False From 7ed7472a7b4a486c8f0cbc3436e98caed380e4a6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 06:33:57 +0200 Subject: [PATCH 542/922] try to minimize dll test failures on windows by picking python dll --- psutil/tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index c813ce463..a1bd53a1d 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -968,7 +968,7 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): libs = [x.path for x in psutil.Process().memory_maps() if os.path.normcase(os.path.splitext(x.path)[1]) == ext] if WINDOWS: - libs = [x for x in libs if 'wow64' not in x.lower()] + libs = [x for x in libs if 'python' in os.path.basename(x).lower()] src = random.choice(libs) cfile = None try: From 46b43f3bd313d2d54cf162fe8e5fde8d4068f704 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 06:34:59 +0200 Subject: [PATCH 543/922] try to minimize dll test failures on windows by picking python dll --- psutil/tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index c813ce463..a1bd53a1d 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -968,7 +968,7 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): libs = [x.path for x in psutil.Process().memory_maps() if os.path.normcase(os.path.splitext(x.path)[1]) == ext] if WINDOWS: - libs = [x for x in libs if 'wow64' not in x.lower()] + libs = [x for x in libs if 'python' in os.path.basename(x).lower()] src = random.choice(libs) cfile = None try: From cd84364e63303dfec6fa4decbf8a78ac631ccf24 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 06:49:44 +0200 Subject: [PATCH 544/922] update doc --- docs/index.rst | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 509d5f9c9..02f93570d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2244,26 +2244,27 @@ Constants Unicode ======= -Starting from version 5.3.0 psutil -`fully supports unicode `__. -The notes below apply to *any* method returning a string such as +Starting from version 5.3.0 psutil fully supports unicode, see +`issue #1040 `__. +The notes below apply to *any* API returning a string such as :meth:`Process.exe` or :meth:`Process.cwd`, including non-filesystem related -methods such as :meth:`Process.username`: +methods such as :meth:`Process.username` or :meth:`WindowsService.description`: * all strings are encoded by using the OS filesystem encoding which varies - depending on the platform you're on (e.g. UTF-8 on Linux, mbcs on Win) + depending on the platform (e.g. UTF-8 on Linux, mbcs on Win) * no API call is supposed to crash with ``UnicodeDecodeError`` -* instead, in case of badly encoded data returned by the OS, the following error handlers are used to replace the bad characters in the string: - * Python 2: ``"replace"`` +* instead, in case of badly encoded data returned by the OS, the following error handlers are used to replace the corrupted characters in the string: * Python 3: ``"surrogatescape"`` on POSIX and ``"replace"`` on Windows + * Python 2: ``"replace"`` * on Python 2 all APIs return bytes (``str`` type), never ``unicode`` -* on Python 2 you can go back to unicode by doing: +* on Python 2, you can go back to ``unicode`` by doing: .. code-block:: python >>> unicode(p.exe(), sys.getdefaultencoding(), errors="replace") -Example which filters processes with a funky name working with Python 2 and 3:: +Example which filters processes with a funky name working with both Python 2 +and 3:: # -*- coding: utf-8 -*- import psutil, sys From cdb1e714b2134f91e337e4e74146f5962291a972 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 06:55:15 +0200 Subject: [PATCH 545/922] try to minimize dll related test failures --- psutil/tests/__init__.py | 16 +++++++++++----- psutil/tests/test_unicode.py | 1 - 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index a1bd53a1d..196278ca9 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -967,13 +967,19 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() if os.path.normcase(os.path.splitext(x.path)[1]) == ext] - if WINDOWS: - libs = [x for x in libs if 'python' in os.path.basename(x).lower()] - src = random.choice(libs) cfile = None try: - shutil.copyfile(src, dst) - cfile = ctypes.CDLL(dst) + for x in range(10): + # ...because sometimes either copyfile() or ctypes fail + # for no apparent reason. + # https://travis-ci.org/giampaolo/psutil/jobs/228599944 + try: + src = random.choice(libs) + shutil.copyfile(src, dst) + cfile = ctypes.CDLL(dst) + break + except Exception as exc: + print("%s - retry" % exc) yield dst finally: if WINDOWS and cfile is not None: diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 47e8d15c3..e2e697862 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -233,7 +233,6 @@ def test_disk_usage(self): psutil.disk_usage(self.funky_name) @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") - @unittest.skipIf(not PY3, "ctypes opens err msg box on Python 2") def test_memory_maps(self): # XXX: on Python 2, using ctypes.CDLL with a unicode path # opens a message box which blocks the test run. From f327bf134674fa1e651aecb6043d9cedc8040755 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 20:48:22 +0200 Subject: [PATCH 546/922] refactoring --- psutil/_psposix.py | 28 +++++++++++++++------------- psutil/tests/__init__.py | 24 +++++++++--------------- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/psutil/_psposix.py b/psutil/_psposix.py index 6ed7694a1..66d81a3d1 100644 --- a/psutil/_psposix.py +++ b/psutil/_psposix.py @@ -127,21 +127,23 @@ def disk_usage(path): total and used disk space whereas "free" and "percent" represent the "free" and "used percent" user disk space. """ - try: + if PY3: st = os.statvfs(path) - except UnicodeEncodeError: - if not PY3 and isinstance(path, unicode): - # this is a bug with os.statvfs() and unicode on - # Python 2, see: - # - https://github.com/giampaolo/psutil/issues/416 - # - http://bugs.python.org/issue18695 - try: - path = path.encode(sys.getfilesystemencoding()) - except UnicodeEncodeError: - pass + else: + # os.statvfs() does not support unicode on Python 2: + # - https://github.com/giampaolo/psutil/issues/416 + # - http://bugs.python.org/issue18695 + try: st = os.statvfs(path) - else: - raise + except UnicodeEncodeError: + if isinstance(path, unicode): + try: + path = path.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + pass + st = os.statvfs(path) + else: + raise # Total space which is only available to root (unless changed # at system level). diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 196278ca9..7f309c03c 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -226,9 +226,10 @@ def get_test_subprocess(cmd=None, **kwds): """Creates a python subprocess which does nothing for 60 secs and return it as subprocess.Popen instance. If "cmd" is specified that is used instead of python. - By default stdout and stderr are redirected to /dev/null. + By default stdin and stdout are redirected to /dev/null. It also attemps to make sure the process is in a reasonably initialized state. + The process is registered for cleanup on reap_children(). """ kwds.setdefault("stdin", DEVNULL) kwds.setdefault("stdout", DEVNULL) @@ -256,7 +257,8 @@ def create_proc_children_pair(): """Create a subprocess which creates another one as in: A (us) -> B (child) -> C (grandchild). Return a (child, grandchild) tuple. - The 2 processes are fully initialized and will live for 60 secs. + The 2 processes are fully initialized and will live for 60 secs + and are registered for cleanup on reap_children(). """ _TESTFN2 = os.path.basename(_TESTFN) + '2' # need to be relative s = textwrap.dedent("""\ @@ -285,7 +287,7 @@ def create_proc_children_pair(): def pyrun(src): - """Run python 'src' code in a separate interpreter. + """Run python 'src' code string in a separate interpreter. Returns a subprocess.Popen instance. """ with tempfile.NamedTemporaryFile( @@ -296,7 +298,7 @@ def pyrun(src): subp = get_test_subprocess([PYTHON, f.name], stdout=None, stderr=None) wait_for_pid(subp.pid) - return subp + return subp def sh(cmd): @@ -969,17 +971,9 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): if os.path.normcase(os.path.splitext(x.path)[1]) == ext] cfile = None try: - for x in range(10): - # ...because sometimes either copyfile() or ctypes fail - # for no apparent reason. - # https://travis-ci.org/giampaolo/psutil/jobs/228599944 - try: - src = random.choice(libs) - shutil.copyfile(src, dst) - cfile = ctypes.CDLL(dst) - break - except Exception as exc: - print("%s - retry" % exc) + src = random.choice(libs) + shutil.copyfile(src, dst) + cfile = ctypes.CDLL(dst) yield dst finally: if WINDOWS and cfile is not None: From a1f2a0970fcf353e01fe5abd5c3a4e7a50037648 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 21:07:31 +0200 Subject: [PATCH 547/922] skip failing test --- psutil/tests/test_unicode.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index e2e697862..0a54198bf 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -233,6 +233,7 @@ def test_disk_usage(self): psutil.disk_usage(self.funky_name) @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") + @unittest.skipIf(not PY3, "ctypes does not support unicode on PY2") def test_memory_maps(self): # XXX: on Python 2, using ctypes.CDLL with a unicode path # opens a message box which blocks the test run. From 982f255b4a14c96482ad1ce455b0f2fb0275e144 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 4 May 2017 22:35:15 +0200 Subject: [PATCH 548/922] #1040: on py 3.6 use sys.getfilesystemencodeerrors() to determined the default error handler instead of guessing it --- docs/index.rst | 8 +++++--- psutil/_common.py | 13 +++++++++++++ psutil/_pslinux.py | 13 ++++++------- psutil/_pswindows.py | 12 +++++++----- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 02f93570d..350af93f8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2250,11 +2250,13 @@ The notes below apply to *any* API returning a string such as :meth:`Process.exe` or :meth:`Process.cwd`, including non-filesystem related methods such as :meth:`Process.username` or :meth:`WindowsService.description`: -* all strings are encoded by using the OS filesystem encoding which varies - depending on the platform (e.g. UTF-8 on Linux, mbcs on Win) +* all strings are encoded by using the OS filesystem encoding + (``sys.getfilesystemencoding()``) which varies depending on the platform + (e.g. "UTF-8" on OSX, "mbcs" on Win) * no API call is supposed to crash with ``UnicodeDecodeError`` * instead, in case of badly encoded data returned by the OS, the following error handlers are used to replace the corrupted characters in the string: - * Python 3: ``"surrogatescape"`` on POSIX and ``"replace"`` on Windows + * Python 3: ``sys.getfilesystemencodeerrors()`` (PY 3.6+) or + ``"surrogatescape"`` on POSIX and ``"replace"`` on Windows * Python 2: ``"replace"`` * on Python 2 all APIs return bytes (``str`` type), never ``unicode`` * on Python 2, you can go back to ``unicode`` by doing: diff --git a/psutil/_common.py b/psutil/_common.py index 54cb1ff5d..e6ba9dc38 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -27,6 +27,8 @@ except ImportError: AF_UNIX = None +from psutil._compat import PY3 + if sys.version_info >= (3, 4): import enum else: @@ -132,6 +134,17 @@ class BatteryTime(enum.IntEnum): globals().update(BatteryTime.__members__) +# --- others + +ENCODING = sys.getfilesystemencoding() +if not PY3: + ENCODING_ERRS = "replace" +else: + try: + ENCODING_ERRS = sys.getfilesystemencodeerrors() # py 3.6 + except AttributeError: + ENCODING_ERRS = "surrogateescape" if POSIX else "replace" + # =================================================================== # --- namedtuples diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 1a890a874..7a03940a5 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -25,13 +25,15 @@ from . import _psposix from . import _psutil_linux as cext from . import _psutil_posix as cext_posix +from ._common import ENCODING +from ._common import ENCODING_ERRS from ._common import isfile_strict from ._common import memoize from ._common import memoize_when_activated -from ._common import parse_environ_block from ._common import NIC_DUPLEX_FULL from ._common import NIC_DUPLEX_HALF from ._common import NIC_DUPLEX_UNKNOWN +from ._common import parse_environ_block from ._common import path_exists_strict from ._common import supports_ipv6 from ._common import usage_percent @@ -84,9 +86,6 @@ BIGGER_FILE_BUFFERING = -1 if PY3 else 8192 LITTLE_ENDIAN = sys.byteorder == 'little' SECTOR_SIZE_FALLBACK = 512 -if PY3: - FS_ENCODING = sys.getfilesystemencoding() - ENCODING_ERRORS_HANDLER = 'surrogateescape' if enum is None: AF_LINK = socket.AF_PACKET else: @@ -200,14 +199,14 @@ def open_text(fname, **kwargs): # See: # https://github.com/giampaolo/psutil/issues/675 # https://github.com/giampaolo/psutil/pull/733 - kwargs.setdefault('encoding', FS_ENCODING) - kwargs.setdefault('errors', ENCODING_ERRORS_HANDLER) + kwargs.setdefault('encoding', ENCODING) + kwargs.setdefault('errors', ENCODING_ERRS) return open(fname, "rt", **kwargs) if PY3: def decode(s): - return s.decode(encoding=FS_ENCODING, errors=ENCODING_ERRORS_HANDLER) + return s.decode(encoding=ENCODING, errors=ENCODING_ERRS) else: def decode(s): return s diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index dd7b07a1f..9f55194f7 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -32,11 +32,13 @@ raise from ._common import conn_tmap +from ._common import ENCODING +from ._common import ENCODING_ERRS from ._common import isfile_strict +from ._common import memoize_when_activated from ._common import parse_environ_block from ._common import sockfam_to_enum from ._common import socktype_to_enum -from ._common import memoize_when_activated from ._common import usage_percent from ._compat import long from ._compat import lru_cache @@ -71,8 +73,6 @@ # --- globals # ===================================================================== -FS_ENCODING = sys.getfilesystemencoding() -PY2_ENCODING_ERRS = "replace" CONN_DELETE_TCB = "DELETE_TCB" WAIT_TIMEOUT = 0x00000102 # 258 in decimal ACCESS_DENIED_SET = frozenset([errno.EPERM, errno.EACCES, @@ -198,7 +198,7 @@ def py2_strencode(s): if isinstance(s, str): return s else: - return s.encode(FS_ENCODING, errors=PY2_ENCODING_ERRS) + return s.encode(ENCODING, errors=ENCODING_ERRS) # ===================================================================== @@ -240,7 +240,9 @@ def swap_memory(): def disk_usage(path): """Return disk usage associated with path.""" if PY3 and isinstance(path, bytes): - path = path.decode(FS_ENCODING) + # XXX: do we want to use "strict"? Probably yes, in order + # to fail immediately. After all we are accepting input here... + path = path.decode(ENCODING, errors="strict") total, free = cext.disk_usage(path) used = total - free percent = usage_percent(used, total, _round=1) From 3977de2616650c1ed2c9bf392c88eb2d0dd837d4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 17:56:06 +0200 Subject: [PATCH 549/922] make.bat: add -p opt --- DEVGUIDE.rst | 22 ++++++++++++++++++++++ make.bat | 2 +- psutil/_common.py | 8 ++++++-- scripts/internal/winmake.py | 35 +++++++++++++++++++++++++++++++++-- setup.py | 17 ++++++----------- 5 files changed, 68 insertions(+), 16 deletions(-) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index af34ee12a..c00a3d5c8 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -46,6 +46,28 @@ Some useful make commands:: $ make coverage # run test coverage $ make flake8 # run PEP8 linter +There are some differences between ``make`` on UNIX and Windows. +For instance, to run a specific Python version. On UNIX:: + + make test PYTHON=python3.5 + +On Windows: + + set PYTHON=C:\python35\python.exe && make test + + # ...or + + make -p 35 test + +If you want to modify psutil and run a script on the fly which uses it do +(on UNIX):: + + make test TSCRIPT=foo.py + +On Windows: + + set TSCRIPT=foo.py && make test + ==================== Adding a new feature ==================== diff --git a/make.bat b/make.bat index 98456457d..cdabe3a61 100644 --- a/make.bat +++ b/make.bat @@ -29,4 +29,4 @@ if "%TSCRIPT%" == "" ( rem Needed to locate the .pypirc file and upload exes on PYPI. set HOME=%USERPROFILE% -%PYTHON% scripts\internal\winmake.py %1 %2 +%PYTHON% scripts\internal\winmake.py %1 %2 %3 %4 %5 %6 diff --git a/psutil/_common.py b/psutil/_common.py index e6ba9dc38..38ffdc094 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -4,6 +4,9 @@ """Common objects shared by __init__.py and _ps*.py modules.""" +# Note: this module is imported by setup.py so it should not import +# psutil or third-party modules. + from __future__ import division import contextlib @@ -27,13 +30,14 @@ except ImportError: AF_UNIX = None -from psutil._compat import PY3 - if sys.version_info >= (3, 4): import enum else: enum = None +# can't take it from _common.py as this script is imported by setup.py +PY3 = sys.version_info[0] == 3 + __all__ = [ # OS constants 'FREEBSD', 'BSD', 'LINUX', 'NETBSD', 'OPENBSD', 'OSX', 'POSIX', 'SUNOS', diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index b8d111a4b..05eae8a66 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -180,10 +180,11 @@ def recursive_rm(*patterns): @cmd def help(): """Print this help""" - safe_print('Run "make " where is one of:') + safe_print('Run "make [-p ] " where is one of:') for name in sorted(_cmds): safe_print( " %-20s %s" % (name.replace('_', '-'), _cmds[name] or '')) + sys.exit(1) @cmd @@ -440,12 +441,42 @@ def bench_oneshot_2(): sh("%s scripts\\internal\\bench_oneshot_2.py" % PYTHON) +def set_python(s): + global PYTHON + if os.path.isabs(s): + PYTHON = s + else: + # try to look for a python installation + orig = s + s = s.replace('.', '') + for v in ('26', '27', '33', '34', '35', '36', '37'): + if s == v: + path = 'C:\\python%s\python.exe' % s + if os.path.isfile(path): + print(path) + PYTHON = path + return + return sys.exit( + "can't find any python installation matching %r" % orig) + + +def parse_cmdline(): + if '-p' in sys.argv: + try: + pos = sys.argv.index('-p') + sys.argv.pop(pos) + py = sys.argv.pop(pos) + except IndexError: + return help() + set_python(py) + + def main(): + parse_cmdline() try: cmd = sys.argv[1].replace('-', '_') except IndexError: return help() - if cmd in _cmds: fun = getattr(sys.modules[__name__], cmd) fun() diff --git a/setup.py b/setup.py index d4fc45915..c4f3bcbcf 100755 --- a/setup.py +++ b/setup.py @@ -22,6 +22,8 @@ from distutils.core import setup, Extension HERE = os.path.abspath(os.path.dirname(__file__)) + +# ...so we can import _common.py sys.path.insert(0, os.path.join(HERE, "psutil")) from _common import BSD # NOQA @@ -61,6 +63,10 @@ def get_version(): raise ValueError("couldn't find version string") +VERSION = get_version() +macros.append(('PSUTIL_VERSION', int(VERSION.replace('.', '')))) + + def get_description(): README = os.path.join(HERE, 'README.rst') with open(README, 'r') as f: @@ -84,10 +90,6 @@ def write(self, s): setattr(sys, stream_name, orig) -VERSION = get_version() -macros.append(('PSUTIL_VERSION', int(VERSION.replace('.', '')))) - -# Windows if WINDOWS: def get_winver(): maj, min = sys.getwindowsversion()[0:2] @@ -130,7 +132,6 @@ def get_winver(): # extra_link_args=["/DEBUG"] ) -# OS X elif OSX: macros.append(("PSUTIL_OSX", 1)) ext = Extension( @@ -144,7 +145,6 @@ def get_winver(): '-framework', 'CoreFoundation', '-framework', 'IOKit' ]) -# FreeBSD elif FREEBSD: macros.append(("PSUTIL_FREEBSD", 1)) ext = Extension( @@ -157,7 +157,6 @@ def get_winver(): define_macros=macros, libraries=["devstat"]) -# OpenBSD elif OPENBSD: macros.append(("PSUTIL_OPENBSD", 1)) ext = Extension( @@ -169,7 +168,6 @@ def get_winver(): define_macros=macros, libraries=["kvm"]) -# NetBSD elif NETBSD: macros.append(("PSUTIL_NETBSD", 1)) ext = Extension( @@ -182,7 +180,6 @@ def get_winver(): define_macros=macros, libraries=["kvm"]) -# Linux elif LINUX: def get_ethtool_macro(): # see: https://github.com/giampaolo/psutil/issues/659 @@ -218,7 +215,6 @@ def get_ethtool_macro(): sources=sources + ['psutil/_psutil_linux.c'], define_macros=macros) -# Solaris elif SUNOS: macros.append(("PSUTIL_SUNOS", 1)) ext = Extension( @@ -230,7 +226,6 @@ def get_ethtool_macro(): else: sys.exit('platform %s is not supported' % sys.platform) -# POSIX if POSIX: posix_extension = Extension( 'psutil._psutil_posix', From a73376529d4423cff81f8cb8feb22da7ed9c79a7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 18:03:28 +0200 Subject: [PATCH 550/922] make.bat: recognize python 64 versions --- scripts/internal/winmake.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 05eae8a66..e3ac1e283 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -449,7 +449,9 @@ def set_python(s): # try to look for a python installation orig = s s = s.replace('.', '') - for v in ('26', '27', '33', '34', '35', '36', '37'): + vers = ('26', '27', '33', '34', '35', '36', '37', + '26-64', '27-64', '33-64', '34-64', '35-64', '36-64', '37-64') + for v in vers: if s == v: path = 'C:\\python%s\python.exe' % s if os.path.isfile(path): From 94734dc40c7df507eaf5fc63ee47dd83cbe5ce14 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 18:30:36 +0200 Subject: [PATCH 551/922] use mock to raise an exc instead of picking up the low level C function --- psutil/tests/test_unicode.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 0a54198bf..ae3c012fd 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -69,6 +69,7 @@ from psutil.tests import get_test_subprocess from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import mock from psutil.tests import reap_children from psutil.tests import run_test_module_by_name from psutil.tests import safe_mkdir @@ -143,8 +144,10 @@ def test_proc_name(self): # On Windows name() is determined from exe() first, because # it's faster; we want to overcome the internal optimization # and test name() instead of exe(). - from psutil._pswindows import py2_strencode - name = py2_strencode(psutil._psplatform.cext.proc_name(subp.pid)) + with mock.patch("psutil._psplatform.cext.proc_exe", + side_effect=psutil.AccessDenied(os.getpid())) as m: + name = psutil.Process(subp.pid).name() + assert m.called else: name = psutil.Process(subp.pid).name() self.assertIsInstance(name, str) @@ -284,9 +287,10 @@ def test_name_type(self): # On Windows name() is determined from exe() first, because # it's faster; we want to overcome the internal optimization # and test name() instead of exe(). - from psutil._pswindows import py2_strencode - name = py2_strencode(psutil._psplatform.cext.proc_name(os.getpid())) - self.assertIsInstance(name, str) + with mock.patch("psutil._psplatform.cext.proc_exe", + side_effect=psutil.AccessDenied(os.getpid())) as m: + self.assertIsInstance(psutil.Process().name(), str) + assert m.called # =================================================================== From a79ab08c3910f73ee17877ef7a2784de4706e55b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 18:50:05 +0200 Subject: [PATCH 552/922] fix failing test --- DEVGUIDE.rst | 6 +++--- psutil/_common.py | 3 ++- psutil/tests/test_contracts.py | 1 - psutil/tests/test_system.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index c00a3d5c8..c4ddc52d7 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -51,11 +51,11 @@ For instance, to run a specific Python version. On UNIX:: make test PYTHON=python3.5 -On Windows: +On Windows:: set PYTHON=C:\python35\python.exe && make test - # ...or + # ...or: make -p 35 test @@ -64,7 +64,7 @@ If you want to modify psutil and run a script on the fly which uses it do make test TSCRIPT=foo.py -On Windows: +On Windows:: set TSCRIPT=foo.py && make test diff --git a/psutil/_common.py b/psutil/_common.py index 38ffdc094..8f3b4f413 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -39,9 +39,10 @@ PY3 = sys.version_info[0] == 3 __all__ = [ - # OS constants + # constants 'FREEBSD', 'BSD', 'LINUX', 'NETBSD', 'OPENBSD', 'OSX', 'POSIX', 'SUNOS', 'WINDOWS', + 'ENCODING', 'ENCODING_ERRS', # connection constants 'CONN_CLOSE', 'CONN_CLOSE_WAIT', 'CONN_CLOSING', 'CONN_ESTABLISHED', 'CONN_FIN_WAIT1', 'CONN_FIN_WAIT2', 'CONN_LAST_ACK', 'CONN_LISTEN', diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index d3c0377a5..7666f2f83 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -553,7 +553,6 @@ def memory_percent(self, ret, proc): def is_running(self, ret, proc): self.assertIsInstance(ret, bool) - assert ret # XXX: racy def cpu_affinity(self, ret, proc): self.assertIsInstance(ret, list) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index bb485296c..6aae894a6 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -453,7 +453,7 @@ def test_disk_usage(self): try: psutil.disk_usage(fname) except OSError as err: - if err.args[0] != errno.ENOENT: + if err.errno != errno.ENOENT: raise else: self.fail("OSError not raised") From fa3a646a597a760283470757fde2c4a3bb98bbab Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 18:51:26 +0200 Subject: [PATCH 553/922] small test refactoring --- psutil/tests/test_system.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 6aae894a6..32f76f57c 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -450,13 +450,9 @@ def test_disk_usage(self): # if path does not exist OSError ENOENT is expected across # all platforms fname = tempfile.mktemp() - try: + with self.assertRaises(OSError) as exc: psutil.disk_usage(fname) - except OSError as err: - if err.errno != errno.ENOENT: - raise - else: - self.fail("OSError not raised") + self.assertEqual(exc.exception.errno, errno.ENOENT) def test_disk_usage_unicode(self): # See: https://github.com/giampaolo/psutil/issues/416 From 810d4eb7132679712adee47cb26c632614f93413 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 18:54:14 +0200 Subject: [PATCH 554/922] try to minimize shared lib test failures on win --- psutil/tests/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 7f309c03c..842152599 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -969,6 +969,10 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() if os.path.normcase(os.path.splitext(x.path)[1]) == ext] + if WINDOWS: + libs = [x for x in libs + if 'python' in x.lower() and 'wow64' not in x.lower()] + assert libs cfile = None try: src = random.choice(libs) From 094eeb53d759884f87abebf9f19c49e2f19aa7ae Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 19:32:06 +0200 Subject: [PATCH 555/922] change var name --- psutil/_pswindows.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 9f55194f7..d18c55de7 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -75,8 +75,11 @@ CONN_DELETE_TCB = "DELETE_TCB" WAIT_TIMEOUT = 0x00000102 # 258 in decimal -ACCESS_DENIED_SET = frozenset([errno.EPERM, errno.EACCES, - cext.ERROR_ACCESS_DENIED]) +ACCESS_DENIED_ERRSET = frozenset([errno.EPERM, errno.EACCES, + cext.ERROR_ACCESS_DENIED]) +NO_SUCH_SERVICE_ERRSET = frozenset(cext.ERROR_INVALID_NAME, + cext.ERROR_SERVICE_DOES_NOT_EXIST) + if enum is None: AF_LINK = -1 @@ -484,15 +487,13 @@ def _wrap_exceptions(self): try: yield except WindowsError as err: - NO_SUCH_SERVICE_SET = (cext.ERROR_INVALID_NAME, - cext.ERROR_SERVICE_DOES_NOT_EXIST) - if err.errno in ACCESS_DENIED_SET: + if err.errno in ACCESS_DENIED_ERRSET: raise AccessDenied( pid=None, name=self._name, msg="service %r is not querable (not enough privileges)" % self._name) - elif err.errno in NO_SUCH_SERVICE_SET or \ - err.winerror in NO_SUCH_SERVICE_SET: + elif err.errno in NO_SUCH_SERVICE_ERRSET or \ + err.winerror in NO_SUCH_SERVICE_ERRSET: raise NoSuchProcess( pid=None, name=self._name, msg="service %r does not exist)" % self._name) @@ -618,7 +619,7 @@ def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) except OSError as err: - if err.errno in ACCESS_DENIED_SET: + if err.errno in ACCESS_DENIED_ERRSET: raise AccessDenied(self.pid, self._name) if err.errno == errno.ESRCH: raise NoSuchProcess(self.pid, self._name) @@ -709,7 +710,7 @@ def _get_raw_meminfo(self): try: return cext.proc_memory_info(self.pid) except OSError as err: - if err.errno in ACCESS_DENIED_SET: + if err.errno in ACCESS_DENIED_ERRSET: # TODO: the C ext can probably be refactored in order # to get this from cext.proc_info() info = self.oneshot_info() @@ -749,7 +750,7 @@ def memory_maps(self): except OSError as err: # XXX - can't use wrap_exceptions decorator as we're # returning a generator; probably needs refactoring. - if err.errno in ACCESS_DENIED_SET: + if err.errno in ACCESS_DENIED_ERRSET: raise AccessDenied(self.pid, self._name) if err.errno == errno.ESRCH: raise NoSuchProcess(self.pid, self._name) @@ -798,7 +799,7 @@ def create_time(self): try: return cext.proc_create_time(self.pid) except OSError as err: - if err.errno in ACCESS_DENIED_SET: + if err.errno in ACCESS_DENIED_ERRSET: return self.oneshot_info()[pinfo_map['create_time']] raise @@ -820,7 +821,7 @@ def cpu_times(self): try: user, system = cext.proc_cpu_times(self.pid) except OSError as err: - if err.errno in ACCESS_DENIED_SET: + if err.errno in ACCESS_DENIED_ERRSET: info = self.oneshot_info() user = info[pinfo_map['user_time']] system = info[pinfo_map['kernel_time']] @@ -901,7 +902,7 @@ def io_counters(self): try: ret = cext.proc_io_counters(self.pid) except OSError as err: - if err.errno in ACCESS_DENIED_SET: + if err.errno in ACCESS_DENIED_ERRSET: info = self.oneshot_info() ret = ( info[pinfo_map['io_rcount']], @@ -960,7 +961,7 @@ def num_handles(self): try: return cext.proc_num_handles(self.pid) except OSError as err: - if err.errno in ACCESS_DENIED_SET: + if err.errno in ACCESS_DENIED_ERRSET: return self.oneshot_info()[pinfo_map['num_handles']] raise From a47c7c977cb04fc0980449f7a26a82f8191b8255 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 20:43:14 +0200 Subject: [PATCH 556/922] wimake: listdir() unicode so that also paths with funky names can be removed --- psutil/tests/__init__.py | 4 ++-- scripts/internal/winmake.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 842152599..ad5c9761c 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -688,8 +688,8 @@ def wrapper(*args, **kwargs): def cleanup(): - for name in os.listdir('.'): - if name.startswith(TESTFILE_PREFIX): + for name in os.listdir(u('.')): + if name.startswith(u(TESTFILE_PREFIX)): try: safe_rmpath(name) except UnicodeEncodeError as exc: diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index e3ac1e283..0408b80cd 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -158,7 +158,7 @@ def onerror(fun, path, excinfo): def recursive_rm(*patterns): """Recursively remove a file or matching a list of patterns.""" - for root, subdirs, subfiles in os.walk('.'): + for root, subdirs, subfiles in os.walk(u'.'): root = os.path.normpath(root) if root.startswith('.git/'): continue From d9dc82a379e14f34fa556d5fc8a0b5a8aa37988e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 21:12:40 +0200 Subject: [PATCH 557/922] add workaround for ctypes python bug bpo-30286 --- psutil/tests/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index ad5c9761c..c03daf7fd 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -12,7 +12,6 @@ import atexit import contextlib -import ctypes import errno import functools import os @@ -965,6 +964,7 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): in memory via ctypes. Return the new absolutized, normcased path. """ + import ctypes ext = ".so" if POSIX else ".dll" dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() @@ -981,5 +981,12 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): yield dst finally: if WINDOWS and cfile is not None: + # See: + # - https://ci.appveyor.com/project/giampaolo/psutil/build/1207/ + # job/o53330pbnri9bcw7 + # - http://bugs.python.org/issue30286 + # - http://stackoverflow.com/questions/23522055 + from ctypes import wintypes + ctypes.windll.kernel32.FreeLibrary.argtypes = [wintypes.HMODULE] ctypes.windll.kernel32.FreeLibrary(cfile._handle) safe_rmpath(dst) From 64a3e0730b7bab98dd2bd35b900131bb28b173e4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 21:53:01 +0200 Subject: [PATCH 558/922] make test file executable --- psutil/tests/__init__.py | 2 +- psutil/tests/test_unicode.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 psutil/tests/test_unicode.py diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index c03daf7fd..2546f98c7 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -981,7 +981,7 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): yield dst finally: if WINDOWS and cfile is not None: - # See: + # Work around ctypes issue introduced in Python 3.4: # - https://ci.appveyor.com/project/giampaolo/psutil/build/1207/ # job/o53330pbnri9bcw7 # - http://bugs.python.org/issue30286 diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py old mode 100644 new mode 100755 From eb3be3d3ab2e55e79121ba8e0f183b60343f73da Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 22:09:34 +0200 Subject: [PATCH 559/922] fix TypeError --- psutil/_pswindows.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index d18c55de7..4b27557c2 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -77,8 +77,8 @@ WAIT_TIMEOUT = 0x00000102 # 258 in decimal ACCESS_DENIED_ERRSET = frozenset([errno.EPERM, errno.EACCES, cext.ERROR_ACCESS_DENIED]) -NO_SUCH_SERVICE_ERRSET = frozenset(cext.ERROR_INVALID_NAME, - cext.ERROR_SERVICE_DOES_NOT_EXIST) +NO_SUCH_SERVICE_ERRSET = frozenset([cext.ERROR_INVALID_NAME, + cext.ERROR_SERVICE_DOES_NOT_EXIST]) if enum is None: From ac81153b664e468428bb9c550dbea03d3087f8cc Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 22:50:53 +0200 Subject: [PATCH 560/922] make.bat uninstall: remove files from site-packages dir --- scripts/internal/winmake.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 0408b80cd..e813f33c2 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -17,6 +17,7 @@ import functools import os import shutil +import site import ssl import subprocess import sys @@ -252,19 +253,13 @@ def install(): @cmd def uninstall(): """Uninstall psutil""" - try: - import psutil - except ImportError: - clean() - return + # Uninstalling psutil on Windows seems to be tricky. + # On "import psutil" tests may import a psutil version living in + # C:\PythonXY\Lib\site-packages which is not what we want, so + # we try both "pip uninstall psutil" and manually remove stuff + # from site-packages. clean() install_pip() - sh("%s -m pip uninstall -y psutil" % PYTHON) - - # Uninstalling psutil on Windows seems to be tricky as we may have - # different versions os psutil installed. Also we don't want to be - # in the main psutil source dir as "import psutil" will always - # succeed so this really removes files from site-packages dir. here = os.getcwd() try: os.chdir('C:\\') @@ -272,12 +267,17 @@ def uninstall(): try: import psutil # NOQA except ImportError: - clean() - return - sh("%s -m pip uninstall -y psutil" % PYTHON) + break + else: + sh("%s -m pip uninstall -y psutil" % PYTHON) finally: os.chdir(here) + for dir in site.getsitepackages(): + for name in os.listdir(dir): + if name.startswith('psutil'): + rm(os.path.join(dir, name)) + @cmd def clean(): From 93a989f7040539ca9d928d635169a0df9ba20b43 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 5 May 2017 23:41:04 +0200 Subject: [PATCH 561/922] make.bat: have subprocesses inherit cwd and environ --- scripts/internal/winmake.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index e813f33c2..aaeaeed56 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -24,8 +24,8 @@ import tempfile -PYTHON = sys.executable -TSCRIPT = os.environ['TSCRIPT'] +PYTHON = os.getenv('PYTHON', sys.executable) +TSCRIPT = os.getenv('TSCRIPT', 'psutil\\tests\\__main__.py') GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" PY3 = sys.version_info[0] == 3 DEPS = [ @@ -77,9 +77,7 @@ def safe_print(text, file=sys.stdout, flush=False): def sh(cmd, nolog=False): if not nolog: safe_print("cmd: " + cmd) - code = os.system(cmd) - if code: - raise SystemExit + subprocess.check_call(cmd, shell=True, env=os.environ, cwd=os.getcwd()) def cmd(fun): @@ -457,6 +455,7 @@ def set_python(s): if os.path.isfile(path): print(path) PYTHON = path + os.putenv('PYTHON', path) return return sys.exit( "can't find any python installation matching %r" % orig) From 9d0d8515c907aae49427c45503f13f0739df8860 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 6 May 2017 21:26:41 +0200 Subject: [PATCH 562/922] fix linux test --- psutil/tests/test_linux.py | 35 ++++++++++++++++++----------------- tox.ini | 2 +- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 053c5f692..7906d64ef 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1556,25 +1556,26 @@ def open_mock(name, *args, **kwargs): with mock.patch(patch_point, side_effect=open_mock): self.assertRaises(psutil.AccessDenied, psutil.Process().threads) - # not sure why (doesn't fail locally) - # https://travis-ci.org/giampaolo/psutil/jobs/108629915 - @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") def test_exe_mocked(self): with mock.patch('psutil._pslinux.readlink', - side_effect=OSError(errno.ENOENT, "")) as m: - # No such file error; might be raised also if /proc/pid/exe - # path actually exists for system processes with low pids - # (about 0-20). In this case psutil is supposed to return - # an empty string. - ret = psutil.Process().exe() - assert m.called - self.assertEqual(ret, "") - - # ...but if /proc/pid no longer exist we're supposed to treat - # it as an alias for zombie process - with mock.patch('psutil._pslinux.os.path.lexists', - return_value=False): - self.assertRaises(psutil.ZombieProcess, psutil.Process().exe) + side_effect=OSError(errno.ENOENT, "")) as m1: + with mock.patch('psutil.Process.cmdline', + side_effect=psutil.AccessDenied(0, "")) as m2: + # No such file error; might be raised also if /proc/pid/exe + # path actually exists for system processes with low pids + # (about 0-20). In this case psutil is supposed to return + # an empty string. + ret = psutil.Process().exe() + assert m1.called + assert m2.called + self.assertEqual(ret, "") + + # ...but if /proc/pid no longer exist we're supposed to treat + # it as an alias for zombie process + with mock.patch('psutil._pslinux.os.path.lexists', + return_value=False): + self.assertRaises( + psutil.ZombieProcess, psutil.Process().exe) def test_issue_1014(self): # Emulates a case where smaps file does not exist. In this case diff --git a/tox.ini b/tox.ini index 20b9f229d..3bee1d5a0 100644 --- a/tox.ini +++ b/tox.ini @@ -5,7 +5,7 @@ # directory. [tox] -envlist = py26, py27, py32, py33, py34 +envlist = py26, py27, py33, py34, py35, py36 [testenv] deps = From 91a043514e32eb3e79f4e44a572da25ea5c6310b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 14:31:49 +0200 Subject: [PATCH 563/922] copyload_shared_lib: split it in 2 functions for POSIX and WIN --- psutil/tests/__init__.py | 65 ++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 23 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 2546f98c7..e8d7fb056 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -12,6 +12,7 @@ import atexit import contextlib +import ctypes import errno import functools import os @@ -957,36 +958,54 @@ def is_namedtuple(x): return all(type(n) == str for n in f) -@contextlib.contextmanager -def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): - """Ctx manager which picks up a random shared so/dll lib used - by this process, copies it in another location and loads it - in memory via ctypes. - Return the new absolutized, normcased path. - """ - import ctypes - ext = ".so" if POSIX else ".dll" - dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) - libs = [x.path for x in psutil.Process().memory_maps() - if os.path.normcase(os.path.splitext(x.path)[1]) == ext] - if WINDOWS: +if POSIX: + @contextlib.contextmanager + def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): + """Ctx manager which picks up a random shared CO lib used + by this process, copies it in another location and loads it + in memory via ctypes. Return the new absolutized path. + """ + ext = ".so" + dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) + libs = [x.path for x in psutil.Process().memory_maps() + if os.path.splitext(x.path)[1] == ext] + src = random.choice(libs) + shutil.copyfile(src, dst) + try: + ctypes.CDLL(dst) + yield dst + finally: + safe_rmpath(dst) +else: + @contextlib.contextmanager + def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): + """Ctx manager which picks up a random shared DLL lib used + by this process, copies it in another location and loads it + in memory via ctypes. + Return the new absolutized, normcased path. + """ + from ctypes import wintypes + ext = ".dll" + dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) + libs = [x.path for x in psutil.Process().memory_maps() + if os.path.splitext(x.path)[1].lower() == ext] libs = [x for x in libs if 'python' in x.lower() and 'wow64' not in x.lower()] assert libs - cfile = None - try: src = random.choice(libs) shutil.copyfile(src, dst) - cfile = ctypes.CDLL(dst) - yield dst - finally: - if WINDOWS and cfile is not None: + cfile = None + try: + cfile = ctypes.CDLL(dst) + yield dst + finally: # Work around ctypes issue introduced in Python 3.4: # - https://ci.appveyor.com/project/giampaolo/psutil/build/1207/ # job/o53330pbnri9bcw7 # - http://bugs.python.org/issue30286 # - http://stackoverflow.com/questions/23522055 - from ctypes import wintypes - ctypes.windll.kernel32.FreeLibrary.argtypes = [wintypes.HMODULE] - ctypes.windll.kernel32.FreeLibrary(cfile._handle) - safe_rmpath(dst) + if cfile is not None: + ctypes.windll.kernel32.FreeLibrary.argtypes = \ + [wintypes.HMODULE] + ctypes.windll.kernel32.FreeLibrary(cfile._handle) + safe_rmpath(dst) From 6de4345b6a37ff9237171698fddaf0255fb9e87b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 14:36:37 +0200 Subject: [PATCH 564/922] use ctypes WinDLL instead of CDLL --- psutil/tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index e8d7fb056..3f6f52905 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -996,7 +996,7 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): shutil.copyfile(src, dst) cfile = None try: - cfile = ctypes.CDLL(dst) + cfile = ctypes.WinDLL(dst) yield dst finally: # Work around ctypes issue introduced in Python 3.4: From ef7edab7cdb519756b015c51f1618576b54f04f8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 14:43:38 +0200 Subject: [PATCH 565/922] FreeLibrary: catch return code and raise exc --- psutil/tests/__init__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 3f6f52905..edbfaa500 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -985,13 +985,12 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): Return the new absolutized, normcased path. """ from ctypes import wintypes + from ctypes import WinError ext = ".dll" dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() - if os.path.splitext(x.path)[1].lower() == ext] - libs = [x for x in libs - if 'python' in x.lower() and 'wow64' not in x.lower()] - assert libs + if os.path.splitext(x.path)[1].lower() == ext and + 'python' in x.lower() and 'wow64' not in x.lower()] src = random.choice(libs) shutil.copyfile(src, dst) cfile = None @@ -999,13 +998,15 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): cfile = ctypes.WinDLL(dst) yield dst finally: - # Work around ctypes issue introduced in Python 3.4: + # Work around OverflowError: # - https://ci.appveyor.com/project/giampaolo/psutil/build/1207/ # job/o53330pbnri9bcw7 # - http://bugs.python.org/issue30286 # - http://stackoverflow.com/questions/23522055 if cfile is not None: - ctypes.windll.kernel32.FreeLibrary.argtypes = \ - [wintypes.HMODULE] - ctypes.windll.kernel32.FreeLibrary(cfile._handle) + FreeLibrary = ctypes.windll.kernel32.FreeLibrary + FreeLibrary.argtypes = [wintypes.HMODULE] + ret = FreeLibrary(cfile._handle) + if ret == 0: + WinError() safe_rmpath(dst) From 86b408d94681c190491b52e57cca1296ccd790c4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 14:45:12 +0200 Subject: [PATCH 566/922] fix AttributeError --- psutil/tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index edbfaa500..0de94fd44 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -990,7 +990,7 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() if os.path.splitext(x.path)[1].lower() == ext and - 'python' in x.lower() and 'wow64' not in x.lower()] + 'python' in x.path.lower() and 'wow64' not in x.path.lower()] src = random.choice(libs) shutil.copyfile(src, dst) cfile = None From 84a9806a2b82483686ee428afa4fa600981601d2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 14:58:11 +0200 Subject: [PATCH 567/922] filter shared libs with looking for 'python' in their name --- psutil/_compat.py | 3 +-- psutil/tests/__init__.py | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/psutil/_compat.py b/psutil/_compat.py index 9f2182c22..a318f70fa 100644 --- a/psutil/_compat.py +++ b/psutil/_compat.py @@ -11,7 +11,7 @@ import sys __all__ = ["PY3", "long", "xrange", "unicode", "basestring", "u", "b", - "callable", "lru_cache", "which"] + "callable", "lru_cache", "which", "nested"] PY3 = sys.version_info[0] == 3 @@ -268,7 +268,6 @@ def nested(*managers): with B as Y: with C as Z: - """ exits = [] vars = [] diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 0de94fd44..97fbfc8f4 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -968,7 +968,8 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): ext = ".so" dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() - if os.path.splitext(x.path)[1] == ext] + if os.path.splitext(x.path)[1] == ext and + 'python' in os.path.basename(x.path)] src = random.choice(libs) shutil.copyfile(src, dst) try: @@ -990,7 +991,8 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() if os.path.splitext(x.path)[1].lower() == ext and - 'python' in x.path.lower() and 'wow64' not in x.path.lower()] + 'python' in os.path.basebaname(x.path).lower() and + 'wow64' not in x.path.lower()] src = random.choice(libs) shutil.copyfile(src, dst) cfile = None From ae5c4378b7a441cd2ed8cb5ab795669206b5c7c4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 15:05:08 +0200 Subject: [PATCH 568/922] fix AttributeError --- psutil/tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 97fbfc8f4..0cf642278 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -991,7 +991,7 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() if os.path.splitext(x.path)[1].lower() == ext and - 'python' in os.path.basebaname(x.path).lower() and + 'python' in os.path.basename(x.path).lower() and 'wow64' not in x.path.lower()] src = random.choice(libs) shutil.copyfile(src, dst) From 46399478463f2a16eb2c611ba74489682be95e1a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 15:48:27 +0200 Subject: [PATCH 569/922] prevent windows tests to open error dialogs/windows --- psutil/tests/__init__.py | 23 ++++++++++++++++++----- psutil/tests/test_process.py | 5 ++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 0cf642278..8bdd9dd62 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -233,6 +233,9 @@ def get_test_subprocess(cmd=None, **kwds): """ kwds.setdefault("stdin", DEVNULL) kwds.setdefault("stdout", DEVNULL) + if WINDOWS: + # Prevents the subprocess to open error dialogs. + kwds.setdefault("creationflags", 0x8000000) # CREATE_NO_WINDOW if cmd is None: safe_rmpath(_TESTFN) pyline = "from time import sleep;" @@ -272,7 +275,13 @@ def create_proc_children_pair(): subprocess.Popen([PYTHON, '-c', s]) time.sleep(60) """ % _TESTFN2) - subp = pyrun(s) + # On Windows if we create a subprocess with CREATE_NO_WINDOW flag + # set (which is the default) a "conhost.exe" extra process will be + # spawned as a child. We don't want that. + if WINDOWS: + subp = pyrun(s, creationflags=0) + else: + subp = pyrun(s) try: child1 = psutil.Process(subp.pid) data = wait_for_file(_TESTFN2, delete=False, empty=False) @@ -286,17 +295,18 @@ def create_proc_children_pair(): raise -def pyrun(src): +def pyrun(src, **kwds): """Run python 'src' code string in a separate interpreter. Returns a subprocess.Popen instance. """ + kwds.setdefault("stdout", None) + kwds.setdefault("stderr", None) with tempfile.NamedTemporaryFile( prefix=TESTFILE_PREFIX, mode="wt", delete=False) as f: _testfiles_created.add(f.name) f.write(src) f.flush() - subp = get_test_subprocess([PYTHON, f.name], stdout=None, - stderr=None) + subp = get_test_subprocess([PYTHON, f.name], **kwds) wait_for_pid(subp.pid) return subp @@ -306,8 +316,11 @@ def sh(cmd): raises RuntimeError on error. """ shell = True if isinstance(cmd, (str, unicode)) else False + # Prevents subprocess to open error dialogs in case of error. + flags = 0x8000000 if WINDOWS and shell else 0 p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True) + stderr=subprocess.PIPE, universal_newlines=True, + creationflags=flags) stdout, stderr = p.communicate() if p.returncode != 0: raise RuntimeError(stderr) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index d1cb96573..b6c418991 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1037,7 +1037,10 @@ def test_children(self): p = psutil.Process() self.assertEqual(p.children(), []) self.assertEqual(p.children(recursive=True), []) - sproc = get_test_subprocess() + # On Windows we set the flag to 0 in order to cancel out the + # CREATE_NO_WINDOW flag (enabled by default) which creates + # an extra "conhost.exe" child. + sproc = get_test_subprocess(creationflags=0) children1 = p.children() children2 = p.children(recursive=True) for children in (children1, children2): From a12e418580e89936f0b95b2ce70990a9c1301e6e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 15:59:13 +0200 Subject: [PATCH 570/922] minor refactoring --- psutil/tests/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 8bdd9dd62..4c60669c2 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -238,9 +238,9 @@ def get_test_subprocess(cmd=None, **kwds): kwds.setdefault("creationflags", 0x8000000) # CREATE_NO_WINDOW if cmd is None: safe_rmpath(_TESTFN) - pyline = "from time import sleep;" - pyline += "open(r'%s', 'w').close();" % _TESTFN - pyline += "sleep(60);" + pyline = "from time import sleep;" \ + "open(r'%s', 'w').close();" \ + "sleep(60);" % _TESTFN cmd = [PYTHON, "-c", pyline] sproc = subprocess.Popen(cmd, **kwds) _subprocesses_started.add(sproc) From 6c1473c8a04b8dc01effeb883901537185fd90fe Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 16:15:05 +0200 Subject: [PATCH 571/922] add _cleanup_on_err decorator for subprocess test functions --- psutil/tests/__init__.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 4c60669c2..826337350 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -222,6 +222,18 @@ def stop(self): # =================================================================== +def _cleanup_on_err(fun): + @functools.wraps(fun) + def wrapper(*args, **kwargs): + try: + return fun(*args, **kwargs) + except Exception: + reap_children() + raise + return wrapper + + +@_cleanup_on_err def get_test_subprocess(cmd=None, **kwds): """Creates a python subprocess which does nothing for 60 secs and return it as subprocess.Popen instance. @@ -244,11 +256,7 @@ def get_test_subprocess(cmd=None, **kwds): cmd = [PYTHON, "-c", pyline] sproc = subprocess.Popen(cmd, **kwds) _subprocesses_started.add(sproc) - try: - wait_for_file(_TESTFN, delete=True, empty=True) - except Exception: - reap_children() - raise + wait_for_file(_TESTFN, delete=True, empty=True) else: sproc = subprocess.Popen(cmd, **kwds) _subprocesses_started.add(sproc) @@ -256,6 +264,7 @@ def get_test_subprocess(cmd=None, **kwds): return sproc +@_cleanup_on_err def create_proc_children_pair(): """Create a subprocess which creates another one as in: A (us) -> B (child) -> C (grandchild). @@ -282,19 +291,16 @@ def create_proc_children_pair(): subp = pyrun(s, creationflags=0) else: subp = pyrun(s) - try: - child1 = psutil.Process(subp.pid) - data = wait_for_file(_TESTFN2, delete=False, empty=False) - os.remove(_TESTFN2) - child2_pid = int(data) - _pids_started.add(child2_pid) - child2 = psutil.Process(child2_pid) - return (child1, child2) - except Exception: - reap_children() - raise + child1 = psutil.Process(subp.pid) + data = wait_for_file(_TESTFN2, delete=False, empty=False) + os.remove(_TESTFN2) + child2_pid = int(data) + _pids_started.add(child2_pid) + child2 = psutil.Process(child2_pid) + return (child1, child2) +@_cleanup_on_err def pyrun(src, **kwds): """Run python 'src' code string in a separate interpreter. Returns a subprocess.Popen instance. @@ -311,6 +317,7 @@ def pyrun(src, **kwds): return subp +@_cleanup_on_err def sh(cmd): """run cmd in a subprocess and return its output. raises RuntimeError on error. From 7666c166054a21a5150f13cd24128398b74239b9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 20:36:48 +0200 Subject: [PATCH 572/922] relax compyload_shared_lib filtering --- psutil/tests/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 826337350..4b9483939 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -988,8 +988,7 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): ext = ".so" dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) libs = [x.path for x in psutil.Process().memory_maps() - if os.path.splitext(x.path)[1] == ext and - 'python' in os.path.basename(x.path)] + if os.path.splitext(x.path)[1] == ext and 'python' in x.path] src = random.choice(libs) shutil.copyfile(src, dst) try: From 5a92ca5c58c4343f0f5bd1a430355e03f791a380 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 7 May 2017 20:45:45 +0200 Subject: [PATCH 573/922] relax compyload_shared_lib filtering --- psutil/tests/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 4b9483939..3e5f743a6 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -987,8 +987,9 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): """ ext = ".so" dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) - libs = [x.path for x in psutil.Process().memory_maps() - if os.path.splitext(x.path)[1] == ext and 'python' in x.path] + libs = [x.path for x in psutil.Process().memory_maps() if + os.path.splitext(x.path)[1] == ext and + 'python' in x.path.lower()] src = random.choice(libs) shutil.copyfile(src, dst) try: @@ -1008,8 +1009,8 @@ def copyload_shared_lib(dst_prefix=TESTFILE_PREFIX): from ctypes import WinError ext = ".dll" dst = tempfile.mktemp(prefix=dst_prefix, suffix=ext) - libs = [x.path for x in psutil.Process().memory_maps() - if os.path.splitext(x.path)[1].lower() == ext and + libs = [x.path for x in psutil.Process().memory_maps() if + os.path.splitext(x.path)[1].lower() == ext and 'python' in os.path.basename(x.path).lower() and 'wow64' not in x.path.lower()] src = random.choice(libs) From a5539d371e368f5505e4ccb746c49345403a552a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 00:21:35 +0200 Subject: [PATCH 574/922] enhance atexit functions --- psutil/_psutil_windows.c | 2 -- psutil/tests/__init__.py | 40 ++++++++++++++++++++++++---------------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 53deadcc3..a3e921c02 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -397,8 +397,6 @@ psutil_proc_wait(PyObject *self, PyObject *args) { return Py_BuildValue("l", WAIT_TIMEOUT); } - // get the exit code; note: subprocess module (erroneously?) uses - // what returned by WaitForSingleObject if (GetExitCodeProcess(hProcess, &ExitCode) == 0) { CloseHandle(hProcess); return PyErr_SetFromWindowsErr(GetLastError()); diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 3e5f743a6..6ffeab150 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -27,6 +27,7 @@ import textwrap import threading import time +import traceback import warnings from socket import AF_INET from socket import AF_INET6 @@ -82,7 +83,7 @@ # threads 'ThreadTask' # test utils - 'unittest', 'cleanup', 'skip_on_access_denied', 'skip_on_not_implemented', + 'unittest', 'skip_on_access_denied', 'skip_on_not_implemented', 'retry_before_failing', 'run_test_module_by_name', # install utils 'install_pip', 'install_test_deps', @@ -176,6 +177,28 @@ _testfiles_created = set() +@atexit.register +def _cleanup_files(): + DEVNULL.close() + for name in os.listdir(u('.')): + if name.startswith(u(TESTFILE_PREFIX)): + try: + safe_rmpath(name) + except Exception: + traceback.print_exc() + for path in _testfiles_created: + try: + safe_rmpath(path) + except Exception: + traceback.print_exc() + + +# this is executed first +@atexit.register +def _cleanup_procs(): + reap_children(recursive=True) + + # =================================================================== # --- threads # =================================================================== @@ -707,21 +730,6 @@ def wrapper(*args, **kwargs): return decorator -def cleanup(): - for name in os.listdir(u('.')): - if name.startswith(u(TESTFILE_PREFIX)): - try: - safe_rmpath(name) - except UnicodeEncodeError as exc: - warn(exc) - for path in _testfiles_created: - safe_rmpath(path) - - -atexit.register(cleanup) -atexit.register(lambda: DEVNULL.close()) - - # =================================================================== # --- network # =================================================================== From 411ec0da2da0e81087564545cdf40b186ef52184 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 00:25:23 +0200 Subject: [PATCH 575/922] refactoring --- psutil/tests/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 6ffeab150..e54677bc7 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -379,9 +379,8 @@ def reap_children(recursive=False): # Terminate subprocess.Popen instances "cleanly" by closing their # fds and wiat()ing for them in order to avoid zombies. - subprocs = _subprocesses_started.copy() - _subprocesses_started.clear() - for subp in subprocs: + while _subprocesses_started: + subp = _subprocesses_started.pop() try: subp.terminate() except OSError as err: @@ -404,16 +403,16 @@ def reap_children(recursive=False): raise # Terminate started pids. - for pid in _pids_started: + while _pids_started: + pid = _pids_started.pop() try: p = psutil.Process(pid) except psutil.NoSuchProcess: pass else: children.add(p) - _pids_started.clear() - # Terminate grandchildren. + # Terminate children. if children: for p in children: try: From 3e06eee00d3717592336191113f3f82c9832ddce Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 00:41:32 +0200 Subject: [PATCH 576/922] make ThreadTask a ctx manager --- psutil/tests/__init__.py | 9 ++++++++- psutil/tests/test_process.py | 12 ++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index e54677bc7..22ccc52d0 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -205,7 +205,7 @@ def _cleanup_procs(): class ThreadTask(threading.Thread): - """A thread object used for running process thread tests.""" + """A thread task which does nothing expect staying alive.""" def __init__(self): threading.Thread.__init__(self) @@ -217,6 +217,13 @@ def __repr__(self): name = self.__class__.__name__ return '<%s running=%s at %#x>' % (name, self._running, id(self)) + def __enter__(self): + self.start() + return self + + def __exit__(self, *args, **kwargs): + self.stop() + def start(self): """Start thread and keep it running until an explicit stop() request. Polls for shutdown every 'timeout' seconds. diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index b6c418991..598180c9c 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -513,13 +513,9 @@ def test_num_threads(self): else: step1 = p.num_threads() - thread = ThreadTask() - thread.start() - try: + with ThreadTask(): step2 = p.num_threads() self.assertEqual(step2, step1 + 1) - finally: - thread.stop() @unittest.skipIf(not WINDOWS, 'WINDOWS only') def test_num_handles(self): @@ -537,9 +533,7 @@ def test_threads(self): else: step1 = p.threads() - thread = ThreadTask() - thread.start() - try: + with ThreadTask(): step2 = p.threads() self.assertEqual(len(step2), len(step1) + 1) # on Linux, first thread id is supposed to be this process @@ -550,8 +544,6 @@ def test_threads(self): self.assertEqual(athread.id, athread[0]) self.assertEqual(athread.user_time, athread[1]) self.assertEqual(athread.system_time, athread[2]) - finally: - thread.stop() @retry_before_failing() @skip_on_access_denied(only_if=OSX) From 49ce1f6de1532fb2be4301f3a62699340cd06a2d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 03:15:21 +0200 Subject: [PATCH 577/922] #1007 / boot_time() / win: consider 1 sec fluctuation between calls acceptable and return always the same value --- HISTORY.rst | 3 +++ psutil/_pswindows.py | 14 +++++++++++++- psutil/tests/test_windows.py | 11 +++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index c1ad8d847..bf7f63671 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,9 @@ **Bug fixes** +- 1007_: [Windows] boot_time() can have a 1 sec fluctuation between calls; the + value of the first call is now cached so that boot_time() always returns the + same value if fluctuation is <= 1 second. - 1014_: [Linux] Process class can mask legitimate ENOENT exceptions as NoSuchProcess. - 1016_: disk_io_counters() raises RuntimeError on a system with no disks. diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 4b27557c2..bd6b091f6 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -401,9 +401,21 @@ def sensors_battery(): # ===================================================================== +_last_btime = 0 + + def boot_time(): """The system boot time expressed in seconds since the epoch.""" - return cext.boot_time() + # This dirty hack is to adjust the precision of the returned + # value which may have a 1 second fluctuation, see: + # https://github.com/giampaolo/psutil/issues/1007 + global _last_btime + ret = cext.boot_time() + if abs(ret - _last_btime) <= 1: + return _last_btime + else: + _last_btime = ret + return ret def users(): diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index ac7872837..a4f32315c 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -7,6 +7,7 @@ """Windows specific tests.""" +import datetime import errno import glob import os @@ -179,6 +180,16 @@ def test_net_if_stats(self): self.assertTrue(ps_names & wmi_names, "no common entries in %s, %s" % (ps_names, wmi_names)) + def test_boot_time(self): + wmi_os = wmi.WMI().Win32_OperatingSystem() + wmi_btime_str = wmi_os[0].LastBootUpTime.split('.')[0] + wmi_btime_dt = datetime.datetime.strptime( + wmi_btime_str, "%Y%m%d%H%M%S") + psutil_dt = datetime.datetime.fromtimestamp(psutil.boot_time()) + diff = abs((wmi_btime_dt - psutil_dt).total_seconds()) + # Wmic time is 2 secs lower for some reason; that's OK. + self.assertLessEqual(diff, 2) + # =================================================================== # sensors_battery() From ef026ada97a4faadd5658472100c5e68b869d962 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 04:30:02 +0200 Subject: [PATCH 578/922] #1007: add note about boot_time() on win whose is may not be the same across different processes --- docs/index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 350af93f8..b1ff16a97 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -749,6 +749,11 @@ Other system info >>> datetime.datetime.fromtimestamp(psutil.boot_time()).strftime("%Y-%m-%d %H:%M:%S") '2014-01-12 22:51:00' + .. note:: + on Windows this function may return a time which is off by 1 second if it's + used across different processes (see + `issue #1007 `__). + .. function:: users() Return users currently connected on the system as a list of named tuples From f56f777229c9931a22859466bf8e4480524918c4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 05:11:32 +0200 Subject: [PATCH 579/922] update doc --- docs/index.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b1ff16a97..31cbd2830 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -536,18 +536,19 @@ Network ...] .. note:: - (OSX) :class:`psutil.AccessDenied` is always raised unless running as root - (lsof does the same). + (OSX) :class:`psutil.AccessDenied` is always raised unless running as root. + This is a limitation of the OS and ``lsof`` does the same. .. note:: (Solaris) UNIX sockets are not supported. .. note:: (Linux, FreeBSD) "raddr" field for UNIX sockets is always set to "". + This is a limitation of the OS. .. note:: (OpenBSD) "laddr" and "raddr" fields for UNIX sockets are always set to - "". + "". This is a limitation of the OS. .. versionadded:: 2.1.0 @@ -1822,10 +1823,11 @@ Process class .. note:: (Linux, FreeBSD) "raddr" field for UNIX sockets is always set to "". + This is a limitation of the OS. .. note:: (OpenBSD) "laddr" and "raddr" fields for UNIX sockets are always set to - "". + "". This is a limitation of the OS. .. method:: is_running() From 65f76c0311901f2e8c77a8f6a719d340d227d017 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 14:17:13 +0200 Subject: [PATCH 580/922] #1007: add test for fluctuation --- psutil/tests/test_windows.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index a4f32315c..e781d1b7d 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -190,6 +190,16 @@ def test_boot_time(self): # Wmic time is 2 secs lower for some reason; that's OK. self.assertLessEqual(diff, 2) + def test_boot_time_fluctuation(self): + with mock('psutil._pswindows.cext.boot_time', return_value=5): + self.assertEqual(psutil.boot_time(), 5) + with mock('psutil._pswindows.cext.boot_time', return_value=4): + self.assertEqual(psutil.boot_time(), 5) + with mock('psutil._pswindows.cext.boot_time', return_value=6): + self.assertEqual(psutil.boot_time(), 5) + with mock('psutil._pswindows.cext.boot_time', return_value=333): + self.assertEqual(psutil.boot_time(), 333) + # =================================================================== # sensors_battery() From 4cdf4058aff5ac4fb044e09e3f4e5f82d1ddaf7c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 14:18:01 +0200 Subject: [PATCH 581/922] add doc --- psutil/tests/test_windows.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index e781d1b7d..6a18311e2 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -191,6 +191,7 @@ def test_boot_time(self): self.assertLessEqual(diff, 2) def test_boot_time_fluctuation(self): + # https://github.com/giampaolo/psutil/issues/1007 with mock('psutil._pswindows.cext.boot_time', return_value=5): self.assertEqual(psutil.boot_time(), 5) with mock('psutil._pswindows.cext.boot_time', return_value=4): From b947416bb783926ef0e6df902bcc6ac6a5a6ffe9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 14:18:58 +0200 Subject: [PATCH 582/922] fix typo --- psutil/tests/test_windows.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 6a18311e2..e01457a49 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -192,13 +192,13 @@ def test_boot_time(self): def test_boot_time_fluctuation(self): # https://github.com/giampaolo/psutil/issues/1007 - with mock('psutil._pswindows.cext.boot_time', return_value=5): + with mock.patch('psutil._pswindows.cext.boot_time', return_value=5): self.assertEqual(psutil.boot_time(), 5) - with mock('psutil._pswindows.cext.boot_time', return_value=4): + with mock.patch('psutil._pswindows.cext.boot_time', return_value=4): self.assertEqual(psutil.boot_time(), 5) - with mock('psutil._pswindows.cext.boot_time', return_value=6): + with mock.patch('psutil._pswindows.cext.boot_time', return_value=6): self.assertEqual(psutil.boot_time(), 5) - with mock('psutil._pswindows.cext.boot_time', return_value=333): + with mock.patch('psutil._pswindows.cext.boot_time', return_value=333): self.assertEqual(psutil.boot_time(), 333) From 950a57d7d3124f7e2f83cd439032848c69a03b33 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 15:09:21 +0200 Subject: [PATCH 583/922] #802: first PoC of (not working) wrap function --- psutil/_common.py | 65 +++++++++++++++++++++++++++++++++++++++ psutil/tests/test_misc.py | 45 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/psutil/_common.py b/psutil/_common.py index 8f3b4f413..1f826a0dc 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -16,6 +16,7 @@ import socket import stat import sys +import threading import warnings from collections import namedtuple from socket import AF_INET @@ -465,3 +466,67 @@ def inner(self, *args, **kwargs): return getattr(self, replacement)(*args, **kwargs) return inner return outer + + +_wrapn_lock = threading.Lock() +_wrapn_cache = {} + + +def wrap_numbers(new_dict, name): + def did_nums_wrap(new_nt, old_nt): + # Return True if one of the numbers of the new ntuple is smaller + # than the number of the old ntuple in the same position. + for i in range(len(new_nt)): + new_value = new_nt[i] + old_value = old_nt[i] + if new_value < old_value: + return True + return False + + def replace_ntuple(new_nt, old_nt): + # Return a new ntuple with the "adjusted" numbers. + bits = [] + for i in range(len(new_nt)): + new_value = new_nt[i] + old_value = old_nt[i] + if new_value < old_value: + # they wrapped! + num = old_value + new_value + else: + num = new_value + bits.append(num) + return new_nt._make(bits) + + with _wrapn_lock: + if name not in _wrapn_cache: + # This was the first call. + _wrapn_cache[name] = new_dict + return new_dict + + old_dict = _wrapn_cache[name] + for key, new_nt in new_dict.items(): + try: + old_nt = old_dict[key] + except KeyError: + # We may get here if net_io_counters() returned a new NIC + # or disk_io_counters() returned a new disk. + continue + else: + assert new_nt._fields == old_nt._fields, (new_nt, old_nt) + if did_nums_wrap(new_nt, old_nt): + # we wrapped! + new_dict[key] = replace_ntuple(new_nt, old_nt) + + _wrapn_cache[name] = new_dict + return new_dict + + +def _wrapn_cache_clear(name=None): + with _wrapn_lock: + if name is None: + _wrapn_cache.clear() + else: + _wrapn_cache.pop(name) + + +wrap_numbers.cache_clear = _wrapn_cache_clear diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 6bc2e28c9..8953e31d1 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -20,6 +20,7 @@ import socket import stat import sys +from collections import namedtuple from psutil import LINUX from psutil import POSIX @@ -27,6 +28,7 @@ from psutil._common import memoize from psutil._common import memoize_when_activated from psutil._common import supports_ipv6 +from psutil._common import wrap_numbers from psutil._compat import PY3 from psutil.tests import APPVEYOR from psutil.tests import bind_socket @@ -377,6 +379,49 @@ def test_sanity_version_check(self): self.assertIn("version conflict", str(cm.exception).lower()) +nt = namedtuple('foo', 'a b c') + + +class TestWrapNumbers(unittest.TestCase): + + def tearDown(self): + wrap_numbers.cache_clear() + + def test_first_call(self): + input = {'foo': nt(5, 5, 5)} + self.assertEqual(wrap_numbers(input, 'funname'), input) + + def test_input_hasnt_changed(self): + input = {'foo': nt(5, 5, 5)} + self.assertEqual(wrap_numbers(input, 'funname'), input) + self.assertEqual(wrap_numbers(input, 'funname'), input) + + def test_increase_but_no_wrap(self): + input = {'foo': nt(5, 5, 5)} + self.assertEqual(wrap_numbers(input, 'funname'), input) + input = {'foo': nt(10, 15, 20)} + self.assertEqual(wrap_numbers(input, 'funname'), input) + + def test_wrap_once(self): + input = {'foo': nt(5, 5, 5)} + self.assertEqual(wrap_numbers(input, 'funname'), input) + input = {'foo': nt(5, 5, 3)} + self.assertEqual(wrap_numbers(input, 'funname'), + {'foo': nt(5, 5, 8)}) + + def test_keep_wrapping(self): + input = {'foo': nt(100, 100, 100)} + self.assertEqual(wrap_numbers(input, 'funname'), input) + # wrap from 5, expect 105 + input = {'foo': nt(100, 100, 5)} + self.assertEqual(wrap_numbers(input, 'funname'), + {'foo': nt(100, 100, 105)}) + # next go to 10, expect 115 + input = {'foo': nt(100, 100, 10)} + self.assertEqual(wrap_numbers(input, 'funname'), + {'foo': nt(100, 100, 115)}) + + # =================================================================== # --- Example script tests # =================================================================== From b566ae2f14d36fc350462f3e927e4c48673d90a5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 15:17:31 +0200 Subject: [PATCH 584/922] #802 more tests --- psutil/tests/test_misc.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 8953e31d1..a55a24ebb 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -403,13 +403,6 @@ def test_increase_but_no_wrap(self): self.assertEqual(wrap_numbers(input, 'funname'), input) def test_wrap_once(self): - input = {'foo': nt(5, 5, 5)} - self.assertEqual(wrap_numbers(input, 'funname'), input) - input = {'foo': nt(5, 5, 3)} - self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(5, 5, 8)}) - - def test_keep_wrapping(self): input = {'foo': nt(100, 100, 100)} self.assertEqual(wrap_numbers(input, 'funname'), input) # wrap from 5, expect 105 @@ -421,6 +414,27 @@ def test_keep_wrapping(self): self.assertEqual(wrap_numbers(input, 'funname'), {'foo': nt(100, 100, 115)}) + def test_wrap_twice(self): + # let's say 100 is the threshold + input = {'foo': nt(100, 100, 100)} + self.assertEqual(wrap_numbers(input, 'funname'), input) + # first wrap restart from 10 + input = {'foo': nt(100, 100, 10)} + self.assertEqual(wrap_numbers(input, 'funname'), + {'foo': nt(100, 100, 110)}) + # then it goes on (90) + input = {'foo': nt(100, 100, 90)} + self.assertEqual(wrap_numbers(input, 'funname'), + {'foo': nt(100, 100, 200)}) + # then it wraps again (5) + input = {'foo': nt(100, 100, 5)} + self.assertEqual(wrap_numbers(input, 'funname'), + {'foo': nt(100, 100, 205)}) + # then another number wraps + input = {'foo': nt(100, 20, 205)} + self.assertEqual(wrap_numbers(input, 'funname'), + {'foo': nt(100, 120, 205)}) + # =================================================================== # --- Example script tests From 26bcd9c85a2250a5ac34b6750615b5ba9cc1c002 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 17:32:28 +0200 Subject: [PATCH 585/922] #802: test driven development ;) --- psutil/tests/test_misc.py | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index a55a24ebb..a34224c7b 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -401,20 +401,12 @@ def test_increase_but_no_wrap(self): self.assertEqual(wrap_numbers(input, 'funname'), input) input = {'foo': nt(10, 15, 20)} self.assertEqual(wrap_numbers(input, 'funname'), input) - - def test_wrap_once(self): - input = {'foo': nt(100, 100, 100)} + input = {'foo': nt(20, 25, 30)} + self.assertEqual(wrap_numbers(input, 'funname'), input) + input = {'foo': nt(20, 25, 30)} self.assertEqual(wrap_numbers(input, 'funname'), input) - # wrap from 5, expect 105 - input = {'foo': nt(100, 100, 5)} - self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(100, 100, 105)}) - # next go to 10, expect 115 - input = {'foo': nt(100, 100, 10)} - self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(100, 100, 115)}) - def test_wrap_twice(self): + def test_wrap(self): # let's say 100 is the threshold input = {'foo': nt(100, 100, 100)} self.assertEqual(wrap_numbers(input, 'funname'), input) @@ -422,18 +414,18 @@ def test_wrap_twice(self): input = {'foo': nt(100, 100, 10)} self.assertEqual(wrap_numbers(input, 'funname'), {'foo': nt(100, 100, 110)}) - # then it goes on (90) + # then it remains the same + input = {'foo': nt(100, 100, 10)} + self.assertEqual(wrap_numbers(input, 'funname'), + {'foo': nt(100, 100, 110)}) + # then it goes up (90, expect 200) input = {'foo': nt(100, 100, 90)} self.assertEqual(wrap_numbers(input, 'funname'), {'foo': nt(100, 100, 200)}) - # then it wraps again (5) - input = {'foo': nt(100, 100, 5)} - self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(100, 100, 205)}) - # then another number wraps - input = {'foo': nt(100, 20, 205)} + # then wrap again (expect 220) + input = {'foo': nt(100, 100, 20)} self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(100, 120, 205)}) + {'foo': nt(100, 100, 220)}) # =================================================================== From 6b2f97b9e4e8bb9bd095ac8efd342b8ea7b22f69 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 18:10:38 +0200 Subject: [PATCH 586/922] test was not correct --- psutil/tests/test_misc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index a34224c7b..795b58e2c 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -418,14 +418,14 @@ def test_wrap(self): input = {'foo': nt(100, 100, 10)} self.assertEqual(wrap_numbers(input, 'funname'), {'foo': nt(100, 100, 110)}) - # then it goes up (90, expect 200) + # then it goes up input = {'foo': nt(100, 100, 90)} self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(100, 100, 200)}) - # then wrap again (expect 220) + {'foo': nt(100, 100, 190)}) + # then it wraps again input = {'foo': nt(100, 100, 20)} self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(100, 100, 220)}) + {'foo': nt(100, 100, 210)}) # =================================================================== From f0d5eed1b365e020358447b4abc3f253552e57dd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 19:17:30 +0200 Subject: [PATCH 587/922] #802: finally start to get it right (tests pass) --- psutil/_common.py | 64 +++++++++++++++++------------------------------ 1 file changed, 23 insertions(+), 41 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 1f826a0dc..9e604b6a1 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -18,6 +18,7 @@ import sys import threading import warnings +from collections import defaultdict from collections import namedtuple from socket import AF_INET from socket import SOCK_DGRAM @@ -470,54 +471,33 @@ def inner(self, *args, **kwargs): _wrapn_lock = threading.Lock() _wrapn_cache = {} +_wrapn_reminders = defaultdict(int) -def wrap_numbers(new_dict, name): - def did_nums_wrap(new_nt, old_nt): - # Return True if one of the numbers of the new ntuple is smaller - # than the number of the old ntuple in the same position. - for i in range(len(new_nt)): - new_value = new_nt[i] - old_value = old_nt[i] - if new_value < old_value: - return True - return False - - def replace_ntuple(new_nt, old_nt): - # Return a new ntuple with the "adjusted" numbers. - bits = [] - for i in range(len(new_nt)): - new_value = new_nt[i] - old_value = old_nt[i] - if new_value < old_value: - # they wrapped! - num = old_value + new_value - else: - num = new_value - bits.append(num) - return new_nt._make(bits) - +def wrap_numbers(input_dict, name): with _wrapn_lock: if name not in _wrapn_cache: # This was the first call. - _wrapn_cache[name] = new_dict - return new_dict + _wrapn_cache[name] = input_dict + return input_dict + new_dict = {} old_dict = _wrapn_cache[name] - for key, new_nt in new_dict.items(): - try: - old_nt = old_dict[key] - except KeyError: - # We may get here if net_io_counters() returned a new NIC - # or disk_io_counters() returned a new disk. - continue - else: - assert new_nt._fields == old_nt._fields, (new_nt, old_nt) - if did_nums_wrap(new_nt, old_nt): - # we wrapped! - new_dict[key] = replace_ntuple(new_nt, old_nt) - - _wrapn_cache[name] = new_dict + for key in input_dict.keys(): + old_nt = old_dict[key] + input_nt = input_dict[key] + + bits = [] + for i in range(len(input_nt)): + old_value = old_nt[i] + input_value = input_nt[i] + remkey = (name, key, i) + if input_value < old_value: + _wrapn_reminders[remkey] += old_value + bits.append(input_value + _wrapn_reminders[remkey]) + new_dict[key] = input_nt._make(bits) + + _wrapn_cache[name] = input_dict return new_dict @@ -525,8 +505,10 @@ def _wrapn_cache_clear(name=None): with _wrapn_lock: if name is None: _wrapn_cache.clear() + _wrapn_reminders.clear() else: _wrapn_cache.pop(name) + # TODO wrap_numbers.cache_clear = _wrapn_cache_clear From 8735a5218548be2c04f8aec4e0153dd4e8ac15f4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 19:52:22 +0200 Subject: [PATCH 588/922] #802: handle the case where dict keys changes between calls --- psutil/_common.py | 8 +++++++- psutil/tests/test_misc.py | 10 +++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 9e604b6a1..202c371a1 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -484,8 +484,14 @@ def wrap_numbers(input_dict, name): new_dict = {} old_dict = _wrapn_cache[name] for key in input_dict.keys(): - old_nt = old_dict[key] input_nt = input_dict[key] + try: + old_nt = old_dict[key] + except KeyError: + # The input dict has a new key (e.g. a new disk or NIC) + # which didn't exist in the previous call. + new_dict[key] = input_nt + continue bits = [] for i in range(len(input_nt)): diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 795b58e2c..799dd6b5d 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -410,7 +410,7 @@ def test_wrap(self): # let's say 100 is the threshold input = {'foo': nt(100, 100, 100)} self.assertEqual(wrap_numbers(input, 'funname'), input) - # first wrap restart from 10 + # first wrap restarts from 10 input = {'foo': nt(100, 100, 10)} self.assertEqual(wrap_numbers(input, 'funname'), {'foo': nt(100, 100, 110)}) @@ -427,6 +427,14 @@ def test_wrap(self): self.assertEqual(wrap_numbers(input, 'funname'), {'foo': nt(100, 100, 210)}) + def test_dict_keys_mismatch(self): + # Emulate a case where the second call to disk_io_counters() + # (or whatever) provides a new disk. + input = {'disk1': nt(5, 5, 5)} + self.assertEqual(wrap_numbers(input, 'funname'), input) + input = {'disk1': nt(5, 5, 5), 'disk2': nt(7, 7, 7)} + self.assertEqual(wrap_numbers(input, 'funname'), input) + # =================================================================== # --- Example script tests From 69aef18a79a8b23c66ade2aaf0234b46e845cf4c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 19:55:41 +0200 Subject: [PATCH 589/922] update tests --- psutil/tests/test_misc.py | 65 ++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 799dd6b5d..545c7af3f 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -388,52 +388,55 @@ def tearDown(self): wrap_numbers.cache_clear() def test_first_call(self): - input = {'foo': nt(5, 5, 5)} - self.assertEqual(wrap_numbers(input, 'funname'), input) + input = {'disk1': nt(5, 5, 5)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) def test_input_hasnt_changed(self): - input = {'foo': nt(5, 5, 5)} - self.assertEqual(wrap_numbers(input, 'funname'), input) - self.assertEqual(wrap_numbers(input, 'funname'), input) + input = {'disk1': nt(5, 5, 5)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) + self.assertEqual(wrap_numbers(input, 'disk_io'), input) def test_increase_but_no_wrap(self): - input = {'foo': nt(5, 5, 5)} - self.assertEqual(wrap_numbers(input, 'funname'), input) - input = {'foo': nt(10, 15, 20)} - self.assertEqual(wrap_numbers(input, 'funname'), input) - input = {'foo': nt(20, 25, 30)} - self.assertEqual(wrap_numbers(input, 'funname'), input) - input = {'foo': nt(20, 25, 30)} - self.assertEqual(wrap_numbers(input, 'funname'), input) + input = {'disk1': nt(5, 5, 5)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) + input = {'disk1': nt(10, 15, 20)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) + input = {'disk1': nt(20, 25, 30)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) + input = {'disk1': nt(20, 25, 30)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) def test_wrap(self): # let's say 100 is the threshold - input = {'foo': nt(100, 100, 100)} - self.assertEqual(wrap_numbers(input, 'funname'), input) + input = {'disk1': nt(100, 100, 100)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) # first wrap restarts from 10 - input = {'foo': nt(100, 100, 10)} - self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(100, 100, 110)}) + input = {'disk1': nt(100, 100, 10)} + self.assertEqual(wrap_numbers(input, 'disk_io'), + {'disk1': nt(100, 100, 110)}) # then it remains the same - input = {'foo': nt(100, 100, 10)} - self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(100, 100, 110)}) + input = {'disk1': nt(100, 100, 10)} + self.assertEqual(wrap_numbers(input, 'disk_io'), + {'disk1': nt(100, 100, 110)}) # then it goes up - input = {'foo': nt(100, 100, 90)} - self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(100, 100, 190)}) + input = {'disk1': nt(100, 100, 90)} + self.assertEqual(wrap_numbers(input, 'disk_io'), + {'disk1': nt(100, 100, 190)}) # then it wraps again - input = {'foo': nt(100, 100, 20)} - self.assertEqual(wrap_numbers(input, 'funname'), - {'foo': nt(100, 100, 210)}) + input = {'disk1': nt(100, 100, 20)} + self.assertEqual(wrap_numbers(input, 'disk_io'), + {'disk1': nt(100, 100, 210)}) - def test_dict_keys_mismatch(self): + def test_diff_keys(self): # Emulate a case where the second call to disk_io_counters() # (or whatever) provides a new disk. input = {'disk1': nt(5, 5, 5)} - self.assertEqual(wrap_numbers(input, 'funname'), input) - input = {'disk1': nt(5, 5, 5), 'disk2': nt(7, 7, 7)} - self.assertEqual(wrap_numbers(input, 'funname'), input) + self.assertEqual(wrap_numbers(input, 'disk_io'), input) + input = {'disk1': nt(5, 5, 5), + 'disk2': nt(7, 7, 7)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) + input = {'disk1': nt(8, 8, 8)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) # =================================================================== From ef42840b3ec30e283e73f0710ccbd9c8b171080f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 20:00:52 +0200 Subject: [PATCH 590/922] rewording --- psutil/tests/test_misc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 545c7af3f..fdeca9b3f 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -427,9 +427,10 @@ def test_wrap(self): self.assertEqual(wrap_numbers(input, 'disk_io'), {'disk1': nt(100, 100, 210)}) - def test_diff_keys(self): - # Emulate a case where the second call to disk_io_counters() - # (or whatever) provides a new disk. + def test_changing_keys(self): + # Emulate a case where the second call to disk_io() + # (or whatever) provides a new disk, then the new disk + # disappears on the third call. input = {'disk1': nt(5, 5, 5)} self.assertEqual(wrap_numbers(input, 'disk_io'), input) input = {'disk1': nt(5, 5, 5), From 726e179629273f7bae3e6b38262b2cd40f224746 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 20:45:52 +0200 Subject: [PATCH 591/922] #802: add test for disappearing keys which wrap --- psutil/tests/test_misc.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index fdeca9b3f..112a3f386 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -439,6 +439,25 @@ def test_changing_keys(self): input = {'disk1': nt(8, 8, 8)} self.assertEqual(wrap_numbers(input, 'disk_io'), input) + def test_changing_keys_w_wrap(self): + input = {'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 100)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) + # disk 2 wraps + input = {'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 10)} + self.assertEqual(wrap_numbers(input, 'disk_io'), + {'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 110)}) + # disk 2 disappears + input = {'disk1': nt(50, 50, 50)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) + # then it appears again; the old wrap is supposed to be + # gone. + input = {'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 100)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) + # =================================================================== # --- Example script tests From 5141848f0e607c5894733297c46755be41af975c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 21:08:51 +0200 Subject: [PATCH 592/922] #802: further tests for disappearing keys (impl is still broken) --- psutil/_common.py | 2 +- psutil/tests/test_misc.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/psutil/_common.py b/psutil/_common.py index 202c371a1..bdfcefcd9 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -495,8 +495,8 @@ def wrap_numbers(input_dict, name): bits = [] for i in range(len(input_nt)): - old_value = old_nt[i] input_value = input_nt[i] + old_value = old_nt[i] remkey = (name, key, i) if input_value < old_value: _wrapn_reminders[remkey] += old_value diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 112a3f386..3e2a3ce81 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -452,11 +452,22 @@ def test_changing_keys_w_wrap(self): # disk 2 disappears input = {'disk1': nt(50, 50, 50)} self.assertEqual(wrap_numbers(input, 'disk_io'), input) + # then it appears again; the old wrap is supposed to be # gone. input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 100)} self.assertEqual(wrap_numbers(input, 'disk_io'), input) + # remains the same + input = {'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 100)} + self.assertEqual(wrap_numbers(input, 'disk_io'), input) + # and then wraps again + input = {'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 10)} + self.assertEqual(wrap_numbers(input, 'disk_io'), + {'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 110)}) # =================================================================== From 09a8258ba06c068f32f8c66518c93666cfed5d0a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 8 May 2017 21:26:33 +0200 Subject: [PATCH 593/922] #802: remove entries from index dict --- psutil/_common.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/psutil/_common.py b/psutil/_common.py index bdfcefcd9..d26d0ff0e 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -481,8 +481,19 @@ def wrap_numbers(input_dict, name): _wrapn_cache[name] = input_dict return input_dict - new_dict = {} + # In case the number of keys changed between calls (e.g. a + # disk disappears) this removes the entry from _wrapn_reminders. + # TODO: this is messy; change the algorithm. old_dict = _wrapn_cache[name] + gone_keys = set(old_dict.keys()) - set(input_dict.keys()) + if gone_keys: + for gone_key in gone_keys: + for k in _wrapn_reminders.keys(): + nam, key, i = k + if nam == name and key == gone_key: + del _wrapn_reminders[k] + + new_dict = {} for key in input_dict.keys(): input_nt = input_dict[key] try: From 98af48a1c675e28c18f3dd05e11db6cb1d44dbaf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 01:30:03 +0200 Subject: [PATCH 594/922] #802: change algorithm --- psutil/_common.py | 22 ++++++++++++++-------- psutil/tests/test_misc.py | 12 ++++++++++++ 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index d26d0ff0e..dc44f3a8b 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -472,6 +472,7 @@ def inner(self, *args, **kwargs): _wrapn_lock = threading.Lock() _wrapn_cache = {} _wrapn_reminders = defaultdict(int) +_wrapn_rmap = defaultdict(list) def wrap_numbers(input_dict, name): @@ -483,15 +484,12 @@ def wrap_numbers(input_dict, name): # In case the number of keys changed between calls (e.g. a # disk disappears) this removes the entry from _wrapn_reminders. - # TODO: this is messy; change the algorithm. old_dict = _wrapn_cache[name] gone_keys = set(old_dict.keys()) - set(input_dict.keys()) - if gone_keys: - for gone_key in gone_keys: - for k in _wrapn_reminders.keys(): - nam, key, i = k - if nam == name and key == gone_key: - del _wrapn_reminders[k] + for gone_key in gone_keys: + for remkey in _wrapn_rmap[name + "-" + gone_key]: + del _wrapn_reminders[remkey] + del _wrapn_rmap[name + "-" + gone_key] new_dict = {} for key in input_dict.keys(): @@ -512,6 +510,8 @@ def wrap_numbers(input_dict, name): if input_value < old_value: _wrapn_reminders[remkey] += old_value bits.append(input_value + _wrapn_reminders[remkey]) + _wrapn_rmap[name + "-" + key].append(remkey) + new_dict[key] = input_nt._make(bits) _wrapn_cache[name] = input_dict @@ -523,9 +523,15 @@ def _wrapn_cache_clear(name=None): if name is None: _wrapn_cache.clear() _wrapn_reminders.clear() + _wrapn_rmap.clear() else: _wrapn_cache.pop(name) - # TODO + _wrapn_rmap.pop(name) + + +def _wrapn_cache_info(name=None): + return (_wrapn_cache, _wrapn_reminders, _wrapn_rmap) wrap_numbers.cache_clear = _wrapn_cache_clear +wrap_numbers.cache_info = _wrapn_cache_info diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 3e2a3ce81..58f0ceb85 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -426,6 +426,18 @@ def test_wrap(self): input = {'disk1': nt(100, 100, 20)} self.assertEqual(wrap_numbers(input, 'disk_io'), {'disk1': nt(100, 100, 210)}) + # now wrap another num + input = {'disk1': nt(50, 100, 20)} + self.assertEqual(wrap_numbers(input, 'disk_io'), + {'disk1': nt(150, 100, 210)}) + # and again + input = {'disk1': nt(40, 100, 20)} + self.assertEqual(wrap_numbers(input, 'disk_io'), + {'disk1': nt(190, 100, 210)}) + # keep it the same + input = {'disk1': nt(40, 100, 20)} + self.assertEqual(wrap_numbers(input, 'disk_io'), + {'disk1': nt(190, 100, 210)}) def test_changing_keys(self): # Emulate a case where the second call to disk_io() From 291870b94b753a22d1efb30059a5600b084fcbe6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 02:11:56 +0200 Subject: [PATCH 595/922] #802: move everything into a class --- psutil/_common.py | 63 +++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index dc44f3a8b..8189a9dd8 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -469,27 +469,28 @@ def inner(self, *args, **kwargs): return outer -_wrapn_lock = threading.Lock() -_wrapn_cache = {} -_wrapn_reminders = defaultdict(int) -_wrapn_rmap = defaultdict(list) +class WrapNumbers: + def __init__(self): + self.lock = threading.Lock() + self.cache = {} + self.reminders = defaultdict(int) + self.rmap = defaultdict(list) -def wrap_numbers(input_dict, name): - with _wrapn_lock: - if name not in _wrapn_cache: + def run(self, input_dict, name): + if name not in self.cache: # This was the first call. - _wrapn_cache[name] = input_dict + self.cache[name] = input_dict return input_dict # In case the number of keys changed between calls (e.g. a - # disk disappears) this removes the entry from _wrapn_reminders. - old_dict = _wrapn_cache[name] + # disk disappears) this removes the entry from self.reminders. + old_dict = self.cache[name] gone_keys = set(old_dict.keys()) - set(input_dict.keys()) for gone_key in gone_keys: - for remkey in _wrapn_rmap[name + "-" + gone_key]: - del _wrapn_reminders[remkey] - del _wrapn_rmap[name + "-" + gone_key] + for remkey in self.rmap[name + "-" + gone_key]: + del self.reminders[remkey] + del self.rmap[name + "-" + gone_key] new_dict = {} for key in input_dict.keys(): @@ -508,30 +509,32 @@ def wrap_numbers(input_dict, name): old_value = old_nt[i] remkey = (name, key, i) if input_value < old_value: - _wrapn_reminders[remkey] += old_value - bits.append(input_value + _wrapn_reminders[remkey]) - _wrapn_rmap[name + "-" + key].append(remkey) + self.reminders[remkey] += old_value + bits.append(input_value + self.reminders[remkey]) + self.rmap[name + "-" + key].append(remkey) new_dict[key] = input_nt._make(bits) - _wrapn_cache[name] = input_dict + self.cache[name] = input_dict return new_dict + def cache_clear(self, name=None): + with self.lock: + if name is None: + self.cache.clear() + self.reminders.clear() + self.rmap.clear() + else: + self.cache.pop(name) + self.rmap.pop(name) -def _wrapn_cache_clear(name=None): - with _wrapn_lock: - if name is None: - _wrapn_cache.clear() - _wrapn_reminders.clear() - _wrapn_rmap.clear() - else: - _wrapn_cache.pop(name) - _wrapn_rmap.pop(name) +wn = WrapNumbers() -def _wrapn_cache_info(name=None): - return (_wrapn_cache, _wrapn_reminders, _wrapn_rmap) + +def wrap_numbers(input_dict, name): + with wn.lock: + return wn.run(input_dict, name) -wrap_numbers.cache_clear = _wrapn_cache_clear -wrap_numbers.cache_info = _wrapn_cache_info +wrap_numbers.cache_clear = wn.cache_clear From 42af6fc151d0bff8ad07e10ed7f11f2151744d0c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 02:19:05 +0200 Subject: [PATCH 596/922] #802: refactoring --- psutil/_common.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 8189a9dd8..350ffe79b 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -469,7 +469,7 @@ def inner(self, *args, **kwargs): return outer -class WrapNumbers: +class _WrapNumbers: def __init__(self): self.lock = threading.Lock() @@ -477,14 +477,10 @@ def __init__(self): self.reminders = defaultdict(int) self.rmap = defaultdict(list) - def run(self, input_dict, name): - if name not in self.cache: - # This was the first call. - self.cache[name] = input_dict - return input_dict - - # In case the number of keys changed between calls (e.g. a - # disk disappears) this removes the entry from self.reminders. + def _remove_dead_reminders(self, input_dict, name): + """In case the number of keys changed between calls (e.g. a + disk disappears) this removes the entry from self.reminders. + """ old_dict = self.cache[name] gone_keys = set(old_dict.keys()) - set(input_dict.keys()) for gone_key in gone_keys: @@ -492,6 +488,14 @@ def run(self, input_dict, name): del self.reminders[remkey] del self.rmap[name + "-" + gone_key] + def run(self, input_dict, name): + if name not in self.cache: + # This was the first call. + self.cache[name] = input_dict + return input_dict + + self._remove_dead_reminders(input_dict, name) + old_dict = self.cache[name] new_dict = {} for key in input_dict.keys(): input_nt = input_dict[key] @@ -529,12 +533,12 @@ def cache_clear(self, name=None): self.rmap.pop(name) -wn = WrapNumbers() +_wn = _WrapNumbers() def wrap_numbers(input_dict, name): - with wn.lock: - return wn.run(input_dict, name) + with _wn.lock: + return _wn.run(input_dict, name) -wrap_numbers.cache_clear = wn.cache_clear +wrap_numbers.cache_clear = _wn.cache_clear From 397f56881ba04a6bcec32bd45616b4382834dab3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 02:33:53 +0200 Subject: [PATCH 597/922] refactoring --- psutil/_common.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 350ffe79b..14b564a00 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -474,7 +474,7 @@ class _WrapNumbers: def __init__(self): self.lock = threading.Lock() self.cache = {} - self.reminders = defaultdict(int) + self.reminders = {} self.rmap = defaultdict(list) def _remove_dead_reminders(self, input_dict, name): @@ -485,13 +485,14 @@ def _remove_dead_reminders(self, input_dict, name): gone_keys = set(old_dict.keys()) - set(input_dict.keys()) for gone_key in gone_keys: for remkey in self.rmap[name + "-" + gone_key]: - del self.reminders[remkey] + del self.reminders[name][remkey] del self.rmap[name + "-" + gone_key] def run(self, input_dict, name): if name not in self.cache: # This was the first call. self.cache[name] = input_dict + self.reminders[name] = defaultdict(int) return input_dict self._remove_dead_reminders(input_dict, name) @@ -513,8 +514,8 @@ def run(self, input_dict, name): old_value = old_nt[i] remkey = (name, key, i) if input_value < old_value: - self.reminders[remkey] += old_value - bits.append(input_value + self.reminders[remkey]) + self.reminders[name][remkey] += old_value + bits.append(input_value + self.reminders[name][remkey]) self.rmap[name + "-" + key].append(remkey) new_dict[key] = input_nt._make(bits) @@ -530,15 +531,14 @@ def cache_clear(self, name=None): self.rmap.clear() else: self.cache.pop(name) + self.reminders[name].clear() self.rmap.pop(name) -_wn = _WrapNumbers() - - def wrap_numbers(input_dict, name): with _wn.lock: return _wn.run(input_dict, name) +_wn = _WrapNumbers() wrap_numbers.cache_clear = _wn.cache_clear From fe8f4c0791547ab9ade78378b1d6021f5838f1ea Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 02:41:24 +0200 Subject: [PATCH 598/922] refactoring --- psutil/_common.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 14b564a00..2d3e5c2f6 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -475,7 +475,15 @@ def __init__(self): self.lock = threading.Lock() self.cache = {} self.reminders = {} - self.rmap = defaultdict(list) + self.rmap = {} + + def _add_dict(self, input_dict, name): + assert name not in self.cache + assert name not in self.reminders + assert name not in self.rmap + self.cache[name] = input_dict + self.reminders[name] = defaultdict(int) + self.rmap[name] = defaultdict(list) def _remove_dead_reminders(self, input_dict, name): """In case the number of keys changed between calls (e.g. a @@ -484,15 +492,14 @@ def _remove_dead_reminders(self, input_dict, name): old_dict = self.cache[name] gone_keys = set(old_dict.keys()) - set(input_dict.keys()) for gone_key in gone_keys: - for remkey in self.rmap[name + "-" + gone_key]: + for remkey in self.rmap[name][gone_key]: del self.reminders[name][remkey] - del self.rmap[name + "-" + gone_key] + del self.rmap[name][gone_key] def run(self, input_dict, name): if name not in self.cache: # This was the first call. - self.cache[name] = input_dict - self.reminders[name] = defaultdict(int) + self._add_dict(input_dict, name) return input_dict self._remove_dead_reminders(input_dict, name) @@ -516,7 +523,7 @@ def run(self, input_dict, name): if input_value < old_value: self.reminders[name][remkey] += old_value bits.append(input_value + self.reminders[name][remkey]) - self.rmap[name + "-" + key].append(remkey) + self.rmap[name][key].append(remkey) new_dict[key] = input_nt._make(bits) @@ -531,7 +538,7 @@ def cache_clear(self, name=None): self.rmap.clear() else: self.cache.pop(name) - self.reminders[name].clear() + self.reminders.pop(name) self.rmap.pop(name) From 4c75cdd2f802cf7cf44f9575da6f064165c4c1cb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 02:47:36 +0200 Subject: [PATCH 599/922] change var names --- psutil/_common.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 2d3e5c2f6..9f63a8ad1 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -475,15 +475,15 @@ def __init__(self): self.lock = threading.Lock() self.cache = {} self.reminders = {} - self.rmap = {} + self.reminder_keys = {} def _add_dict(self, input_dict, name): assert name not in self.cache assert name not in self.reminders - assert name not in self.rmap + assert name not in self.reminder_keys self.cache[name] = input_dict self.reminders[name] = defaultdict(int) - self.rmap[name] = defaultdict(list) + self.reminder_keys[name] = defaultdict(list) def _remove_dead_reminders(self, input_dict, name): """In case the number of keys changed between calls (e.g. a @@ -492,9 +492,9 @@ def _remove_dead_reminders(self, input_dict, name): old_dict = self.cache[name] gone_keys = set(old_dict.keys()) - set(input_dict.keys()) for gone_key in gone_keys: - for remkey in self.rmap[name][gone_key]: + for remkey in self.reminder_keys[name][gone_key]: del self.reminders[name][remkey] - del self.rmap[name][gone_key] + del self.reminder_keys[name][gone_key] def run(self, input_dict, name): if name not in self.cache: @@ -503,6 +503,7 @@ def run(self, input_dict, name): return input_dict self._remove_dead_reminders(input_dict, name) + old_dict = self.cache[name] new_dict = {} for key in input_dict.keys(): @@ -523,7 +524,7 @@ def run(self, input_dict, name): if input_value < old_value: self.reminders[name][remkey] += old_value bits.append(input_value + self.reminders[name][remkey]) - self.rmap[name][key].append(remkey) + self.reminder_keys[name][key].append(remkey) new_dict[key] = input_nt._make(bits) @@ -535,11 +536,11 @@ def cache_clear(self, name=None): if name is None: self.cache.clear() self.reminders.clear() - self.rmap.clear() + self.reminder_keys.clear() else: self.cache.pop(name) self.reminders.pop(name) - self.rmap.pop(name) + self.reminder_keys.pop(name) def wrap_numbers(input_dict, name): From 53dbbfce60a92a36b571945c36958878fd50bcaf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 02:54:49 +0200 Subject: [PATCH 600/922] remove useless key item --- psutil/_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/_common.py b/psutil/_common.py index 9f63a8ad1..47d8cd09b 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -520,7 +520,7 @@ def run(self, input_dict, name): for i in range(len(input_nt)): input_value = input_nt[i] old_value = old_nt[i] - remkey = (name, key, i) + remkey = (key, i) if input_value < old_value: self.reminders[name][remkey] += old_value bits.append(input_value + self.reminders[name][remkey]) From 7828b9c1e4dd4fa5df5b0681850d81d24afdf015 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 03:07:05 +0200 Subject: [PATCH 601/922] expose cache_info() method --- psutil/_common.py | 5 +++++ psutil/tests/test_misc.py | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/psutil/_common.py b/psutil/_common.py index 47d8cd09b..67150a782 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -542,6 +542,10 @@ def cache_clear(self, name=None): self.reminders.pop(name) self.reminder_keys.pop(name) + def cache_info(self): + with self.lock: + return (self.cache, self.reminders, self.reminder_keys) + def wrap_numbers(input_dict, name): with _wn.lock: @@ -550,3 +554,4 @@ def wrap_numbers(input_dict, name): _wn = _WrapNumbers() wrap_numbers.cache_clear = _wn.cache_clear +wrap_numbers.cache_info = _wn.cache_info diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 58f0ceb85..7f22ee548 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -384,9 +384,11 @@ def test_sanity_version_check(self): class TestWrapNumbers(unittest.TestCase): - def tearDown(self): + def setUp(self): wrap_numbers.cache_clear() + tearDown = setUp + def test_first_call(self): input = {'disk1': nt(5, 5, 5)} self.assertEqual(wrap_numbers(input, 'disk_io'), input) From 247d3a446955ddbd35ffb3db66896d6013894cb3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 03:15:55 +0200 Subject: [PATCH 602/922] avoid to unnecessarily populate reminder_keys dict --- psutil/_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/_common.py b/psutil/_common.py index 67150a782..5e11b9525 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -523,8 +523,8 @@ def run(self, input_dict, name): remkey = (key, i) if input_value < old_value: self.reminders[name][remkey] += old_value + self.reminder_keys[name][key].append(remkey) bits.append(input_value + self.reminders[name][remkey]) - self.reminder_keys[name][key].append(remkey) new_dict[key] = input_nt._make(bits) From 956c1cdc627e9e9f35b56eac13f3ce0c63f953da Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 04:10:46 +0200 Subject: [PATCH 603/922] #802 add tests --- psutil/_common.py | 4 +-- psutil/tests/test_misc.py | 74 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 5e11b9525..3bca07993 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -483,7 +483,7 @@ def _add_dict(self, input_dict, name): assert name not in self.reminder_keys self.cache[name] = input_dict self.reminders[name] = defaultdict(int) - self.reminder_keys[name] = defaultdict(list) + self.reminder_keys[name] = defaultdict(set) def _remove_dead_reminders(self, input_dict, name): """In case the number of keys changed between calls (e.g. a @@ -523,7 +523,7 @@ def run(self, input_dict, name): remkey = (key, i) if input_value < old_value: self.reminders[name][remkey] += old_value - self.reminder_keys[name][key].append(remkey) + self.reminder_keys[name][key].add(remkey) bits.append(input_value + self.reminders[name][remkey]) new_dict[key] = input_nt._make(bits) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 7f22ee548..d59bd1591 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -426,6 +426,10 @@ def test_wrap(self): {'disk1': nt(100, 100, 190)}) # then it wraps again input = {'disk1': nt(100, 100, 20)} + self.assertEqual(wrap_numbers(input, 'disk_io'), + {'disk1': nt(100, 100, 210)}) + # and remains the same + input = {'disk1': nt(100, 100, 20)} self.assertEqual(wrap_numbers(input, 'disk_io'), {'disk1': nt(100, 100, 210)}) # now wrap another num @@ -483,6 +487,76 @@ def test_changing_keys_w_wrap(self): {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 110)}) + # --- cache tests + + def test_cache_first_call(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + self.assertEqual(cache[0], {'disk_io': input}) + self.assertEqual(cache[1], {'disk_io': {}}) + self.assertEqual(cache[2], {'disk_io': {}}) + + def test_cache_call_twice(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + input = {'disk1': nt(10, 10, 10)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + self.assertEqual(cache[0], {'disk_io': input}) + self.assertEqual( + cache[1], + {'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 0}}) + self.assertEqual(cache[2], {'disk_io': {}}) + + def test_cache_wrap(self): + # let's say 100 is the threshold + input = {'disk1': nt(100, 100, 100)} + wrap_numbers(input, 'disk_io') + + # first wrap restarts from 10 + input = {'disk1': nt(100, 100, 10)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + self.assertEqual(cache[0], {'disk_io': input}) + self.assertEqual( + cache[1], + {'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 100}}) + self.assertEqual(cache[2], {'disk_io': {'disk1': set([('disk1', 2)])}}) + + def assert_(): + cache = wrap_numbers.cache_info() + self.assertEqual( + cache[1], + {'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, + ('disk1', 2): 100}}) + self.assertEqual(cache[2], + {'disk_io': {'disk1': set([('disk1', 2)])}}) + + # then it remains the same + input = {'disk1': nt(100, 100, 10)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + self.assertEqual(cache[0], {'disk_io': input}) + assert_() + + # then it goes up + input = {'disk1': nt(100, 100, 90)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + self.assertEqual(cache[0], {'disk_io': input}) + assert_() + + # then it wraps again + input = {'disk1': nt(100, 100, 20)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + self.assertEqual(cache[0], {'disk_io': input}) + self.assertEqual( + cache[1], + {'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 190}}) + self.assertEqual(cache[2], {'disk_io': {'disk1': set([('disk1', 2)])}}) + # =================================================================== # --- Example script tests From f56a5e8102fe2c2e37792ea1171507a242421651 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 04:18:44 +0200 Subject: [PATCH 604/922] addd test --- psutil/tests/test_misc.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index d59bd1591..676c7554e 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -557,6 +557,19 @@ def assert_(): {'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 190}}) self.assertEqual(cache[2], {'disk_io': {'disk1': set([('disk1', 2)])}}) + def test_cache_changing_keys(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + input = {'disk1': nt(5, 5, 5), + 'disk2': nt(7, 7, 7)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + self.assertEqual(cache[0], {'disk_io': input}) + self.assertEqual( + cache[1], + {'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 0}}) + self.assertEqual(cache[2], {'disk_io': {}}) + # =================================================================== # --- Example script tests From 29ca4c2e11975decb7035582922ecada67a64c49 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 04:25:33 +0200 Subject: [PATCH 605/922] add test --- psutil/_common.py | 6 +++--- psutil/tests/test_misc.py | 20 +++++++++++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 3bca07993..6988eb8d1 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -538,9 +538,9 @@ def cache_clear(self, name=None): self.reminders.clear() self.reminder_keys.clear() else: - self.cache.pop(name) - self.reminders.pop(name) - self.reminder_keys.pop(name) + self.cache.pop(name, None) + self.reminders.pop(name, None) + self.reminder_keys.pop(name, None) def cache_info(self): with self.lock: diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 676c7554e..fe582a776 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -68,8 +68,12 @@ import psutil.tests +# =================================================================== +# --- Misc / generic tests. +# =================================================================== + + class TestMisc(unittest.TestCase): - """Misc / generic tests.""" def test_process__repr__(self, func=repr): p = psutil.Process() @@ -379,6 +383,11 @@ def test_sanity_version_check(self): self.assertIn("version conflict", str(cm.exception).lower()) +# =================================================================== +# --- Tests for wrap_numbers() function. +# =================================================================== + + nt = namedtuple('foo', 'a b c') @@ -570,6 +579,15 @@ def test_cache_changing_keys(self): {'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 0}}) self.assertEqual(cache[2], {'disk_io': {}}) + def test_cache_clear(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + wrap_numbers(input, 'disk_io') + wrap_numbers.cache_clear('disk_io') + self.assertEqual(wrap_numbers.cache_info(), ({}, {}, {})) + wrap_numbers.cache_clear('disk_io') + wrap_numbers.cache_clear('?!?') + # =================================================================== # --- Example script tests From 9566f65471458116c6db4c4c286b468e054334fe Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 10:41:00 +0200 Subject: [PATCH 606/922] change var name --- psutil/_common.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 6988eb8d1..46ccf0862 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -507,26 +507,26 @@ def run(self, input_dict, name): old_dict = self.cache[name] new_dict = {} for key in input_dict.keys(): - input_nt = input_dict[key] + input_tuple = input_dict[key] try: - old_nt = old_dict[key] + old_tuple = old_dict[key] except KeyError: # The input dict has a new key (e.g. a new disk or NIC) # which didn't exist in the previous call. - new_dict[key] = input_nt + new_dict[key] = input_tuple continue bits = [] - for i in range(len(input_nt)): - input_value = input_nt[i] - old_value = old_nt[i] + for i in range(len(input_tuple)): + input_value = input_tuple[i] + old_value = old_tuple[i] remkey = (key, i) if input_value < old_value: self.reminders[name][remkey] += old_value self.reminder_keys[name][key].add(remkey) bits.append(input_value + self.reminders[name][remkey]) - new_dict[key] = input_nt._make(bits) + new_dict[key] = tuple(bits) self.cache[name] = input_dict return new_dict From de8734dff352b6b18e4e557aa5cfe244ccc32f00 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 10:44:58 +0200 Subject: [PATCH 607/922] update disk_io_counters() doc --- docs/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 31cbd2830..4b0c28210 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -410,6 +410,10 @@ Disks `issue #802 `__. Applications should be prepared to deal with that. + .. note:: + on Windows ``"diskperf -y"`` command may need to be executed first + otherwise this function won't find any disk. + .. versionchanged:: 4.0.0 added *busy_time* (Linux, FreeBSD), *read_merged_count* and *write_merged_count* (Linux) fields. From 30ca61ad5385e83d3c114587da5aea2d303c4f1f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 11:54:39 +0200 Subject: [PATCH 608/922] memleaks: warm up before starting --- psutil/tests/test_memory_leaks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 28a083f26..4f764dbd4 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -112,9 +112,12 @@ def call_many_times(): loops = kwargs.pop('loops_', None) or self.loops retry_for = kwargs.pop('retry_for_', None) or self.retry_for - self._call(fun, *args, **kwargs) + # warm up + for x in range(10): + self._call(fun, *args, **kwargs) self.assertEqual(gc.garbage, []) self.assertEqual(threading.active_count(), 1) + self.assertEqual(thisproc.children(), []) # Get 2 distinct memory samples, before and after having # called fun repeadetly. From 9db34af12cc5e39092400bddad842328bed406da Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 14:13:15 +0200 Subject: [PATCH 609/922] add test --- psutil/tests/test_misc.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index fe582a776..f73812a9c 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -588,6 +588,23 @@ def test_cache_clear(self): wrap_numbers.cache_clear('disk_io') wrap_numbers.cache_clear('?!?') + # --- + + def test_real_data(self): + d = {'nvme0n1': (300, 508, 640, 1571, 5970, 1987, 2049, 451751, 47048), + 'nvme0n1p1': (1171, 2, 5600256, 1024, 516, 0, 0, 0, 8), + 'nvme0n1p2': (54, 54, 2396160, 5165056, 4, 24, 30, 1207, 28), + 'nvme0n1p3': (2389, 4539, 5154, 150, 4828, 1844, 2019, 398, 348)} + self.assertEqual(wrap_numbers(d, 'disk_io'), d) + self.assertEqual(wrap_numbers(d, 'disk_io'), d) + # decrease this ↓ + d = {'nvme0n1': (100, 508, 640, 1571, 5970, 1987, 2049, 451751, 47048), + 'nvme0n1p1': (1171, 2, 5600256, 1024, 516, 0, 0, 0, 8), + 'nvme0n1p2': (54, 54, 2396160, 5165056, 4, 24, 30, 1207, 28), + 'nvme0n1p3': (2389, 4539, 5154, 150, 4828, 1844, 2019, 398, 348)} + out = wrap_numbers(d, 'disk_io') + self.assertEqual(out['nvme0n1'][0], 400) + # =================================================================== # --- Example script tests From 7786cd8eff5680938c260520ae5b81a9cdc762ff Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 15:50:07 +0200 Subject: [PATCH 610/922] update DEVGUIDE --- DEVGUIDE.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index c4ddc52d7..6a0f08fc8 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -101,6 +101,16 @@ Typical process occurring when adding a new functionality (API): - update ``README.rst`` (if necessary). - make a pull request. +=================== +Make a pull request +=================== + +- fork psutil +- create your feature branch (``git checkout -b my-new-feature``) +- commit your changes (``git commit -am 'add some feature'``) +- push to the branch (``git push origin my-new-feature``) +- create a new pull request + ====================== Continuous integration ====================== From d1d20cc92ddfe3b054dc9d406852f230e3f6ef81 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 17:17:27 +0200 Subject: [PATCH 611/922] #802: update doc + apply wrap function to disk and net io functions --- HISTORY.rst | 2 ++ docs/index.rst | 30 ++++++++++++++++++------------ psutil/__init__.py | 9 +++++++-- psutil/_common.py | 8 ++++++++ psutil/tests/test_linux.py | 9 +++++---- psutil/tests/test_memory_leaks.py | 4 ++-- 6 files changed, 42 insertions(+), 20 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bf7f63671..bc9096081 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,8 @@ **Enhancements** +- 802_: disk_io_counters() and net_io_counters() numbers no longer wrap + (restart from 0). Introduced a new "nowrap" argument. - 1015_: swap_memory() now relies on /proc/meminfo instead of sysinfo() syscall so that it can be used in conjunction with PROCFS_PATH in order to retrieve memory info about Linux containers such as Docker and Heroku. diff --git a/docs/index.rst b/docs/index.rst index 4b0c28210..3ad544bbe 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -366,7 +366,7 @@ Disks .. versionchanged:: 4.3.0 *percent* value takes root reserved space into account. -.. function:: disk_io_counters(perdisk=False) +.. function:: disk_io_counters(perdisk=False, nowrap=True) Return system-wide disk I/O statistics as a named tuple including the following fields: @@ -394,6 +394,11 @@ Disks the named tuple described above as the values. See `iotop.py `__ for an example application. + On some systems such as Linux, on a very busy or long-lived system, the + numbers returned by the kernel may wrap (restart from zero). + If *nowrap* is ``True`` psutil will detect and adjust those numbers across + function calls and add "old value" to "new value" so that the numbers will + always be increasing or remain the same, but never decrease. >>> import psutil >>> psutil.disk_io_counters() @@ -404,16 +409,14 @@ Disks 'sda2': sdiskio(read_count=18707, write_count=8830, read_bytes=6060, write_bytes=3443, read_time=24585, write_time=1572), 'sdb1': sdiskio(read_count=161, write_count=0, read_bytes=786432, write_bytes=0, read_time=44, write_time=0)} - .. warning:: - on some systems such as Linux, on a very busy or long-lived system these - numbers may wrap (restart from zero), see - `issue #802 `__. - Applications should be prepared to deal with that. - .. note:: on Windows ``"diskperf -y"`` command may need to be executed first otherwise this function won't find any disk. + .. versionchanged:: + 5.3.0 numbers no longer wrap (restart from zero) across calls thanks to new + *nowrap* argument. + .. versionchanged:: 4.0.0 added *busy_time* (Linux, FreeBSD), *read_merged_count* and *write_merged_count* (Linux) fields. @@ -442,6 +445,11 @@ Network If *pernic* is ``True`` return the same information for every network interface installed on the system as a dictionary with network interface names as the keys and the named tuple described above as the values. + On some systems such as Linux, on a very busy or long-lived system, the + numbers returned by the kernel may wrap (restart from zero). + If *nowrap* is ``True`` psutil will detect and adjust those numbers across + function calls and add "old value" to "new value" so that the numbers will + always be increasing or remain the same, but never decrease. >>> import psutil >>> psutil.net_io_counters() @@ -455,11 +463,9 @@ Network and `ifconfig.py `__ for an example application. - .. warning:: - on some systems such as Linux, on a very busy or long-lived system these - numbers may wrap (restart from zero), see - `issues #802 `__. - Applications should be prepared to deal with that. + .. versionchanged:: + 5.3.0 numbers no longer wrap (restart from zero) across calls thanks to new + *nowrap* argument. .. function:: net_connections(kind='inet') diff --git a/psutil/__init__.py b/psutil/__init__.py index d39f54f1d..710bfd67d 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -30,6 +30,7 @@ from ._common import deprecated_method from ._common import memoize from ._common import memoize_when_activated +from ._common import wrap_numbers as _wrap_numbers from ._compat import callable from ._compat import long from ._compat import PY3 as _PY3 @@ -2032,7 +2033,7 @@ def disk_partitions(all=False): return _psplatform.disk_partitions(all) -def disk_io_counters(perdisk=False): +def disk_io_counters(perdisk=False, nowrap=True): """Return system disk I/O statistics as a namedtuple including the following fields: @@ -2052,6 +2053,8 @@ def disk_io_counters(perdisk=False): executed first otherwise this function won't find any disk. """ rawdict = _psplatform.disk_io_counters() + if nowrap: + rawdict = _wrap_numbers(rawdict, 'psutil.disk_io_counters') nt = getattr(_psplatform, "sdiskio", _common.sdiskio) if perdisk: for disk, fields in rawdict.items(): @@ -2066,7 +2069,7 @@ def disk_io_counters(perdisk=False): # ===================================================================== -def net_io_counters(pernic=False): +def net_io_counters(pernic=False, nowrap=True): """Return network I/O statistics as a namedtuple including the following fields: @@ -2086,6 +2089,8 @@ def net_io_counters(pernic=False): described above as the values. """ rawdict = _psplatform.net_io_counters() + if nowrap: + rawdict = _wrap_numbers(rawdict, 'psutil.net_io_counters') if pernic: for nic, fields in rawdict.items(): rawdict[nic] = _common.snetio(*fields) diff --git a/psutil/_common.py b/psutil/_common.py index 46ccf0862..e5351e972 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -522,6 +522,7 @@ def run(self, input_dict, name): old_value = old_tuple[i] remkey = (key, i) if input_value < old_value: + # it wrapped! self.reminders[name][remkey] += old_value self.reminder_keys[name][key].add(remkey) bits.append(input_value + self.reminders[name][remkey]) @@ -532,6 +533,7 @@ def run(self, input_dict, name): return new_dict def cache_clear(self, name=None): + """Clear the internal cache, optionally only for function 'name'.""" with self.lock: if name is None: self.cache.clear() @@ -543,11 +545,17 @@ def cache_clear(self, name=None): self.reminder_keys.pop(name, None) def cache_info(self): + """Return internal cache dicts as a tuple of 3 elements.""" with self.lock: return (self.cache, self.reminders, self.reminder_keys) def wrap_numbers(input_dict, name): + """Given an `input_dict` and a function `name`, adjust the numbers + which "wrap" (restart from zero) across different calls by adding + "old value" to "new value". + + """ with _wn.lock: return _wn.run(input_dict, name) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 7906d64ef..691e69f52 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -728,7 +728,8 @@ def ifconfig(nic): ret['bytes_sent'] = int(re.findall('TX bytes:(\d+)', out)[0]) return ret - for name, stats in psutil.net_io_counters(pernic=True).items(): + nio = psutil.net_io_counters(pernic=True, nowrap=False) + for name, stats in nio.items(): try: ifconfig_ret = ifconfig(name) except RuntimeError: @@ -873,7 +874,7 @@ def open_mock(name, *args, **kwargs): orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock) as m: - ret = psutil.disk_io_counters() + ret = psutil.disk_io_counters(nowrap=False) assert m.called self.assertEqual(ret.read_count, 1) self.assertEqual(ret.read_merged_count, 2) @@ -905,7 +906,7 @@ def open_mock(name, *args, **kwargs): orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock) as m: - ret = psutil.disk_io_counters() + ret = psutil.disk_io_counters(nowrap=False) assert m.called self.assertEqual(ret.read_count, 1) self.assertEqual(ret.read_merged_count, 2) @@ -939,7 +940,7 @@ def open_mock(name, *args, **kwargs): orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock) as m: - ret = psutil.disk_io_counters() + ret = psutil.disk_io_counters(nowrap=False) assert m.called self.assertEqual(ret.read_count, 1) self.assertEqual(ret.read_bytes, 2 * SECTOR_SIZE) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 28a083f26..be270ab20 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -503,7 +503,7 @@ def test_disk_partitions(self): '/proc/diskstats not available on this Linux version') @skip_if_linux() def test_disk_io_counters(self): - self.execute(psutil.disk_io_counters) + self.execute(psutil.disk_io_counters, nowrap=False) # --- proc @@ -515,7 +515,7 @@ def test_pids(self): @skip_if_linux() def test_net_io_counters(self): - self.execute(psutil.net_io_counters) + self.execute(psutil.net_io_counters, nowrap=False) @unittest.skipIf(LINUX, "worthless on Linux (pure python)") From f647e5f0d3a561eddf7f2f2c598472ee1860f1d9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 18:06:27 +0200 Subject: [PATCH 612/922] #802: expose clear_cache and attach it to disk and net io primary function objects --- docs/index.rst | 12 ++++++++---- psutil/__init__.py | 10 ++++++++++ psutil/tests/test_misc.py | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 3ad544bbe..accefa708 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -397,8 +397,10 @@ Disks On some systems such as Linux, on a very busy or long-lived system, the numbers returned by the kernel may wrap (restart from zero). If *nowrap* is ``True`` psutil will detect and adjust those numbers across - function calls and add "old value" to "new value" so that the numbers will - always be increasing or remain the same, but never decrease. + function calls and add "old value" to "new value" so that the returned + numbers will always be increasing or remain the same, but never decrease. + ``disk_io_counters.cache_clear()`` can be used to invalidate the *nowrap* + cache. >>> import psutil >>> psutil.disk_io_counters() @@ -448,8 +450,10 @@ Network On some systems such as Linux, on a very busy or long-lived system, the numbers returned by the kernel may wrap (restart from zero). If *nowrap* is ``True`` psutil will detect and adjust those numbers across - function calls and add "old value" to "new value" so that the numbers will - always be increasing or remain the same, but never decrease. + function calls and add "old value" to "new value" so that the returned + numbers will always be increasing or remain the same, but never decrease. + ``net_io_counters.cache_clear()`` can be used to invalidate the *nowrap* + cache. >>> import psutil >>> psutil.net_io_counters() diff --git a/psutil/__init__.py b/psutil/__init__.py index 710bfd67d..e63f1c512 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2064,6 +2064,11 @@ def disk_io_counters(perdisk=False, nowrap=True): return nt(*[sum(x) for x in zip(*rawdict.values())]) +disk_io_counters.cache_clear = functools.partial( + _wrap_numbers.cache_clear, 'psutil.disk_io_counters') +disk_io_counters.cache_clear.__doc__ = "Clears nowrap argument cache" + + # ===================================================================== # --- network related functions # ===================================================================== @@ -2099,6 +2104,11 @@ def net_io_counters(pernic=False, nowrap=True): return _common.snetio(*[sum(x) for x in zip(*rawdict.values())]) +net_io_counters.cache_clear = functools.partial( + _wrap_numbers.cache_clear, 'psutil.net_io_counters') +net_io_counters.cache_clear.__doc__ = "Clears nowrap argument cache" + + def net_connections(kind='inet'): """Return system-wide connections as a list of (fd, family, type, laddr, raddr, status, pid) namedtuples. diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index f73812a9c..5f32df36c 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -588,6 +588,24 @@ def test_cache_clear(self): wrap_numbers.cache_clear('disk_io') wrap_numbers.cache_clear('?!?') + def test_cache_clear_public_apis(self): + psutil.disk_io_counters() + psutil.net_io_counters() + caches = wrap_numbers.cache_info() + for cache in caches: + self.assertIn('psutil.disk_io_counters', cache) + self.assertIn('psutil.net_io_counters', cache) + + psutil.disk_io_counters.cache_clear() + caches = wrap_numbers.cache_info() + for cache in caches: + self.assertIn('psutil.net_io_counters', cache) + self.assertNotIn('psutil.disk_io_counters', cache) + + psutil.net_io_counters.cache_clear() + caches = wrap_numbers.cache_info() + self.assertEqual(caches, ({}, {}, {})) + # --- def test_real_data(self): From 8009476329f56b2e85f4e4a4de6d602854139019 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 19:17:31 +0200 Subject: [PATCH 613/922] move tests around --- psutil/_common.py | 2 +- psutil/tests/test_misc.py | 35 ++++++++++++++++------------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index e5351e972..9be96fe2a 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -470,6 +470,7 @@ def inner(self, *args, **kwargs): class _WrapNumbers: + """Watches numbers so that they don't overlap.""" def __init__(self): self.lock = threading.Lock() @@ -554,7 +555,6 @@ def wrap_numbers(input_dict, name): """Given an `input_dict` and a function `name`, adjust the numbers which "wrap" (restart from zero) across different calls by adding "old value" to "new value". - """ with _wn.lock: return _wn.run(input_dict, name) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 5f32df36c..28c40ed79 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -20,7 +20,6 @@ import socket import stat import sys -from collections import namedtuple from psutil import LINUX from psutil import POSIX @@ -388,7 +387,7 @@ def test_sanity_version_check(self): # =================================================================== -nt = namedtuple('foo', 'a b c') +nt = collections.namedtuple('foo', 'a b c') class TestWrapNumbers(unittest.TestCase): @@ -496,6 +495,21 @@ def test_changing_keys_w_wrap(self): {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 110)}) + def test_real_data(self): + d = {'nvme0n1': (300, 508, 640, 1571, 5970, 1987, 2049, 451751, 47048), + 'nvme0n1p1': (1171, 2, 5600256, 1024, 516, 0, 0, 0, 8), + 'nvme0n1p2': (54, 54, 2396160, 5165056, 4, 24, 30, 1207, 28), + 'nvme0n1p3': (2389, 4539, 5154, 150, 4828, 1844, 2019, 398, 348)} + self.assertEqual(wrap_numbers(d, 'disk_io'), d) + self.assertEqual(wrap_numbers(d, 'disk_io'), d) + # decrease this ↓ + d = {'nvme0n1': (100, 508, 640, 1571, 5970, 1987, 2049, 451751, 47048), + 'nvme0n1p1': (1171, 2, 5600256, 1024, 516, 0, 0, 0, 8), + 'nvme0n1p2': (54, 54, 2396160, 5165056, 4, 24, 30, 1207, 28), + 'nvme0n1p3': (2389, 4539, 5154, 150, 4828, 1844, 2019, 398, 348)} + out = wrap_numbers(d, 'disk_io') + self.assertEqual(out['nvme0n1'][0], 400) + # --- cache tests def test_cache_first_call(self): @@ -606,23 +620,6 @@ def test_cache_clear_public_apis(self): caches = wrap_numbers.cache_info() self.assertEqual(caches, ({}, {}, {})) - # --- - - def test_real_data(self): - d = {'nvme0n1': (300, 508, 640, 1571, 5970, 1987, 2049, 451751, 47048), - 'nvme0n1p1': (1171, 2, 5600256, 1024, 516, 0, 0, 0, 8), - 'nvme0n1p2': (54, 54, 2396160, 5165056, 4, 24, 30, 1207, 28), - 'nvme0n1p3': (2389, 4539, 5154, 150, 4828, 1844, 2019, 398, 348)} - self.assertEqual(wrap_numbers(d, 'disk_io'), d) - self.assertEqual(wrap_numbers(d, 'disk_io'), d) - # decrease this ↓ - d = {'nvme0n1': (100, 508, 640, 1571, 5970, 1987, 2049, 451751, 47048), - 'nvme0n1p1': (1171, 2, 5600256, 1024, 516, 0, 0, 0, 8), - 'nvme0n1p2': (54, 54, 2396160, 5165056, 4, 24, 30, 1207, 28), - 'nvme0n1p3': (2389, 4539, 5154, 150, 4828, 1844, 2019, 398, 348)} - out = wrap_numbers(d, 'disk_io') - self.assertEqual(out['nvme0n1'][0], 400) - # =================================================================== # --- Example script tests From 89b7e8a0ddcc569d52ecc1880e172c530338559c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 19:21:28 +0200 Subject: [PATCH 614/922] update doc --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index accefa708..b31f9d73c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -395,7 +395,7 @@ Disks See `iotop.py `__ for an example application. On some systems such as Linux, on a very busy or long-lived system, the - numbers returned by the kernel may wrap (restart from zero). + numbers returned by the kernel may overflow and wrap (restart from zero). If *nowrap* is ``True`` psutil will detect and adjust those numbers across function calls and add "old value" to "new value" so that the returned numbers will always be increasing or remain the same, but never decrease. @@ -448,7 +448,7 @@ Network interface installed on the system as a dictionary with network interface names as the keys and the named tuple described above as the values. On some systems such as Linux, on a very busy or long-lived system, the - numbers returned by the kernel may wrap (restart from zero). + numbers returned by the kernel may overflow and wrap (restart from zero). If *nowrap* is ``True`` psutil will detect and adjust those numbers across function calls and add "old value" to "new value" so that the returned numbers will always be increasing or remain the same, but never decrease. From 14d0d376af1ddf1694536084e63ab9096f3484b5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 19:27:45 +0200 Subject: [PATCH 615/922] update docstrings --- psutil/_common.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 9be96fe2a..d58dac6b0 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -63,7 +63,7 @@ # utility functions 'conn_tmap', 'deprecated_method', 'isfile_strict', 'memoize', 'parse_environ_block', 'path_exists_strict', 'usage_percent', - 'supports_ipv6', 'sockfam_to_enum', 'socktype_to_enum', + 'supports_ipv6', 'sockfam_to_enum', 'socktype_to_enum', "wrap_numbers", ] @@ -470,7 +470,9 @@ def inner(self, *args, **kwargs): class _WrapNumbers: - """Watches numbers so that they don't overlap.""" + """Watches numbers so that they don't overflow and wrap + (reset to zero). + """ def __init__(self): self.lock = threading.Lock() @@ -498,6 +500,9 @@ def _remove_dead_reminders(self, input_dict, name): del self.reminder_keys[name][gone_key] def run(self, input_dict, name): + """Cache dict and sum numbers which overflow and wrap. + Return an updated copy of `input_dict` + """ if name not in self.cache: # This was the first call. self._add_dict(input_dict, name) @@ -554,7 +559,7 @@ def cache_info(self): def wrap_numbers(input_dict, name): """Given an `input_dict` and a function `name`, adjust the numbers which "wrap" (restart from zero) across different calls by adding - "old value" to "new value". + "old value" to "new value" and return an updated dict. """ with _wn.lock: return _wn.run(input_dict, name) From 50e95732be8bfe25ef6717b331939c664b8c2ecb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 9 May 2017 19:34:19 +0200 Subject: [PATCH 616/922] update docstring --- psutil/__init__.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index e63f1c512..8de1cac2c 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2041,14 +2041,27 @@ def disk_io_counters(perdisk=False, nowrap=True): - write_count: number of writes - read_bytes: number of bytes read - write_bytes: number of bytes written - - read_time: time spent reading from disk (in milliseconds) - - write_time: time spent writing to disk (in milliseconds) + - read_time: time spent reading from disk (in ms) + - write_time: time spent writing to disk (in ms) - If perdisk is True return the same information for every + Platform specific: + + - busy_time: (Linux, FreeBSD) time spent doing actual I/Os (in ms) + - read_merged_count (Linux): number of merged reads + - write_merged_count (Linux): number of merged writes + + If *perdisk* is True return the same information for every physical disk installed on the system as a dictionary with partition names as the keys and the namedtuple described above as the values. + If *nowrap* is True it detects and adjust the numbers which overflow + and wrap (restart from 0) and add "old value" to "new value" so that + the returned numbers will always be increasing or remain the same, + but never decrease. + "disk_io_counters.cache_clear()" can be used to invalidate the + cache. + On recent Windows versions 'diskperf -y' command may need to be executed first otherwise this function won't find any disk. """ @@ -2088,10 +2101,17 @@ def net_io_counters(pernic=False, nowrap=True): - dropout: total number of outgoing packets which were dropped (always 0 on OSX and BSD) - If pernic is True return the same information for every + If *pernic* is True return the same information for every network interface installed on the system as a dictionary with network interface names as the keys and the namedtuple described above as the values. + + If *nowrap* is True it detects and adjust the numbers which overflow + and wrap (restart from 0) and add "old value" to "new value" so that + the returned numbers will always be increasing or remain the same, + but never decrease. + "disk_io_counters.cache_clear()" can be used to invalidate the + cache. """ rawdict = _psplatform.net_io_counters() if nowrap: From d975b23a2f4f8d9d46939d238c381f31fdc72634 Mon Sep 17 00:00:00 2001 From: Yannick Gingras Date: Tue, 9 May 2017 14:58:20 -0700 Subject: [PATCH 617/922] Marked regexp patterns as raw strings and bytes to address the invalid escape warning on Python 3.6+ --- psutil/_pslinux.py | 18 +++++++++--------- psutil/tests/__init__.py | 4 ++-- psutil/tests/test_bsd.py | 2 +- psutil/tests/test_linux.py | 24 ++++++++++++------------ psutil/tests/test_osx.py | 4 ++-- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 7a03940a5..f834f19fe 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -596,7 +596,7 @@ def cpu_count_logical(): # https://github.com/giampaolo/psutil/issues/200 # try to parse /proc/stat as a last resort if num == 0: - search = re.compile('cpu\d') + search = re.compile(r'cpu\d') with open_text('%s/stat' % get_procfs_path()) as f: for line in f: line = line.split(' ')[0] @@ -1562,9 +1562,9 @@ def memory_info(self): @wrap_exceptions def memory_full_info( self, - _private_re=re.compile(b"Private.*:\s+(\d+)"), - _pss_re=re.compile(b"Pss.*:\s+(\d+)"), - _swap_re=re.compile(b"Swap.*:\s+(\d+)")): + _private_re=re.compile(br"Private.*:\s+(\d+)"), + _pss_re=re.compile(br"Pss.*:\s+(\d+)"), + _swap_re=re.compile(br"Swap.*:\s+(\d+)")): basic_mem = self.memory_info() # Note: using 3 regexes is faster than reading the file # line by line. @@ -1677,7 +1677,7 @@ def cwd(self): raise @wrap_exceptions - def num_ctx_switches(self, _ctxsw_re=re.compile(b'ctxt_switches:\t(\d+)')): + def num_ctx_switches(self, _ctxsw_re=re.compile(br'ctxt_switches:\t(\d+)')): data = self._read_status_file() ctxsw = _ctxsw_re.findall(data) if not ctxsw: @@ -1690,7 +1690,7 @@ def num_ctx_switches(self, _ctxsw_re=re.compile(b'ctxt_switches:\t(\d+)')): return _common.pctxsw(int(ctxsw[0]), int(ctxsw[1])) @wrap_exceptions - def num_threads(self, _num_threads_re=re.compile(b'Threads:\t(\d+)')): + def num_threads(self, _num_threads_re=re.compile(br'hreads:\t(\d+)')): # Note: on Python 3 using a re is faster than iterating over file # line by line. On Python 2 is the exact opposite, and iterating # over a file on Python 3 is slower than on Python 2. @@ -1746,7 +1746,7 @@ def cpu_affinity_get(self): return cext.proc_cpu_affinity_get(self.pid) def _get_eligible_cpus( - self, _re=re.compile(b"Cpus_allowed_list:\t(\d+)-(\d+)")): + self, _re=re.compile(br"Cpus_allowed_list:\t(\d+)-(\d+)")): # See: https://github.com/giampaolo/psutil/issues/956 data = self._read_status_file() match = _re.findall(data) @@ -1919,13 +1919,13 @@ def ppid(self): return int(self._parse_stat_file()[2]) @wrap_exceptions - def uids(self, _uids_re=re.compile(b'Uid:\t(\d+)\t(\d+)\t(\d+)')): + def uids(self, _uids_re=re.compile(br'Uid:\t(\d+)\t(\d+)\t(\d+)')): data = self._read_status_file() real, effective, saved = _uids_re.findall(data)[0] return _common.puids(int(real), int(effective), int(saved)) @wrap_exceptions - def gids(self, _gids_re=re.compile(b'Gid:\t(\d+)\t(\d+)\t(\d+)')): + def gids(self, _gids_re=re.compile(br'Gid:\t(\d+)\t(\d+)\t(\d+)')): data = self._read_status_file() real, effective, saved = _gids_re.findall(data)[0] return _common.pgids(int(real), int(effective), int(saved)) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 22ccc52d0..e30beec1a 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -479,7 +479,7 @@ def get_winver(): if hasattr(wv, 'service_pack_major'): # python >= 2.7 sp = wv.service_pack_major or 0 else: - r = re.search("\s\d$", wv[4]) + r = re.search(r"\s\d$", wv[4]) if r: sp = int(r.group(0)) else: @@ -893,7 +893,7 @@ def check_net_address(addr, family): addr = unicode(addr) ipaddress.IPv6Address(addr) elif family == psutil.AF_LINK: - assert re.match('([a-fA-F0-9]{2}[:|\-]?){6}', addr) is not None, addr + assert re.match(r'([a-fA-F0-9]{2}[:|\-]?){6}', addr) is not None, addr else: raise ValueError("unknown family %r", family) diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 8fd7fe1d5..50b34c092 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -138,7 +138,7 @@ def test_net_if_stats(self): self.assertEqual(stats.isup, 'RUNNING' in out, msg=out) if "mtu" in out: self.assertEqual(stats.mtu, - int(re.findall('mtu (\d+)', out)[0])) + int(re.findall(r'mtu (\d+)', out)[0])) # ===================================================================== diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 7906d64ef..8ded6cc45 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -506,7 +506,7 @@ class TestSystemCPU(unittest.TestCase): @unittest.skipIf(TRAVIS, "unknown failure on travis") def test_cpu_times(self): fields = psutil.cpu_times()._fields - kernel_ver = re.findall('\d+\.\d+\.\d+', os.uname()[2])[0] + kernel_ver = re.findall(r'\d+\.\d+\.\d+', os.uname()[2])[0] kernel_ver_info = tuple(map(int, kernel_ver.split('.'))) if kernel_ver_info >= (2, 6, 11): self.assertIn('steal', fields) @@ -534,7 +534,7 @@ def test_cpu_count_logical_w_sysdev_cpu_online(self): "/sys/devices/system/cpu does not exist") def test_cpu_count_logical_w_sysdev_cpu_num(self): ls = os.listdir("/sys/devices/system/cpu") - count = len([x for x in ls if re.search("cpu\d+$", x) is not None]) + count = len([x for x in ls if re.search(r"cpu\d+$", x) is not None]) self.assertEqual(psutil.cpu_count(), count) @unittest.skipIf(not which("nproc"), "nproc utility not available") @@ -711,21 +711,21 @@ def test_net_if_stats(self): # Not always reliable. # self.assertEqual(stats.isup, 'RUNNING' in out, msg=out) self.assertEqual(stats.mtu, - int(re.findall('MTU:(\d+)', out)[0])) + int(re.findall(r'MTU:(\d+)', out)[0])) @retry_before_failing() def test_net_io_counters(self): def ifconfig(nic): ret = {} out = sh("ifconfig %s" % name) - ret['packets_recv'] = int(re.findall('RX packets:(\d+)', out)[0]) - ret['packets_sent'] = int(re.findall('TX packets:(\d+)', out)[0]) - ret['errin'] = int(re.findall('errors:(\d+)', out)[0]) - ret['errout'] = int(re.findall('errors:(\d+)', out)[1]) - ret['dropin'] = int(re.findall('dropped:(\d+)', out)[0]) - ret['dropout'] = int(re.findall('dropped:(\d+)', out)[1]) - ret['bytes_recv'] = int(re.findall('RX bytes:(\d+)', out)[0]) - ret['bytes_sent'] = int(re.findall('TX bytes:(\d+)', out)[0]) + ret['packets_recv'] = int(re.findall(r'RX packets:(\d+)', out)[0]) + ret['packets_sent'] = int(re.findall(r'TX packets:(\d+)', out)[0]) + ret['errin'] = int(re.findall(r'errors:(\d+)', out)[0]) + ret['errout'] = int(re.findall(r'errors:(\d+)', out)[1]) + ret['dropin'] = int(re.findall(r'dropped:(\d+)', out)[0]) + ret['dropout'] = int(re.findall(r'dropped:(\d+)', out)[1]) + ret['bytes_recv'] = int(re.findall(r'RX bytes:(\d+)', out)[0]) + ret['bytes_sent'] = int(re.findall(r'TX bytes:(\d+)', out)[0]) return ret for name, stats in psutil.net_io_counters(pernic=True).items(): @@ -758,7 +758,7 @@ def test_net_if_names(self): found = 0 for line in out.split('\n'): line = line.strip() - if re.search("^\d+:", line): + if re.search(r"^\d+:", line): found += 1 name = line.split(':')[1].strip() self.assertIn(name, nics) diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 8ba949b0a..225f95db2 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -44,7 +44,7 @@ def vm_stat(field): break else: raise ValueError("line not found") - return int(re.search('\d+', line).group(0)) * PAGESIZE + return int(re.search(r'\d+', line).group(0)) * PAGESIZE # http://code.activestate.com/recipes/578019/ @@ -221,7 +221,7 @@ def test_net_if_stats(self): else: self.assertEqual(stats.isup, 'RUNNING' in out, msg=out) self.assertEqual(stats.mtu, - int(re.findall('mtu (\d+)', out)[0])) + int(re.findall(r'mtu (\d+)', out)[0])) if __name__ == '__main__': From 09a1277540db204d3925e612f7c99326848e4ad4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 10 May 2017 14:18:44 +0200 Subject: [PATCH 618/922] update docstrings --- docs/index.rst | 2 +- psutil/__init__.py | 201 ++++++++++++++++++++++++--------------------- 2 files changed, 109 insertions(+), 94 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index b31f9d73c..ff38af981 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -894,7 +894,7 @@ Functions argument). This tunction will return as soon as all processes terminate or when *timeout* occurs, if specified. - Differently from :meth:`Process.wait` it does not raise + Differently from :meth:`Process.wait` it will not raise :class:`TimeoutExpired` if timeout occurs. A typical use case may be: diff --git a/psutil/__init__.py b/psutil/__init__.py index 8de1cac2c..509fcf823 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -5,8 +5,18 @@ # found in the LICENSE file. """psutil is a cross-platform library for retrieving information on -running processes and system utilization (CPU, memory, disks, network) -in Python. +running processes and system utilization (CPU, memory, disks, network, +sensors) in Python. Supported platforms: + + - Linux + - Windows + - OSX + - Sun Solaris + - FreeBSD + - OpenBSD + - NetBSD + +Works with Python versions from 2.6 to 3.X. """ from __future__ import division @@ -372,10 +382,10 @@ class Process(object): - kill() To prevent this problem for all other methods you can: - - use is_running() before querying the process - - if you're continuously iterating over a set of Process - instances use process_iter() which pre-emptively checks - process identity for every yielded instance + - use is_running() before querying the process + - if you're continuously iterating over a set of Process + instances use process_iter() which pre-emptively checks + process identity for every yielded instance """ def __init__(self, pid=None): @@ -533,11 +543,11 @@ def oneshot(self): def as_dict(self, attrs=None, ad_value=None): """Utility method returning process information as a hashable dictionary. - If 'attrs' is specified it must be a list of strings + If *attrs* is specified it must be a list of strings reflecting available Process class' attribute names (e.g. ['cpu_times', 'name']) else all public (read only) attributes are assumed. - 'ad_value' is the value which gets assigned in case + *ad_value* is the value which gets assigned in case AccessDenied or ZombieProcess exception is raised when retrieving that particular process information. """ @@ -793,11 +803,11 @@ def io_counters(self): def ionice(self, ioclass=None, value=None): """Get or set process I/O niceness (priority). - On Linux 'ioclass' is one of the IOPRIO_CLASS_* constants. - 'value' is a number which goes from 0 to 7. The higher the + On Linux *ioclass* is one of the IOPRIO_CLASS_* constants. + *value* is a number which goes from 0 to 7. The higher the value, the lower the I/O priority of the process. - On Windows only 'ioclass' is used and it can be set to 2 + On Windows only *ioclass* is used and it can be set to 2 (normal), 1 (low) or 0 (very low). Available on Linux and Windows > Vista only. @@ -816,8 +826,8 @@ def rlimit(self, resource, limits=None): """Get or set process resource limits as a (soft, hard) tuple. - 'resource' is one of the RLIMIT_* constants. - 'limits' is supposed to be a (soft, hard) tuple. + *resource* is one of the RLIMIT_* constants. + *limits* is supposed to be a (soft, hard) tuple. See "man prlimit" for further info. Available on Linux only. @@ -832,7 +842,7 @@ def rlimit(self, resource, limits=None): def cpu_affinity(self, cpus=None): """Get or set process CPU affinity. - If specified 'cpus' must be a list of CPUs for which you + If specified, *cpus* must be a list of CPUs for which you want to set the affinity (e.g. [0, 1]). If an empty list is passed, all egible CPUs are assumed (and set). @@ -902,7 +912,7 @@ def threads(self): def children(self, recursive=False): """Return the children of this process as a list of Process instances, pre-emptively checking whether PID has been reused. - If recursive is True return all the parent descendants. + If *recursive* is True return all the parent descendants. Example (A == this process): @@ -998,12 +1008,12 @@ def cpu_percent(self, interval=None): """Return a float representing the current process CPU utilization as a percentage. - When interval is 0.0 or None (default) compares process times + When *interval* is 0.0 or None (default) compares process times to system CPU times elapsed since last call, returning immediately (non-blocking). That means that the first time this is called it will return a meaningful 0.0 value. - When interval is > 0.0 compares process times to system CPU + When *interval* is > 0.0 compares process times to system CPU times elapsed before and after the interval (blocking). In this case is recommended for accuracy that this function @@ -1012,7 +1022,7 @@ def cpu_percent(self, interval=None): A value > 100.0 can be returned in case of processes running multiple threads on different CPU cores. - The returned value is explicitly *not* split evenly between + The returned value is explicitly NOT split evenly between all available logical CPUs. This means that a busy loop process running on a system with 2 logical CPUs will be reported as having 100% CPU utilization instead of 50%. @@ -1131,7 +1141,7 @@ def memory_full_info(self): def memory_percent(self, memtype="rss"): """Compare process memory to total physical system memory and calculate process memory utilization as a percentage. - 'memtype' argument is a string that dictates what type of + *memtype* argument is a string that dictates what type of process memory you want to compare against (defaults to "rss"). The list of available strings can be obtained like this: @@ -1166,10 +1176,10 @@ def memory_maps(self, grouped=True): """Return process' mapped memory regions as a list of namedtuples whose fields are variable depending on the platform. - If 'grouped' is True the mapped regions with the same 'path' + If *grouped* is True the mapped regions with the same 'path' are grouped together and the different memory fields are summed. - If 'grouped' is False every mapped region is shown as a single + If *grouped* is False every mapped region is shown as a single entity and the namedtuple will also include the mapped region's address space ('addr') and permission set ('perms'). """ @@ -1197,23 +1207,26 @@ def open_files(self): return self._proc.open_files() def connections(self, kind='inet'): - """Return connections opened by process as a list of + """Return socket connections opened by process as a list of (fd, family, type, laddr, raddr, status) namedtuples. - The 'kind' parameter filters for connections that match the + The *kind* parameter filters for connections that match the following criteria: - Kind Value Connections using - inet IPv4 and IPv6 - inet4 IPv4 - inet6 IPv6 - tcp TCP - tcp4 TCP over IPv4 - tcp6 TCP over IPv6 - udp UDP - udp4 UDP over IPv4 - udp6 UDP over IPv6 - unix UNIX socket (both UDP and TCP protocols) - all the sum of all the possible families and protocols + +------------+----------------------------------------------------+ + | Kind Value | Connections using | + +------------+----------------------------------------------------+ + | inet | IPv4 and IPv6 | + | inet4 | IPv4 | + | inet6 | IPv6 | + | tcp | TCP | + | tcp4 | TCP over IPv4 | + | tcp6 | TCP over IPv6 | + | udp | UDP | + | udp4 | UDP over IPv4 | + | udp6 | UDP over IPv6 | + | unix | UNIX socket (both UDP and TCP protocols) | + | all | the sum of all the possible families and protocols | + +------------+----------------------------------------------------+ """ return self._proc.connections(kind) @@ -1245,8 +1258,8 @@ def _send_signal(self, sig): @_assert_pid_not_reused def send_signal(self, sig): - """Send a signal to process pre-emptively checking whether - PID has been reused (see signal module constants) . + """Send a signal *sig* to process pre-emptively checking + whether PID has been reused (see signal module constants) . On Windows only SIGTERM is valid and is treated as an alias for kill(). """ @@ -1314,8 +1327,8 @@ def wait(self, timeout=None): If the process is already terminated immediately return None instead of raising NoSuchProcess. - If timeout (in seconds) is specified and process is still alive - raise TimeoutExpired. + If *timeout* (in seconds) is specified and process is still + alive raise TimeoutExpired. To wait for multiple Process(es) use psutil.wait_procs(). """ @@ -1466,11 +1479,11 @@ def process_iter(attrs=None, ad_value=None): The sorting order in which processes are yielded is based on their PIDs. - "attrs" and "ad_value" have the same meaning as in - Process.as_dict(). If "attrs" is specified as_dict() is called + *attrs* and *ad_value* have the same meaning as in + Process.as_dict(). If *attrs* is specified as_dict() is called and the resulting dict is stored as a 'info' attribute attached to returned Process instance. - If "attrs" is an empty list it will retrieve all process info + If *attrs* is an empty list it will retrieve all process info (slow). """ def add(pid): @@ -1529,14 +1542,16 @@ def wait_procs(procs, timeout=None, callback=None): Return a (gone, alive) tuple indicating which processes are gone and which ones are still alive. - The gone ones will have a new 'returncode' attribute indicating + The gone ones will have a new *returncode* attribute indicating process exit status (may be None). - 'callback' is a function which gets called every time a process + *callback* is a function which gets called every time a process terminates (a Process instance is passed as callback argument). Function will return as soon as all processes terminate or when - timeout occurs. + *timeout* occurs. + Differently from Process.wait() it will not raise TimeoutExpired if + *timeout* occurs. Typical use case is: @@ -1618,7 +1633,7 @@ def cpu_count(logical=True): """Return the number of logical CPUs in the system (same as os.cpu_count() in Python 3.4). - If logical is False return the number of physical cores only + If *logical* is False return the number of physical cores only (e.g. hyper thread CPUs are excluded). Return None if undetermined. @@ -1636,8 +1651,10 @@ def cpu_count(logical=True): def cpu_times(percpu=False): """Return system-wide CPU times as a namedtuple. - Every CPU time represents the seconds the CPU has spent in the given mode. - The namedtuple's fields availability varies depending on the platform: + Every CPU time represents the seconds the CPU has spent in the + given mode. The namedtuple's fields availability varies depending on the + platform: + - user - system - idle @@ -1649,7 +1666,7 @@ def cpu_times(percpu=False): - guest (Linux >= 2.6.24) - guest_nice (Linux >= 3.2.0) - When percpu is True return a list of namedtuples for each CPU. + When *percpu* is True return a list of namedtuples for each CPU. First element of the list refers to first CPU, second element to second CPU and so on. The order of the list is consistent across calls. @@ -1714,17 +1731,17 @@ def cpu_percent(interval=None, percpu=False): """Return a float representing the current system-wide CPU utilization as a percentage. - When interval is > 0.0 compares system CPU times elapsed before + When *interval* is > 0.0 compares system CPU times elapsed before and after the interval (blocking). - When interval is 0.0 or None compares system CPU times elapsed + When *interval* is 0.0 or None compares system CPU times elapsed since last call or module import, returning immediately (non blocking). That means the first time this is called it will return a meaningless 0.0 value which you should ignore. In this case is recommended for accuracy that this function be called with at least 0.1 seconds between calls. - When percpu is True returns a list of floats representing the + When *percpu* is True returns a list of floats representing the utilization as a percentage for each CPU. First element of the list refers to first CPU, second element to second CPU and so on. @@ -1821,7 +1838,7 @@ def cpu_times_percent(interval=None, percpu=False): irq=0.0, softirq=0.0, steal=0.0, guest=0.0, guest_nice=0.0) >>> - interval and percpu arguments have the same meaning as in + *interval* and *percpu* arguments have the same meaning as in cpu_percent(). """ global _last_cpu_times_2 @@ -1901,7 +1918,7 @@ def cpu_freq(percpu=False): """Return CPU frequency as a nameduple including current, min and max frequency expressed in Mhz. - If percpu is True and the system supports per-cpu frequency + If *percpu* is True and the system supports per-cpu frequency retrieval (Linux only) a list of frequencies is returned for each CPU. If not a list with one element is returned. """ @@ -1951,8 +1968,8 @@ def virtual_memory(): the percentage usage calculated as (total - available) / total * 100 - used: - memory used, calculated differently depending on the platform and - designed for informational purposes only: + memory used, calculated differently depending on the platform and + designed for informational purposes only: OSX: active + inactive + wired BSD: active + wired + cached LINUX: total - free @@ -2014,9 +2031,9 @@ def swap_memory(): def disk_usage(path): - """Return disk usage statistics about the given path as a namedtuple - including total, used and free space expressed in bytes plus the - percentage usage. + """Return disk usage statistics about the given *path* as a + namedtuple including total, used and free space expressed in bytes + plus the percentage usage. """ return _psplatform.disk_usage(path) @@ -2027,7 +2044,7 @@ def disk_partitions(all=False): 'opts' field is a raw string separated by commas indicating mount options which may vary depending on the platform. - If "all" parameter is False return physical devices only and ignore + If *all* parameter is False return physical devices only and ignore all others. """ return _psplatform.disk_partitions(all) @@ -2130,25 +2147,28 @@ def net_io_counters(pernic=False, nowrap=True): def net_connections(kind='inet'): - """Return system-wide connections as a list of + """Return system-wide socket connections as a list of (fd, family, type, laddr, raddr, status, pid) namedtuples. In case of limited privileges 'fd' and 'pid' may be set to -1 and None respectively. - The 'kind' parameter filters for connections that fit the + The *kind* parameter filters for connections that fit the following criteria: - Kind Value Connections using - inet IPv4 and IPv6 - inet4 IPv4 - inet6 IPv6 - tcp TCP - tcp4 TCP over IPv4 - tcp6 TCP over IPv6 - udp UDP - udp4 UDP over IPv4 - udp6 UDP over IPv6 - unix UNIX socket (both UDP and TCP protocols) - all the sum of all the possible families and protocols + +------------+----------------------------------------------------+ + | Kind Value | Connections using | + +------------+----------------------------------------------------+ + | inet | IPv4 and IPv6 | + | inet4 | IPv4 | + | inet6 | IPv6 | + | tcp | TCP | + | tcp4 | TCP over IPv4 | + | tcp6 | TCP over IPv6 | + | udp | UDP | + | udp4 | UDP over IPv4 | + | udp6 | UDP over IPv6 | + | unix | UNIX socket (both UDP and TCP protocols) | + | all | the sum of all the possible families and protocols | + +------------+----------------------------------------------------+ On OSX this function requires root privileges. """ @@ -2161,19 +2181,14 @@ def net_if_addrs(): NIC names and value is a list of namedtuples for each address assigned to the NIC. Each namedtuple includes 5 fields: - - family - - address - - netmask - - broadcast - - ptp - - 'family' can be either socket.AF_INET, socket.AF_INET6 or - psutil.AF_LINK, which refers to a MAC address. - 'address' is the primary address and it is always set. - 'netmask' and 'broadcast' and 'ptp' may be None. - 'ptp' stands for "point to point" and references the destination - address on a point to point interface (typically a VPN). - 'broadcast' and 'ptp' are mutually exclusive. + - family: can be either socket.AF_INET, socket.AF_INET6 or + psutil.AF_LINK, which refers to a MAC address. + - address: is the primary address and it is always set. + - netmask: and 'broadcast' and 'ptp' may be None. + - ptp: stands for "point to point" and references the + destination address on a point to point interface + (typically a VPN). + - broadcast: and *ptp* are mutually exclusive. Note: you can have more than one address of the same family associated with each interface. @@ -2286,11 +2301,11 @@ def sensors_battery(): """Return battery information. If no battery is installed returns None. - - percent: battery power left as a percentage. - - secsleft: a rough approximation of how many seconds are left - before the battery runs out of power. - May be POWER_TIME_UNLIMITED or POWER_TIME_UNLIMITED. - - power_plugged: True if the AC power cable is connected. + - percent: battery power left as a percentage. + - secsleft: a rough approximation of how many seconds are left + before the battery runs out of power. May be + POWER_TIME_UNLIMITED or POWER_TIME_UNLIMITED. + - power_plugged: True if the AC power cable is connected. """ return _psplatform.sensors_battery() @@ -2336,7 +2351,7 @@ def win_service_iter(): return _psplatform.win_service_iter() def win_service_get(name): - """Get a Windows service by name. + """Get a Windows service by *name*. Raise NoSuchProcess if no service with such name exists. """ return _psplatform.win_service_get(name) From 3ed70c10f263653b299575d28faded1a966eb629 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 10 May 2017 15:31:51 +0200 Subject: [PATCH 619/922] enable all python warnings by default during tests - re #1057 --- psutil/tests/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 22ccc52d0..85075d121 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -103,6 +103,11 @@ ] +# Enable all warnings by default. +if 'PYTHONWARNINGS' not in os.environ: + warnings.simplefilter('always') + + # =================================================================== # --- constants # =================================================================== From 596b43381711248093caf1c1df3b5d93d0c70bb6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 10 May 2017 17:24:35 +0200 Subject: [PATCH 620/922] fix re, fix, flake 8, give CREDITS to @ygingras for #1057 --- CREDITS | 4 ++++ psutil/_pslinux.py | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CREDITS b/CREDITS index 148c7bbc3..4c2b6c703 100644 --- a/CREDITS +++ b/CREDITS @@ -468,3 +468,7 @@ W: https://github.com/alexanha N: Himanshu Shekhar W: https://github.com/himanshub16 I: 1036 + +N: Yannick Gingras +W: https://github.com/ygingras +I: 1057 diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index f834f19fe..7c075f419 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1677,7 +1677,8 @@ def cwd(self): raise @wrap_exceptions - def num_ctx_switches(self, _ctxsw_re=re.compile(br'ctxt_switches:\t(\d+)')): + def num_ctx_switches(self, + _ctxsw_re=re.compile(br'ctxt_switches:\t(\d+)')): data = self._read_status_file() ctxsw = _ctxsw_re.findall(data) if not ctxsw: @@ -1690,7 +1691,7 @@ def num_ctx_switches(self, _ctxsw_re=re.compile(br'ctxt_switches:\t(\d+)')): return _common.pctxsw(int(ctxsw[0]), int(ctxsw[1])) @wrap_exceptions - def num_threads(self, _num_threads_re=re.compile(br'hreads:\t(\d+)')): + def num_threads(self, _num_threads_re=re.compile(br'Threads:\t(\d+)')): # Note: on Python 3 using a re is faster than iterating over file # line by line. On Python 2 is the exact opposite, and iterating # over a file on Python 3 is slower than on Python 2. From 58aef46d74ff577f60eb9e0478d6de3361c8a4e9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 10 May 2017 20:38:05 +0200 Subject: [PATCH 621/922] 1058 enable fix warnings (#1059) * #1058: have Makefile use PYTHONWARNINGS=all by default for (almost) all commands * #1058 fix linux tests warnings * #1058: try not to use imp module * #1058: get rid of imp module completely * #1058: ignore unicode warnings * #1058: ignore stderr from procsmem.py * #1058: fix resource warning from Popen * #1058: get rid of contextlib.nested (deprecated) --- HISTORY.rst | 2 ++ Makefile | 56 ++++++++++++++--------------- psutil/_compat.py | 50 +------------------------- psutil/tests/__init__.py | 60 +++++++++++++++++++++++--------- psutil/tests/test_connections.py | 11 ++++-- psutil/tests/test_linux.py | 24 +++++++++---- psutil/tests/test_misc.py | 15 ++++---- psutil/tests/test_posix.py | 3 +- psutil/tests/test_process.py | 4 ++- psutil/tests/test_unicode.py | 10 +++++- setup.py | 11 +++--- 11 files changed, 129 insertions(+), 117 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bc9096081..3e5f0f475 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -19,6 +19,7 @@ comprehensions. - 1040_: implemented full unicode support. - 1051_: disk_usage() on Python 3 is now able to accept bytes. +- 1058_: test suite now enables all warnings by default. **Bug fixes** @@ -41,6 +42,7 @@ - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. - 1048_: [Windows] users()'s host field report an invalid IP address. +- 1058_: fixed Python warnings. **Porting notes** diff --git a/Makefile b/Makefile index 953225e3e..12469d4c7 100644 --- a/Makefile +++ b/Makefile @@ -63,11 +63,11 @@ _: # Compile without installing. build: _ - $(PYTHON) setup.py build + PYTHONWARNINGS=all $(PYTHON) setup.py build @# copies compiled *.so files in ./psutil directory in order to allow @# "import psutil" when using the interactive interpreter from within @# this directory. - $(PYTHON) setup.py build_ext -i + PYTHONWARNINGS=all $(PYTHON) setup.py build_ext -i rm -rf tmp # Install this package + GIT hooks. Install is done: @@ -77,7 +77,7 @@ install: ${MAKE} build # make sure setuptools is installed (needed for 'develop' / edit mode) $(PYTHON) -c "import setuptools" - $(PYTHON) setup.py develop $(INSTALL_OPTS) + PYTHONWARNINGS=all $(PYTHON) setup.py develop $(INSTALL_OPTS) rm -rf tmp # Uninstall this package via pip. @@ -86,7 +86,7 @@ uninstall: # Install PIP (only if necessary). install-pip: - $(PYTHON) -c \ + PYTHONWARNINGS=all $(PYTHON) -c \ "import sys, ssl, os, pkgutil, tempfile, atexit; \ sys.exit(0) if pkgutil.find_loader('pip') else None; \ pyexc = 'from urllib.request import urlopen' if sys.version_info[0] == 3 else 'from urllib2 import urlopen'; \ @@ -118,65 +118,65 @@ setup-dev-env: # Run all tests. test: ${MAKE} install - $(PYTHON) $(TSCRIPT) + PYTHONWARNINGS=all $(PYTHON) $(TSCRIPT) # Run process-related API tests. test-process: ${MAKE} install - $(PYTHON) -m unittest -v psutil.tests.test_process + PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_process # Run system-related API tests. test-system: ${MAKE} install - $(PYTHON) -m unittest -v psutil.tests.test_system + PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_system # Run miscellaneous tests. test-misc: ${MAKE} install - $(PYTHON) psutil/tests/test_misc.py + PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_misc.py # Test APIs dealing with strings. test-unicode: ${MAKE} install - $(PYTHON) psutil/tests/test_unicode.py + PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_unicode.py # APIs sanity tests. test-contracts: ${MAKE} install - $(PYTHON) psutil/tests/test_contracts.py + PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_contracts.py # Test net_connections() and Process.connections(). test-connections: ${MAKE} install - $(PYTHON) psutil/tests/test_connections.py + PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_connections.py # POSIX specific tests. test-posix: ${MAKE} install - $(PYTHON) psutil/tests/test_posix.py + PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_posix.py # Run specific platform tests only. test-platform: ${MAKE} install - $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py + PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py # Memory leak tests. test-memleaks: ${MAKE} install - $(PYTHON) psutil/tests/test_memory_leaks.py + PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_memory_leaks.py # Run a specific test by name, e.g. # make test-by-name psutil.tests.test_system.TestSystemAPIs.test_cpu_times test-by-name: ${MAKE} install - @$(PYTHON) -m unittest -v $(ARGS) + @PYTHONWARNINGS=all $(PYTHON) -m unittest -v $(ARGS) # Run test coverage. coverage: ${MAKE} install # Note: coverage options are controlled by .coveragerc file rm -rf .coverage htmlcov - $(PYTHON) -m coverage run $(TSCRIPT) + PYTHONWARNINGS=all $(PYTHON) -m coverage run $(TSCRIPT) $(PYTHON) -m coverage report @echo "writing results to htmlcov/index.html" $(PYTHON) -m coverage html @@ -197,7 +197,7 @@ flake8: @git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8 check-manifest: - $(PYTHON) -m check_manifest -v $(ARGS) + PYTHONWARNINGS=all $(PYTHON) -m check_manifest -v $(ARGS) # =================================================================== # GIT @@ -220,22 +220,22 @@ install-git-hooks: # Upload source tarball on https://pypi.python.org/pypi/psutil. upload-src: ${MAKE} clean - $(PYTHON) setup.py sdist upload + PYTHONWARNINGS=all $(PYTHON) setup.py sdist upload # Download exes/wheels hosted on appveyor. win-download-exes: - $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil + PYTHONWARNINGS=all $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil # Upload exes/wheels in dist/* directory to PYPI. win-upload-exes: - $(PYTHON) -m twine upload dist/*.exe - $(PYTHON) -m twine upload dist/*.whl + PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/*.exe + PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/*.whl # All the necessary steps before making a release. pre-release: ${MAKE} clean ${MAKE} install # to import psutil from download_exes.py - $(PYTHON) -c \ + PYTHONWARNINGS=all $(PYTHON) -c \ "from psutil import __version__ as ver; \ doc = open('docs/index.rst').read(); \ history = open('HISTORY.rst').read(); \ @@ -244,18 +244,18 @@ pre-release: assert 'XXXX' not in history; \ " ${MAKE} win-download-exes - $(PYTHON) setup.py sdist + PYTHONWARNINGS=all $(PYTHON) setup.py sdist # Create a release: creates tar.gz and exes/wheels, uploads them, # upload doc, git tag release. release: ${MAKE} pre-release - $(PYTHON) -m twine upload dist/* # upload tar.gz, exes, wheels on PYPI + PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/* # upload tar.gz, exes, wheels on PYPI ${MAKE} git-tag-release # Print announce of new release. print-announce: - @$(PYTHON) scripts/internal/print_announce.py + @PYTHONWARNINGS=all $(PYTHON) scripts/internal/print_announce.py # =================================================================== # Misc @@ -267,12 +267,12 @@ grep-todos: # run script which benchmarks oneshot() ctx manager (see #799) bench-oneshot: ${MAKE} install - $(PYTHON) scripts/internal/bench_oneshot.py + PYTHONWARNINGS=all $(PYTHON) scripts/internal/bench_oneshot.py # same as above but using perf module (supposed to be more precise) bench-oneshot-2: ${MAKE} install - $(PYTHON) scripts/internal/bench_oneshot_2.py + PYTHONWARNINGS=all $(PYTHON) scripts/internal/bench_oneshot_2.py # generate a doc.zip file and manually upload it to PYPI. doc: @@ -282,4 +282,4 @@ doc: # check whether the links mentioned in some files are valid. check-broken-links: - git ls-files | grep \\.rst$ | xargs $(PYTHON) scripts/internal/check_broken_links.py + git ls-files | grep \\.rst$ | xargs PYTHONWARNINGS=all $(PYTHON) scripts/internal/check_broken_links.py diff --git a/psutil/_compat.py b/psutil/_compat.py index a318f70fa..de91638f6 100644 --- a/psutil/_compat.py +++ b/psutil/_compat.py @@ -5,13 +5,12 @@ """Module which provides compatibility with older Python versions.""" import collections -import contextlib import functools import os import sys __all__ = ["PY3", "long", "xrange", "unicode", "basestring", "u", "b", - "callable", "lru_cache", "which", "nested"] + "callable", "lru_cache", "which"] PY3 = sys.version_info[0] == 3 @@ -248,50 +247,3 @@ def _access_check(fn, mode): if _access_check(name, mode): return name return None - - -# A backport of contextlib.nested for Python 3. -nested = getattr(contextlib, "nested", None) -if nested is None: - @contextlib.contextmanager - def nested(*managers): - """Support multiple context managers in a single with-statement. - - Code like this: - - with nested(A, B, C) as (X, Y, Z): - - - is equivalent to this: - - with A as X: - with B as Y: - with C as Z: - - """ - exits = [] - vars = [] - exc = (None, None, None) - try: - for mgr in managers: - exit = mgr.__exit__ - enter = mgr.__enter__ - vars.append(enter()) - exits.append(exit) - yield vars - except: # NOQA - exc = sys.exc_info() - finally: - while exits: - exit = exits.pop() - try: - if exit(*exc): - exc = (None, None, None) - except: # NOQA - exc = sys.exc_info() - if exc != (None, None, None): - # Don't rely on sys.exc_info() still containing - # the right information. Another exception may - # have been raised and caught by an exit method - # exc[1] already has the __traceback__ attribute populated - raise exc[1] diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index a77b5f5d7..03ad9553b 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -58,14 +58,6 @@ else: enum = None -if PY3: - import importlib - # python <=3.3 - if not hasattr(importlib, 'reload'): - import imp as importlib -else: - import imp as importlib - __all__ = [ # constants @@ -98,16 +90,13 @@ 'check_connection_ntuple', 'check_net_address', 'get_free_port', 'unix_socket_path', 'bind_socket', 'bind_unix_socket', 'tcp_socketpair', 'unix_socketpair', 'create_sockets', + # compat + 'reload_module', 'import_module_by_path', # others 'warn', 'copyload_shared_lib', 'is_namedtuple', ] -# Enable all warnings by default. -if 'PYTHONWARNINGS' not in os.environ: - warnings.simplefilter('always') - - # =================================================================== # --- constants # =================================================================== @@ -353,16 +342,19 @@ def pyrun(src, **kwds): @_cleanup_on_err -def sh(cmd): +def sh(cmd, **kwds): """run cmd in a subprocess and return its output. raises RuntimeError on error. """ shell = True if isinstance(cmd, (str, unicode)) else False # Prevents subprocess to open error dialogs in case of error. flags = 0x8000000 if WINDOWS and shell else 0 - p = subprocess.Popen(cmd, shell=shell, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, universal_newlines=True, - creationflags=flags) + kwds.setdefault("shell", shell) + kwds.setdefault("stdout", subprocess.PIPE) + kwds.setdefault("stderr", subprocess.PIPE) + kwds.setdefault("universal_newlines", True) + kwds.setdefault("creationflags", flags) + p = subprocess.Popen(cmd, **kwds) stdout, stderr = p.communicate() if p.returncode != 0: raise RuntimeError(stderr) @@ -975,6 +967,40 @@ def check_connection_ntuple(conn): assert conn.status in valids, conn +# =================================================================== +# --- compatibility +# =================================================================== + + +def reload_module(module): + """Backport of importlib.reload of Python 3.3+.""" + try: + import importlib + if not hasattr(importlib, 'reload'): # python <=3.3 + raise ImportError + except ImportError: + import imp + return imp.reload(module) + else: + return importlib.reload(module) + + +def import_module_by_path(path): + name = os.path.splitext(os.path.basename(path))[0] + if sys.version_info[0] == 2: + import imp + return imp.load_source(name, path) + elif sys.version_info[:2] <= (3, 4): + from importlib.machinery import SourceFileLoader + return SourceFileLoader(name, path).load_module() + else: + import importlib.util + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + # =================================================================== # --- others # =================================================================== diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index d0c5445a6..906706a56 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -26,7 +26,6 @@ from psutil import WINDOWS from psutil._common import pconn from psutil._common import supports_ipv6 -from psutil._compat import nested from psutil._compat import PY3 from psutil.tests import AF_UNIX from psutil.tests import bind_socket @@ -207,7 +206,7 @@ def test_tcp(self): addr = ("127.0.0.1", get_free_port()) assert not thisproc.connections(kind='tcp4') server, client = tcp_socketpair(AF_INET, addr=addr) - with nested(closing(server), closing(client)): + try: cons = thisproc.connections(kind='tcp4') self.assertEqual(len(cons), 2) self.assertEqual(cons[0].status, psutil.CONN_ESTABLISHED) @@ -218,12 +217,15 @@ def test_tcp(self): # cons = thisproc.connections(kind='all') # self.assertEqual(len(cons), 1) # self.assertEqual(cons[0].status, psutil.CONN_CLOSE_WAIT) + finally: + server.close() + client.close() @unittest.skipIf(not POSIX, 'POSIX only') def test_unix(self): with unix_socket_path() as name: server, client = unix_socketpair(name) - with nested(closing(server), closing(client)): + try: cons = thisproc.connections(kind='unix') assert not (cons[0].laddr and cons[0].raddr) assert not (cons[1].laddr and cons[1].raddr) @@ -248,6 +250,9 @@ def test_unix(self): # of both peers are set. self.assertEqual(cons[0].laddr or cons[1].laddr, name) self.assertEqual(cons[0].raddr or cons[1].raddr, name) + finally: + server.close() + client.close() @skip_on_access_denied(only_if=OSX) def test_combos(self): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 4adcb3761..2054da8b2 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -30,12 +30,12 @@ from psutil.tests import call_until from psutil.tests import HAS_BATTERY from psutil.tests import HAS_RLIMIT -from psutil.tests import importlib from psutil.tests import MEMORY_TOLERANCE from psutil.tests import mock from psutil.tests import PYPY from psutil.tests import pyrun from psutil.tests import reap_children +from psutil.tests import reload_module from psutil.tests import retry_before_failing from psutil.tests import run_test_module_by_name from psutil.tests import safe_rmpath @@ -324,9 +324,13 @@ def open_mock(name, *args, **kwargs): orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, create=True, side_effect=open_mock) as m: - ret = psutil.virtual_memory() + with warnings.catch_warnings(record=True) as ws: + ret = psutil.virtual_memory() assert m.called self.assertEqual(ret.available, 6574984 * 1024) + w = ws[0] + self.assertIn( + "inactive memory stats couldn't be determined", str(w.message)) def test_avail_old_missing_fields(self): # Remove Active(file), Inactive(file) and SReclaimable @@ -351,9 +355,13 @@ def open_mock(name, *args, **kwargs): orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, create=True, side_effect=open_mock) as m: - ret = psutil.virtual_memory() + with warnings.catch_warnings(record=True) as ws: + ret = psutil.virtual_memory() assert m.called self.assertEqual(ret.available, 2057400 * 1024 + 4818144 * 1024) + w = ws[0] + self.assertIn( + "inactive memory stats couldn't be determined", str(w.message)) def test_avail_old_missing_zoneinfo(self): # Remove /proc/zoneinfo file. Make sure fallback is used @@ -382,9 +390,13 @@ def open_mock(name, *args, **kwargs): orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, create=True, side_effect=open_mock) as m: - ret = psutil.virtual_memory() + with warnings.catch_warnings(record=True) as ws: + ret = psutil.virtual_memory() assert m.called self.assertEqual(ret.available, 2057400 * 1024 + 4818144 * 1024) + w = ws[0] + self.assertIn( + "inactive memory stats couldn't be determined", str(w.message)) # ===================================================================== @@ -986,7 +998,7 @@ def open_mock(name, *args, **kwargs): patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock): - importlib.reload(psutil) + reload_module(psutil) assert tb.called self.assertRaises(IOError, psutil.cpu_times) @@ -1025,7 +1037,7 @@ def open_mock(name, *args, **kwargs): sum(map(sum, psutil.cpu_times_percent(percpu=True))), 0) finally: shutil.rmtree(my_procfs) - importlib.reload(psutil) + reload_module(psutil) self.assertEqual(psutil.PROCFS_PATH, '/proc') diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 28c40ed79..0e1ce3509 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -13,7 +13,6 @@ import collections import contextlib import errno -import imp import json import os import pickle @@ -36,6 +35,7 @@ from psutil.tests import chdir from psutil.tests import create_proc_children_pair from psutil.tests import create_sockets +from psutil.tests import DEVNULL from psutil.tests import get_free_port from psutil.tests import get_test_subprocess from psutil.tests import HAS_BATTERY @@ -44,10 +44,11 @@ from psutil.tests import HAS_SENSORS_BATTERY from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES -from psutil.tests import importlib +from psutil.tests import import_module_by_path from psutil.tests import is_namedtuple from psutil.tests import mock from psutil.tests import reap_children +from psutil.tests import reload_module from psutil.tests import retry from psutil.tests import ROOT_DIR from psutil.tests import run_test_module_by_name @@ -352,7 +353,7 @@ def check(ret): def test_setup_script(self): setup_py = os.path.join(ROOT_DIR, 'setup.py') - module = imp.load_source('setup', setup_py) + module = import_module_by_path(setup_py) self.assertRaises(SystemExit, module.setup) self.assertEqual(module.get_version(), psutil.__version__) @@ -378,7 +379,7 @@ def test_sanity_version_check(self): with mock.patch( "psutil._psplatform.cext.version", return_value="0.0.0"): with self.assertRaises(ImportError) as cm: - importlib.reload(psutil) + reload_module(psutil) self.assertIn("version conflict", str(cm.exception).lower()) @@ -631,12 +632,12 @@ class TestScripts(unittest.TestCase): """Tests for scripts in the "scripts" directory.""" @staticmethod - def assert_stdout(exe, args=None): + def assert_stdout(exe, args=None, **kwds): exe = '"%s"' % os.path.join(SCRIPTS_DIR, exe) if args: exe = exe + ' ' + args try: - out = sh(sys.executable + ' ' + exe).strip() + out = sh(sys.executable + ' ' + exe, **kwds).strip() except RuntimeError as err: if 'AccessDenied' in str(err): return str(err) @@ -712,7 +713,7 @@ def test_pmap(self): @unittest.skipIf(not HAS_MEMORY_FULL_INFO, "not supported") def test_procsmem(self): - self.assert_stdout('procsmem.py') + self.assert_stdout('procsmem.py', stderr=DEVNULL) def test_killall(self): self.assert_syntax('killall.py') diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 3274c02ca..d3d2d5b19 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -224,7 +224,8 @@ def call(p, attr): p = psutil.Process(os.getpid()) failures = [] ignored_names = ['terminate', 'kill', 'suspend', 'resume', 'nice', - 'send_signal', 'wait', 'children', 'as_dict'] + 'send_signal', 'wait', 'children', 'as_dict', + 'memory_info_ex'] if LINUX and get_kernel_version() < (2, 6, 36): ignored_names.append('rlimit') if LINUX and get_kernel_version() < (2, 6, 23): diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 598180c9c..86ad136f7 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1397,7 +1397,9 @@ def test_Popen(self): self.assertTrue(dir(proc)) self.assertRaises(AttributeError, getattr, proc, 'foo') finally: - proc.kill() + proc.terminate() + proc.stdout.close() + proc.stderr.close() proc.wait() def test_Popen_ctx_manager(self): diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index ae3c012fd..159ccdff8 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -53,6 +53,7 @@ """ import os +import warnings from contextlib import closing from psutil import BSD @@ -61,6 +62,7 @@ from psutil import POSIX from psutil import WINDOWS from psutil._compat import PY3 +from psutil._compat import u from psutil.tests import ASCII_FS from psutil.tests import bind_unix_socket from psutil.tests import chdir @@ -264,7 +266,13 @@ class TestFSAPIs(_BaseFSAPIsTests, unittest.TestCase): def expect_exact_path_match(cls): # Do not expect psutil to correctly handle unicode paths on # Python 2 if os.listdir() is not able either. - return PY3 or cls.funky_name in os.listdir('.') + if PY3: + return True + else: + here = '.' if isinstance(cls.funky_name, str) else u('.') + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return cls.funky_name in os.listdir(here) @unittest.skipIf(OSX and TRAVIS, "unreliable on TRAVIS") # TODO diff --git a/setup.py b/setup.py index c4f3bcbcf..05c212ab7 100755 --- a/setup.py +++ b/setup.py @@ -16,10 +16,13 @@ import sys import tempfile import warnings -try: - from setuptools import setup, Extension -except ImportError: - from distutils.core import setup, Extension + +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + from setuptools import setup, Extension + except ImportError: + from distutils.core import setup, Extension HERE = os.path.abspath(os.path.dirname(__file__)) From 80f43ab6595c63567ed55e1b56fda640f83a122f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 10 May 2017 22:19:10 +0200 Subject: [PATCH 622/922] add a script to print releases timeline in RST format; also show a diff between versions in the timeline section of the doc --- Makefile | 4 + docs/index.rst | 281 +++++++++++++++++++++++++++++++++++++++---------- 2 files changed, 228 insertions(+), 57 deletions(-) diff --git a/Makefile b/Makefile index 12469d4c7..319e77ca5 100644 --- a/Makefile +++ b/Makefile @@ -257,6 +257,10 @@ release: print-announce: @PYTHONWARNINGS=all $(PYTHON) scripts/internal/print_announce.py +# Print releases' timeline. +print-timeline: + @PYTHONWARNINGS=all $(PYTHON) scripts/internal/print_timeline.py + # =================================================================== # Misc # =================================================================== diff --git a/docs/index.rst b/docs/index.rst index ff38af981..2bbcde7b0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2552,60 +2552,227 @@ take a look at the Timeline ======== -- 2017-04-17: `5.2.2 `__ - `what's new `__ -- 2017-03-24: `5.2.1 `__ - `what's new `__ -- 2017-03-05: `5.2.0 `__ - `what's new `__ -- 2017-02-07: `5.1.3 `__ - `what's new `__ -- 2017-02-03: `5.1.2 `__ - `what's new `__ -- 2017-02-03: `5.1.1 `__ - `what's new `__ -- 2017-02-01: `5.1.0 `__ - `what's new `__ -- 2016-12-21: `5.0.1 `__ - `what's new `__ -- 2016-11-06: `5.0.0 `__ - `what's new `__ -- 2016-10-26: `4.4.2 `__ - `what's new `__ -- 2016-10-25: `4.4.1 `__ - `what's new `__ -- 2016-10-23: `4.4.0 `__ - `what's new `__ -- 2016-09-01: `4.3.1 `__ - `what's new `__ -- 2016-06-18: `4.3.0 `__ - `what's new `__ -- 2016-05-15: `4.2.0 `__ - `what's new `__ -- 2016-03-12: `4.1.0 `__ - `what's new `__ -- 2016-02-17: `4.0.0 `__ - `what's new `__ -- 2016-01-20: `3.4.2 `__ - `what's new `__ -- 2016-01-15: `3.4.1 `__ - `what's new `__ -- 2015-11-25: `3.3.0 `__ - `what's new `__ -- 2015-10-04: `3.2.2 `__ - `what's new `__ -- 2015-09-03: `3.2.1 `__ - `what's new `__ -- 2015-09-02: `3.2.0 `__ - `what's new `__ -- 2015-07-15: `3.1.1 `__ - `what's new `__ -- 2015-07-15: `3.1.0 `__ - `what's new `__ -- 2015-06-18: `3.0.1 `__ - `what's new `__ -- 2015-06-13: `3.0.0 `__ - `what's new `__ -- 2015-02-02: `2.2.1 `__ - `what's new `__ -- 2015-01-06: `2.2.0 `__ - `what's new `__ -- 2014-09-26: `2.1.3 `__ - `what's new `__ -- 2014-09-21: `2.1.2 `__ - `what's new `__ -- 2014-04-30: `2.1.1 `__ - `what's new `__ -- 2014-04-08: `2.1.0 `__ - `what's new `__ -- 2014-03-10: `2.0.0 `__ - `what's new `__ -- 2013-11-25: `1.2.1 `__ - `what's new `__ -- 2013-11-20: `1.2.0 `__ - `what's new `__ -- 2013-11-07: `1.1.3 `__ - `what's new `__ -- 2013-10-22: `1.1.2 `__ - `what's new `__ -- 2013-10-08: `1.1.1 `__ - `what's new `__ -- 2013-09-28: `1.1.0 `__ - `what's new `__ -- 2013-07-12: `1.0.1 `__ - `what's new `__ -- 2013-07-10: `1.0.0 `__ - `what's new `__ -- 2013-05-03: `0.7.1 `__ - `what's new `__ -- 2013-04-12: `0.7.0 `__ - `what's new `__ -- 2012-08-16: `0.6.1 `__ - `what's new `__ -- 2012-08-13: `0.6.0 `__ - `what's new `__ -- 2012-06-29: `0.5.1 `__ - `what's new `__ -- 2012-06-27: `0.5.0 `__ - `what's new `__ -- 2011-12-14: `0.4.1 `__ - `what's new `__ -- 2011-10-29: `0.4.0 `__ - `what's new `__ -- 2011-07-08: `0.3.0 `__ - `what's new `__ -- 2011-03-20: `0.2.1 `__ - `what's new `__ -- 2010-11-13: `0.2.0 `__ - `what's new `__ -- 2010-03-02: `0.1.3 `__ - `what's new `__ -- 2009-05-06: `0.1.2 `__ - `what's new `__ -- 2009-03-06: `0.1.1 `__ - `what's new `__ -- 2009-01-27: `0.1.0 `__ - `what's new `__ +- 2017-04-10: + `5.2.2 `__ - + `what's new `__ - + `diff `__ +- 2017-03-24: + `5.2.1 `__ - + `what's new `__ - + `diff `__ +- 2017-03-05: + `5.2.0 `__ - + `what's new `__ - + `diff `__ +- 2017-02-07: + `5.1.3 `__ - + `what's new `__ - + `diff `__ +- 2017-02-03: + `5.1.2 `__ - + `what's new `__ - + `diff `__ +- 2017-02-03: + `5.1.1 `__ - + `what's new `__ - + `diff `__ +- 2017-02-01: + `5.1.0 `__ - + `what's new `__ - + `diff `__ +- 2016-12-21: + `5.0.1 `__ - + `what's new `__ - + `diff `__ +- 2016-11-06: + `5.0.0 `__ - + `what's new `__ - + `diff `__ +- 2016-10-05: + `4.4.2 `__ - + `what's new `__ - + `diff `__ +- 2016-10-25: + `4.4.1 `__ - + `what's new `__ - + `diff `__ +- 2016-10-23: + `4.4.0 `__ - + `what's new `__ - + `diff `__ +- 2016-09-01: + `4.3.1 `__ - + `what's new `__ - + `diff `__ +- 2016-06-18: + `4.3.0 `__ - + `what's new `__ - + `diff `__ +- 2016-05-14: + `4.2.0 `__ - + `what's new `__ - + `diff `__ +- 2016-03-12: + `4.1.0 `__ - + `what's new `__ - + `diff `__ +- 2016-02-17: + `4.0.0 `__ - + `what's new `__ - + `diff `__ +- 2016-01-20: + `3.4.2 `__ - + `what's new `__ - + `diff `__ +- 2016-01-15: + `3.4.1 `__ - + `what's new `__ - + `diff `__ +- 2015-11-25: + `3.3.0 `__ - + `what's new `__ - + `diff `__ +- 2015-10-04: + `3.2.2 `__ - + `what's new `__ - + `diff `__ +- 2015-09-03: + `3.2.1 `__ - + `what's new `__ - + `diff `__ +- 2015-09-02: + `3.2.0 `__ - + `what's new `__ - + `diff `__ +- 2015-07-15: + `3.1.1 `__ - + `what's new `__ - + `diff `__ +- 2015-07-15: + `3.1.0 `__ - + `what's new `__ - + `diff `__ +- 2015-06-18: + `3.0.1 `__ - + `what's new `__ - + `diff `__ +- 2015-06-13: + `3.0.0 `__ - + `what's new `__ - + `diff `__ +- 2015-02-02: + `2.2.1 `__ - + `what's new `__ - + `diff `__ +- 2015-01-06: + `2.2.0 `__ - + `what's new `__ - + `diff `__ +- 2014-09-26: + `2.1.3 `__ - + `what's new `__ - + `diff `__ +- 2014-09-21: + `2.1.2 `__ - + `what's new `__ - + `diff `__ +- 2014-04-30: + `2.1.1 `__ - + `what's new `__ - + `diff `__ +- 2014-04-08: + `2.1.0 `__ - + `what's new `__ - + `diff `__ +- 2014-03-10: + `2.0.0 `__ - + `what's new `__ - + `diff `__ +- 2013-11-25: + `1.2.1 `__ - + `what's new `__ - + `diff `__ +- 2013-11-20: + `1.2.0 `__ - + `what's new `__ - + `diff `__ +- 2013-10-22: + `1.1.2 `__ - + `what's new `__ - + `diff `__ +- 2013-10-08: + `1.1.1 `__ - + `what's new `__ - + `diff `__ +- 2013-09-28: + `1.1.0 `__ - + `what's new `__ - + `diff `__ +- 2013-07-12: + `1.0.1 `__ - + `what's new `__ - + `diff `__ +- 2013-07-10: + `1.0.0 `__ - + `what's new `__ - + `diff `__ +- 2013-05-03: + `0.7.1 `__ - + `what's new `__ - + `diff `__ +- 2013-04-12: + `0.7.0 `__ - + `what's new `__ - + `diff `__ +- 2012-08-16: + `0.6.1 `__ - + `what's new `__ - + `diff `__ +- 2012-08-13: + `0.6.0 `__ - + `what's new `__ - + `diff `__ +- 2012-06-29: + `0.5.1 `__ - + `what's new `__ - + `diff `__ +- 2012-06-27: + `0.5.0 `__ - + `what's new `__ - + `diff `__ +- 2011-12-14: + `0.4.1 `__ - + `what's new `__ - + `diff `__ +- 2011-10-29: + `0.4.0 `__ - + `what's new `__ - + `diff `__ +- 2011-07-08: + `0.3.0 `__ - + `what's new `__ - + `diff `__ +- 2011-03-20: + `0.2.1 `__ - + `what's new `__ - + `diff `__ +- 2010-11-13: + `0.2.0 `__ - + `what's new `__ - + `diff `__ +- 2010-03-02: + `0.1.3 `__ - + `what's new `__ - + `diff `__ +- 2009-05-06: + `0.1.2 `__ - + `what's new `__ - + `diff `__ +- 2009-03-06: + `0.1.1 `__ - + `what's new `__ - + `diff `__ +- 2009-01-27: + `0.1.0 `__ - + `what's new `__ - + `diff `__ From 959bddc9da20c7d35e154c9eabc746b202dc0ec7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 10 May 2017 22:19:26 +0200 Subject: [PATCH 623/922] add a script to print releases timeline in RST format; also show a diff between versions in the timeline section of the doc --- scripts/internal/print_timeline.py | 53 ++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 scripts/internal/print_timeline.py diff --git a/scripts/internal/print_timeline.py b/scripts/internal/print_timeline.py new file mode 100644 index 000000000..ffcb8fe85 --- /dev/null +++ b/scripts/internal/print_timeline.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Prints releases' timeline in RST format. +""" + +import subprocess + + +entry = """\ +- {date}: + `{ver} `__ - + `what's new `__ - + `diff `__""" # NOQA + + +def sh(cmd): + return subprocess.check_output( + cmd, shell=True, universal_newlines=True).strip() + + +def get_tag_date(tag): + out = sh(r"git log -1 --format=%ai {}".format(tag)) + return out.split(' ')[0] + + +def main(): + releases = [] + out = sh("git tags") + for line in out.split('\n'): + tag = line.split(' ')[0] + ver = tag.replace('release-', '') + nodotver = ver.replace('.', '') + date = get_tag_date(tag) + releases.append((tag, ver, nodotver, date)) + releases.sort(reverse=True) + + for i, rel in enumerate(releases): + tag, ver, nodotver, date = rel + try: + prevtag = releases[i + 1][0] + except IndexError: + # get first commit + prevtag = sh("git rev-list --max-parents=0 HEAD") + print(entry.format(**locals())) + + +if __name__ == '__main__': + main() From 2d32a26a257c7ce17806f0ec150bcbba5f39f914 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 12:17:53 +0200 Subject: [PATCH 624/922] fix 1062: avoid TypeError on disk|net_io_counters() if no disks or NICs are installed on the system --- HISTORY.rst | 2 ++ psutil/__init__.py | 4 ++++ psutil/tests/test_system.py | 18 ++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 3e5f0f475..ea4d1e5b8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -43,6 +43,8 @@ - 1047_: [Windows] Process username(): memory leak in case exception is thrown. - 1048_: [Windows] users()'s host field report an invalid IP address. - 1058_: fixed Python warnings. +- 1062_: disk_io_counters() and net_io_counters() raise TypeError if no disks + or NICs are installed on the system. **Porting notes** diff --git a/psutil/__init__.py b/psutil/__init__.py index 509fcf823..88f7e158e 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2083,6 +2083,8 @@ def disk_io_counters(perdisk=False, nowrap=True): executed first otherwise this function won't find any disk. """ rawdict = _psplatform.disk_io_counters() + if not rawdict: + return {} if perdisk else None if nowrap: rawdict = _wrap_numbers(rawdict, 'psutil.disk_io_counters') nt = getattr(_psplatform, "sdiskio", _common.sdiskio) @@ -2131,6 +2133,8 @@ def net_io_counters(pernic=False, nowrap=True): cache. """ rawdict = _psplatform.net_io_counters() + if not rawdict: + return {} if pernic else None if nowrap: rawdict = _wrap_numbers(rawdict, 'psutil.net_io_counters') if pernic: diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 32f76f57c..fed7a222a 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -553,6 +553,15 @@ def check_ntuple(nt): self.assertIsInstance(key, str) check_ntuple(ret[key]) + def test_net_io_counters_no_nics(self): + # Emulate a case where no NICs are installed, see: + # https://github.com/giampaolo/psutil/issues/1062 + with mock.patch('psutil._psplatform.net_io_counters', + return_value={}) as m: + self.assertIsNone(psutil.net_io_counters(pernic=False)) + self.assertEqual(psutil.net_io_counters(pernic=True), {}) + assert m.called + def test_net_if_addrs(self): nics = psutil.net_if_addrs() assert nics, nics @@ -682,6 +691,15 @@ def check_ntuple(nt): key = key[:-1] self.assertNotIn(key, ret.keys()) + def test_disk_io_counters_no_disks(self): + # Emulate a case where no disks are installed, see: + # https://github.com/giampaolo/psutil/issues/1062 + with mock.patch('psutil._psplatform.disk_io_counters', + return_value={}) as m: + self.assertIsNone(psutil.disk_io_counters(perdisk=False)) + self.assertEqual(psutil.disk_io_counters(perdisk=True), {}) + assert m.called + # can't find users on APPVEYOR or TRAVIS @unittest.skipIf(APPVEYOR or TRAVIS and not psutil.users(), "unreliable on APPVEYOR or TRAVIS") From 8db7365fec353ed4465bc9b3a03b7a6ef0ea5991 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 12:18:18 +0200 Subject: [PATCH 625/922] try to avoid failures on win --- psutil/tests/test_unicode.py | 16 ++++++++++++++++ psutil/tests/test_windows.py | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 159ccdff8..4c2181d49 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -52,7 +52,9 @@ - https://pythonhosted.org/psutil/#unicode """ +import errno import os +import traceback import warnings from contextlib import closing @@ -127,6 +129,20 @@ def tearDown(self): reap_children() safe_rmpath(self.funky_name) + def safe_rmpath(self, name): + if POSIX: + safe_rmpath(name) + else: + # https://ci.appveyor.com/project/giampaolo/psutil/build/ + # 1225/job/1yec67sr6e9rl217 + try: + safe_rmpath(name) + except OSError as err: + if err.errno in (errno.EACCES, errno.EPERM): + traceback.print_exc() + else: + raise + def expect_exact_path_match(self): raise NotImplementedError("must be implemented in subclass") diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index e01457a49..abb208b3d 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -187,8 +187,8 @@ def test_boot_time(self): wmi_btime_str, "%Y%m%d%H%M%S") psutil_dt = datetime.datetime.fromtimestamp(psutil.boot_time()) diff = abs((wmi_btime_dt - psutil_dt).total_seconds()) - # Wmic time is 2 secs lower for some reason; that's OK. - self.assertLessEqual(diff, 2) + # Wmic time is 2-3 secs lower for some reason; that's OK. + self.assertLessEqual(diff, 3) def test_boot_time_fluctuation(self): # https://github.com/giampaolo/psutil/issues/1007 From b258d829066c21869a8993535798122ab57812e6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 12:29:14 +0200 Subject: [PATCH 626/922] #1058: enable warnings on make.bat, appveyor and travis --- .ci/travis/run.sh | 8 ++++---- appveyor.yml | 2 +- scripts/internal/winmake.py | 30 +++++++++++++++--------------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index a0cdd1b67..05bc60c25 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -19,16 +19,16 @@ python setup.py develop # run tests (with coverage) if [[ $PYVER == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then - coverage run psutil/tests/__main__.py + python -Wa -m coverage run psutil/tests/__main__.py else - python psutil/tests/__main__.py + python -Wa psutil/tests/__main__.py fi if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then # run mem leaks test - python psutil/tests/test_memory_leaks.py + python -Wa psutil/tests/test_memory_leaks.py # run linter (on Linux only) if [[ "$(uname -s)" != 'Darwin' ]]; then - python -m flake8 + python -Wa -m flake8 fi fi diff --git a/appveyor.yml b/appveyor.yml index af7a63192..78169616d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -90,7 +90,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "%WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" + - "%WITH_COMPILER% %PYTHON%/python -Wa psutil/tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index aaeaeed56..d0c2c0a13 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -320,14 +320,14 @@ def flake8(): py_files = py_files.decode() py_files = [x for x in py_files.split() if x.endswith('.py')] py_files = ' '.join(py_files) - sh("%s -m flake8 %s" % (PYTHON, py_files), nolog=True) + sh("%s -Wa -m flake8 %s" % (PYTHON, py_files), nolog=True) @cmd def test(): """Run tests""" install() - sh("%s %s" % (PYTHON, TSCRIPT)) + sh("%s -Wa %s" % (PYTHON, TSCRIPT)) @cmd @@ -335,7 +335,7 @@ def coverage(): """Run coverage tests.""" # Note: coverage options are controlled by .coveragerc file install() - sh("%s -m coverage run %s" % (PYTHON, TSCRIPT)) + sh("%s -Wa -m coverage run %s" % (PYTHON, TSCRIPT)) sh("%s -m coverage report" % PYTHON) sh("%s -m coverage html" % PYTHON) sh("%s -m webbrowser -t htmlcov/index.html" % PYTHON) @@ -345,49 +345,49 @@ def coverage(): def test_process(): """Run process tests""" install() - sh("%s -m unittest -v psutil.tests.test_process" % PYTHON) + sh("%s -Wa -m unittest -v psutil.tests.test_process" % PYTHON) @cmd def test_system(): """Run system tests""" install() - sh("%s -m unittest -v psutil.tests.test_system" % PYTHON) + sh("%s -Wa -m unittest -v psutil.tests.test_system" % PYTHON) @cmd def test_platform(): """Run windows only tests""" install() - sh("%s -m unittest -v psutil.tests.test_windows" % PYTHON) + sh("%s -Wa -m unittest -v psutil.tests.test_windows" % PYTHON) @cmd def test_misc(): """Run misc tests""" install() - sh("%s -m unittest -v psutil.tests.test_misc" % PYTHON) + sh("%s -Wa -m unittest -v psutil.tests.test_misc" % PYTHON) @cmd def test_unicode(): """Run unicode tests""" install() - sh("%s -m unittest -v psutil.tests.test_unicode" % PYTHON) + sh("%s -Wa -m unittest -v psutil.tests.test_unicode" % PYTHON) @cmd def test_connections(): """Run connections tests""" install() - sh("%s -m unittest -v psutil.tests.test_connections" % PYTHON) + sh("%s -Wa -m unittest -v psutil.tests.test_connections" % PYTHON) @cmd def test_contracts(): """Run contracts tests""" install() - sh("%s -m unittest -v psutil.tests.test_contracts" % PYTHON) + sh("%s -Wa -m unittest -v psutil.tests.test_contracts" % PYTHON) @cmd @@ -399,7 +399,7 @@ def test_by_name(): except IndexError: sys.exit('second arg missing') install() - sh("%s -m unittest -v %s" % (PYTHON, name)) + sh("%s -Wa -m unittest -v %s" % (PYTHON, name)) @cmd @@ -411,14 +411,14 @@ def test_script(): except IndexError: sys.exit('second arg missing') install() - sh("%s %s" % (PYTHON, name)) + sh("%s -Wa %s" % (PYTHON, name)) @cmd def test_memleaks(): """Run memory leaks tests""" install() - sh("%s psutil\\tests\\test_memory_leaks.py" % PYTHON) + sh("%s -Wa psutil\\tests\\test_memory_leaks.py" % PYTHON) @cmd @@ -430,13 +430,13 @@ def install_git_hooks(): @cmd def bench_oneshot(): install() - sh("%s scripts\\internal\\bench_oneshot.py" % PYTHON) + sh("%s -Wa scripts\\internal\\bench_oneshot.py" % PYTHON) @cmd def bench_oneshot_2(): install() - sh("%s scripts\\internal\\bench_oneshot_2.py" % PYTHON) + sh("%s -Wa scripts\\internal\\bench_oneshot_2.py" % PYTHON) def set_python(s): From 57bc1c55e891e09b5807c89586f629bbac275b26 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 13:15:50 +0200 Subject: [PATCH 627/922] fix #1060: dynamically generate MANIFEST.in --- HISTORY.rst | 2 + MANIFEST.in | 130 +++++++++++++++++++++++--- Makefile | 14 ++- scripts/internal/generate_manifest.py | 31 ++++++ 4 files changed, 159 insertions(+), 18 deletions(-) create mode 100755 scripts/internal/generate_manifest.py diff --git a/HISTORY.rst b/HISTORY.rst index ea4d1e5b8..7e292b3b3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -20,6 +20,8 @@ - 1040_: implemented full unicode support. - 1051_: disk_usage() on Python 3 is now able to accept bytes. - 1058_: test suite now enables all warnings by default. +- 1060_: source distribution is dynamically generated so that it only includes + relevant files. **Bug fixes** diff --git a/MANIFEST.in b/MANIFEST.in index b0c156457..6c4b2b93b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,18 +1,120 @@ -include *.bat -include *.rst include .coveragerc -include CREDITS* +include .git-pre-commit +include .gitignore +include CREDITS +include DEVGUIDE.rst +include HISTORY.rst include IDEAS -include INSTALL* -include LICENSE* -include HISTORY* +include INSTALL.rst +include LICENSE +include MANIFEST.in include Makefile +include README.rst +include docs/Makefile +include docs/README +include docs/_static/copybutton.js +include docs/_static/favicon.ico +include docs/_static/sidebar.js +include docs/_template/globaltoc.html +include docs/_template/indexcontent.html +include docs/_template/indexsidebar.html +include docs/_template/page.html +include docs/_themes/pydoctheme/static/pydoctheme.css +include docs/_themes/pydoctheme/theme.conf +include docs/conf.py +include docs/index.rst +include docs/make.bat +include make.bat +include psutil/__init__.py +include psutil/_common.py +include psutil/_compat.py +include psutil/_psbsd.py +include psutil/_pslinux.py +include psutil/_psosx.py +include psutil/_psposix.py +include psutil/_pssunos.py +include psutil/_psutil_bsd.c +include psutil/_psutil_common.c +include psutil/_psutil_common.h +include psutil/_psutil_linux.c +include psutil/_psutil_osx.c +include psutil/_psutil_posix.c +include psutil/_psutil_posix.h +include psutil/_psutil_sunos.c +include psutil/_psutil_windows.c +include psutil/_pswindows.py +include psutil/arch/bsd/freebsd.c +include psutil/arch/bsd/freebsd.h +include psutil/arch/bsd/freebsd_socks.c +include psutil/arch/bsd/freebsd_socks.h +include psutil/arch/bsd/netbsd.c +include psutil/arch/bsd/netbsd.h +include psutil/arch/bsd/netbsd_socks.c +include psutil/arch/bsd/netbsd_socks.h +include psutil/arch/bsd/openbsd.c +include psutil/arch/bsd/openbsd.h +include psutil/arch/osx/process_info.c +include psutil/arch/osx/process_info.h +include psutil/arch/solaris/v10/ifaddrs.c +include psutil/arch/solaris/v10/ifaddrs.h +include psutil/arch/windows/glpi.h +include psutil/arch/windows/inet_ntop.c +include psutil/arch/windows/inet_ntop.h +include psutil/arch/windows/ntextapi.h +include psutil/arch/windows/process_handles.c +include psutil/arch/windows/process_handles.h +include psutil/arch/windows/process_info.c +include psutil/arch/windows/process_info.h +include psutil/arch/windows/security.c +include psutil/arch/windows/security.h +include psutil/arch/windows/services.c +include psutil/arch/windows/services.h +include psutil/tests/README.rst +include psutil/tests/__init__.py +include psutil/tests/__main__.py +include psutil/tests/test_bsd.py +include psutil/tests/test_connections.py +include psutil/tests/test_contracts.py +include psutil/tests/test_linux.py +include psutil/tests/test_memory_leaks.py +include psutil/tests/test_misc.py +include psutil/tests/test_osx.py +include psutil/tests/test_posix.py +include psutil/tests/test_process.py +include psutil/tests/test_sunos.py +include psutil/tests/test_system.py +include psutil/tests/test_unicode.py +include psutil/tests/test_windows.py +include scripts/battery.py +include scripts/cpu_distribution.py +include scripts/disk_usage.py +include scripts/fans.py +include scripts/free.py +include scripts/ifconfig.py +include scripts/internal/README +include scripts/internal/bench_oneshot.py +include scripts/internal/bench_oneshot_2.py +include scripts/internal/check_broken_links.py +include scripts/internal/download_exes.py +include scripts/internal/generate_manifest.py +include scripts/internal/print_announce.py +include scripts/internal/print_timeline.py +include scripts/internal/winmake.py +include scripts/iotop.py +include scripts/killall.py +include scripts/meminfo.py +include scripts/netstat.py +include scripts/nettop.py +include scripts/pidof.py +include scripts/pmap.py +include scripts/procinfo.py +include scripts/procsmem.py +include scripts/ps.py +include scripts/pstree.py +include scripts/sensors.py +include scripts/temperatures.py +include scripts/top.py +include scripts/who.py +include scripts/winservices.py +include setup.py include tox.ini - -recursive-include psutil *.py *.c *.h *.rst -recursive-include scripts *.py -recursive-include README* - -recursive-include docs *.conf *.rst *.js *.html *.css *.py *.bat *Makefile* README* -recursive-exclude docs/_build * -recursive-exclude .ci * diff --git a/Makefile b/Makefile index 319e77ca5..3e67cdf6e 100644 --- a/Makefile +++ b/Makefile @@ -196,9 +196,6 @@ pyflakes: flake8: @git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8 -check-manifest: - PYTHONWARNINGS=all $(PYTHON) -m check_manifest -v $(ARGS) - # =================================================================== # GIT # =================================================================== @@ -243,8 +240,9 @@ pre-release: assert ver in history, '%r not in HISTORY.rst' % ver; \ assert 'XXXX' not in history; \ " - ${MAKE} win-download-exes + ${MAKE} generate-manifest PYTHONWARNINGS=all $(PYTHON) setup.py sdist + ${MAKE} win-download-exes # Create a release: creates tar.gz and exes/wheels, uploads them, # upload doc, git tag release. @@ -261,6 +259,14 @@ print-announce: print-timeline: @PYTHONWARNINGS=all $(PYTHON) scripts/internal/print_timeline.py +# Inspect MANIFEST.in file. +check-manifest: + PYTHONWARNINGS=all $(PYTHON) -m check_manifest -v $(ARGS) + +# Generates MANIFEST.in file. +generate-manifest: + @PYTHONWARNINGS=all $(PYTHON) scripts/internal/generate_manifest.py > MANIFEST.in + # =================================================================== # Misc # =================================================================== diff --git a/scripts/internal/generate_manifest.py b/scripts/internal/generate_manifest.py new file mode 100755 index 000000000..8b6b4f5fa --- /dev/null +++ b/scripts/internal/generate_manifest.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +# Copyright (c) 2009 Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +""" +Generate MANIFEST.in file. +""" + +import os +import subprocess + + +def sh(cmd): + return subprocess.check_output( + cmd, shell=True, universal_newlines=True).strip() + + +def main(): + files = sh("git ls-files").split('\n') + for file in files: + if file.startswith('.ci/') or \ + os.path.splitext(file)[1] in ('.png', '.jpg') or \ + file in ('.travis.yml', 'appveyor.yml'): + continue + print("include " + file) + + +if __name__ == '__main__': + main() From 75622070daeddbc5715e2c35480c2acd37695fe8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 13:30:42 +0200 Subject: [PATCH 628/922] add make sdist --- Makefile | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 3e67cdf6e..ac3c699ff 100644 --- a/Makefile +++ b/Makefile @@ -214,9 +214,15 @@ install-git-hooks: # Distribution # =================================================================== +# Generate tar.gz source distribution. +sdist: + ${MAKE} clean + ${MAKE} generate-manifest + PYTHONWARNINGS=all $(PYTHON) setup.py sdist + # Upload source tarball on https://pypi.python.org/pypi/psutil. upload-src: - ${MAKE} clean + ${MAKE} sdist PYTHONWARNINGS=all $(PYTHON) setup.py sdist upload # Download exes/wheels hosted on appveyor. @@ -230,18 +236,17 @@ win-upload-exes: # All the necessary steps before making a release. pre-release: - ${MAKE} clean - ${MAKE} install # to import psutil from download_exes.py - PYTHONWARNINGS=all $(PYTHON) -c \ + ${MAKE} sdist + # Make sure MANIFEST.in has no uncommitted changes. + PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff MANIFEST.in', shell=True).strip(); sys.exit('MANIFEST.in has uncommitted changes') if out else sys.exit(0);" + ${MAKE} install + @PYTHONWARNINGS=all $(PYTHON) -c \ "from psutil import __version__ as ver; \ doc = open('docs/index.rst').read(); \ history = open('HISTORY.rst').read(); \ assert ver in doc, '%r not in docs/index.rst' % ver; \ assert ver in history, '%r not in HISTORY.rst' % ver; \ - assert 'XXXX' not in history; \ - " - ${MAKE} generate-manifest - PYTHONWARNINGS=all $(PYTHON) setup.py sdist + assert 'XXXX' not in history, 'XXXX in HISTORY.rst';" ${MAKE} win-download-exes # Create a release: creates tar.gz and exes/wheels, uploads them, From 8de6ee7c645eac0814db66463267610301c89cc1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 13:39:12 +0200 Subject: [PATCH 629/922] exit make pre-release if there are uncommitted changes --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ac3c699ff..5f0242ef5 100644 --- a/Makefile +++ b/Makefile @@ -236,9 +236,8 @@ win-upload-exes: # All the necessary steps before making a release. pre-release: + git diff-index --quiet HEAD -- || echo "err: there are uncommitted changes"; exit 1 ${MAKE} sdist - # Make sure MANIFEST.in has no uncommitted changes. - PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff MANIFEST.in', shell=True).strip(); sys.exit('MANIFEST.in has uncommitted changes') if out else sys.exit(0);" ${MAKE} install @PYTHONWARNINGS=all $(PYTHON) -c \ "from psutil import __version__ as ver; \ From 296d1a6e1a244eeb61ccd48015079f862d1dd57d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 13:43:01 +0200 Subject: [PATCH 630/922] better way to check if there are uncommitted changes --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5f0242ef5..d14d9897a 100644 --- a/Makefile +++ b/Makefile @@ -236,7 +236,7 @@ win-upload-exes: # All the necessary steps before making a release. pre-release: - git diff-index --quiet HEAD -- || echo "err: there are uncommitted changes"; exit 1 + git diff --quiet --exit-code || echo "err: there are uncommitted changes"; exit 1 ${MAKE} sdist ${MAKE} install @PYTHONWARNINGS=all $(PYTHON) -c \ From 92d53d211c482fda49adf13db11dfca89cbc7a2b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 13:46:19 +0200 Subject: [PATCH 631/922] better way to check if there are uncommitted changes --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d14d9897a..d1d9fcffd 100644 --- a/Makefile +++ b/Makefile @@ -236,7 +236,7 @@ win-upload-exes: # All the necessary steps before making a release. pre-release: - git diff --quiet --exit-code || echo "err: there are uncommitted changes"; exit 1 + @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff', shell=True).strip(); sys.exit('there are uncommitted changes') if out else sys.exit(0);" ${MAKE} sdist ${MAKE} install @PYTHONWARNINGS=all $(PYTHON) -c \ From fc5b475c1b70ee3198023aa9daf26d7fc285bc36 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 13:49:52 +0200 Subject: [PATCH 632/922] better way to check if there are uncommitted changes --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d1d9fcffd..c72e4b9f3 100644 --- a/Makefile +++ b/Makefile @@ -236,7 +236,7 @@ win-upload-exes: # All the necessary steps before making a release. pre-release: - @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff', shell=True).strip(); sys.exit('there are uncommitted changes') if out else sys.exit(0);" + @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes') if out else sys.exit(0);" ${MAKE} sdist ${MAKE} install @PYTHONWARNINGS=all $(PYTHON) -c \ From 63b43b31da9c90b77258fd69be9261591cd52b70 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 13:53:04 +0200 Subject: [PATCH 633/922] fix make check-broken-links --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c72e4b9f3..31b84d1bf 100644 --- a/Makefile +++ b/Makefile @@ -236,7 +236,7 @@ win-upload-exes: # All the necessary steps before making a release. pre-release: - @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes') if out else sys.exit(0);" + @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes') if out else sys.exit(0);" ${MAKE} sdist ${MAKE} install @PYTHONWARNINGS=all $(PYTHON) -c \ @@ -296,4 +296,4 @@ doc: # check whether the links mentioned in some files are valid. check-broken-links: - git ls-files | grep \\.rst$ | xargs PYTHONWARNINGS=all $(PYTHON) scripts/internal/check_broken_links.py + git ls-files | grep \\.rst$ | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py From ec6fbeb15cf5bf12352fde6ebcfbe3f74529f336 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 15:24:06 +0200 Subject: [PATCH 634/922] check_urls.py refactoring --- scripts/internal/check_broken_links.py | 33 ++++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index ec492f612..3cc78ec8c 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -60,10 +60,9 @@ RETRY_STATUSES = [503, 401, 403] -def get_urls(filename): - """Extracts all URLs available in specified filename.""" - with open(filename) as fs: - text = fs.read() +def get_urls_rst(filename): + with open(filename) as f: + text = f.read() urls = re.findall(REGEX, text) # remove duplicates, list for sets are not iterable urls = list(set(urls)) @@ -73,6 +72,14 @@ def get_urls(filename): return urls +def get_urls(filename): + """Extracts all URLs available in specified filename.""" + if filename.endswith('.rst'): + return get_urls_rst(filename) + else: + return [] + + def validate_url(url): """Validate the URL by attempting an HTTP connection. Makes an HTTP-HEAD request for each URL. @@ -113,21 +120,24 @@ def parallel_validator(urls): else: if not ok: fails.append((fname, url)) - if fails: - print() + + print() return fails def main(): files = sys.argv[1:] if not files: - return sys.exit("usage: %s " % __name__) + print("usage: %s " % sys.argv[0], file=sys.stderr) + return sys.exit(1) all_urls = [] for fname in files: urls = get_urls(fname) - for url in urls: - all_urls.append((fname, url)) + if urls: + print("%4s %s" % (len(urls), fname)) + for url in urls: + all_urls.append((fname, url)) fails = parallel_validator(all_urls) if not fails: @@ -142,4 +152,7 @@ def main(): if __name__ == '__main__': - main() + try: + main() + except (KeyboardInterrupt, SystemExit): + os._exit(0) From 3775929b7b6dc802abd32028f17585f82fbb12be Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 15:30:35 +0200 Subject: [PATCH 635/922] faster regex --- scripts/internal/check_broken_links.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 3cc78ec8c..cd9875daa 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -41,11 +41,11 @@ from __future__ import print_function +import concurrent.futures import os import re import sys import traceback -import concurrent.futures import requests @@ -60,10 +60,10 @@ RETRY_STATUSES = [503, 401, 403] -def get_urls_rst(filename): +def get_urls_rst(filename, _regex=re.compile(REGEX)): with open(filename) as f: text = f.read() - urls = re.findall(REGEX, text) + urls = _regex.findall(text) # remove duplicates, list for sets are not iterable urls = list(set(urls)) # correct urls which are between < and/or > From 971d4ebcb0f8991f3eafdaf543589298f19dcf93 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 16:27:52 +0200 Subject: [PATCH 636/922] check broken links: also inspect py files --- Makefile | 2 +- psutil/_pslinux.py | 3 ++- scripts/internal/check_broken_links.py | 29 ++++++++++++++++++++++++-- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 31b84d1bf..8b40d8c24 100644 --- a/Makefile +++ b/Makefile @@ -296,4 +296,4 @@ doc: # check whether the links mentioned in some files are valid. check-broken-links: - git ls-files | grep \\.rst$ | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py + git ls-files | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 7c075f419..92e6c22b2 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -319,7 +319,8 @@ def cat(fname, fallback=_DEFAULT, binary=True): def calculate_avail_vmem(mems): """Fallback for kernels < 3.14 where /proc/meminfo does not provide - "MemAvailable:" column (see: https://blog.famzah.net/2014/09/24/). + "MemAvailable:" column, see: + https://blog.famzah.net/2014/09/24/ This code reimplements the algorithm outlined here: https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/ commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773 diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index cd9875daa..d01db28fb 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -53,7 +53,7 @@ HERE = os.path.abspath(os.path.dirname(__file__)) REGEX = r'(?:http|ftp|https)?://' \ r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' -REQUEST_TIMEOUT = 30 +REQUEST_TIMEOUT = 10 # There are some status codes sent by websites on HEAD request. # Like 503 by Microsoft, and 401 by Apple # They need to be sent GET request @@ -66,16 +66,41 @@ def get_urls_rst(filename, _regex=re.compile(REGEX)): urls = _regex.findall(text) # remove duplicates, list for sets are not iterable urls = list(set(urls)) + # HISTORY file has a lot of dead links. + if filename == 'HISTORY.rst': + urls = [ + x for x in urls if + not x.startswith('https://github.com/giampaolo/psutil/issues/')] # correct urls which are between < and/or > for i, url in enumerate(urls): urls[i] = re.sub("[\*<>\(\)\)]", '', url) return urls +def get_urls_py(filename, _regex=re.compile(REGEX)): + with open(filename) as f: + lines = f.readlines() + urls = set() + for i, line in enumerate(lines): + line = line.strip() + match = _regex.findall(line) + if match: + url = match[0] + if line.startswith('# '): + nextline = lines[i + 1].strip() + if re.match('^# .+', nextline): + url += nextline[1:].strip() + url = re.sub("[\*<>\(\)\)]", '', url) + urls.add(url) + return urls + + def get_urls(filename): """Extracts all URLs available in specified filename.""" if filename.endswith('.rst'): return get_urls_rst(filename) + elif filename.endswith('.py'): + return get_urls_py(filename) else: return [] @@ -145,7 +170,7 @@ def main(): else: for fail in fails: fname, url = fail - print("%s : %s " % (url, fname)) + print("%-30s: %s " % (fname, url)) print('-' * 20) print("total: %s fails!" % len(fails)) sys.exit(1) From 4563f78c34aa163722a0bdc148fbeaf2a9511cd3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 16:44:51 +0200 Subject: [PATCH 637/922] check broken links: use memoize decorator --- scripts/internal/check_broken_links.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index d01db28fb..a7c42e8dc 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -42,6 +42,7 @@ from __future__ import print_function import concurrent.futures +import functools import os import re import sys @@ -60,6 +61,21 @@ RETRY_STATUSES = [503, 401, 403] +def memoize(fun): + """A memoize decorator.""" + @functools.wraps(fun) + def wrapper(*args, **kwargs): + key = (args, frozenset(sorted(kwargs.items()))) + try: + return cache[key] + except KeyError: + ret = cache[key] = fun(*args, **kwargs) + return ret + + cache = {} + return wrapper + + def get_urls_rst(filename, _regex=re.compile(REGEX)): with open(filename) as f: text = f.read() @@ -105,6 +121,7 @@ def get_urls(filename): return [] +@memoize def validate_url(url): """Validate the URL by attempting an HTTP connection. Makes an HTTP-HEAD request for each URL. From 4fdfd4a45bf9d2187efed4c9453607fe3ed5a6e0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 17:05:27 +0200 Subject: [PATCH 638/922] fix broken link --- psutil/_pslinux.py | 2 +- psutil/_psosx.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 92e6c22b2..eb7d11262 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -122,7 +122,7 @@ class IOPriority(enum.IntEnum): "W": _common.STATUS_WAKING } -# http://students.mimuw.edu.pl/lxr/source/include/net/tcp_states.h +# https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h TCP_STATUSES = { "01": _common.CONN_ESTABLISHED, "02": _common.CONN_SYN_SENT, diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 872d3a144..14cdb1e1f 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -33,7 +33,6 @@ PAGESIZE = os.sysconf("SC_PAGE_SIZE") AF_LINK = cext_posix.AF_LINK -# http://students.mimuw.edu.pl/lxr/source/include/net/tcp_states.h TCP_STATUSES = { cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, From e80a005de6de339e1abad140c0ed2cfa0ef7dbaf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 17:35:43 +0200 Subject: [PATCH 639/922] refactor broken links script --- HISTORY.rst | 3 +- scripts/internal/check_broken_links.py | 46 +++++++++++++++----------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7e292b3b3..24b8c7bfe 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -555,8 +555,7 @@ https://ci.appveyor.com/project/giampaolo/psutil. - 647_: new dev guide: https://github.com/giampaolo/psutil/blob/master/DEVGUIDE.rst -- 651_: continuous code quality test integration with - https://scrutinizer-ci.com/g/giampaolo/psutil/ +- 651_: continuous code quality test integration with scrutinizer-ci.com **Bug fixes** diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index a7c42e8dc..2d2d9d30a 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -52,8 +52,9 @@ HERE = os.path.abspath(os.path.dirname(__file__)) -REGEX = r'(?:http|ftp|https)?://' \ - r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+' +REGEX = re.compile( + r'(?:http|ftp|https)?://' + r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+') REQUEST_TIMEOUT = 10 # There are some status codes sent by websites on HEAD request. # Like 503 by Microsoft, and 401 by Apple @@ -76,39 +77,44 @@ def wrapper(*args, **kwargs): return wrapper -def get_urls_rst(filename, _regex=re.compile(REGEX)): +def sanitize_url(url): + return \ + url.strip('(').strip(')').strip('[').strip(']').strip('<').strip('>') + + +def find_urls(s): + matches = REGEX.findall(s) + if matches: + return list(set([sanitize_url(x) for x in matches])) + + +def get_urls_rst(filename): with open(filename) as f: text = f.read() - urls = _regex.findall(text) - # remove duplicates, list for sets are not iterable - urls = list(set(urls)) + urls = find_urls(text) # HISTORY file has a lot of dead links. - if filename == 'HISTORY.rst': + if filename == 'HISTORY.rst' and urls: urls = [ x for x in urls if - not x.startswith('https://github.com/giampaolo/psutil/issues/')] - # correct urls which are between < and/or > - for i, url in enumerate(urls): - urls[i] = re.sub("[\*<>\(\)\)]", '', url) + not x.startswith('https://github.com/giampaolo/psutil/issues')] return urls -def get_urls_py(filename, _regex=re.compile(REGEX)): +def get_urls_py(filename): with open(filename) as f: lines = f.readlines() - urls = set() + ret = set() for i, line in enumerate(lines): - line = line.strip() - match = _regex.findall(line) - if match: - url = match[0] + urls = find_urls(line) + if urls: + assert len(urls) == 1, urls + url = urls[0] if line.startswith('# '): nextline = lines[i + 1].strip() if re.match('^# .+', nextline): url += nextline[1:].strip() - url = re.sub("[\*<>\(\)\)]", '', url) - urls.add(url) - return urls + ret.add(url) + return list(ret) def get_urls(filename): From 762fa897d2abe72412f9d16948ac2f172ce8b721 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 17:51:55 +0200 Subject: [PATCH 640/922] parse comment blocks --- scripts/internal/check_broken_links.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 2d2d9d30a..70f368e06 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -109,10 +109,16 @@ def get_urls_py(filename): if urls: assert len(urls) == 1, urls url = urls[0] - if line.startswith('# '): - nextline = lines[i + 1].strip() - if re.match('^# .+', nextline): - url += nextline[1:].strip() + # comment block + if line.lstrip().startswith('# '): + subidx = i + 1 + while 1: + nextline = lines[subidx].strip() + if re.match('^# .+', nextline): + url += nextline[1:].strip() + else: + break + subidx += 1 ret.add(url) return list(ret) From e988ae62abf8ea588046312f4935121643691ef7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 18:14:52 +0200 Subject: [PATCH 641/922] refactor broken links script --- psutil/tests/test_posix.py | 2 -- scripts/internal/check_broken_links.py | 35 +++++++++++++------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index d3d2d5b19..a84c3bb6e 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -70,8 +70,6 @@ def setUpClass(cls): def tearDownClass(cls): reap_children() - # for ps -o arguments see: http://unixhelp.ed.ac.uk/CGI/man-cgi?ps - def test_ppid(self): ppid_ps = ps("ps --no-headers -o ppid -p %s" % self.pid) ppid_psutil = psutil.Process(self.pid).ppid() diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 70f368e06..3d1766b35 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -83,12 +83,12 @@ def sanitize_url(url): def find_urls(s): - matches = REGEX.findall(s) - if matches: - return list(set([sanitize_url(x) for x in matches])) + matches = REGEX.findall(s) or [] + return list(set([sanitize_url(x) for x in matches])) -def get_urls_rst(filename): +def parse_rst(filename): + """Look for links in a .rst file.""" with open(filename) as f: text = f.read() urls = find_urls(text) @@ -100,35 +100,34 @@ def get_urls_rst(filename): return urls -def get_urls_py(filename): +def parse_py(filename): + """Look for links in a .py file.""" with open(filename) as f: lines = f.readlines() - ret = set() + urls = set() for i, line in enumerate(lines): - urls = find_urls(line) - if urls: - assert len(urls) == 1, urls + for url in find_urls(line): url = urls[0] # comment block if line.lstrip().startswith('# '): subidx = i + 1 - while 1: + while True: nextline = lines[subidx].strip() if re.match('^# .+', nextline): url += nextline[1:].strip() else: break subidx += 1 - ret.add(url) - return list(ret) + urls.add(url) + return list(urls) -def get_urls(filename): - """Extracts all URLs available in specified filename.""" - if filename.endswith('.rst'): - return get_urls_rst(filename) - elif filename.endswith('.py'): - return get_urls_py(filename) +def get_urls(fname): + """Extracts all URLs available in specified fname.""" + if fname.endswith('.rst'): + return parse_rst(fname) + elif fname.endswith('.py'): + return parse_py(fname) else: return [] From bb6ffa8b7f2a0865c8ecda05c98f51794fcfef50 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 18:59:05 +0200 Subject: [PATCH 642/922] broken links: also inspect C and H files --- psutil/_psutil_bsd.c | 10 +++--- psutil/_psutil_sunos.c | 4 +-- psutil/arch/bsd/freebsd.c | 4 +-- psutil/arch/bsd/freebsd_socks.c | 3 +- psutil/arch/bsd/openbsd.c | 3 +- psutil/arch/solaris/v10/ifaddrs.c | 2 +- scripts/internal/check_broken_links.py | 46 ++++++++++++++++++++++---- 7 files changed, 52 insertions(+), 20 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 43189a219..3fa93d4bb 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -7,7 +7,7 @@ * Platform-specific module methods for FreeBSD and OpenBSD. * OpenBSD references: - * - OpenBSD source code: http://anoncvs.spacehopper.org/openbsd-src/ + * - OpenBSD source code: https://github.com/openbsd/src * * OpenBSD / NetBSD: missing APIs compared to FreeBSD implementation: * - psutil.net_connections() @@ -234,13 +234,13 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { rss = (long)kp.p_vm_rssize * pagesize; #ifdef PSUTIL_OPENBSD // VMS, this is how ps determines it on OpenBSD: - // http://anoncvs.spacehopper.org/openbsd-src/tree/bin/ps/print.c#n461 - // vms + // https://github.com/openbsd/src/blob/ + // 588f7f8c69786211f2d16865c552afb91b1c7cba/bin/ps/print.c#L505 vms = (long)(kp.p_vm_dsize + kp.p_vm_ssize + kp.p_vm_tsize) * pagesize; #elif PSUTIL_NETBSD // VMS, this is how top determines it on NetBSD: - // ftp://ftp.iij.ad.jp/pub/NetBSD/NetBSD-release-6/src/external/bsd/ - // top/dist/machine/m_netbsd.c + // https://github.com/IIJ-NetBSD/netbsd-src/blob/master/external/ + // bsd/top/dist/machine/m_netbsd.c vms = (long)kp.p_vm_msize * pagesize; #endif memtext = (long)kp.p_vm_tsize * pagesize; diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index c205a3ac5..422d48c7b 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -306,7 +306,7 @@ psutil_proc_num_ctx_switches(PyObject *self, PyObject *args) { * - 'pr_ioch' is a sum of chars read and written, with no distinction * - 'pr_inblk' and 'pr_oublk', which should be the number of bytes * read and written, hardly increase and according to: - * http://www.brendangregg.com/Perf/paper_diskubyp1.pdf + * http://www.brendangregg.com/Solaris/paper_diskubyp1.pdf * ...they should be meaningless anyway. * static PyObject* @@ -326,7 +326,7 @@ proc_io_counters(PyObject* self, PyObject* args) { // *and* written. // 'pr_inblk' and 'pr_oublk' should be expressed in blocks of // 8KB according to: - // http://www.brendangregg.com/Perf/paper_diskubyp1.pdf (pag. 8) + // http://www.brendangregg.com/Solaris/paper_diskubyp1.pdf (pag. 8) return Py_BuildValue("kkkk", info.pr_ioch, info.pr_ioch, diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index 11594b9d6..a45690012 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -324,8 +324,8 @@ psutil_proc_threads(PyObject *self, PyObject *args) { // Retrieves all threads used by process returning a list of tuples // including thread id, user time and system time. // Thanks to Robert N. M. Watson: - // http://fxr.googlebit.com/source/usr.bin/procstat/ - // procstat_threads.c?v=8-CURRENT + // http://code.metager.de/source/xref/freebsd/usr.bin/procstat/ + // procstat_threads.c long pid; int mib[4]; struct kinfo_proc *kip = NULL; diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index c7a263238..4f36ef113 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -202,8 +202,7 @@ psutil_get_pid_from_sock(int sock_hash) { // Reference: -// https://gitorious.org/freebsd/freebsd/source/ -// f1d6f4778d2044502209708bc167c05f9aa48615:usr.bin/sockstat/sockstat.c +// https://github.com/freebsd/freebsd/blob/master/usr.bin/sockstat/sockstat.c int psutil_gather_inet(int proto, PyObject *py_retlist) { struct xinpgen *xig, *exig; struct xinpcb *xip; diff --git a/psutil/arch/bsd/openbsd.c b/psutil/arch/bsd/openbsd.c index 8891c4611..3b3f4449e 100644 --- a/psutil/arch/bsd/openbsd.c +++ b/psutil/arch/bsd/openbsd.c @@ -415,7 +415,8 @@ psutil_proc_num_fds(PyObject *self, PyObject *args) { PyObject * psutil_proc_cwd(PyObject *self, PyObject *args) { // Reference: - // http://anoncvs.spacehopper.org/openbsd-src/tree/bin/ps/print.c#n179 + // https://github.com/openbsd/src/blob/ + // 588f7f8c69786211f2d16865c552afb91b1c7cba/bin/ps/print.c#L191 long pid; struct kinfo_proc kp; char path[MAXPATHLEN]; diff --git a/psutil/arch/solaris/v10/ifaddrs.c b/psutil/arch/solaris/v10/ifaddrs.c index 59529e6af..2eb36a3ad 100644 --- a/psutil/arch/solaris/v10/ifaddrs.c +++ b/psutil/arch/solaris/v10/ifaddrs.c @@ -1,7 +1,7 @@ /* Refrences: * https://lists.samba.org/archive/samba-technical/2009-February/063079.html * http://stackoverflow.com/questions/4139405/#4139811 - * https://code.google.com/p/openpgm/source/browse/trunk/openpgm/pgm/getifaddrs.c + * https://github.com/steve-o/openpgm/blob/master/openpgm/pgm/getifaddrs.c */ #include diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 3d1766b35..0ae2b323e 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -55,7 +55,7 @@ REGEX = re.compile( r'(?:http|ftp|https)?://' r'(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+') -REQUEST_TIMEOUT = 10 +REQUEST_TIMEOUT = 15 # There are some status codes sent by websites on HEAD request. # Like 503 by Microsoft, and 401 by Apple # They need to be sent GET request @@ -87,27 +87,26 @@ def find_urls(s): return list(set([sanitize_url(x) for x in matches])) -def parse_rst(filename): +def parse_rst(fname): """Look for links in a .rst file.""" - with open(filename) as f: + with open(fname) as f: text = f.read() urls = find_urls(text) # HISTORY file has a lot of dead links. - if filename == 'HISTORY.rst' and urls: + if fname == 'HISTORY.rst' and urls: urls = [ x for x in urls if not x.startswith('https://github.com/giampaolo/psutil/issues')] return urls -def parse_py(filename): +def parse_py(fname): """Look for links in a .py file.""" - with open(filename) as f: + with open(fname) as f: lines = f.readlines() urls = set() for i, line in enumerate(lines): for url in find_urls(line): - url = urls[0] # comment block if line.lstrip().startswith('# '): subidx = i + 1 @@ -122,12 +121,45 @@ def parse_py(filename): return list(urls) +def parse_c(fname): + """Look for links in a .py file.""" + with open(fname) as f: + lines = f.readlines() + urls = set() + for i, line in enumerate(lines): + for url in find_urls(line): + # comment block // + if line.lstrip().startswith('// '): + subidx = i + 1 + while True: + nextline = lines[subidx].strip() + if re.match('^// .+', nextline): + url += nextline[2:].strip() + else: + break + subidx += 1 + # comment block /* + elif line.lstrip().startswith('* '): + subidx = i + 1 + while True: + nextline = lines[subidx].strip() + if re.match('^\* .+', nextline): + url += nextline[1:].strip() + else: + break + subidx += 1 + urls.add(url) + return list(urls) + + def get_urls(fname): """Extracts all URLs available in specified fname.""" if fname.endswith('.rst'): return parse_rst(fname) elif fname.endswith('.py'): return parse_py(fname) + elif fname.endswith('.c') or fname.endswith('.h'): + return parse_c(fname) else: return [] From c7251d25c63fb00f068f740d4138a0fdc53e4399 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 19:14:08 +0200 Subject: [PATCH 643/922] broken links: also inspect generic files --- scripts/internal/check_broken_links.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scripts/internal/check_broken_links.py b/scripts/internal/check_broken_links.py index 0ae2b323e..7cf1e4898 100755 --- a/scripts/internal/check_broken_links.py +++ b/scripts/internal/check_broken_links.py @@ -78,8 +78,15 @@ def wrapper(*args, **kwargs): def sanitize_url(url): - return \ - url.strip('(').strip(')').strip('[').strip(']').strip('<').strip('>') + url = url.rstrip(',') + url = url.rstrip('.') + url = url.lstrip('(') + url = url.rstrip(')') + url = url.lstrip('[') + url = url.rstrip(']') + url = url.lstrip('<') + url = url.rstrip('>') + return url def find_urls(s): @@ -152,8 +159,14 @@ def parse_c(fname): return list(urls) +def parse_generic(fname): + with open(fname) as f: + text = f.read() + return find_urls(text) + + def get_urls(fname): - """Extracts all URLs available in specified fname.""" + """Extracts all URLs in fname and return them as a list.""" if fname.endswith('.rst'): return parse_rst(fname) elif fname.endswith('.py'): @@ -161,7 +174,10 @@ def get_urls(fname): elif fname.endswith('.c') or fname.endswith('.h'): return parse_c(fname) else: - return [] + with open(fname) as f: + if f.readline().strip().startswith('#!/usr/bin/env python'): + return parse_py(fname) + return parse_generic(fname) @memoize From fd59c26170519ad759040b559f33cdca1b5715fa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 19:29:52 +0200 Subject: [PATCH 644/922] #1055: don't cache cpu_count() --- HISTORY.rst | 1 + psutil/__init__.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 24b8c7bfe..879982219 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -44,6 +44,7 @@ - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. - 1048_: [Windows] users()'s host field report an invalid IP address. +- 1055_: cpu_count() is no longer cached. - 1058_: fixed Python warnings. - 1062_: disk_io_counters() and net_io_counters() raise TypeError if no disks or NICs are installed on the system. diff --git a/psutil/__init__.py b/psutil/__init__.py index 88f7e158e..a05d62498 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -209,6 +209,7 @@ POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED POWER_TIME_UNKNOWN = _common.POWER_TIME_UNKNOWN _TOTAL_PHYMEM = None +_NUM_CPUS = None _timer = getattr(time, 'monotonic', time.time) @@ -1042,7 +1043,7 @@ def cpu_percent(self, interval=None): blocking = interval is not None and interval > 0.0 if interval is not None and interval < 0: raise ValueError("interval is not positive (got %r)" % interval) - num_cpus = cpu_count() or 1 + num_cpus = _NUM_CPUS or cpu_count() def timer(): return _timer() * num_cpus @@ -1628,7 +1629,6 @@ def check_gone(proc, timeout): # ===================================================================== -@memoize def cpu_count(logical=True): """Return the number of logical CPUs in the system (same as os.cpu_count() in Python 3.4). @@ -1643,8 +1643,10 @@ def cpu_count(logical=True): >>> psutil.cpu_count.cache_clear() """ + global _NUM_CPUS if logical: - return _psplatform.cpu_count_logical() + _NUM_CPUS = _psplatform.cpu_count_logical() + return _NUM_CPUS else: return _psplatform.cpu_count_physical() From b003ad50fed327ff65efd06f9181cec457b2b1e7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 19:55:25 +0200 Subject: [PATCH 645/922] #1039 freebsd / connections() / UNIX: set laddr to empty string instead of None --- README.rst | 2 +- psutil/arch/bsd/freebsd_socks.c | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 9f849d6ec..ed55ea13d 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,7 @@ Projects using psutil ===================== At the time of writing there are over -`5000 open source projects `__ +`5200 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 4f36ef113..54c09ab10 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -420,21 +420,20 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { if (! py_lpath) goto error; - py_tuple = Py_BuildValue("(iiiOOii)", - -1, - AF_UNIX, - proto, - py_lpath, - Py_None, - PSUTIL_CONN_NONE, - pid); + py_tuple = Py_BuildValue("(iiiOsii)", + -1, // fd + AF_UNIX, // family + proto, // type + py_lpath, // lpath + "", // rath + PSUTIL_CONN_NONE, // status + pid); // pid if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; Py_DECREF(py_lpath); Py_DECREF(py_tuple); - Py_INCREF(Py_None); } free(buf); From 976ac81375a88274e6b6b7bb0aed4cccb2facd46 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 19:58:42 +0200 Subject: [PATCH 646/922] freebsd: fix memory leak in open_files() --- psutil/_psutil_bsd.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 3fa93d4bb..258f03763 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -499,10 +499,10 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { // XXX - it appears path is not exposed in the kinfo_file struct. path = ""; #endif - py_path = PyUnicode_DecodeFSDefault(path); - if (! py_path) - goto error; if (regular == 1) { + py_path = PyUnicode_DecodeFSDefault(path); + if (! py_path) + goto error; py_tuple = Py_BuildValue("(Oi)", py_path, fd); if (py_tuple == NULL) goto error; From 33db0e996863ec5c044a8eb10404401bba017570 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 20:07:59 +0200 Subject: [PATCH 647/922] fix compilation err on netbsd --- psutil/_psutil_bsd.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 258f03763..99fccb727 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -938,11 +938,12 @@ PsutilMethods[] = { #endif #if defined(PSUTIL_FREEBSD) || defined(PSUTIL_NETBSD) - {"proc_exe", psutil_proc_exe, METH_VARARGS, - "Return process pathname executable"}, {"proc_num_threads", psutil_proc_num_threads, METH_VARARGS, "Return number of threads used by process"}, +#endif #if defined(PSUTIL_FREEBSD) + {"proc_exe", psutil_proc_exe, METH_VARARGS, + "Return process pathname executable"}, {"proc_memory_maps", psutil_proc_memory_maps, METH_VARARGS, "Return a list of tuples for every process's memory map"}, {"proc_cpu_affinity_get", psutil_proc_cpu_affinity_get, METH_VARARGS, @@ -951,7 +952,6 @@ PsutilMethods[] = { "Set process CPU affinity."}, {"cpu_count_phys", psutil_cpu_count_phys, METH_VARARGS, "Return an XML string to determine the number physical CPUs."}, -#endif #endif // --- system-related functions From 445c23e27eaf5f4ea899323abbd00f9a2bf2593b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 12 May 2017 20:23:42 +0200 Subject: [PATCH 648/922] refactor ifdefs --- psutil/_psutil_bsd.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 99fccb727..d1a27f348 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -916,23 +916,19 @@ PsutilMethods[] = { "Return multiple info about a process"}, {"proc_name", psutil_proc_name, METH_VARARGS, "Return process name"}, -#if !defined(PSUTIL_NETBSD) - {"proc_connections", psutil_proc_connections, METH_VARARGS, - "Return connections opened by process"}, -#endif {"proc_cmdline", psutil_proc_cmdline, METH_VARARGS, "Return process cmdline as a list of cmdline arguments"}, {"proc_threads", psutil_proc_threads, METH_VARARGS, "Return process threads"}, #if defined(PSUTIL_FREEBSD) || defined(PSUTIL_OPENBSD) + {"proc_connections", psutil_proc_connections, METH_VARARGS, + "Return connections opened by process"}, {"proc_cwd", psutil_proc_cwd, METH_VARARGS, "Return process current working directory."}, #endif #if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 || PSUTIL_OPENBSD || defined(PSUTIL_NETBSD) {"proc_num_fds", psutil_proc_num_fds, METH_VARARGS, "Return the number of file descriptors opened by this process"}, -#endif -#if defined(__FreeBSD_version) && __FreeBSD_version >= 800000 || PSUTIL_OPENBSD || defined(PSUTIL_NETBSD) {"proc_open_files", psutil_proc_open_files, METH_VARARGS, "Return files opened by process as a list of (path, fd) tuples"}, #endif From 47379de2dde66d7aa3d64d920b5ab20998299c79 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 16:42:45 +0200 Subject: [PATCH 649/922] fix failing tests on netbsd --- psutil/_psutil_bsd.c | 1 - psutil/tests/test_bsd.py | 1 + psutil/tests/test_posix.py | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index d1a27f348..4b6df9c71 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -932,7 +932,6 @@ PsutilMethods[] = { {"proc_open_files", psutil_proc_open_files, METH_VARARGS, "Return files opened by process as a list of (path, fd) tuples"}, #endif - #if defined(PSUTIL_FREEBSD) || defined(PSUTIL_NETBSD) {"proc_num_threads", psutil_proc_num_threads, METH_VARARGS, "Return number of threads used by process"}, diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 50b34c092..3d644c92e 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -83,6 +83,7 @@ def setUpClass(cls): def tearDownClass(cls): reap_children() + @unittest.skipIf(NETBSD, "-o lstart doesn't work on NETBSD") def test_process_create_time(self): output = sh("ps -o lstart -p %s" % self.pid) start_ps = output.replace('STARTED', '').strip() diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index a84c3bb6e..f810a09e7 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -36,6 +36,7 @@ from psutil.tests import TRAVIS from psutil.tests import unittest from psutil.tests import wait_for_pid +from psutil.tests import which def ps(cmd): @@ -295,6 +296,7 @@ def test_pids(self): # returned by psutil @unittest.skipIf(SUNOS, "unreliable on SUNOS") @unittest.skipIf(TRAVIS, "unreliable on TRAVIS") + @unittest.skipIf(not which('ifconfig'), "no ifconfig cmd") def test_nic_names(self): output = sh("ifconfig -a") for nic in psutil.net_io_counters(pernic=True).keys(): From 337e59331e4df84eeb57528a5cc95965220aaf3d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 16:49:30 +0200 Subject: [PATCH 650/922] fix windows test; also do make clean on CI services in order to show python warnings --- .ci/travis/run.sh | 1 + appveyor.yml | 1 + psutil/tests/test_misc.py | 3 +++ psutil/tests/test_windows.py | 2 +- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index 05bc60c25..b8526dffc 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -14,6 +14,7 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then fi # install psutil +make clean python setup.py build python setup.py develop diff --git a/appveyor.yml b/appveyor.yml index 78169616d..603d2e159 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -80,6 +80,7 @@ install: - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32 wmi wheel" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" + - make clean - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build build_ext -i" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py develop" diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 0e1ce3509..2cb4dbb1f 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -603,6 +603,9 @@ def test_cache_clear(self): wrap_numbers.cache_clear('disk_io') wrap_numbers.cache_clear('?!?') + @unittest.skipIf( + not psutil.disk_io_counters() or not psutil.net_io_counters(), + "no disks or NICs available") def test_cache_clear_public_apis(self): psutil.disk_io_counters() psutil.net_io_counters() diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index abb208b3d..c1b080cf5 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -338,7 +338,7 @@ def call(p, attr): if name.startswith('_') \ or name in ('terminate', 'kill', 'suspend', 'resume', 'nice', 'send_signal', 'wait', 'children', - 'as_dict'): + 'as_dict', 'memory_info_ex'): continue else: try: From 4d13160c9b1fabf49f6222414e981ccc95df189c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 17:06:22 +0200 Subject: [PATCH 651/922] netbsd: fix compiler warnings --- psutil/arch/bsd/netbsd.c | 7 +------ psutil/arch/bsd/netbsd_socks.c | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index 5748000fa..dfef0006c 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -269,13 +269,9 @@ psutil_get_proc_list(kinfo_proc **procList, size_t *procCount) { // On success, the function returns 0. // On error, the function returns a BSD errno value. kinfo_proc *result; - int done; - static const int name[] = { CTL_KERN, KERN_PROC, KERN_PROC, 0 }; // Declaring name as const requires us to cast it when passing it to // sysctl because the prototype doesn't include the const modifier. - size_t length; char errbuf[_POSIX2_LINE_MAX]; - kinfo_proc *x; int cnt; kvm_t *kd; @@ -519,7 +515,6 @@ psutil_proc_num_fds(PyObject *self, PyObject *args) { PyObject * psutil_per_cpu_times(PyObject *self, PyObject *args) { // XXX: why static? - static int maxcpus; int mib[3]; int ncpu; size_t len; @@ -579,7 +574,7 @@ PyObject * psutil_disk_io_counters(PyObject *self, PyObject *args) { int i, dk_ndrive, mib[3]; size_t len; - struct io_sysctl *stats; + struct io_sysctl *stats = NULL; PyObject *py_disk_info = NULL; PyObject *py_retdict = PyDict_New(); diff --git a/psutil/arch/bsd/netbsd_socks.c b/psutil/arch/bsd/netbsd_socks.c index 06b7c7556..dcc25d013 100644 --- a/psutil/arch/bsd/netbsd_socks.c +++ b/psutil/arch/bsd/netbsd_socks.c @@ -169,7 +169,7 @@ static int psutil_get_sockets(const char *name) { size_t namelen; int mib[8]; - int ret, j; + int j; struct kinfo_pcb *pcb; size_t len; From 826c0bfbdf6675eb6be1db956cb8b6c2c6cc30ea Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 17:24:43 +0200 Subject: [PATCH 652/922] setup.py: do not use setuptools opts if setuptools is not installed (avoid warnings) --- setup.py | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 05c212ab7..87c82ce22 100755 --- a/setup.py +++ b/setup.py @@ -20,8 +20,10 @@ with warnings.catch_warnings(): warnings.simplefilter("ignore") try: + import setuptools from setuptools import setup, Extension except ImportError: + setuptools = None from distutils.core import setup, Extension HERE = os.path.abspath(os.path.dirname(__file__)) @@ -52,6 +54,18 @@ if POSIX: sources.append('psutil/_psutil_posix.c') +tests_require = [] +if sys.version_info[:2] <= (2, 6): + tests_require.append('unittest2') +if sys.version_info[:2] <= (2, 7): + tests_require.append('mock') +if sys.version_info[:2] <= (3, 2): + tests_require.append('ipaddress') + +extras_require = {} +if sys.version_info[:2] <= (3, 3): + extras_require.update(dict(enum='enum34')) + def get_version(): INIT = os.path.join(HERE, 'psutil/__init__.py') @@ -193,8 +207,8 @@ def get_ethtool_macro(): suffix='.c', delete=False, mode="wt") as f: f.write("#include ") - compiler = UnixCCompiler() try: + compiler = UnixCCompiler() with silenced_output('stderr'): with silenced_output('stdout'): compiler.compile([f.name]) @@ -208,9 +222,8 @@ def get_ethtool_macro(): except OSError: pass - ETHTOOL_MACRO = get_ethtool_macro() - macros.append(("PSUTIL_LINUX", 1)) + ETHTOOL_MACRO = get_ethtool_macro() if ETHTOOL_MACRO is not None: macros.append(ETHTOOL_MACRO) ext = Extension( @@ -246,7 +259,7 @@ def get_ethtool_macro(): def main(): - setup( + kwargs = dict( name='psutil', version=VERSION, description=__doc__ .replace('\n', '').strip() if __doc__ else '', @@ -264,9 +277,6 @@ def main(): license='BSD', packages=['psutil', 'psutil.tests'], ext_modules=extensions, - test_suite="psutil.tests.get_suite", - tests_require=['ipaddress', 'mock', 'unittest2'], - zip_safe=False, # http://stackoverflow.com/questions/19548957 # see: python setup.py register --list-classifiers classifiers=[ 'Development Status :: 5 - Production/Stable', @@ -314,6 +324,14 @@ def main(): 'Topic :: Utilities', ], ) + if setuptools is not None: + kwargs.update( + test_suite="psutil.tests.get_suite", + tests_require=tests_require, + extras_require=extras_require, + zip_safe=False, + ) + setup(**kwargs) if __name__ == '__main__': From 89962e2c719bdfd687f5587b036591491881ae86 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 17:27:49 +0200 Subject: [PATCH 653/922] netbsd: fix compiler warnings --- psutil/arch/bsd/netbsd_socks.c | 14 +++++++------- setup.py | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/psutil/arch/bsd/netbsd_socks.c b/psutil/arch/bsd/netbsd_socks.c index dcc25d013..76b546c1d 100644 --- a/psutil/arch/bsd/netbsd_socks.c +++ b/psutil/arch/bsd/netbsd_socks.c @@ -312,11 +312,16 @@ psutil_get_info(int aff) { */ PyObject * psutil_net_connections(PyObject *self, PyObject *args) { - PyObject *py_retlist = PyList_New(0); + char laddr[PATH_MAX]; + char raddr[PATH_MAX]; + int32_t lport; + int32_t rport; + int32_t status; + pid_t pid; PyObject *py_tuple = NULL; PyObject *py_laddr = NULL; PyObject *py_raddr = NULL; - pid_t pid; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; @@ -339,11 +344,6 @@ psutil_net_connections(PyObject *self, PyObject *args) { SLIST_FOREACH(kp, &kpcbhead, kpcbs) { if (k->kif->ki_fdata != kp->kpcb->ki_sockaddr) continue; - char laddr[PATH_MAX]; - char raddr[PATH_MAX]; - int32_t lport; - int32_t rport; - int32_t status; // IPv4 or IPv6 if ((kp->kpcb->ki_family == AF_INET) || diff --git a/setup.py b/setup.py index 87c82ce22..d51b82feb 100755 --- a/setup.py +++ b/setup.py @@ -242,6 +242,7 @@ def get_ethtool_macro(): else: sys.exit('platform %s is not supported' % sys.platform) + if POSIX: posix_extension = Extension( 'psutil._psutil_posix', From 8b0c184ec44c4f4e7d49708bb01c4db14d97d9ee Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 17:40:54 +0200 Subject: [PATCH 654/922] fix #1063 / NetBSD / net_connections: 'continue;' if family is not in AF_INET|6 or AF_UNIX --- HISTORY.rst | 1 + psutil/arch/bsd/netbsd_socks.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 879982219..32dba95f0 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -67,6 +67,7 @@ - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. - 1050_: [Windows] Process.memory_maps memory() leaks memory. +- 1063_: [NetBSD] net_connections() may list incorrect sockets. *2017-04-10* diff --git a/psutil/arch/bsd/netbsd_socks.c b/psutil/arch/bsd/netbsd_socks.c index 76b546c1d..8ea72ae61 100644 --- a/psutil/arch/bsd/netbsd_socks.c +++ b/psutil/arch/bsd/netbsd_socks.c @@ -413,6 +413,9 @@ psutil_net_connections(PyObject *self, PyObject *args) { if (! py_raddr) goto error; } + else { + continue; + } // append tuple to list py_tuple = Py_BuildValue( From 0eeebc282dba7c3e2877c42698900da0e816d921 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 17:45:12 +0200 Subject: [PATCH 655/922] netbsd: fix compiler warnings --- psutil/arch/bsd/netbsd.c | 2 +- psutil/arch/bsd/netbsd_socks.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index dfef0006c..66bad193a 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -360,7 +360,7 @@ psutil_get_cmd_args(pid_t pid, size_t *argsize) { PyObject * psutil_get_cmdline(pid_t pid) { char *argstr = NULL; - int pos = 0; + size_t pos = 0; size_t argsize = 0; PyObject *py_arg = NULL; PyObject *py_retlist = PyList_New(0); diff --git a/psutil/arch/bsd/netbsd_socks.c b/psutil/arch/bsd/netbsd_socks.c index 8ea72ae61..bd260ad35 100644 --- a/psutil/arch/bsd/netbsd_socks.c +++ b/psutil/arch/bsd/netbsd_socks.c @@ -112,10 +112,10 @@ psutil_kpcblist_clear(void) { static int psutil_get_files(void) { size_t len; + size_t j; int mib[6]; char *buf; off_t offset; - int j; mib[0] = CTL_KERN; mib[1] = KERN_FILE2; @@ -169,9 +169,9 @@ static int psutil_get_sockets(const char *name) { size_t namelen; int mib[8]; - int j; struct kinfo_pcb *pcb; size_t len; + size_t j; memset(mib, 0, sizeof(mib)); @@ -339,7 +339,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { struct kif *k; SLIST_FOREACH(k, &kihead, kifs) { struct kpcb *kp; - if ((pid != -1) && (k->kif->ki_pid != pid)) + if ((pid != -1) && (k->kif->ki_pid != (unsigned int)pid)) continue; SLIST_FOREACH(kp, &kpcbhead, kpcbs) { if (k->kif->ki_fdata != kp->kpcb->ki_sockaddr) From 137e7edf60918196e9338789f27a31fd5920a956 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 17:48:27 +0200 Subject: [PATCH 656/922] #1064 / netbsd: swap_memory() may segfault as it does not in case of error --- HISTORY.rst | 1 + psutil/arch/bsd/netbsd.c | 1 + 2 files changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 32dba95f0..cfc5aeb74 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -68,6 +68,7 @@ - 1047_: [Windows] Process username(): memory leak in case exception is thrown. - 1050_: [Windows] Process.memory_maps memory() leaks memory. - 1063_: [NetBSD] net_connections() may list incorrect sockets. +- 1064_: [NetBSD] swap_memory() may segfault in case of error. *2017-04-10* diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index 66bad193a..48f7b07ca 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -487,6 +487,7 @@ psutil_swap_mem(PyObject *self, PyObject *args) { error: free(swdev); + return NULL; } From 42ced547443877db7da9502d7d4ca8e3f3dd8594 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 17:55:22 +0200 Subject: [PATCH 657/922] freebsd: fix compiler warning --- psutil/arch/bsd/freebsd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index a45690012..e2e77f9c8 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -222,7 +222,7 @@ static char PyObject * psutil_get_cmdline(long pid) { char *argstr = NULL; - int pos = 0; + size_t pos = 0; size_t argsize = 0; PyObject *py_retlist = Py_BuildValue("[]"); PyObject *py_arg = NULL; From e10a3c5f1ad110ba0832c5c50d2eaedbefefcf02 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 17:58:42 +0200 Subject: [PATCH 658/922] openbsd: fix compiler warning --- psutil/arch/bsd/openbsd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/arch/bsd/openbsd.c b/psutil/arch/bsd/openbsd.c index 3b3f4449e..fe9c289be 100644 --- a/psutil/arch/bsd/openbsd.c +++ b/psutil/arch/bsd/openbsd.c @@ -709,7 +709,7 @@ PyObject * psutil_disk_io_counters(PyObject *self, PyObject *args) { int i, dk_ndrive, mib[3]; size_t len; - struct diskstats *stats; + struct diskstats *stats = NULL; PyObject *py_retdict = PyDict_New(); PyObject *py_disk_info = NULL; From 2d0fb7ea58990a08834cfe07cf505676484bdc14 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 18:35:06 +0200 Subject: [PATCH 659/922] fix #1065: cmdline() on OpenBSD may raise SystemError; also set a limit of retries to realloc() the space to store the cmdline to max 8k so that it won't loop forever --- HISTORY.rst | 9 ++++----- psutil/_psutil_bsd.c | 7 ++----- psutil/arch/bsd/freebsd.c | 10 +++++----- psutil/arch/bsd/netbsd.c | 3 +-- psutil/arch/bsd/openbsd.c | 19 +++++++++++-------- 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index cfc5aeb74..788ad0647 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -43,11 +43,15 @@ Python 3 + UNIX and invalid encoded data on Windows. - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. +- 1050_: [Windows] Process.memory_maps memory() leaks memory. - 1048_: [Windows] users()'s host field report an invalid IP address. - 1055_: cpu_count() is no longer cached. - 1058_: fixed Python warnings. - 1062_: disk_io_counters() and net_io_counters() raise TypeError if no disks or NICs are installed on the system. +- 1063_: [NetBSD] net_connections() may list incorrect sockets. +- 1064_: [NetBSD] swap_memory() may segfault in case of error. +- 1065_: [OpenBSD] Process.cmdline() may raise SystemError. **Porting notes** @@ -64,11 +68,6 @@ - WindosService.description() - WindosService.display_name() - WindosService.username() -- 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. -- 1047_: [Windows] Process username(): memory leak in case exception is thrown. -- 1050_: [Windows] Process.memory_maps memory() leaks memory. -- 1063_: [NetBSD] net_connections() may list incorrect sockets. -- 1064_: [NetBSD] swap_memory() may segfault in case of error. *2017-04-10* diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 4b6df9c71..6478cc65c 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -382,12 +382,9 @@ psutil_proc_cmdline(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; - py_retlist = psutil_get_cmdline(pid); - // psutil_get_cmdline() returns NULL only if psutil_cmd_args - // failed with ESRCH (no process with that PID) - if (NULL == py_retlist) - return PyErr_SetFromErrno(PyExc_OSError); + if (py_retlist == NULL) + return NULL; return Py_BuildValue("N", py_retlist); } diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index e2e77f9c8..98751c6d9 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -179,7 +179,8 @@ psutil_get_proc_list(struct kinfo_proc **procList, size_t *procCount) { */ static char *psutil_get_cmd_args(long pid, size_t *argsize) { - int mib[4], argmax; + int mib[4]; + int argmax; size_t size = sizeof(argmax); char *procargs = NULL; @@ -198,9 +199,7 @@ static char return NULL; } - /* - * Make a sysctl() call to get the raw argument space of the process. - */ + // Make a sysctl() call to get the raw argument space of the process. mib[0] = CTL_KERN; mib[1] = KERN_PROC; mib[2] = KERN_PROC_ARGS; @@ -209,7 +208,8 @@ static char size = argmax; if (sysctl(mib, 4, procargs, &size, NULL, 0) == -1) { free(procargs); - return NULL; // Insufficient privileges + PyErr_SetFromErrno(PyExc_OSError); + return NULL; } // return string and set the length of arguments diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index 48f7b07ca..a2a9dd5b7 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -326,7 +326,6 @@ psutil_get_cmd_args(pid_t pid, size_t *argsize) { size = sizeof(argmax); st = sysctl(mib, 2, &argmax, &size, NULL, 0); if (st == -1) { - warn("failed to get kern.argmax"); PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -344,7 +343,6 @@ psutil_get_cmd_args(pid_t pid, size_t *argsize) { st = sysctl(mib, 4, procargs, &argmax, NULL, 0); if (st == -1) { - warn("failed to get kern.procargs"); PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -353,6 +351,7 @@ psutil_get_cmd_args(pid_t pid, size_t *argsize) { return procargs; } + // Return the command line as a python list object. // XXX - most of the times sysctl() returns a truncated string. // Also /proc/pid/cmdline behaves the same so it looks like this diff --git a/psutil/arch/bsd/openbsd.c b/psutil/arch/bsd/openbsd.c index fe9c289be..75679cf23 100644 --- a/psutil/arch/bsd/openbsd.c +++ b/psutil/arch/bsd/openbsd.c @@ -170,18 +170,21 @@ _psutil_get_argv(long pid) { static char **argv; int argv_mib[] = {CTL_KERN, KERN_PROC_ARGS, pid, KERN_PROC_ARGV}; size_t argv_size = 128; - /* Loop and reallocate until we have enough space to fit argv. */ + // Loop and reallocate until we have enough space to fit argv. for (;; argv_size *= 2) { + if (argv_size >= 8192) { + PyErr_SetString(PyExc_RuntimeError, + "can't allocate enough space for KERN_PROC_ARGV"); + return NULL; + } if ((argv = realloc(argv, argv_size)) == NULL) - err(1, NULL); + continue; if (sysctl(argv_mib, 4, argv, &argv_size, NULL, 0) == 0) return argv; - if (errno == ESRCH) { - PyErr_SetFromErrno(PyExc_OSError); - return NULL; - } - if (errno != ENOMEM) - err(1, NULL); + if (errno == ENOMEM) + continue; + PyErr_SetFromErrno(PyExc_OSError); + return NULL; } } From 820179eeefcee1c74c7a39bb8a64555acc42dc1b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 18:39:53 +0200 Subject: [PATCH 660/922] openbsd: skip memleak test failing because of access denied --- psutil/tests/__init__.py | 4 +--- psutil/tests/test_memory_leaks.py | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 03ad9553b..583919869 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -708,9 +708,7 @@ def wrapper(*args, **kwargs): if only_if is not None: if not only_if: raise - msg = "%r was skipped because it raised AccessDenied" \ - % fun.__name__ - raise unittest.SkipTest(msg) + raise unittest.SkipTest("raises AccessDenied") return wrapper return decorator diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 5e9cfbf65..3e3087dcc 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -44,6 +44,7 @@ from psutil.tests import reap_children from psutil.tests import run_test_module_by_name from psutil.tests import safe_rmpath +from psutil.tests import skip_on_access_denied from psutil.tests import TESTFN from psutil.tests import TRAVIS from psutil.tests import unittest @@ -268,6 +269,7 @@ def test_create_time(self): self.execute(self.proc.create_time) @skip_if_linux() + @skip_on_access_denied(only_if=OPENBSD) def test_num_threads(self): self.execute(self.proc.num_threads) @@ -285,6 +287,7 @@ def test_num_ctx_switches(self): self.execute(self.proc.num_ctx_switches) @skip_if_linux() + @skip_on_access_denied(only_if=OPENBSD) def test_threads(self): self.execute(self.proc.threads) From e03113737ddd668be08b72bfcc3fde8b86ac7dee Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 21:40:46 +0200 Subject: [PATCH 661/922] test-memleaks: skip memory_maps test if not supported by platform --- psutil/tests/test_memory_leaks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 3e3087dcc..9d05834cb 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -29,12 +29,13 @@ from psutil import SUNOS from psutil import WINDOWS from psutil._compat import xrange +from psutil.tests import create_sockets from psutil.tests import get_test_subprocess from psutil.tests import HAS_CPU_AFFINITY from psutil.tests import HAS_CPU_FREQ -from psutil.tests import create_sockets from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_IONICE +from psutil.tests import HAS_MEMORY_MAPS from psutil.tests import HAS_PROC_CPU_NUM from psutil.tests import HAS_PROC_IO_COUNTERS from psutil.tests import HAS_RLIMIT @@ -341,7 +342,7 @@ def test_open_files(self): # OSX implementation is unbelievably slow @unittest.skipIf(OSX, "too slow on OSX") - @unittest.skipIf(OPENBSD, "not supported") + @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") @skip_if_linux() def test_memory_maps(self): self.execute(self.proc.memory_maps) From c0550c79cf304bd753f4199c40eaef45b8179bdd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 22:02:02 +0200 Subject: [PATCH 662/922] fix #1067: cmdline() memleak on NetBSD --- HISTORY.rst | 1 + psutil/arch/bsd/netbsd.c | 1 + 2 files changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 788ad0647..ee18f0e0d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -52,6 +52,7 @@ - 1063_: [NetBSD] net_connections() may list incorrect sockets. - 1064_: [NetBSD] swap_memory() may segfault in case of error. - 1065_: [OpenBSD] Process.cmdline() may raise SystemError. +- 1067_: [NetBSD] Process.cmdline() memory leak. **Porting notes** diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index a2a9dd5b7..972418ff3 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -343,6 +343,7 @@ psutil_get_cmd_args(pid_t pid, size_t *argsize) { st = sysctl(mib, 4, procargs, &argmax, NULL, 0); if (st == -1) { + free(procargs); PyErr_SetFromErrno(PyExc_OSError); return NULL; } From 0fad89c3448400229d5f94ea1952373e5246e87e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 22:08:29 +0200 Subject: [PATCH 663/922] update HISTORY --- HISTORY.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index ee18f0e0d..a80d11231 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -52,7 +52,7 @@ - 1063_: [NetBSD] net_connections() may list incorrect sockets. - 1064_: [NetBSD] swap_memory() may segfault in case of error. - 1065_: [OpenBSD] Process.cmdline() may raise SystemError. -- 1067_: [NetBSD] Process.cmdline() memory leak. +- 1067_: [NetBSD] Process.cmdline() leaks memory if proces has terminated. **Porting notes** From 90df05ccc3800cab3e5e313567237610b23761a1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 22:31:21 +0200 Subject: [PATCH 664/922] #1068 / openbsd / net_connections(): free() mem if sysctl() fails --- psutil/arch/bsd/openbsd.c | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/arch/bsd/openbsd.c b/psutil/arch/bsd/openbsd.c index 75679cf23..4caf6ed5e 100644 --- a/psutil/arch/bsd/openbsd.c +++ b/psutil/arch/bsd/openbsd.c @@ -102,6 +102,7 @@ kinfo_getfile(long pid, int* cnt) { } mib[5] = (int)(len / sizeof(struct kinfo_file)); if (sysctl(mib, 6, kf, &len, NULL, 0) < 0) { + free(kf); PyErr_SetFromErrno(PyExc_OSError); return NULL; } From 41879e900901b371ec319eac6fddaad6e7d73930 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 22:55:38 +0200 Subject: [PATCH 665/922] memleaks tests: show extra proc memory --- psutil/tests/test_memory_leaks.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 9d05834cb..680fe7803 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -13,10 +13,12 @@ for some reason). """ +from __future__ import print_function import errno import functools import gc import os +import sys import threading import time @@ -152,12 +154,14 @@ def call_many_times(): if mem3 > mem2: # failure - self.fail("+%s after %s calls, +%s after another %s calls" % ( - bytes2human(diff1), - loops, - bytes2human(diff2), - ncalls - )) + extra_proc_mem = bytes2human(diff1 + diff2) + print("exta proc mem: %s" % extra_proc_mem, file=sys.stderr) + msg = "+%s after %s calls, +%s after another %s calls, " + msg += "+%s extra proc mem" + msg = msg % ( + bytes2human(diff1), loops, bytes2human(diff2), ncalls, + extra_proc_mem) + self.fail(msg) def execute_w_exc(self, exc, fun, *args, **kwargs): """Convenience function which tests a callable raising From cedafdfed944d22f5c195293ca813a3134b6efdb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 23:12:31 +0200 Subject: [PATCH 666/922] fix appveyor build --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 603d2e159..ccc7eb48d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -80,7 +80,7 @@ install: - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32 wmi wheel" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - - make clean + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py clean - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build build_ext -i" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py develop" From 010e0c0ed99fce8ac6a403103af3e10b63f1b555 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 13 May 2017 23:13:08 +0200 Subject: [PATCH 667/922] fix appveyor build --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index ccc7eb48d..2e6735bba 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -80,7 +80,7 @@ install: - "%WITH_COMPILER% %PYTHON%/python.exe -m pip --version" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip install --upgrade --user unittest2 ipaddress pypiwin32 wmi wheel" - "%WITH_COMPILER% %PYTHON%/python.exe -m pip freeze" - - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py clean + - "%WITH_COMPILER% %PYTHON%/python.exe scripts/internal/winmake.py clean" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py build build_ext -i" - "%WITH_COMPILER% %PYTHON%/python.exe setup.py develop" From 06ded749e398c1dbec32548302d73b2832b1c5a2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 14 May 2017 13:46:53 +0200 Subject: [PATCH 668/922] fix #1042: psutil won't compile on FreeBSD 12; applied patch by @glebius --- CREDITS | 4 ++++ HISTORY.rst | 1 + psutil/arch/bsd/freebsd_socks.c | 31 +++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/CREDITS b/CREDITS index 4c2b6c703..a514d7ee1 100644 --- a/CREDITS +++ b/CREDITS @@ -472,3 +472,7 @@ I: 1036 N: Yannick Gingras W: https://github.com/ygingras I: 1057 + +N: Gleb Smirnoff +W: https://github.com/glebius +I: 1042 diff --git a/HISTORY.rst b/HISTORY.rst index a80d11231..2ada3621e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -41,6 +41,7 @@ Process.connections() when retrieving UNIX sockets (kind='unix'). - 1040_: fixed many unicode related issues such as UnicodeDecodeError on Python 3 + UNIX and invalid encoded data on Windows. +- 1042_: [FreeBSD] psutil won't compile on FreeBSD 12. - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. - 1050_: [Windows] Process.memory_maps memory() leaks memory. diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 54c09ab10..5c4a47df8 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -112,10 +112,17 @@ psutil_sockaddr_matches(int family, int port, void *pcb_addr, } +#if __FreeBSD_version >= 1200026 +static struct xtcpcb * +psutil_search_tcplist(char *buf, struct kinfo_file *kif) { + struct xtcpcb *tp; + struct xinpcb *inp; +#else static struct tcpcb * psutil_search_tcplist(char *buf, struct kinfo_file *kif) { struct tcpcb *tp; struct inpcb *inp; +#endif struct xinpgen *xig, *oxig; struct xsocket *so; @@ -123,9 +130,16 @@ psutil_search_tcplist(char *buf, struct kinfo_file *kif) { for (xig = (struct xinpgen *)((char *)xig + xig->xig_len); xig->xig_len > sizeof(struct xinpgen); xig = (struct xinpgen *)((char *)xig + xig->xig_len)) { + +#if __FreeBSD_version >= 1200026 + tp = (struct xtcpcb *)xig; + inp = &tp->xt_inp; + so = &inp->xi_socket; +#else tp = &((struct xtcpcb *)xig)->xt_tp; inp = &((struct xtcpcb *)xig)->xt_inp; so = &((struct xtcpcb *)xig)->xt_socket; +#endif if (so->so_type != kif->kf_sock_type || so->xso_family != kif->kf_sock_domain || @@ -207,7 +221,11 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { struct xinpgen *xig, *exig; struct xinpcb *xip; struct xtcpcb *xtp; +#if __FreeBSD_version >= 1200026 + struct xinpcb *inp; +#else struct inpcb *inp; +#endif struct xsocket *so; const char *varname = NULL; size_t len, bufsize; @@ -272,8 +290,13 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { goto error; } inp = &xtp->xt_inp; +#if __FreeBSD_version >= 1200026 + so = &inp->xi_socket; + status = xtp->t_state; +#else so = &xtp->xt_socket; status = xtp->xt_tp.t_state; +#endif break; case IPPROTO_UDP: xip = (struct xinpcb *)xig; @@ -282,7 +305,11 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { "struct xinpcb size mismatch"); goto error; } +#if __FreeBSD_version >= 1200026 + inp = xip; +#else inp = &xip->xi_inp; +#endif so = &xip->xi_socket; status = PSUTIL_CONN_NONE; break; @@ -483,7 +510,11 @@ psutil_proc_connections(PyObject *self, PyObject *args) { struct kinfo_file *freep = NULL; struct kinfo_file *kif; char *tcplist = NULL; +#if __FreeBSD_version >= 1200026 + struct xtcpcb *tcp; +#else struct tcpcb *tcp; +#endif PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; From fd2205bad09db0efab8958c359897fc1772dc187 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 14 May 2017 14:17:46 +0200 Subject: [PATCH 669/922] fix #1069 / freebsd: cpu_num() may return 255; now returns -1 --- HISTORY.rst | 2 ++ docs/index.rst | 1 + psutil/_psutil_bsd.c | 7 ++++--- psutil/tests/test_contracts.py | 4 +++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2ada3621e..eeb38bec7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -54,6 +54,8 @@ - 1064_: [NetBSD] swap_memory() may segfault in case of error. - 1065_: [OpenBSD] Process.cmdline() may raise SystemError. - 1067_: [NetBSD] Process.cmdline() leaks memory if proces has terminated. +- 1069_: [FreeBSD] Process.cpu_num() may return 255 for certain kernel + processes. **Porting notes** diff --git a/docs/index.rst b/docs/index.rst index 2bbcde7b0..4c0bb597e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1472,6 +1472,7 @@ Process class Return what CPU this process is currently running on. The returned number should be ``<=`` :func:`psutil.cpu_count()`. + On FreeBSD certain kernel process may return ``-1``. It may be used in conjunction with ``psutil.cpu_percent(percpu=True)`` to observe the system workload distributed across multiple CPUs as shown by `cpu_distribution.py `__ example script. diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 6478cc65c..03bdd94fb 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -197,7 +197,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { long memtext; long memdata; long memstack; - unsigned char oncpu; + int oncpu; kinfo_proc kp; long pagesize = sysconf(_SC_PAGESIZE); char str[1000]; @@ -252,6 +252,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { // what CPU we're on; top was used as an example: // https://svnweb.freebsd.org/base/head/usr.bin/top/machine.c? // view=markup&pathrev=273835 + // XXX - note: for "intr" PID this is -1. if (kp.ki_stat == SRUN && kp.ki_oncpu != NOCPU) oncpu = kp.ki_oncpu; else @@ -300,7 +301,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { memdata, // (long) mem data memstack, // (long) mem stack // others - oncpu, // (unsigned char) the CPU we are on + oncpu, // (int) the CPU we are on #elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) // (long)kp.p_ppid, // (long) ppid @@ -336,7 +337,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { memdata, // (long) mem data memstack, // (long) mem stack // others - oncpu, // (unsigned char) the CPU we are on + oncpu, // (int) the CPU we are on #endif py_name // (pystr) name ); diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 7666f2f83..7dd832597 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -468,10 +468,12 @@ def cpu_percent(self, ret, proc): def cpu_num(self, ret, proc): self.assertIsInstance(ret, int) + if FREEBSD and ret == -1: + return self.assertGreaterEqual(ret, 0) if psutil.cpu_count() == 1: self.assertEqual(ret, 0) - self.assertIn(ret, range(psutil.cpu_count())) + self.assertIn(ret, list(range(psutil.cpu_count()))) def memory_info(self, ret, proc): assert is_namedtuple(ret) From 2bf6c88fc3ed3b5ec9d67a9ab35d20bf2fd43d54 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 15 May 2017 20:09:52 +0200 Subject: [PATCH 670/922] add connections tests --- docs/index.rst | 3 +++ psutil/tests/test_connections.py | 46 +++++++++++++++++++++++++------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 4c0bb597e..d6488e348 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1836,6 +1836,9 @@ Process class pconn(fd=119, family=, type=, laddr=('10.0.0.1', 60759), raddr=('72.14.234.104', 80), status='ESTABLISHED'), pconn(fd=123, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT')] + .. note:: + (Solaris) UNIX sockets are not supported. + .. note:: (Linux, FreeBSD) "raddr" field for UNIX sockets is always set to "". This is a limitation of the OS. diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 906706a56..83bf1c539 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -31,6 +31,7 @@ from psutil.tests import bind_socket from psutil.tests import bind_unix_socket from psutil.tests import check_connection_ntuple +from psutil.tests import create_sockets from psutil.tests import get_free_port from psutil.tests import pyrun from psutil.tests import reap_children @@ -352,6 +353,30 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): # err self.assertRaises(ValueError, p.connections, kind='???') + def test_multi_sockets(self): + with create_sockets() as socks: + cons = thisproc.connections(kind='all') + self.assertEqual(len(socks), len(cons)) + cons = thisproc.connections(kind='tcp') + self.assertEqual(len(cons), 2) + cons = thisproc.connections(kind='tcp4') + self.assertEqual(len(cons), 1) + cons = thisproc.connections(kind='tcp6') + self.assertEqual(len(cons), 1) + cons = thisproc.connections(kind='udp') + self.assertEqual(len(cons), 2) + cons = thisproc.connections(kind='udp4') + self.assertEqual(len(cons), 1) + cons = thisproc.connections(kind='udp6') + self.assertEqual(len(cons), 1) + cons = thisproc.connections(kind='inet') + self.assertEqual(len(cons), 4) + cons = thisproc.connections(kind='inet6') + self.assertEqual(len(cons), 2) + if POSIX and not SUNOS: + cons = thisproc.connections(kind='unix') + self.assertEqual(len(cons), 3) + # ===================================================================== # --- Miscellaneous tests @@ -371,16 +396,17 @@ def check(cons, families, types_): self.assertIn(conn.type, types_, msg=conn) check_connection_ntuple(conn) - from psutil._common import conn_tmap - for kind, groups in conn_tmap.items(): - if SUNOS and kind == 'unix': - continue - families, types_ = groups - cons = psutil.net_connections(kind) - self.assertEqual(len(cons), len(set(cons))) - check(cons, families, types_) - - self.assertRaises(ValueError, psutil.net_connections, kind='???') + with create_sockets(): + from psutil._common import conn_tmap + for kind, groups in conn_tmap.items(): + if SUNOS and kind == 'unix': + continue + families, types_ = groups + cons = psutil.net_connections(kind) + self.assertEqual(len(cons), len(set(cons))) + check(cons, families, types_) + + self.assertRaises(ValueError, psutil.net_connections, kind='???') # ===================================================================== From e4bdc5932c12168720b45e5fe0abb14e8a38821f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 15 May 2017 20:17:40 +0200 Subject: [PATCH 671/922] add test case which exposes #1013 --- psutil/tests/test_connections.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 83bf1c539..8de7e17bb 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -408,6 +408,13 @@ def check(cons, families, types_): self.assertRaises(ValueError, psutil.net_connections, kind='???') + @skip_on_access_denied() + def test_multi_socks(self): + with create_sockets() as socks: + cons = [x for x in psutil.net_connections(kind='all') + if x.pid == os.getpid()] + self.assertEqual(len(socks), len(cons)) + # ===================================================================== # --- Miscellaneous tests From 029711d94d684cb92985666f91ecbd0889218e76 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 12:08:03 +0200 Subject: [PATCH 672/922] small refactoring --- psutil/_pslinux.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index eb7d11262..cf32bbd50 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1070,9 +1070,9 @@ def get_partitions(): raise ValueError("not sure how to interpret line %r" % line) if name in partitions: - sector_size = get_sector_size(name) - rbytes = rbytes * sector_size - wbytes = wbytes * sector_size + ssize = get_sector_size(name) + rbytes *= ssize + wbytes *= ssize retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime, reads_merged, writes_merged, busy_time) return retdict From ec3e2efea85b95c42ab3eb497337e6e597e0f9b6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 13:19:23 +0200 Subject: [PATCH 673/922] add another test case which exposes #1013 --- psutil/tests/test_connections.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 8de7e17bb..4ad68a4ec 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -415,6 +415,36 @@ def test_multi_socks(self): if x.pid == os.getpid()] self.assertEqual(len(socks), len(cons)) + def test_multi_sockets_procs(self): + # Creates multiple sub processes, each creating different + # sockets. For each process check that proc.connections() + # and net_connections() return the same results. + # This is done mainly to check whether net_connections()'s + # pid is properly set, see: + # https://github.com/giampaolo/psutil/issues/1013 + with create_sockets() as socks: + expected = len(socks) + pids = [] + src = textwrap.dedent("""\ + import time + from psutil.tests import create_sockets + with create_sockets(): + open('%s', 'w').close() + time.sleep(60) + """ % TESTFN) + for x in range(10): + sproc = pyrun(src) + wait_for_file(TESTFN, empty=True) + pids.append(sproc.pid) + + syscons = [x for x in psutil.net_connections(kind='all') if x.pid + in pids] + for pid in pids: + p = psutil.Process(pid) + self.assertEqual(len(p.connections('all')), expected) + self.assertEqual(len([x for x in syscons if x.pid == pid]), + expected) + # ===================================================================== # --- Miscellaneous tests From f0c00548fcc02c46a4f1f7e95a6d13deb97f6af5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 13:33:05 +0200 Subject: [PATCH 674/922] write more tests for connections() filtering --- psutil/tests/test_connections.py | 34 +++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 4ad68a4ec..cb9ccf60d 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -353,29 +353,61 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): # err self.assertRaises(ValueError, p.connections, kind='???') - def test_multi_sockets(self): + def test_multi_sockets_filtering(self): with create_sockets() as socks: cons = thisproc.connections(kind='all') self.assertEqual(len(socks), len(cons)) + # tcp cons = thisproc.connections(kind='tcp') self.assertEqual(len(cons), 2) + for conn in cons: + self.assertIn(conn.family, (AF_INET, AF_INET6)) + self.assertEqual(conn.type, SOCK_STREAM) + # tcp4 cons = thisproc.connections(kind='tcp4') self.assertEqual(len(cons), 1) + self.assertEqual(cons[0].family, AF_INET) + self.assertEqual(cons[0].type, SOCK_STREAM) + # tcp6 cons = thisproc.connections(kind='tcp6') self.assertEqual(len(cons), 1) + self.assertEqual(cons[0].family, AF_INET6) + self.assertEqual(cons[0].type, SOCK_STREAM) + # udp cons = thisproc.connections(kind='udp') self.assertEqual(len(cons), 2) + for conn in cons: + self.assertIn(conn.family, (AF_INET, AF_INET6)) + self.assertEqual(conn.type, SOCK_DGRAM) + # udp4 cons = thisproc.connections(kind='udp4') self.assertEqual(len(cons), 1) + self.assertEqual(cons[0].family, AF_INET) + self.assertEqual(cons[0].type, SOCK_DGRAM) + # udp6 cons = thisproc.connections(kind='udp6') self.assertEqual(len(cons), 1) + self.assertEqual(cons[0].family, AF_INET6) + self.assertEqual(cons[0].type, SOCK_DGRAM) + # inet cons = thisproc.connections(kind='inet') self.assertEqual(len(cons), 4) + for conn in cons: + self.assertIn(conn.family, (AF_INET, AF_INET6)) + self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) + # inet6 cons = thisproc.connections(kind='inet6') self.assertEqual(len(cons), 2) + for conn in cons: + self.assertEqual(conn.family, AF_INET6) + self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) + # unix if POSIX and not SUNOS: cons = thisproc.connections(kind='unix') self.assertEqual(len(cons), 3) + for conn in cons: + self.assertEqual(conn.family, AF_UNIX) + self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) # ===================================================================== From 5b6463868f824e566d98d35ce62a4ccc6e1e3c33 Mon Sep 17 00:00:00 2001 From: Gleb Smirnoff Date: Tue, 16 May 2017 04:49:13 -0700 Subject: [PATCH 675/922] Fix socket to PID translation on FreeBSD. (#1070) This file was derived from FreeBSD usr.bin/sockstat/sockstat.c. The logic of socket to PID translation was copied incorrectly. The hash, that sockstat(1) utility has, is completely internal feature, it isn't part of FreeBSD API, it is just to speed things up. So, to use this hash one actually needs to create it: declare array of buckets, populate it with sockets. In the freebsd_socks.c this wasn't done. This fix doesn't create the hash, instead it removes remnants of hashing that was there in sockstat.c. It makes code more simple, but of course slower than original sockstat(1) in case if machine is running zillions of sockets. I decided to go this way simply because I am low on time to invest into psutil, and also because better first provide correct and simple implementation and then improve it, rather than jump for complexity. --- psutil/arch/bsd/freebsd_socks.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 5c4a47df8..d60dc898f 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -28,7 +28,6 @@ #include "../../_psutil_common.h" #include "../../_psutil_posix.h" -#define HASHSIZE 1009 // a signaler for connections without an actual status static int PSUTIL_CONN_NONE = 128; static struct xfile *psutil_xfiles; @@ -201,14 +200,12 @@ psutil_populate_xfiles() { int -psutil_get_pid_from_sock(int sock_hash) { +psutil_get_pid_from_sock(void *sock) { struct xfile *xf; - int hash, n; + int n; + for (xf = psutil_xfiles, n = 0; n < psutil_nxfiles; ++n, ++xf) { - if (xf->xf_data == NULL) - continue; - hash = (int)((uintptr_t)xf->xf_data % HASHSIZE); - if (sock_hash == hash) + if (xf->xf_data == sock) return xf->xf_pid; } return -1; @@ -230,7 +227,6 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { const char *varname = NULL; size_t len, bufsize; void *buf; - int hash; int retry; int type; @@ -320,8 +316,7 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { char lip[200], rip[200]; - hash = (int)((uintptr_t)so->xso_so % HASHSIZE); - pid = psutil_get_pid_from_sock(hash); + pid = psutil_get_pid_from_sock(so->xso_so); if (pid < 0) continue; lport = ntohs(inp->inp_lport); @@ -377,7 +372,6 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { size_t len; size_t bufsize; void *buf; - int hash; int retry; int pid; struct sockaddr_un *sun; @@ -434,8 +428,7 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { if (xup->xu_len != sizeof *xup) goto error; - hash = (int)((uintptr_t) xup->xu_socket.xso_so % HASHSIZE); - pid = psutil_get_pid_from_sock(hash); + pid = psutil_get_pid_from_sock(xup->xu_socket.xso_so); if (pid < 0) continue; From 0bcd10144db472682b2bb0c0b3cba3e193094a0b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 13:53:03 +0200 Subject: [PATCH 676/922] give CREDITS to @glebius --- CREDITS | 2 ++ HISTORY.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/CREDITS b/CREDITS index a514d7ee1..515ba38d9 100644 --- a/CREDITS +++ b/CREDITS @@ -30,6 +30,8 @@ Github usernames of people to CC on github when in need of help. - ryoqun, Ryo Onodera - OpenBSD: - landryb, Landry Breuil +- FreeNBSD: + - glebius, Gleb Smirnoff (#1013) - OSX: - whitlockjc, Jeremy Whitlock - Windows: diff --git a/HISTORY.rst b/HISTORY.rst index eeb38bec7..33f8db8ec 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,6 +28,8 @@ - 1007_: [Windows] boot_time() can have a 1 sec fluctuation between calls; the value of the first call is now cached so that boot_time() always returns the same value if fluctuation is <= 1 second. +- 1013_: [FreeBSD] psutil.net_connections() may return incorrect PID. (patch + by Gleb Smirnoff) - 1014_: [Linux] Process class can mask legitimate ENOENT exceptions as NoSuchProcess. - 1016_: disk_io_counters() raises RuntimeError on a system with no disks. From 25e0672735e0465f905105d7ad865a3069b0c637 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 15:05:52 +0200 Subject: [PATCH 677/922] style --- psutil/arch/bsd/freebsd_socks.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index d60dc898f..8fcb45e5e 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -343,8 +343,15 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { py_raddr = Py_BuildValue("()"); if (!py_raddr) goto error; - py_tuple = Py_BuildValue("(iiiNNii)", -1, family, type, py_laddr, - py_raddr, status, pid); + py_tuple = Py_BuildValue( + "(iiiNNii)", + -1, // fd + family, // family + type, // type + py_laddr, // laddr + py_raddr, // raddr + status, // status + pid); // pid if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) From ee05d1851b77846d408df0ef64e5d0db41939a45 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 15:42:35 +0200 Subject: [PATCH 678/922] ignore me --- psutil/tests/test_connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index cb9ccf60d..4ead7d164 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -445,7 +445,7 @@ def test_multi_socks(self): with create_sockets() as socks: cons = [x for x in psutil.net_connections(kind='all') if x.pid == os.getpid()] - self.assertEqual(len(socks), len(cons)) + self.assertEqual(len(cons), len(socks)) def test_multi_sockets_procs(self): # Creates multiple sub processes, each creating different From 82760460d2f7f5dab3374b5e95a035cf69880ba8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 16:58:48 +0200 Subject: [PATCH 679/922] BSD: mv arch C files (#1072) * move freebsd C arch files * rename C file * move openbsd C arch files * move netbsd C arch files --- psutil/_psutil_bsd.c | 10 +++++----- psutil/arch/{bsd/freebsd_socks.c => freebsd/socks.c} | 0 psutil/arch/{bsd/freebsd_socks.h => freebsd/socks.h} | 0 psutil/arch/{bsd/freebsd.c => freebsd/specific.c} | 2 +- psutil/arch/{bsd/freebsd.h => freebsd/specific.h} | 0 psutil/arch/{bsd/netbsd_socks.c => netbsd/socks.c} | 0 psutil/arch/{bsd/netbsd_socks.h => netbsd/socks.h} | 0 psutil/arch/{bsd/netbsd.c => netbsd/specific.c} | 4 +--- psutil/arch/{bsd/netbsd.h => netbsd/specific.h} | 0 psutil/arch/{bsd/openbsd.c => openbsd/specific.c} | 0 psutil/arch/{bsd/openbsd.h => openbsd/specific.h} | 0 setup.py | 10 +++++----- 12 files changed, 12 insertions(+), 14 deletions(-) rename psutil/arch/{bsd/freebsd_socks.c => freebsd/socks.c} (100%) rename psutil/arch/{bsd/freebsd_socks.h => freebsd/socks.h} (100%) rename psutil/arch/{bsd/freebsd.c => freebsd/specific.c} (99%) rename psutil/arch/{bsd/freebsd.h => freebsd/specific.h} (100%) rename psutil/arch/{bsd/netbsd_socks.c => netbsd/socks.c} (100%) rename psutil/arch/{bsd/netbsd_socks.h => netbsd/socks.h} (100%) rename psutil/arch/{bsd/netbsd.c => netbsd/specific.c} (99%) rename psutil/arch/{bsd/netbsd.h => netbsd/specific.h} (100%) rename psutil/arch/{bsd/openbsd.c => openbsd/specific.c} (100%) rename psutil/arch/{bsd/openbsd.h => openbsd/specific.h} (100%) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 03bdd94fb..506666b4d 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -63,8 +63,8 @@ #include "_psutil_posix.h" #ifdef PSUTIL_FREEBSD - #include "arch/bsd/freebsd.h" - #include "arch/bsd/freebsd_socks.h" + #include "arch/freebsd/specific.h" + #include "arch/freebsd/socks.h" #include #include // get io counters @@ -75,7 +75,7 @@ #include #endif #elif PSUTIL_OPENBSD - #include "arch/bsd/openbsd.h" + #include "arch/openbsd/specific.h" #include #include // for VREG @@ -84,8 +84,8 @@ #undef _KERNEL #include // for CPUSTATES & CP_* #elif PSUTIL_NETBSD - #include "arch/bsd/netbsd.h" - #include "arch/bsd/netbsd_socks.h" + #include "arch/netbsd/specific.h" + #include "arch/netbsd/socks.h" #include #include // for VREG diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/freebsd/socks.c similarity index 100% rename from psutil/arch/bsd/freebsd_socks.c rename to psutil/arch/freebsd/socks.c diff --git a/psutil/arch/bsd/freebsd_socks.h b/psutil/arch/freebsd/socks.h similarity index 100% rename from psutil/arch/bsd/freebsd_socks.h rename to psutil/arch/freebsd/socks.h diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/freebsd/specific.c similarity index 99% rename from psutil/arch/bsd/freebsd.c rename to psutil/arch/freebsd/specific.c index 98751c6d9..46bf38688 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/freebsd/specific.c @@ -3,7 +3,7 @@ * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. * - * Helper functions related to fetching process information. + * Helper functions specific to FreeBSD. * Used by _psutil_bsd module methods. */ diff --git a/psutil/arch/bsd/freebsd.h b/psutil/arch/freebsd/specific.h similarity index 100% rename from psutil/arch/bsd/freebsd.h rename to psutil/arch/freebsd/specific.h diff --git a/psutil/arch/bsd/netbsd_socks.c b/psutil/arch/netbsd/socks.c similarity index 100% rename from psutil/arch/bsd/netbsd_socks.c rename to psutil/arch/netbsd/socks.c diff --git a/psutil/arch/bsd/netbsd_socks.h b/psutil/arch/netbsd/socks.h similarity index 100% rename from psutil/arch/bsd/netbsd_socks.h rename to psutil/arch/netbsd/socks.h diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/netbsd/specific.c similarity index 99% rename from psutil/arch/bsd/netbsd.c rename to psutil/arch/netbsd/specific.c index 972418ff3..1dc2080ee 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/netbsd/specific.c @@ -38,9 +38,7 @@ #include #include - -#include "netbsd_socks.h" -#include "netbsd.h" +#include "specific.h" #include "../../_psutil_common.h" #include "../../_psutil_posix.h" diff --git a/psutil/arch/bsd/netbsd.h b/psutil/arch/netbsd/specific.h similarity index 100% rename from psutil/arch/bsd/netbsd.h rename to psutil/arch/netbsd/specific.h diff --git a/psutil/arch/bsd/openbsd.c b/psutil/arch/openbsd/specific.c similarity index 100% rename from psutil/arch/bsd/openbsd.c rename to psutil/arch/openbsd/specific.c diff --git a/psutil/arch/bsd/openbsd.h b/psutil/arch/openbsd/specific.h similarity index 100% rename from psutil/arch/bsd/openbsd.h rename to psutil/arch/openbsd/specific.h diff --git a/setup.py b/setup.py index d51b82feb..af96cf4b6 100755 --- a/setup.py +++ b/setup.py @@ -168,8 +168,8 @@ def get_winver(): 'psutil._psutil_bsd', sources=sources + [ 'psutil/_psutil_bsd.c', - 'psutil/arch/bsd/freebsd.c', - 'psutil/arch/bsd/freebsd_socks.c', + 'psutil/arch/freebsd/specific.c', + 'psutil/arch/freebsd/socks.c', ], define_macros=macros, libraries=["devstat"]) @@ -180,7 +180,7 @@ def get_winver(): 'psutil._psutil_bsd', sources=sources + [ 'psutil/_psutil_bsd.c', - 'psutil/arch/bsd/openbsd.c', + 'psutil/arch/openbsd/specific.c', ], define_macros=macros, libraries=["kvm"]) @@ -191,8 +191,8 @@ def get_winver(): 'psutil._psutil_bsd', sources=sources + [ 'psutil/_psutil_bsd.c', - 'psutil/arch/bsd/netbsd.c', - 'psutil/arch/bsd/netbsd_socks.c', + 'psutil/arch/netbsd/specific.c', + 'psutil/arch/netbsd/socks.c', ], define_macros=macros, libraries=["kvm"]) From 59749a716a18678a8296ce346652cf9bc8f75842 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 17:32:09 +0200 Subject: [PATCH 680/922] Freebsd socks refactoring (#1073) * freebsd: split socks.c into 2 separate files * remove unused h files * freebsd: split socks.c into 2 separate files --- psutil/_psutil_bsd.c | 3 +- psutil/arch/freebsd/{socks.c => proc_socks.c} | 347 +---------------- psutil/arch/freebsd/proc_socks.h | 9 + psutil/arch/freebsd/sys_socks.c | 362 ++++++++++++++++++ psutil/arch/freebsd/{socks.h => sys_socks.h} | 1 - setup.py | 3 +- 6 files changed, 380 insertions(+), 345 deletions(-) rename psutil/arch/freebsd/{socks.c => proc_socks.c} (51%) create mode 100644 psutil/arch/freebsd/proc_socks.h create mode 100644 psutil/arch/freebsd/sys_socks.c rename psutil/arch/freebsd/{socks.h => sys_socks.h} (79%) diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 506666b4d..217a95de5 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -64,7 +64,8 @@ #ifdef PSUTIL_FREEBSD #include "arch/freebsd/specific.h" - #include "arch/freebsd/socks.h" + #include "arch/freebsd/sys_socks.h" + #include "arch/freebsd/proc_socks.h" #include #include // get io counters diff --git a/psutil/arch/freebsd/socks.c b/psutil/arch/freebsd/proc_socks.c similarity index 51% rename from psutil/arch/freebsd/socks.c rename to psutil/arch/freebsd/proc_socks.c index 8fcb45e5e..de4142be4 100644 --- a/psutil/arch/freebsd/socks.c +++ b/psutil/arch/freebsd/proc_socks.c @@ -3,35 +3,27 @@ * All rights reserved. * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. + * + * Retrieves per-process open socket connections. */ #include #include -#include #include // for struct xsocket #include -#include -#include #include #include // for xinpcb struct -#include -#include #include -#include -#include #include // for struct xtcpcb -#include // for TCP connection states #include // for inet_ntop() -#include #include #include "../../_psutil_common.h" #include "../../_psutil_posix.h" + // a signaler for connections without an actual status static int PSUTIL_CONN_NONE = 128; -static struct xfile *psutil_xfiles; -static int psutil_nxfiles; // The tcplist fetching and walking is borrowed from netstat/inet.c. @@ -171,342 +163,13 @@ psutil_search_tcplist(char *buf, struct kinfo_file *kif) { } -int -psutil_populate_xfiles() { - size_t len; - - if ((psutil_xfiles = malloc(len = sizeof *psutil_xfiles)) == NULL) { - PyErr_NoMemory(); - return 0; - } - while (sysctlbyname("kern.file", psutil_xfiles, &len, 0, 0) == -1) { - if (errno != ENOMEM) { - PyErr_SetFromErrno(0); - return 0; - } - len *= 2; - if ((psutil_xfiles = realloc(psutil_xfiles, len)) == NULL) { - PyErr_NoMemory(); - return 0; - } - } - if (len > 0 && psutil_xfiles->xf_size != sizeof *psutil_xfiles) { - PyErr_Format(PyExc_RuntimeError, "struct xfile size mismatch"); - return 0; - } - psutil_nxfiles = len / sizeof *psutil_xfiles; - return 1; -} - - -int -psutil_get_pid_from_sock(void *sock) { - struct xfile *xf; - int n; - - for (xf = psutil_xfiles, n = 0; n < psutil_nxfiles; ++n, ++xf) { - if (xf->xf_data == sock) - return xf->xf_pid; - } - return -1; -} - - -// Reference: -// https://github.com/freebsd/freebsd/blob/master/usr.bin/sockstat/sockstat.c -int psutil_gather_inet(int proto, PyObject *py_retlist) { - struct xinpgen *xig, *exig; - struct xinpcb *xip; - struct xtcpcb *xtp; -#if __FreeBSD_version >= 1200026 - struct xinpcb *inp; -#else - struct inpcb *inp; -#endif - struct xsocket *so; - const char *varname = NULL; - size_t len, bufsize; - void *buf; - int retry; - int type; - - PyObject *py_tuple = NULL; - PyObject *py_laddr = NULL; - PyObject *py_raddr = NULL; - - switch (proto) { - case IPPROTO_TCP: - varname = "net.inet.tcp.pcblist"; - type = SOCK_STREAM; - break; - case IPPROTO_UDP: - varname = "net.inet.udp.pcblist"; - type = SOCK_DGRAM; - break; - } - - buf = NULL; - bufsize = 8192; - retry = 5; - do { - for (;;) { - buf = realloc(buf, bufsize); - if (buf == NULL) - continue; // XXX - len = bufsize; - if (sysctlbyname(varname, buf, &len, NULL, 0) == 0) - break; - if (errno != ENOMEM) { - PyErr_SetFromErrno(0); - goto error; - } - bufsize *= 2; - } - xig = (struct xinpgen *)buf; - exig = (struct xinpgen *)(void *)((char *)buf + len - sizeof *exig); - if (xig->xig_len != sizeof *xig || exig->xig_len != sizeof *exig) { - PyErr_Format(PyExc_RuntimeError, "struct xinpgen size mismatch"); - goto error; - } - } while (xig->xig_gen != exig->xig_gen && retry--); - - for (;;) { - int lport, rport, pid, status, family; - - xig = (struct xinpgen *)(void *)((char *)xig + xig->xig_len); - if (xig >= exig) - break; - - switch (proto) { - case IPPROTO_TCP: - xtp = (struct xtcpcb *)xig; - if (xtp->xt_len != sizeof *xtp) { - PyErr_Format(PyExc_RuntimeError, - "struct xtcpcb size mismatch"); - goto error; - } - inp = &xtp->xt_inp; -#if __FreeBSD_version >= 1200026 - so = &inp->xi_socket; - status = xtp->t_state; -#else - so = &xtp->xt_socket; - status = xtp->xt_tp.t_state; -#endif - break; - case IPPROTO_UDP: - xip = (struct xinpcb *)xig; - if (xip->xi_len != sizeof *xip) { - PyErr_Format(PyExc_RuntimeError, - "struct xinpcb size mismatch"); - goto error; - } -#if __FreeBSD_version >= 1200026 - inp = xip; -#else - inp = &xip->xi_inp; -#endif - so = &xip->xi_socket; - status = PSUTIL_CONN_NONE; - break; - default: - PyErr_Format(PyExc_RuntimeError, "invalid proto"); - goto error; - } - - char lip[200], rip[200]; - - pid = psutil_get_pid_from_sock(so->xso_so); - if (pid < 0) - continue; - lport = ntohs(inp->inp_lport); - rport = ntohs(inp->inp_fport); - - if (inp->inp_vflag & INP_IPV4) { - family = AF_INET; - inet_ntop(AF_INET, &inp->inp_laddr.s_addr, lip, sizeof(lip)); - inet_ntop(AF_INET, &inp->inp_faddr.s_addr, rip, sizeof(rip)); - } - else if (inp->inp_vflag & INP_IPV6) { - family = AF_INET6; - inet_ntop(AF_INET6, &inp->in6p_laddr.s6_addr, lip, sizeof(lip)); - inet_ntop(AF_INET6, &inp->in6p_faddr.s6_addr, rip, sizeof(rip)); - } - - // construct python tuple/list - py_laddr = Py_BuildValue("(si)", lip, lport); - if (!py_laddr) - goto error; - if (rport != 0) - py_raddr = Py_BuildValue("(si)", rip, rport); - else - py_raddr = Py_BuildValue("()"); - if (!py_raddr) - goto error; - py_tuple = Py_BuildValue( - "(iiiNNii)", - -1, // fd - family, // family - type, // type - py_laddr, // laddr - py_raddr, // raddr - status, // status - pid); // pid - if (!py_tuple) - goto error; - if (PyList_Append(py_retlist, py_tuple)) - goto error; - Py_DECREF(py_tuple); - } - - free(buf); - return 1; - -error: - Py_XDECREF(py_tuple); - Py_XDECREF(py_laddr); - Py_XDECREF(py_raddr); - free(buf); - return 0; -} - - -int psutil_gather_unix(int proto, PyObject *py_retlist) { - struct xunpgen *xug, *exug; - struct xunpcb *xup; - const char *varname = NULL; - const char *protoname = NULL; - size_t len; - size_t bufsize; - void *buf; - int retry; - int pid; - struct sockaddr_un *sun; - char path[PATH_MAX]; - - PyObject *py_tuple = NULL; - PyObject *py_lpath = NULL; - - switch (proto) { - case SOCK_STREAM: - varname = "net.local.stream.pcblist"; - protoname = "stream"; - break; - case SOCK_DGRAM: - varname = "net.local.dgram.pcblist"; - protoname = "dgram"; - break; - } - - buf = NULL; - bufsize = 8192; - retry = 5; - - do { - for (;;) { - buf = realloc(buf, bufsize); - if (buf == NULL) { - PyErr_NoMemory(); - goto error; - } - len = bufsize; - if (sysctlbyname(varname, buf, &len, NULL, 0) == 0) - break; - if (errno != ENOMEM) { - PyErr_SetFromErrno(0); - goto error; - } - bufsize *= 2; - } - xug = (struct xunpgen *)buf; - exug = (struct xunpgen *)(void *) - ((char *)buf + len - sizeof *exug); - if (xug->xug_len != sizeof *xug || exug->xug_len != sizeof *exug) { - PyErr_Format(PyExc_RuntimeError, "struct xinpgen size mismatch"); - goto error; - } - } while (xug->xug_gen != exug->xug_gen && retry--); - - for (;;) { - xug = (struct xunpgen *)(void *)((char *)xug + xug->xug_len); - if (xug >= exug) - break; - xup = (struct xunpcb *)xug; - if (xup->xu_len != sizeof *xup) - goto error; - - pid = psutil_get_pid_from_sock(xup->xu_socket.xso_so); - if (pid < 0) - continue; - - sun = (struct sockaddr_un *)&xup->xu_addr; - snprintf(path, sizeof(path), "%.*s", - (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), - sun->sun_path); - py_lpath = PyUnicode_DecodeFSDefault(path); - if (! py_lpath) - goto error; - - py_tuple = Py_BuildValue("(iiiOsii)", - -1, // fd - AF_UNIX, // family - proto, // type - py_lpath, // lpath - "", // rath - PSUTIL_CONN_NONE, // status - pid); // pid - if (!py_tuple) - goto error; - if (PyList_Append(py_retlist, py_tuple)) - goto error; - Py_DECREF(py_lpath); - Py_DECREF(py_tuple); - } - - free(buf); - return 1; - -error: - Py_XDECREF(py_tuple); - Py_XDECREF(py_lpath); - free(buf); - return 0; -} - - -PyObject* -psutil_net_connections(PyObject* self, PyObject* args) { - // Return system-wide open connections. - PyObject *py_retlist = PyList_New(0); - - if (py_retlist == NULL) - return NULL; - if (psutil_populate_xfiles() != 1) - goto error; - if (psutil_gather_inet(IPPROTO_TCP, py_retlist) == 0) - goto error; - if (psutil_gather_inet(IPPROTO_UDP, py_retlist) == 0) - goto error; - if (psutil_gather_unix(SOCK_STREAM, py_retlist) == 0) - goto error; - if (psutil_gather_unix(SOCK_DGRAM, py_retlist) == 0) - goto error; - - free(psutil_xfiles); - return py_retlist; - -error: - Py_DECREF(py_retlist); - free(psutil_xfiles); - return NULL; -} - PyObject * psutil_proc_connections(PyObject *self, PyObject *args) { // Return connections opened by process. long pid; - int i, cnt; + int i; + int cnt; struct kinfo_file *freep = NULL; struct kinfo_file *kif; char *tcplist = NULL; diff --git a/psutil/arch/freebsd/proc_socks.h b/psutil/arch/freebsd/proc_socks.h new file mode 100644 index 000000000..a7996b107 --- /dev/null +++ b/psutil/arch/freebsd/proc_socks.h @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include + +PyObject* psutil_proc_connections(PyObject* self, PyObject* args); diff --git a/psutil/arch/freebsd/sys_socks.c b/psutil/arch/freebsd/sys_socks.c new file mode 100644 index 000000000..4104c27d4 --- /dev/null +++ b/psutil/arch/freebsd/sys_socks.c @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola'. + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Retrieves system-wide open socket connections. This is based off of + * sockstat utility source code: + * https://github.com/freebsd/freebsd/blob/master/usr.bin/sockstat/sockstat.c + */ + +#include +#include +#include +#include // for struct xsocket +#include +#include +#include +#include // for xinpcb struct +#include +#include +#include // for struct xtcpcb +#include // for inet_ntop() + +#include "../../_psutil_common.h" +#include "../../_psutil_posix.h" + +// a signaler for connections without an actual status +static int PSUTIL_CONN_NONE = 128; +static struct xfile *psutil_xfiles; +static int psutil_nxfiles; + + +int +psutil_populate_xfiles() { + size_t len; + + if ((psutil_xfiles = malloc(len = sizeof *psutil_xfiles)) == NULL) { + PyErr_NoMemory(); + return 0; + } + while (sysctlbyname("kern.file", psutil_xfiles, &len, 0, 0) == -1) { + if (errno != ENOMEM) { + PyErr_SetFromErrno(0); + return 0; + } + len *= 2; + if ((psutil_xfiles = realloc(psutil_xfiles, len)) == NULL) { + PyErr_NoMemory(); + return 0; + } + } + if (len > 0 && psutil_xfiles->xf_size != sizeof *psutil_xfiles) { + PyErr_Format(PyExc_RuntimeError, "struct xfile size mismatch"); + return 0; + } + psutil_nxfiles = len / sizeof *psutil_xfiles; + return 1; +} + + +int +psutil_get_pid_from_sock(void *sock) { + struct xfile *xf; + int n; + + for (xf = psutil_xfiles, n = 0; n < psutil_nxfiles; ++n, ++xf) { + if (xf->xf_data == sock) + return xf->xf_pid; + } + return -1; +} + + +// Reference: +// https://github.com/freebsd/freebsd/blob/master/usr.bin/sockstat/sockstat.c +int psutil_gather_inet(int proto, PyObject *py_retlist) { + struct xinpgen *xig, *exig; + struct xinpcb *xip; + struct xtcpcb *xtp; +#if __FreeBSD_version >= 1200026 + struct xinpcb *inp; +#else + struct inpcb *inp; +#endif + struct xsocket *so; + const char *varname = NULL; + size_t len, bufsize; + void *buf; + int retry; + int type; + + PyObject *py_tuple = NULL; + PyObject *py_laddr = NULL; + PyObject *py_raddr = NULL; + + switch (proto) { + case IPPROTO_TCP: + varname = "net.inet.tcp.pcblist"; + type = SOCK_STREAM; + break; + case IPPROTO_UDP: + varname = "net.inet.udp.pcblist"; + type = SOCK_DGRAM; + break; + } + + buf = NULL; + bufsize = 8192; + retry = 5; + do { + for (;;) { + buf = realloc(buf, bufsize); + if (buf == NULL) + continue; // XXX + len = bufsize; + if (sysctlbyname(varname, buf, &len, NULL, 0) == 0) + break; + if (errno != ENOMEM) { + PyErr_SetFromErrno(0); + goto error; + } + bufsize *= 2; + } + xig = (struct xinpgen *)buf; + exig = (struct xinpgen *)(void *)((char *)buf + len - sizeof *exig); + if (xig->xig_len != sizeof *xig || exig->xig_len != sizeof *exig) { + PyErr_Format(PyExc_RuntimeError, "struct xinpgen size mismatch"); + goto error; + } + } while (xig->xig_gen != exig->xig_gen && retry--); + + for (;;) { + int lport, rport, pid, status, family; + + xig = (struct xinpgen *)(void *)((char *)xig + xig->xig_len); + if (xig >= exig) + break; + + switch (proto) { + case IPPROTO_TCP: + xtp = (struct xtcpcb *)xig; + if (xtp->xt_len != sizeof *xtp) { + PyErr_Format(PyExc_RuntimeError, + "struct xtcpcb size mismatch"); + goto error; + } + inp = &xtp->xt_inp; +#if __FreeBSD_version >= 1200026 + so = &inp->xi_socket; + status = xtp->t_state; +#else + so = &xtp->xt_socket; + status = xtp->xt_tp.t_state; +#endif + break; + case IPPROTO_UDP: + xip = (struct xinpcb *)xig; + if (xip->xi_len != sizeof *xip) { + PyErr_Format(PyExc_RuntimeError, + "struct xinpcb size mismatch"); + goto error; + } +#if __FreeBSD_version >= 1200026 + inp = xip; +#else + inp = &xip->xi_inp; +#endif + so = &xip->xi_socket; + status = PSUTIL_CONN_NONE; + break; + default: + PyErr_Format(PyExc_RuntimeError, "invalid proto"); + goto error; + } + + char lip[200], rip[200]; + + pid = psutil_get_pid_from_sock(so->xso_so); + if (pid < 0) + continue; + lport = ntohs(inp->inp_lport); + rport = ntohs(inp->inp_fport); + + if (inp->inp_vflag & INP_IPV4) { + family = AF_INET; + inet_ntop(AF_INET, &inp->inp_laddr.s_addr, lip, sizeof(lip)); + inet_ntop(AF_INET, &inp->inp_faddr.s_addr, rip, sizeof(rip)); + } + else if (inp->inp_vflag & INP_IPV6) { + family = AF_INET6; + inet_ntop(AF_INET6, &inp->in6p_laddr.s6_addr, lip, sizeof(lip)); + inet_ntop(AF_INET6, &inp->in6p_faddr.s6_addr, rip, sizeof(rip)); + } + + // construct python tuple/list + py_laddr = Py_BuildValue("(si)", lip, lport); + if (!py_laddr) + goto error; + if (rport != 0) + py_raddr = Py_BuildValue("(si)", rip, rport); + else + py_raddr = Py_BuildValue("()"); + if (!py_raddr) + goto error; + py_tuple = Py_BuildValue( + "(iiiNNii)", + -1, // fd + family, // family + type, // type + py_laddr, // laddr + py_raddr, // raddr + status, // status + pid); // pid + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + + free(buf); + return 1; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_laddr); + Py_XDECREF(py_raddr); + free(buf); + return 0; +} + + +int psutil_gather_unix(int proto, PyObject *py_retlist) { + struct xunpgen *xug, *exug; + struct xunpcb *xup; + const char *varname = NULL; + const char *protoname = NULL; + size_t len; + size_t bufsize; + void *buf; + int retry; + int pid; + struct sockaddr_un *sun; + char path[PATH_MAX]; + + PyObject *py_tuple = NULL; + PyObject *py_lpath = NULL; + + switch (proto) { + case SOCK_STREAM: + varname = "net.local.stream.pcblist"; + protoname = "stream"; + break; + case SOCK_DGRAM: + varname = "net.local.dgram.pcblist"; + protoname = "dgram"; + break; + } + + buf = NULL; + bufsize = 8192; + retry = 5; + + do { + for (;;) { + buf = realloc(buf, bufsize); + if (buf == NULL) { + PyErr_NoMemory(); + goto error; + } + len = bufsize; + if (sysctlbyname(varname, buf, &len, NULL, 0) == 0) + break; + if (errno != ENOMEM) { + PyErr_SetFromErrno(0); + goto error; + } + bufsize *= 2; + } + xug = (struct xunpgen *)buf; + exug = (struct xunpgen *)(void *) + ((char *)buf + len - sizeof *exug); + if (xug->xug_len != sizeof *xug || exug->xug_len != sizeof *exug) { + PyErr_Format(PyExc_RuntimeError, "struct xinpgen size mismatch"); + goto error; + } + } while (xug->xug_gen != exug->xug_gen && retry--); + + for (;;) { + xug = (struct xunpgen *)(void *)((char *)xug + xug->xug_len); + if (xug >= exug) + break; + xup = (struct xunpcb *)xug; + if (xup->xu_len != sizeof *xup) + goto error; + + pid = psutil_get_pid_from_sock(xup->xu_socket.xso_so); + if (pid < 0) + continue; + + sun = (struct sockaddr_un *)&xup->xu_addr; + snprintf(path, sizeof(path), "%.*s", + (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), + sun->sun_path); + py_lpath = PyUnicode_DecodeFSDefault(path); + if (! py_lpath) + goto error; + + py_tuple = Py_BuildValue("(iiiOsii)", + -1, // fd + AF_UNIX, // family + proto, // type + py_lpath, // lpath + "", // rath + PSUTIL_CONN_NONE, // status + pid); // pid + if (!py_tuple) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_lpath); + Py_DECREF(py_tuple); + } + + free(buf); + return 1; + +error: + Py_XDECREF(py_tuple); + Py_XDECREF(py_lpath); + free(buf); + return 0; +} + + +PyObject* +psutil_net_connections(PyObject* self, PyObject* args) { + // Return system-wide open connections. + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + if (psutil_populate_xfiles() != 1) + goto error; + if (psutil_gather_inet(IPPROTO_TCP, py_retlist) == 0) + goto error; + if (psutil_gather_inet(IPPROTO_UDP, py_retlist) == 0) + goto error; + if (psutil_gather_unix(SOCK_STREAM, py_retlist) == 0) + goto error; + if (psutil_gather_unix(SOCK_DGRAM, py_retlist) == 0) + goto error; + + free(psutil_xfiles); + return py_retlist; + +error: + Py_DECREF(py_retlist); + free(psutil_xfiles); + return NULL; +} diff --git a/psutil/arch/freebsd/socks.h b/psutil/arch/freebsd/sys_socks.h similarity index 79% rename from psutil/arch/freebsd/socks.h rename to psutil/arch/freebsd/sys_socks.h index 15ccb0b3b..752479265 100644 --- a/psutil/arch/freebsd/socks.h +++ b/psutil/arch/freebsd/sys_socks.h @@ -7,5 +7,4 @@ #include -PyObject* psutil_proc_connections(PyObject* self, PyObject* args); PyObject* psutil_net_connections(PyObject* self, PyObject* args); diff --git a/setup.py b/setup.py index af96cf4b6..55d86b47a 100755 --- a/setup.py +++ b/setup.py @@ -169,7 +169,8 @@ def get_winver(): sources=sources + [ 'psutil/_psutil_bsd.c', 'psutil/arch/freebsd/specific.c', - 'psutil/arch/freebsd/socks.c', + 'psutil/arch/freebsd/sys_socks.c', + 'psutil/arch/freebsd/proc_socks.c', ], define_macros=macros, libraries=["devstat"]) From 96e1881c535dc7518cb4dc211a0c26aee5ad2b1c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 17:40:48 +0200 Subject: [PATCH 681/922] declare PSUTIL_CONN_NONE as a static shared constant --- psutil/_psutil_common.h | 3 +++ psutil/_psutil_osx.c | 3 --- psutil/_psutil_sunos.c | 2 -- psutil/_psutil_windows.c | 3 --- psutil/arch/freebsd/proc_socks.c | 4 ---- psutil/arch/freebsd/sys_socks.c | 2 -- psutil/arch/netbsd/socks.c | 2 -- psutil/arch/openbsd/specific.c | 3 --- 8 files changed, 3 insertions(+), 19 deletions(-) diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 0507458aa..aa634ad37 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -6,6 +6,9 @@ #include +// a signaler for connections without an actual status +static const int PSUTIL_CONN_NONE = 128; + PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); #if PY_MAJOR_VERSION < 3 diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index a831441a4..20ece694b 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1184,9 +1184,6 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { } -// a signaler for connections without an actual status -static int PSUTIL_CONN_NONE = 128; - /* * Return process TCP and UDP connections as a list of tuples. * References: diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 422d48c7b..6c152eed5 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -49,8 +49,6 @@ #ifndef EXPER_IP_AND_ALL_IRES #define EXPER_IP_AND_ALL_IRES (1024+4) #endif -// a signaler for connections without an actual status -static int PSUTIL_CONN_NONE = 128; /* diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index a3e921c02..2772cab31 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -48,9 +48,6 @@ * ============================================================================ */ - // a flag for connections without an actual status -static int PSUTIL_CONN_NONE = 128; - #define MALLOC(x) HeapAlloc(GetProcessHeap(), 0, (x)) #define FREE(x) HeapFree(GetProcessHeap(), 0, (x)) #define LO_T ((float)1e-7) diff --git a/psutil/arch/freebsd/proc_socks.c b/psutil/arch/freebsd/proc_socks.c index de4142be4..9b03e0591 100644 --- a/psutil/arch/freebsd/proc_socks.c +++ b/psutil/arch/freebsd/proc_socks.c @@ -22,10 +22,6 @@ #include "../../_psutil_posix.h" -// a signaler for connections without an actual status -static int PSUTIL_CONN_NONE = 128; - - // The tcplist fetching and walking is borrowed from netstat/inet.c. static char * psutil_fetch_tcplist(void) { diff --git a/psutil/arch/freebsd/sys_socks.c b/psutil/arch/freebsd/sys_socks.c index 4104c27d4..3387838e9 100644 --- a/psutil/arch/freebsd/sys_socks.c +++ b/psutil/arch/freebsd/sys_socks.c @@ -25,8 +25,6 @@ #include "../../_psutil_common.h" #include "../../_psutil_posix.h" -// a signaler for connections without an actual status -static int PSUTIL_CONN_NONE = 128; static struct xfile *psutil_xfiles; static int psutil_nxfiles; diff --git a/psutil/arch/netbsd/socks.c b/psutil/arch/netbsd/socks.c index bd260ad35..f370f0946 100644 --- a/psutil/arch/netbsd/socks.c +++ b/psutil/arch/netbsd/socks.c @@ -24,8 +24,6 @@ #include "../../_psutil_common.h" #include "../../_psutil_posix.h" -// a signaler for connections without an actual status -int PSUTIL_CONN_NONE = 128; // address family filter enum af_filter { diff --git a/psutil/arch/openbsd/specific.c b/psutil/arch/openbsd/specific.c index 4caf6ed5e..de30c4d7d 100644 --- a/psutil/arch/openbsd/specific.c +++ b/psutil/arch/openbsd/specific.c @@ -41,9 +41,6 @@ #define PSUTIL_KPT2DOUBLE(t) (t ## _sec + t ## _usec / 1000000.0) // #define PSUTIL_TV2DOUBLE(t) ((t).tv_sec + (t).tv_usec / 1000000.0) -// a signaler for connections without an actual status -int PSUTIL_CONN_NONE = 128; - // ============================================================================ // Utility functions From ee3b71211b40c97e0d8fcac4a323b18fabc7c080 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 19:54:23 +0200 Subject: [PATCH 682/922] fix #1074: [FreeBSD] sensors_battery() raises OSError in case of no battery. --- HISTORY.rst | 1 + psutil/_psbsd.py | 6 +++++- psutil/arch/freebsd/specific.c | 7 ++++++- psutil/tests/test_bsd.py | 13 +++++++++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 33f8db8ec..080aaa860 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -58,6 +58,7 @@ - 1067_: [NetBSD] Process.cmdline() leaks memory if proces has terminated. - 1069_: [FreeBSD] Process.cpu_num() may return 255 for certain kernel processes. +- 1074_: [FreeBSD] sensors_battery() raises OSError in case of no battery. **Porting notes** diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 8b44deeb7..7de6b73d0 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -408,7 +408,11 @@ def net_connections(kind): def sensors_battery(): """Return battery info.""" - percent, minsleft, power_plugged = cext.sensors_battery() + try: + percent, minsleft, power_plugged = cext.sensors_battery() + except NotImplementedError: + # see: https://github.com/giampaolo/psutil/issues/1074 + return None power_plugged = power_plugged == 1 if power_plugged: secsleft = _common.POWER_TIME_UNLIMITED diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index 46bf38688..8d09ad89a 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -1000,5 +1000,10 @@ psutil_sensors_battery(PyObject *self, PyObject *args) { return Py_BuildValue("iii", percent, minsleft, power_plugged); error: - return PyErr_SetFromErrno(PyExc_OSError); + // see: https://github.com/giampaolo/psutil/issues/1074 + if (errno == ENOENT) + PyErr_SetString(PyExc_NotImplementedError, "no battery"); + else + PyErr_SetFromErrno(PyExc_OSError); + return NULL; } diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index 3d644c92e..aa7ad3d23 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -21,6 +21,7 @@ from psutil import NETBSD from psutil import OPENBSD from psutil.tests import get_test_subprocess +from psutil.tests import HAS_BATTERY from psutil.tests import MEMORY_TOLERANCE from psutil.tests import reap_children from psutil.tests import retry_before_failing @@ -354,6 +355,7 @@ def test_boot_time(self): # --- sensors_battery + @unittest.skipIf(not HAS_BATTERY, "no battery") def test_sensors_battery(self): def secs2hours(secs): m, s = divmod(secs, 60) @@ -372,6 +374,7 @@ def secs2hours(secs): else: self.assertEqual(secs2hours(metrics.secsleft), remaining_time) + @unittest.skipIf(not HAS_BATTERY, "no battery") def test_sensors_battery_against_sysctl(self): self.assertEqual(psutil.sensors_battery().percent, sysctl("hw.acpi.battery.life")) @@ -383,6 +386,16 @@ def test_sensors_battery_against_sysctl(self): else: self.assertEqual(secsleft, sysctl("hw.acpi.battery.time") * 60) + @unittest.skipIf(HAS_BATTERY, "has battery") + def test_sensors_battery_no_battery(self): + # If no battery is present one of these calls is supposed + # to fail, see: + # https://github.com/giampaolo/psutil/issues/1074 + with self.assertRaises(RuntimeError): + sysctl("hw.acpi.battery.life") + sysctl("hw.acpi.battery.time") + sysctl("hw.acpi.acline") + # ===================================================================== # --- OpenBSD From c964dc92424c19a588e348a6cbbb2d53fecec0b5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 19:57:26 +0200 Subject: [PATCH 683/922] add extra assert --- psutil/_psbsd.py | 2 +- psutil/tests/test_bsd.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 7de6b73d0..407d5a957 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -411,7 +411,7 @@ def sensors_battery(): try: percent, minsleft, power_plugged = cext.sensors_battery() except NotImplementedError: - # see: https://github.com/giampaolo/psutil/issues/1074 + # See: https://github.com/giampaolo/psutil/issues/1074 return None power_plugged = power_plugged == 1 if power_plugged: diff --git a/psutil/tests/test_bsd.py b/psutil/tests/test_bsd.py index aa7ad3d23..d3868ada1 100755 --- a/psutil/tests/test_bsd.py +++ b/psutil/tests/test_bsd.py @@ -395,6 +395,7 @@ def test_sensors_battery_no_battery(self): sysctl("hw.acpi.battery.life") sysctl("hw.acpi.battery.time") sysctl("hw.acpi.acline") + self.assertIsNone(psutil.sensors_battery()) # ===================================================================== From be82ee662cd5a983c246a97c70a7da664c2bb567 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 20:16:57 +0200 Subject: [PATCH 684/922] prevent AccessDenied on OSX --- psutil/tests/test_connections.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 4ead7d164..927d5bfb0 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -447,6 +447,7 @@ def test_multi_socks(self): if x.pid == os.getpid()] self.assertEqual(len(cons), len(socks)) + @skip_on_access_denied() def test_multi_sockets_procs(self): # Creates multiple sub processes, each creating different # sockets. For each process check that proc.connections() @@ -472,10 +473,10 @@ def test_multi_sockets_procs(self): syscons = [x for x in psutil.net_connections(kind='all') if x.pid in pids] for pid in pids: - p = psutil.Process(pid) - self.assertEqual(len(p.connections('all')), expected) self.assertEqual(len([x for x in syscons if x.pid == pid]), expected) + p = psutil.Process(pid) + self.assertEqual(len(p.connections('all')), expected) # ===================================================================== From e0ff6ee99f624a1d11a2ab6b833867fb7898d975 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 20:23:29 +0200 Subject: [PATCH 685/922] speed up multi socks tests by waiting for multiple test files in parallel --- psutil/tests/test_connections.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 927d5bfb0..00d94a6af 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -458,17 +458,25 @@ def test_multi_sockets_procs(self): with create_sockets() as socks: expected = len(socks) pids = [] - src = textwrap.dedent("""\ - import time - from psutil.tests import create_sockets - with create_sockets(): - open('%s', 'w').close() - time.sleep(60) - """ % TESTFN) - for x in range(10): + times = 10 + for i in range(times): + fname = TESTFN + str(i) + src = textwrap.dedent("""\ + import time, os + from psutil.tests import create_sockets + with create_sockets(): + with open('%s', 'w') as f: + f.write(str(os.getpid())) + time.sleep(60) + """ % fname) sproc = pyrun(src) - wait_for_file(TESTFN, empty=True) pids.append(sproc.pid) + self.addCleanup(safe_rmpath, fname) + + # sync + for i in range(times): + fname = TESTFN + str(i) + wait_for_file(fname) syscons = [x for x in psutil.net_connections(kind='all') if x.pid in pids] From da35c14ae7bbaab35f1152f16d2ef01f1e713a7a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 20:32:33 +0200 Subject: [PATCH 686/922] win: fix C compiler warning --- psutil/_psutil_windows.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 2772cab31..ea9fd387f 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -251,7 +251,7 @@ psutil_boot_time(PyObject *self, PyObject *args) { // Windows XP. // GetTickCount() time will wrap around to zero if the // system is run continuously for 49.7 days. - uptime = GetTickCount() / 1000.00f; + uptime = GetTickCount() / (LONGLONG)1000.00f; } return Py_BuildValue("d", (double)pt - (double)uptime); From 7005b9310f155411c4d53c02c4701868c306964f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 20:34:22 +0200 Subject: [PATCH 687/922] win: fix C compiler warning --- psutil/arch/windows/services.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/arch/windows/services.c b/psutil/arch/windows/services.c index e82f2887a..62a12861f 100644 --- a/psutil/arch/windows/services.c +++ b/psutil/arch/windows/services.c @@ -216,7 +216,7 @@ psutil_winservice_query_config(PyObject *self, PyObject *args) { PyErr_SetFromWindowsErr(0); goto error; } - qsc = (QUERY_SERVICE_CONFIG *)malloc(bytesNeeded); + qsc = (QUERY_SERVICE_CONFIGW *)malloc(bytesNeeded); ok = QueryServiceConfigW(hService, qsc, bytesNeeded, &bytesNeeded); if (ok == 0) { PyErr_SetFromWindowsErr(0); From 44d433db2de6c3834fbad79e0e657437c1bc7b99 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 20:37:23 +0200 Subject: [PATCH 688/922] win: fix C compiler warning --- psutil/arch/windows/process_info.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index a871282cf..13226b7c0 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -93,7 +93,8 @@ psutil_get_pids(DWORD *numberOfReturnedPIDs) { } if (! EnumProcesses(procArray, procArrayByteSz, &enumReturnSz)) { free(procArray); - return PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(0); + return NULL; } } while (enumReturnSz == procArraySz * sizeof(DWORD)); @@ -137,7 +138,8 @@ psutil_pid_is_running(DWORD pid) { // Be strict and raise an exception; the caller is supposed // to take -1 into account. else { - return PyErr_SetFromWindowsErr(err); + PyErr_SetFromWindowsErr(err); + return -1; } } @@ -154,8 +156,11 @@ psutil_pid_is_running(DWORD pid) { // a process to deny access to. if (err == ERROR_ACCESS_DENIED) return 1; - else - return PyErr_SetFromWindowsErr(err); + else { + PyErr_SetFromWindowsErr(err); + return -1; + } + } } From 68c7a70728a31d8b8b58f4be6c4c0baa2f449eda Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 20:39:38 +0200 Subject: [PATCH 689/922] win: fix C compiler warning --- psutil/arch/windows/process_info.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 13226b7c0..544dcfd0e 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -116,7 +116,7 @@ int psutil_pid_is_running(DWORD pid) { HANDLE hProcess; DWORD exitCode; - DWORD WINAPI err; + DWORD err; // Special case for PID 0 System Idle Process if (pid == 0) From 2e05a1748b67ca4c24289ca72963679284e2344c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 20:44:18 +0200 Subject: [PATCH 690/922] win: fix C compiler warning --- psutil/_psutil_windows.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index ea9fd387f..657ff8b53 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2845,7 +2845,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { HANDLE hProcess = NULL; PVOID baseAddress; PVOID previousAllocationBase; - LPWSTR mappedFileName[MAX_PATH]; + WCHAR mappedFileName[MAX_PATH]; SYSTEM_INFO system_info; LPVOID maxAddr; PyObject *py_retlist = PyList_New(0); From cf57b3550aa1d0d3c5698f6bb1922ffb60242a7e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 21:32:50 +0200 Subject: [PATCH 691/922] #1075 / win / net_if_addrs(): inet_ntop() return value isn't checked --- HISTORY.rst | 1 + psutil/_psutil_windows.c | 10 ++++++---- psutil/arch/windows/inet_ntop.c | 26 +++++++++++++++++--------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 080aaa860..8bf147f60 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -59,6 +59,7 @@ - 1069_: [FreeBSD] Process.cpu_num() may return 255 for certain kernel processes. - 1074_: [FreeBSD] sensors_battery() raises OSError in case of no battery. +- 1075_: [Windows] net_if_addrs(): inet_ntop() return value is not checked. **Porting notes** diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 657ff8b53..b4ba665fc 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -3064,6 +3064,8 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { pUnicast->Address.lpSockaddr; intRet = inet_ntop(AF_INET, &(sa_in->sin_addr), buff, bufflen); + if (!intRet) + goto error; #if (_WIN32_WINNT >= 0x0600) // Windows Vista and above netmask_bits = pUnicast->OnLinkPrefixLength; dwRetVal = ConvertLengthToIpv4Mask(netmask_bits, &converted_netmask); @@ -3071,6 +3073,8 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { in_netmask.s_addr = converted_netmask; netmaskIntRet = inet_ntop(AF_INET, &in_netmask, netmask_buff, netmask_bufflen); + if (!netmaskIntRet) + goto error; } #endif } @@ -3079,6 +3083,8 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { pUnicast->Address.lpSockaddr; intRet = inet_ntop(AF_INET6, &(sa_in6->sin6_addr), buff, bufflen); + if (!intRet) + goto error; } else { // we should never get here @@ -3086,10 +3092,6 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { continue; } - if (intRet == NULL) { - PyErr_SetFromWindowsErr(GetLastError()); - goto error; - } #if PY_MAJOR_VERSION >= 3 py_address = PyUnicode_FromString(buff); #else diff --git a/psutil/arch/windows/inet_ntop.c b/psutil/arch/windows/inet_ntop.c index 50dfb6aed..4b6c1dfec 100644 --- a/psutil/arch/windows/inet_ntop.c +++ b/psutil/arch/windows/inet_ntop.c @@ -1,9 +1,15 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola', Jeff Tang. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include #include "inet_ntop.h" // From: https://memset.wordpress.com/2010/10/09/inet_ntop-for-win32/ -PCSTR -WSAAPI -inet_ntop(__in INT Family, +PCSTR WSAAPI +inet_ntop(__in INT family, __in PVOID pAddr, __out_ecount(StringBufSize) PSTR pStringBuf, __in size_t StringBufSize) { @@ -13,17 +19,18 @@ inet_ntop(__in INT Family, struct sockaddr_in6 *srcaddr6 = (struct sockaddr_in6*) &srcaddr; memset(&srcaddr, 0, sizeof(struct sockaddr_storage)); - srcaddr.ss_family = Family; + srcaddr.ss_family = family; - if (Family == AF_INET) - { + if (family == AF_INET) { dwAddressLength = sizeof(struct sockaddr_in); memcpy(&(srcaddr4->sin_addr), pAddr, sizeof(struct in_addr)); - } else if (Family == AF_INET6) - { + } + else if (family == AF_INET6) { dwAddressLength = sizeof(struct sockaddr_in6); memcpy(&(srcaddr6->sin6_addr), pAddr, sizeof(struct in6_addr)); - } else { + } + else { + PyErr_SetString(PyExc_ValueError, "invalid family"); return NULL; } @@ -32,6 +39,7 @@ inet_ntop(__in INT Family, 0, pStringBuf, (LPDWORD) &StringBufSize) != 0) { + PyErr_SetExcFromWindowsErr(PyExc_OSError, WSAGetLastError()); return NULL; } return pStringBuf; From 36a9d1a32435d87a639f9eebfe2708f273943715 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 21:40:20 +0200 Subject: [PATCH 692/922] refactor C win code --- psutil/_psutil_windows.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index b4ba665fc..55eeba379 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2974,10 +2974,8 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { PCTSTR intRet; PCTSTR netmaskIntRet; char *ptr; - char buff[100]; - DWORD bufflen = 100; - char netmask_buff[100]; - DWORD netmask_bufflen = 100; + char buff[1024]; + char netmask_buff[1024]; DWORD dwRetVal = 0; #if (_WIN32_WINNT >= 0x0600) // Windows Vista and above ULONG converted_netmask; @@ -3063,7 +3061,7 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { struct sockaddr_in *sa_in = (struct sockaddr_in *) pUnicast->Address.lpSockaddr; intRet = inet_ntop(AF_INET, &(sa_in->sin_addr), buff, - bufflen); + sizeof(buff)); if (!intRet) goto error; #if (_WIN32_WINNT >= 0x0600) // Windows Vista and above @@ -3072,7 +3070,7 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { if (dwRetVal == NO_ERROR) { in_netmask.s_addr = converted_netmask; netmaskIntRet = inet_ntop(AF_INET, &in_netmask, netmask_buff, - netmask_bufflen); + sizeof(netmask_buff)); if (!netmaskIntRet) goto error; } @@ -3082,7 +3080,7 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { struct sockaddr_in6 *sa_in6 = (struct sockaddr_in6 *) pUnicast->Address.lpSockaddr; intRet = inet_ntop(AF_INET6, &(sa_in6->sin6_addr), - buff, bufflen); + buff, sizeof(buff)); if (!intRet) goto error; } From ff8d4457dbc2df40604fd974b97df785abfbe967 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 21:53:28 +0200 Subject: [PATCH 693/922] win: refactor net_if_addrs C code --- psutil/_psutil_windows.c | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 55eeba379..436dd76b5 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2974,8 +2974,9 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { PCTSTR intRet; PCTSTR netmaskIntRet; char *ptr; - char buff[1024]; - char netmask_buff[1024]; + char buff_addr[1024]; + char buff_macaddr[1024]; + char buff_netmask[1024]; DWORD dwRetVal = 0; #if (_WIN32_WINNT >= 0x0600) // Windows Vista and above ULONG converted_netmask; @@ -3014,22 +3015,22 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { // MAC address if (pCurrAddresses->PhysicalAddressLength != 0) { - ptr = buff; + ptr = buff_macaddr; *ptr = '\0'; for (i = 0; i < (int) pCurrAddresses->PhysicalAddressLength; i++) { if (i == (pCurrAddresses->PhysicalAddressLength - 1)) { - sprintf_s(ptr, _countof(buff), "%.2X\n", + sprintf_s(ptr, _countof(buff_macaddr), "%.2X\n", (int)pCurrAddresses->PhysicalAddress[i]); } else { - sprintf_s(ptr, _countof(buff), "%.2X-", + sprintf_s(ptr, _countof(buff_macaddr), "%.2X-", (int)pCurrAddresses->PhysicalAddress[i]); } ptr += 3; } *--ptr = '\0'; - py_mac_address = Py_BuildValue("s", buff); + py_mac_address = Py_BuildValue("s", buff_macaddr); if (py_mac_address == NULL) goto error; @@ -3060,8 +3061,8 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { if (family == AF_INET) { struct sockaddr_in *sa_in = (struct sockaddr_in *) pUnicast->Address.lpSockaddr; - intRet = inet_ntop(AF_INET, &(sa_in->sin_addr), buff, - sizeof(buff)); + intRet = inet_ntop(AF_INET, &(sa_in->sin_addr), buff_addr, + sizeof(buff_addr)); if (!intRet) goto error; #if (_WIN32_WINNT >= 0x0600) // Windows Vista and above @@ -3069,8 +3070,9 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { dwRetVal = ConvertLengthToIpv4Mask(netmask_bits, &converted_netmask); if (dwRetVal == NO_ERROR) { in_netmask.s_addr = converted_netmask; - netmaskIntRet = inet_ntop(AF_INET, &in_netmask, netmask_buff, - sizeof(netmask_buff)); + netmaskIntRet = inet_ntop( + AF_INET, &in_netmask, buff_netmask, + sizeof(buff_netmask)); if (!netmaskIntRet) goto error; } @@ -3080,7 +3082,7 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { struct sockaddr_in6 *sa_in6 = (struct sockaddr_in6 *) pUnicast->Address.lpSockaddr; intRet = inet_ntop(AF_INET6, &(sa_in6->sin6_addr), - buff, sizeof(buff)); + buff_addr, sizeof(buff_addr)); if (!intRet) goto error; } @@ -3091,18 +3093,18 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { } #if PY_MAJOR_VERSION >= 3 - py_address = PyUnicode_FromString(buff); + py_address = PyUnicode_FromString(buff_addr); #else - py_address = PyString_FromString(buff); + py_address = PyString_FromString(buff_addr); #endif if (py_address == NULL) goto error; if (netmaskIntRet != NULL) { #if PY_MAJOR_VERSION >= 3 - py_netmask = PyUnicode_FromString(netmask_buff); + py_netmask = PyUnicode_FromString(buff_netmask); #else - py_netmask = PyString_FromString(netmask_buff); + py_netmask = PyString_FromString(buff_netmask); #endif } else { Py_INCREF(Py_None); From 2899b4ca814aea82f9190a8a40837dfda897d854 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 22:14:09 +0200 Subject: [PATCH 694/922] fix win C warnings --- psutil/_pswindows.py | 2 +- psutil/tests/test_windows.py | 21 ++++++++++++--------- setup.py | 1 - 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index bd6b091f6..6d0679d63 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -181,7 +181,7 @@ class Priority(enum.IntEnum): @lru_cache(maxsize=512) def convert_dos_path(s): - """Convert paths using native DOS format like: + r"""Convert paths using native DOS format like: "\Device\HarddiskVolume1\Windows\systemew\file.txt" into: "C:\Windows\systemew\file.txt" diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index c1b080cf5..e4a719ea4 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -17,15 +17,7 @@ import subprocess import sys import time - -try: - import win32api # requires "pip install pypiwin32" / "make setup-dev-env" - import win32con - import win32process - import wmi # requires "pip install wmi" / "make setup-dev-env" -except ImportError: - if os.name == 'nt': - raise +import warnings import psutil from psutil import WINDOWS @@ -40,6 +32,17 @@ from psutil.tests import sh from psutil.tests import unittest +with warnings.catch_warnings(): + warnings.simplefilter("ignore") + try: + import win32api # requires "pip install pypiwin32" + import win32con + import win32process + import wmi # requires "pip install wmi" / "make setup-dev-env" + except ImportError: + if os.name == 'nt': + raise + cext = psutil._psplatform.cext diff --git a/setup.py b/setup.py index 55d86b47a..7c6ffc9e4 100755 --- a/setup.py +++ b/setup.py @@ -133,7 +133,6 @@ def get_winver(): 'psutil._psutil_windows', sources=sources + [ 'psutil/_psutil_windows.c', - 'psutil/_psutil_common.c', 'psutil/arch/windows/process_info.c', 'psutil/arch/windows/process_handles.c', 'psutil/arch/windows/security.c', From 8d9aa7848e76544d54750e535b90027632876630 Mon Sep 17 00:00:00 2001 From: Gleb Smirnoff Date: Tue, 16 May 2017 13:57:23 -0700 Subject: [PATCH 695/922] Some fixes to IPv6 testing. (#1076) * Make test_multi_sockets_filtering not fail if kernel doesn't support IPv6. * Fix test_udp_v6 to actually test IPv6. --- psutil/tests/test_connections.py | 37 +++++++++++++++++--------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 00d94a6af..32fb87ab9 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -171,7 +171,7 @@ def test_udp_v4(self): def test_udp_v6(self): addr = ("127.0.0.1", get_free_port()) - with closing(bind_socket(AF_INET, SOCK_DGRAM, addr=addr)) as sock: + with closing(bind_socket(AF_INET6, SOCK_DGRAM, addr=addr)) as sock: conn = self.check_socket(sock) assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_NONE) @@ -359,7 +359,7 @@ def test_multi_sockets_filtering(self): self.assertEqual(len(socks), len(cons)) # tcp cons = thisproc.connections(kind='tcp') - self.assertEqual(len(cons), 2) + self.assertEqual(len(cons), 2 if supports_ipv6() else 1) for conn in cons: self.assertIn(conn.family, (AF_INET, AF_INET6)) self.assertEqual(conn.type, SOCK_STREAM) @@ -369,13 +369,14 @@ def test_multi_sockets_filtering(self): self.assertEqual(cons[0].family, AF_INET) self.assertEqual(cons[0].type, SOCK_STREAM) # tcp6 - cons = thisproc.connections(kind='tcp6') - self.assertEqual(len(cons), 1) - self.assertEqual(cons[0].family, AF_INET6) - self.assertEqual(cons[0].type, SOCK_STREAM) + if supports_ipv6(): + cons = thisproc.connections(kind='tcp6') + self.assertEqual(len(cons), 1) + self.assertEqual(cons[0].family, AF_INET6) + self.assertEqual(cons[0].type, SOCK_STREAM) # udp cons = thisproc.connections(kind='udp') - self.assertEqual(len(cons), 2) + self.assertEqual(len(cons), 2 if supports_ipv6() else 1) for conn in cons: self.assertIn(conn.family, (AF_INET, AF_INET6)) self.assertEqual(conn.type, SOCK_DGRAM) @@ -385,22 +386,24 @@ def test_multi_sockets_filtering(self): self.assertEqual(cons[0].family, AF_INET) self.assertEqual(cons[0].type, SOCK_DGRAM) # udp6 - cons = thisproc.connections(kind='udp6') - self.assertEqual(len(cons), 1) - self.assertEqual(cons[0].family, AF_INET6) - self.assertEqual(cons[0].type, SOCK_DGRAM) + if supports_ipv6(): + cons = thisproc.connections(kind='udp6') + self.assertEqual(len(cons), 1) + self.assertEqual(cons[0].family, AF_INET6) + self.assertEqual(cons[0].type, SOCK_DGRAM) # inet cons = thisproc.connections(kind='inet') - self.assertEqual(len(cons), 4) + self.assertEqual(len(cons), 4 if supports_ipv6() else 2) for conn in cons: self.assertIn(conn.family, (AF_INET, AF_INET6)) self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) # inet6 - cons = thisproc.connections(kind='inet6') - self.assertEqual(len(cons), 2) - for conn in cons: - self.assertEqual(conn.family, AF_INET6) - self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) + if supports_ipv6(): + cons = thisproc.connections(kind='inet6') + self.assertEqual(len(cons), 2) + for conn in cons: + self.assertEqual(conn.family, AF_INET6) + self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) # unix if POSIX and not SUNOS: cons = thisproc.connections(kind='unix') From 7d687e94c8244ac0634491405e1b197b80fac49e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 23:13:30 +0200 Subject: [PATCH 696/922] define a new HAS_CONNECTIONS_UNIX constant --- psutil/_common.py | 1 + psutil/tests/__init__.py | 2 ++ psutil/tests/test_connections.py | 10 ++++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index d58dac6b0..c08c60c13 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -384,6 +384,7 @@ def path_exists_strict(path): return True +@memoize def supports_ipv6(): """Return True if IPv6 is supported on this platform.""" if not socket.has_ipv6 or not hasattr(socket, "AF_INET6"): diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 583919869..4aae1980f 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -36,6 +36,7 @@ import psutil from psutil import POSIX +from psutil import SUNOS from psutil import WINDOWS from psutil._common import supports_ipv6 from psutil._compat import PY3 @@ -145,6 +146,7 @@ HAS_CPU_AFFINITY = hasattr(psutil.Process, "cpu_affinity") HAS_CPU_FREQ = hasattr(psutil, "cpu_freq") +HAS_CONNECTIONS_UNIX = POSIX and not SUNOS HAS_ENVIRON = hasattr(psutil.Process, "environ") HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters") HAS_IONICE = hasattr(psutil.Process, "ionice") diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 32fb87ab9..57851a341 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -33,6 +33,7 @@ from psutil.tests import check_connection_ntuple from psutil.tests import create_sockets from psutil.tests import get_free_port +from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import pyrun from psutil.tests import reap_children from psutil.tests import run_test_module_by_name @@ -110,7 +111,7 @@ def check_socket(self, sock, conn=None): self.assertEqual(conn.laddr, laddr) # XXX Solaris can't retrieve system-wide UNIX sockets - if not (SUNOS and sock.family == AF_UNIX): + if sock.family == AF_UNIX and HAS_CONNECTIONS_UNIX: cons = thisproc.connections(kind='all') self.compare_procsys_connections(os.getpid(), cons) return conn @@ -275,7 +276,7 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): # compare against system-wide connections # XXX Solaris can't retrieve system-wide UNIX # sockets. - if not SUNOS: + if HAS_CONNECTIONS_UNIX: self.compare_procsys_connections(proc.pid, [conn]) tcp_template = textwrap.dedent(""" @@ -405,7 +406,7 @@ def test_multi_sockets_filtering(self): self.assertEqual(conn.family, AF_INET6) self.assertIn(conn.type, (SOCK_STREAM, SOCK_DGRAM)) # unix - if POSIX and not SUNOS: + if HAS_CONNECTIONS_UNIX: cons = thisproc.connections(kind='unix') self.assertEqual(len(cons), 3) for conn in cons: @@ -434,7 +435,8 @@ def check(cons, families, types_): with create_sockets(): from psutil._common import conn_tmap for kind, groups in conn_tmap.items(): - if SUNOS and kind == 'unix': + # XXX: SunOS does not retrieve UNIX sockets. + if kind == 'unix' and not HAS_CONNECTIONS_UNIX: continue families, types_ = groups cons = psutil.net_connections(kind) From da603cf9b996166d94b7dd2bf233cbc474916715 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 23:17:19 +0200 Subject: [PATCH 697/922] #1076: fix UPDv4 address --- psutil/tests/test_connections.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 57851a341..f7dd2ec26 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -171,7 +171,7 @@ def test_udp_v4(self): self.assertEqual(conn.status, psutil.CONN_NONE) def test_udp_v6(self): - addr = ("127.0.0.1", get_free_port()) + addr = ("::1", get_free_port()) with closing(bind_socket(AF_INET6, SOCK_DGRAM, addr=addr)) as sock: conn = self.check_socket(sock) assert not conn.raddr From a667eb13c13375b8ca492cd363e0670a3efcc769 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 23:42:03 +0200 Subject: [PATCH 698/922] fix supports_ipv6 test --- psutil/tests/test_misc.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 2cb4dbb1f..f0bec81ad 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -290,21 +290,27 @@ def k(s): self.assertEqual(parse_environ_block("a=1\0b=2"), {k("a"): "1"}) def test_supports_ipv6(self): + self.addCleanup(supports_ipv6.cache_clear) + supports_ipv6.cache_clear() if supports_ipv6(): with mock.patch('psutil._common.socket') as s: s.has_ipv6 = False assert not supports_ipv6() + supports_ipv6.cache_clear() with mock.patch('psutil._common.socket.socket', side_effect=socket.error) as s: assert not supports_ipv6() + supports_ipv6.cache_clear() assert s.called with mock.patch('psutil._common.socket.socket', side_effect=socket.gaierror) as s: assert not supports_ipv6() + supports_ipv6.cache_clear() assert s.called with mock.patch('psutil._common.socket.socket.bind', side_effect=socket.gaierror) as s: assert not supports_ipv6() + supports_ipv6.cache_clear() assert s.called else: with self.assertRaises(Exception): From 4938d05685e0d506f91196b3beb934853045b2dd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 May 2017 23:52:56 +0200 Subject: [PATCH 699/922] fix supports_ipv6 test --- psutil/tests/test_misc.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index f0bec81ad..57423dd80 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -291,22 +291,26 @@ def k(s): def test_supports_ipv6(self): self.addCleanup(supports_ipv6.cache_clear) - supports_ipv6.cache_clear() if supports_ipv6(): with mock.patch('psutil._common.socket') as s: s.has_ipv6 = False - assert not supports_ipv6() supports_ipv6.cache_clear() + assert not supports_ipv6() + + supports_ipv6.cache_clear() with mock.patch('psutil._common.socket.socket', side_effect=socket.error) as s: assert not supports_ipv6() - supports_ipv6.cache_clear() assert s.called + + supports_ipv6.cache_clear() with mock.patch('psutil._common.socket.socket', side_effect=socket.gaierror) as s: assert not supports_ipv6() supports_ipv6.cache_clear() assert s.called + + supports_ipv6.cache_clear() with mock.patch('psutil._common.socket.socket.bind', side_effect=socket.gaierror) as s: assert not supports_ipv6() From 0a19931c05b9fdcd42bcd47cf1e1c5028116efeb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 00:08:00 +0200 Subject: [PATCH 700/922] Try to fix arcan appveyor err: - https://ci.appveyor.com/project/giampaolo/psutil/build/1240/job/91n29tt3es7os2ut --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 2e6735bba..1390f962d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -91,7 +91,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "%WITH_COMPILER% %PYTHON%/python -Wa psutil/tests/__main__.py" + - "%WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" From b26193933a8d68d278468ba331e357327d82f420 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 01:31:45 +0200 Subject: [PATCH 701/922] Ignore arcane safe_rmpath error on APPVEYOR. See: https://ci.appveyor.com/project/giampaolo/psutil/build/job/jiq2cgd6stsbtn60 This is weird, and I'm afraid it means wait_procs() on Win may be broken (because of STILL_ACTIVE?). Add signalers in reap_children() that may help figure out whether this is the case. --- psutil/tests/__init__.py | 7 +++++++ psutil/tests/test_unicode.py | 24 +++++++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 4aae1980f..1a0e168b1 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -437,6 +437,13 @@ def reap_children(recursive=False): for p in alive: warn("process %r survived kill()" % p) + # TODO: this is temporary and here only to investigate: + # https://ci.appveyor.com/project/giampaolo/psutil/build/job/ + # jiq2cgd6stsbtn60 + for p in children: + assert not psutil.pid_exists(p.pid), p + assert p.pid not in psutil.pids() + # =================================================================== # --- OS diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 4c2181d49..7de15db5b 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -65,6 +65,7 @@ from psutil import WINDOWS from psutil._compat import PY3 from psutil._compat import u +from psutil.tests import APPVEYOR from psutil.tests import ASCII_FS from psutil.tests import bind_unix_socket from psutil.tests import chdir @@ -77,7 +78,7 @@ from psutil.tests import reap_children from psutil.tests import run_test_module_by_name from psutil.tests import safe_mkdir -from psutil.tests import safe_rmpath +from psutil.tests import safe_rmpath as _safe_rmpath from psutil.tests import skip_on_access_denied from psutil.tests import TESTFILE_PREFIX from psutil.tests import TESTFN @@ -89,6 +90,27 @@ import psutil.tests +def safe_rmpath(path): + if APPVEYOR: + # TODO - this is quite random and I'm not sure why it happens, + # nor I can reproduce it locally: + # https://ci.appveyor.com/project/giampaolo/psutil/build/job/ + # jiq2cgd6stsbtn60 + # safe_rmpath() happens after reap_children() so this is weird + # Perhaps wait_procs() on Windows is broken? Maybe because + # of STILL_ACTIVE? + # https://github.com/giampaolo/psutil/blob/ + # 68c7a70728a31d8b8b58f4be6c4c0baa2f449eda/psutil/arch/ + # windows/process_info.c#L146 + try: + return _safe_rmpath(path) + except WindowsError: + # + traceback.print_exc() + else: + return _safe_rmpath(path) + + def subprocess_supports_unicode(name): """Return True if both the fs and the subprocess module can deal with a unicode file name. From 76f567838a4b58a139500a5622db26ecad3dd162 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 01:40:54 +0200 Subject: [PATCH 702/922] smarter appveyor commit files filtering --- appveyor.yml | 1 + psutil/tests/test_unicode.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 1390f962d..2c16ef4be 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -109,6 +109,7 @@ skip_commits: # run build only if one of the following files is modified on commit only_commits: files: + *win* .ci/appveyor/* appveyor.yml psutil/__init__.py diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 7de15db5b..c59237d50 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -105,7 +105,6 @@ def safe_rmpath(path): try: return _safe_rmpath(path) except WindowsError: - # traceback.print_exc() else: return _safe_rmpath(path) From 46eaa8d3e0348c4afd2f7ec1c6bb31c576046eef Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 01:41:47 +0200 Subject: [PATCH 703/922] fix appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 2c16ef4be..7cef39d91 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -109,7 +109,7 @@ skip_commits: # run build only if one of the following files is modified on commit only_commits: files: - *win* + "*win*" .ci/appveyor/* appveyor.yml psutil/__init__.py From 609a02a11a0f9f5c5a2fd3d77386c6b72b31e564 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 02:03:44 +0200 Subject: [PATCH 704/922] appveyor: try smarter filtering --- appveyor.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 7cef39d91..7bd71fefb 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -103,9 +103,6 @@ artifacts: # on_success: # - might want to upload the content of dist/*.whl to a public wheelhouse -skip_commits: - message: skip-ci - # run build only if one of the following files is modified on commit only_commits: files: @@ -119,12 +116,14 @@ only_commits: psutil/_psutil_windows.* psutil/_pswindows.py psutil/arch/windows/* - psutil/tests/__init__.py - psutil/tests/__main__.py - psutil/tests/test_memory_leaks.py - psutil/tests/test_misc.py - psutil/tests/test_process.py - psutil/tests/test_system.py - psutil/tests/test_windows.py + psutil/tests/*.py scripts/* setup.py + +skip_commits: + files: + psutil/tests/test_bsd.py + psutil/tests/test_linux.py + psutil/tests/test_osx.py + psutil/tests/test_posix.py + psutil/tests/test_sunos.py From cb51168c942fcbfea194191cc7e318283fb8dde2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 02:47:52 +0200 Subject: [PATCH 705/922] pass cwd and env to get_test_subprocess; reason: try to figure this out https://travis-ci.org/giampaolo/psutil/jobs/233019876 --- psutil/tests/__init__.py | 2 ++ psutil/tests/test_connections.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 1a0e168b1..b329d0ff1 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -271,6 +271,8 @@ def get_test_subprocess(cmd=None, **kwds): """ kwds.setdefault("stdin", DEVNULL) kwds.setdefault("stdout", DEVNULL) + kwds.setdefault("cwd", os.getcwd()) + kwds.setdefault("env", os.environ) if WINDOWS: # Prevents the subprocess to open error dialogs. kwds.setdefault("creationflags", 0x8000000) # CREATE_NO_WINDOW diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index f7dd2ec26..c4d896eee 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -465,7 +465,7 @@ def test_multi_sockets_procs(self): pids = [] times = 10 for i in range(times): - fname = TESTFN + str(i) + fname = os.path.realpath(TESTFN) + str(i) src = textwrap.dedent("""\ import time, os from psutil.tests import create_sockets From 66e7efe5643e9f0125d9edc52bfa918108570fda Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 02:51:04 +0200 Subject: [PATCH 706/922] revert appveyor.yml --- appveyor.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 7bd71fefb..1390f962d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -103,10 +103,12 @@ artifacts: # on_success: # - might want to upload the content of dist/*.whl to a public wheelhouse +skip_commits: + message: skip-ci + # run build only if one of the following files is modified on commit only_commits: files: - "*win*" .ci/appveyor/* appveyor.yml psutil/__init__.py @@ -116,14 +118,12 @@ only_commits: psutil/_psutil_windows.* psutil/_pswindows.py psutil/arch/windows/* - psutil/tests/*.py + psutil/tests/__init__.py + psutil/tests/__main__.py + psutil/tests/test_memory_leaks.py + psutil/tests/test_misc.py + psutil/tests/test_process.py + psutil/tests/test_system.py + psutil/tests/test_windows.py scripts/* setup.py - -skip_commits: - files: - psutil/tests/test_bsd.py - psutil/tests/test_linux.py - psutil/tests/test_osx.py - psutil/tests/test_posix.py - psutil/tests/test_sunos.py From 91041c4e1f20b48d5841a2aaacc90a106c4b4f6c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 13:54:48 +0200 Subject: [PATCH 707/922] test utils refactoring --- psutil/tests/__init__.py | 77 +++++++++++++++++------------------- psutil/tests/test_unicode.py | 2 + 2 files changed, 39 insertions(+), 40 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index b329d0ff1..2f852ef36 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -359,6 +359,7 @@ def sh(cmd, **kwds): kwds.setdefault("universal_newlines", True) kwds.setdefault("creationflags", flags) p = subprocess.Popen(cmd, **kwds) + _subprocesses_started.add(p) stdout, stderr = p.communicate() if p.returncode != 0: raise RuntimeError(stderr) @@ -452,47 +453,43 @@ def reap_children(recursive=False): # =================================================================== -if not POSIX: - def get_kernel_version(): - return () -else: - def get_kernel_version(): - """Return a tuple such as (2, 6, 36).""" - s = "" - uname = os.uname()[2] - for c in uname: - if c.isdigit() or c == '.': - s += c - else: - break - if not s: - raise ValueError("can't parse %r" % uname) - minor = 0 - micro = 0 - nums = s.split('.') - major = int(nums[0]) - if len(nums) >= 2: - minor = int(nums[1]) - if len(nums) >= 3: - micro = int(nums[2]) - return (major, minor, micro) - - -if not WINDOWS: - def get_winver(): - raise NotImplementedError("not a Windows OS") -else: - def get_winver(): - wv = sys.getwindowsversion() - if hasattr(wv, 'service_pack_major'): # python >= 2.7 - sp = wv.service_pack_major or 0 +def get_kernel_version(): + """Return a tuple such as (2, 6, 36).""" + if not POSIX: + raise NotImplementedError("not POSIX") + s = "" + uname = os.uname()[2] + for c in uname: + if c.isdigit() or c == '.': + s += c else: - r = re.search(r"\s\d$", wv[4]) - if r: - sp = int(r.group(0)) - else: - sp = 0 - return (wv[0], wv[1], sp) + break + if not s: + raise ValueError("can't parse %r" % uname) + minor = 0 + micro = 0 + nums = s.split('.') + major = int(nums[0]) + if len(nums) >= 2: + minor = int(nums[1]) + if len(nums) >= 3: + micro = int(nums[2]) + return (major, minor, micro) + + +def get_winver(): + if not WINDOWS: + raise NotImplementedError("not WINDOWS") + wv = sys.getwindowsversion() + if hasattr(wv, 'service_pack_major'): # python >= 2.7 + sp = wv.service_pack_major or 0 + else: + r = re.search(r"\s\d$", wv[4]) + if r: + sp = int(r.group(0)) + else: + sp = 0 + return (wv[0], wv[1], sp) # =================================================================== diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index c59237d50..7c87a3f2f 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -91,6 +91,8 @@ def safe_rmpath(path): + # XXX + return _safe_rmpath(path) if APPVEYOR: # TODO - this is quite random and I'm not sure why it happens, # nor I can reproduce it locally: From b4b3e59f0f95ebf7f138763ce259c1d7ea9ffb5f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 13:59:59 +0200 Subject: [PATCH 708/922] test utils refactoring --- psutil/tests/__init__.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 2f852ef36..1206abff0 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -378,6 +378,20 @@ def reap_children(recursive=False): If resursive is True it also tries to terminate and wait() all grandchildren started by this process. """ + # This is here to make sure wait_procs() behaves properly and + # investigate: + # https://ci.appveyor.com/project/giampaolo/psutil/build/job/ + # jiq2cgd6stsbtn60 + def assert_gone(pid): + assert not psutil.pid_exists(pid), pid + assert pid not in psutil.pids(), pid + try: + psutil.Process(pid) + except psutil.NoSuchProcess: + pass + else: + assert 0, "pid %s is not gone" % pid + # Get the children here, before terminating the children sub # processes as we don't want to lose the intermediate reference # in case of grandchildren. @@ -410,6 +424,7 @@ def reap_children(recursive=False): except OSError as err: if err.errno != errno.ECHILD: raise + assert_gone(subp.pid) # Terminate started pids. while _pids_started: @@ -440,12 +455,8 @@ def reap_children(recursive=False): for p in alive: warn("process %r survived kill()" % p) - # TODO: this is temporary and here only to investigate: - # https://ci.appveyor.com/project/giampaolo/psutil/build/job/ - # jiq2cgd6stsbtn60 for p in children: - assert not psutil.pid_exists(p.pid), p - assert p.pid not in psutil.pids() + assert_gone(p.pid) # =================================================================== From 0c2eb2ab9e644df8e012dd36e3d2994695bab1cc Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 14:56:58 +0200 Subject: [PATCH 709/922] reap_children(): enforce checks to make sure the process is gone --- psutil/tests/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 1206abff0..f14b1c98c 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -386,7 +386,8 @@ def assert_gone(pid): assert not psutil.pid_exists(pid), pid assert pid not in psutil.pids(), pid try: - psutil.Process(pid) + p = psutil.Process(pid) + assert not p.is_running(), pid except psutil.NoSuchProcess: pass else: @@ -404,6 +405,7 @@ def assert_gone(pid): # fds and wiat()ing for them in order to avoid zombies. while _subprocesses_started: subp = _subprocesses_started.pop() + _pids_started.add(subp.pid) try: subp.terminate() except OSError as err: @@ -424,7 +426,6 @@ def assert_gone(pid): except OSError as err: if err.errno != errno.ECHILD: raise - assert_gone(subp.pid) # Terminate started pids. while _pids_started: @@ -432,7 +433,7 @@ def assert_gone(pid): try: p = psutil.Process(pid) except psutil.NoSuchProcess: - pass + assert_gone(pid) else: children.add(p) From 2c56220ab9c86353f8db109a884ed3efd9c2579c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 17:42:43 +0200 Subject: [PATCH 710/922] small refactoring --- psutil/arch/windows/process_info.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 544dcfd0e..24f7271c6 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -147,7 +147,10 @@ psutil_pid_is_running(DWORD pid) { CloseHandle(hProcess); // XXX - maybe STILL_ACTIVE is not fully reliable as per: // http://stackoverflow.com/questions/1591342/#comment47830782_1591379 - return (exitCode == STILL_ACTIVE); + if (exitCode == STILL_ACTIVE) + return 1; + else + return 0; } else { err = GetLastError(); From 3b50e3a31b5c759e824b7212e51e986862883874 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 18:01:44 +0200 Subject: [PATCH 711/922] move psutil.Popen tests in their own class --- psutil/tests/test_process.py | 70 +++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 86ad136f7..b9b13c58d 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1383,35 +1383,6 @@ def test_pid_0(self): self.assertIn(0, psutil.pids()) self.assertTrue(psutil.pid_exists(0)) - def test_Popen(self): - # XXX this test causes a ResourceWarning on Python 3 because - # psutil.__subproc instance doesn't get propertly freed. - # Not sure what to do though. - cmd = [PYTHON, "-c", "import time; time.sleep(60);"] - proc = psutil.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - try: - proc.name() - proc.cpu_times() - proc.stdin - self.assertTrue(dir(proc)) - self.assertRaises(AttributeError, getattr, proc, 'foo') - finally: - proc.terminate() - proc.stdout.close() - proc.stderr.close() - proc.wait() - - def test_Popen_ctx_manager(self): - with psutil.Popen([PYTHON, "-V"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE) as proc: - pass - assert proc.stdout.closed - assert proc.stderr.closed - assert proc.stdin.closed - @unittest.skipIf(not HAS_ENVIRON, "not supported") def test_environ(self): self.maxDiff = None @@ -1522,5 +1493,46 @@ def test_zombie_process(self): pass +# =================================================================== +# --- psutil.Popen tests +# =================================================================== + + +class TestPopen(unittest.TestCase): + """Tests for psutil.Popen class.""" + + def tearDown(self): + reap_children() + + def test_it(self): + # XXX this test causes a ResourceWarning on Python 3 because + # psutil.__subproc instance doesn't get propertly freed. + # Not sure what to do though. + cmd = [PYTHON, "-c", "import time; time.sleep(60);"] + proc = psutil.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + proc.name() + proc.cpu_times() + proc.stdin + self.assertTrue(dir(proc)) + self.assertRaises(AttributeError, getattr, proc, 'foo') + finally: + proc.terminate() + proc.stdout.close() + proc.stderr.close() + proc.wait() + + def test_ctx_manager(self): + with psutil.Popen([PYTHON, "-V"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE) as proc: + pass + assert proc.stdout.closed + assert proc.stderr.closed + assert proc.stdin.closed + + if __name__ == '__main__': run_test_module_by_name(__file__) From f698d84da2d880fcb9b27f16e599517b09cea5b8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 18:11:23 +0200 Subject: [PATCH 712/922] add Popen test for making sure NSP is raised on kill() and terminate() --- psutil/tests/test_process.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index b9b13c58d..7f3581704 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1504,34 +1504,49 @@ class TestPopen(unittest.TestCase): def tearDown(self): reap_children() - def test_it(self): + def test_misc(self): # XXX this test causes a ResourceWarning on Python 3 because # psutil.__subproc instance doesn't get propertly freed. # Not sure what to do though. cmd = [PYTHON, "-c", "import time; time.sleep(60);"] - proc = psutil.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - try: + with psutil.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: proc.name() proc.cpu_times() proc.stdin self.assertTrue(dir(proc)) self.assertRaises(AttributeError, getattr, proc, 'foo') - finally: proc.terminate() - proc.stdout.close() - proc.stderr.close() - proc.wait() def test_ctx_manager(self): with psutil.Popen([PYTHON, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) as proc: - pass + proc.communicate() assert proc.stdout.closed assert proc.stderr.closed assert proc.stdin.closed + self.assertEqual(proc.returncode, 0) + + def test_kill_terminate(self): + # subprocess.Popen()'s terminate(), kill() and send_signal() do + # not raise exception after the process is gone. psutil.Popen + # diverges from that. + cmd = [PYTHON, "-c", "import time; time.sleep(60);"] + with psutil.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as proc: + proc.terminate() + proc.wait() + self.assertRaises(psutil.NoSuchProcess, proc.terminate) + self.assertRaises(psutil.NoSuchProcess, proc.kill) + self.assertRaises(psutil.NoSuchProcess, proc.send_signal, + signal.SIGTERM) + if WINDOWS: + self.assertRaises(psutil.NoSuchProcess, proc.send_signal, + signal.CTRL_C_EVENT) + self.assertRaises(psutil.NoSuchProcess, proc.send_signal, + signal.CTRL_BREAK_EVENT) if __name__ == '__main__': From c630eff0991116c036723a80531e85d8d599e6c8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 17 May 2017 18:12:49 +0200 Subject: [PATCH 713/922] skip signal.CTRL_ test on py 2.6 --- psutil/tests/test_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 7f3581704..b1f2508f3 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1542,7 +1542,7 @@ def test_kill_terminate(self): self.assertRaises(psutil.NoSuchProcess, proc.kill) self.assertRaises(psutil.NoSuchProcess, proc.send_signal, signal.SIGTERM) - if WINDOWS: + if WINDOWS and sys.version_info >= (2, 7): self.assertRaises(psutil.NoSuchProcess, proc.send_signal, signal.CTRL_C_EVENT) self.assertRaises(psutil.NoSuchProcess, proc.send_signal, From 5250b63787dec4a4cefbc1c93d84b58ac874405a Mon Sep 17 00:00:00 2001 From: Gleb Smirnoff Date: Wed, 17 May 2017 19:17:40 -0700 Subject: [PATCH 714/922] Fixes to net_connections() on FreeBSD. (#1079) * File descriptor 0 is a valid value, for example for a daemon. * On FreeBSD fill in file descriptor values. This not only removes ugly workaround from test_connections.py, but also fixes several failures in this test. Without file descriptor value, two local sockets connected to each other, would be equal objects. Since in the _psbsd.py:net_connections(), the returned value is a Python set, it will exclude any duplicates, resulting in shrinked list of sockets. --- psutil/arch/freebsd/sys_socks.c | 30 ++++++++++++++++-------------- psutil/tests/__init__.py | 2 +- psutil/tests/test_connections.py | 4 ---- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/psutil/arch/freebsd/sys_socks.c b/psutil/arch/freebsd/sys_socks.c index 3387838e9..e0e2046be 100644 --- a/psutil/arch/freebsd/sys_socks.c +++ b/psutil/arch/freebsd/sys_socks.c @@ -57,16 +57,16 @@ psutil_populate_xfiles() { } -int -psutil_get_pid_from_sock(void *sock) { +struct xfile * +psutil_get_file_from_sock(void *sock) { struct xfile *xf; int n; for (xf = psutil_xfiles, n = 0; n < psutil_nxfiles; ++n, ++xf) { if (xf->xf_data == sock) - return xf->xf_pid; + return xf; } - return -1; + return NULL; } @@ -129,7 +129,8 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { } while (xig->xig_gen != exig->xig_gen && retry--); for (;;) { - int lport, rport, pid, status, family; + struct xfile *xf; + int lport, rport, status, family; xig = (struct xinpgen *)(void *)((char *)xig + xig->xig_len); if (xig >= exig) @@ -174,8 +175,8 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { char lip[200], rip[200]; - pid = psutil_get_pid_from_sock(so->xso_so); - if (pid < 0) + xf = psutil_get_file_from_sock(so->xso_so); + if (xf == NULL) continue; lport = ntohs(inp->inp_lport); rport = ntohs(inp->inp_fport); @@ -203,13 +204,13 @@ int psutil_gather_inet(int proto, PyObject *py_retlist) { goto error; py_tuple = Py_BuildValue( "(iiiNNii)", - -1, // fd + xf->xf_fd, // fd family, // family type, // type py_laddr, // laddr py_raddr, // raddr status, // status - pid); // pid + xf->xf_pid); // pid if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) @@ -238,7 +239,6 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { size_t bufsize; void *buf; int retry; - int pid; struct sockaddr_un *sun; char path[PATH_MAX]; @@ -286,6 +286,8 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { } while (xug->xug_gen != exug->xug_gen && retry--); for (;;) { + struct xfile *xf; + xug = (struct xunpgen *)(void *)((char *)xug + xug->xug_len); if (xug >= exug) break; @@ -293,8 +295,8 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { if (xup->xu_len != sizeof *xup) goto error; - pid = psutil_get_pid_from_sock(xup->xu_socket.xso_so); - if (pid < 0) + xf = psutil_get_file_from_sock(xup->xu_socket.xso_so); + if (xf == NULL) continue; sun = (struct sockaddr_un *)&xup->xu_addr; @@ -306,13 +308,13 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { goto error; py_tuple = Py_BuildValue("(iiiOsii)", - -1, // fd + xf->xf_fd, // fd AF_UNIX, // family proto, // type py_lpath, // lpath "", // rath PSUTIL_CONN_NONE, // status - pid); // pid + xf->xf_pid); // pid if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index f14b1c98c..927e8cf64 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -930,7 +930,7 @@ def check_connection_ntuple(conn): # check fd if has_fd: - assert conn.fd > 0, conn + assert conn.fd >= 0, conn if hasattr(socket, 'fromfd') and not WINDOWS: try: dupsock = socket.fromfd(conn.fd, conn.family, conn.type) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index c4d896eee..27822a807 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -132,10 +132,6 @@ def compare_procsys_connections(self, pid, proc_cons, kind='all'): raise # Filter for this proc PID and exlucde PIDs from the tuple. sys_cons = [c[:-1] for c in sys_cons if c.pid == pid] - if FREEBSD: - # On FreeBSD all fds are set to -1 so exclude them - # from comparison. - proc_cons = [pconn(*[-1] + list(x[1:])) for x in proc_cons] sys_cons.sort() proc_cons.sort() self.assertEqual(proc_cons, sys_cons) From 759b8a88ead6caf294aa45e84578056dc79440df Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 04:41:36 +0200 Subject: [PATCH 715/922] update HISTORY with #1079; give CREDITS to @glebius --- CREDITS | 2 +- HISTORY.rst | 4 ++++ docs/index.rst | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CREDITS b/CREDITS index 515ba38d9..5483653a5 100644 --- a/CREDITS +++ b/CREDITS @@ -477,4 +477,4 @@ I: 1057 N: Gleb Smirnoff W: https://github.com/glebius -I: 1042 +I: 1042, 1079 diff --git a/HISTORY.rst b/HISTORY.rst index 8bf147f60..974af6171 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,8 @@ - 1058_: test suite now enables all warnings by default. - 1060_: source distribution is dynamically generated so that it only includes relevant files. +- 1079_: [FreeBSD] net_connections()'s fd number is now being set for real + (instead of -1). (patch by Gleb Smirnoff) **Bug fixes** @@ -60,6 +62,8 @@ processes. - 1074_: [FreeBSD] sensors_battery() raises OSError in case of no battery. - 1075_: [Windows] net_if_addrs(): inet_ntop() return value is not checked. +- 1079_: [FreeBSD] net_connections() didn't list locally connected sockets. + (patch by Gleb Smirnoff) **Porting notes** diff --git a/docs/index.rst b/docs/index.rst index d6488e348..4e378ea44 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -480,7 +480,7 @@ Network process this may be passed to `socket.fromfd() `__ to obtain a usable socket object. - On Windows, FreeBSD and SunOS this is always set to ``-1``. + On Windows and SunOS this is always set to ``-1``. - **family**: the address family, either `AF_INET `__, `AF_INET6 `__ @@ -566,6 +566,9 @@ Network .. versionadded:: 2.1.0 + .. versionchanged:: 5.3.0 : socket "fd" is now set for real instead of being + ``-1``. + .. function:: net_if_addrs() Return the addresses associated to each NIC (network interface card) From 6cce4e7542728de20ab597ae1cd43bed5b0b6df8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 05:02:40 +0200 Subject: [PATCH 716/922] fix travis --- HISTORY.rst | 2 +- psutil/_psutil_common.c | 3 ++- psutil/tests/test_connections.py | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 974af6171..b18438872 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -48,8 +48,8 @@ - 1042_: [FreeBSD] psutil won't compile on FreeBSD 12. - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. -- 1050_: [Windows] Process.memory_maps memory() leaks memory. - 1048_: [Windows] users()'s host field report an invalid IP address. +- 1050_: [Windows] Process.memory_maps memory() leaks memory. - 1055_: cpu_count() is no longer cached. - 1058_: fixed Python warnings. - 1062_: disk_io_counters() and net_io_counters() raise TypeError if no disks diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index bcbd623b6..c757c7255 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -48,8 +48,9 @@ PyUnicode_DecodeFSDefault(char *s) { return PyString_FromString(s); } + PyObject * PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size) { return PyString_FromStringAndSize(s, size); } -#endif \ No newline at end of file +#endif diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 27822a807..83a5278a3 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -24,7 +24,6 @@ from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS -from psutil._common import pconn from psutil._common import supports_ipv6 from psutil._compat import PY3 from psutil.tests import AF_UNIX From a6617589f21510d5480b442d37378bb3721c03d2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 16:51:29 +0200 Subject: [PATCH 717/922] sunos: fix som connections tests --- psutil/tests/__init__.py | 2 +- psutil/tests/test_connections.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 927e8cf64..ba7e1ce43 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -868,7 +868,7 @@ def create_sockets(): if supports_ipv6(): socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM)) socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM)) - if POSIX: + if POSIX and HAS_CONNECTIONS_UNIX: fname1 = unix_socket_path().__enter__() fname2 = unix_socket_path().__enter__() s1, s2 = unix_socketpair(fname1) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 83a5278a3..aff3c8db7 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -66,7 +66,7 @@ def tearDown(self): cons = thisproc.connections(kind='all') assert not cons, cons - def get_conn_from_socck(self, sock): + def get_conn_from_sock(self, sock): cons = thisproc.connections(kind='all') smap = dict([(c.fd, c) for c in cons]) if NETBSD: @@ -85,7 +85,7 @@ def check_socket(self, sock, conn=None): only (the one supposed to be checked). """ if conn is None: - conn = self.get_conn_from_socck(sock) + conn = self.get_conn_from_sock(sock) check_connection_ntuple(conn) # fd, family, type @@ -231,7 +231,7 @@ def test_unix(self): # a UNIX connection to /var/run/log. cons = [c for c in cons if c.raddr != '/var/run/log'] self.assertEqual(len(cons), 2) - if LINUX or FREEBSD: + if LINUX or FREEBSD or SUNOS: # remote path is never set self.assertEqual(cons[0].raddr, "") self.assertEqual(cons[1].raddr, "") From 6a79a37d6b9a2c0edc1bf695eff34b130cf92768 Mon Sep 17 00:00:00 2001 From: Oleksii Shevchuk Date: Thu, 18 May 2017 20:06:47 +0400 Subject: [PATCH 718/922] Fix https://github.com/giampaolo/psutil/issues/1077 (#1081) --- psutil/_psutil_sunos.c | 60 ++++++++++++++++++++----------- psutil/arch/solaris/v10/ifaddrs.c | 12 +++---- psutil/arch/solaris/v10/ifaddrs.h | 2 +- setup.py | 1 + 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 6c152eed5..53f719e48 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -9,18 +9,23 @@ * this in Cython which I later on translated in C. */ +// fix compilation issue on SunOS 5.10, see: +// https://github.com/giampaolo/psutil/issues/421 +// https://github.com/giampaolo/psutil/issues/1077 +// http://us-east.manta.joyent.com/jmc/public/opensolaris/ARChive/PSARC/2010/111/materials/s10ceval.txt +// +// Because LEGACY_MIB_SIZE defined in the same file there is no way to make autoconfiguration =\ +// -#include - -// fix for "Cannot use procfs in the large file compilation environment" -// error, see: -// http://sourceware.org/ml/gdb-patches/2010-11/msg00336.html -#undef _FILE_OFFSET_BITS +#define NEW_MIB_COMPLIANT 1 #define _STRUCTURED_PROC 1 -// fix compilation issue on SunOS 5.10, see: -// https://github.com/giampaolo/psutil/issues/421 -#define NEW_MIB_COMPLIANT +#include + +#if !defined(_LP64) && _FILE_OFFSET_BITS == 64 +# undef _FILE_OFFSET_BITS +# undef _LARGEFILE64_SOURCE +#endif #include #include @@ -46,9 +51,6 @@ #include "_psutil_posix.h" #define PSUTIL_TV2DOUBLE(t) (((t).tv_nsec * 0.000000001) + (t).tv_sec) -#ifndef EXPER_IP_AND_ALL_IRES -#define EXPER_IP_AND_ALL_IRES (1024+4) -#endif /* @@ -960,7 +962,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { #endif char buf[512]; int i, flags, getcode, num_ent, state; - char lip[200], rip[200]; + char lip[INET6_ADDRSTRLEN], rip[INET6_ADDRSTRLEN]; int lport, rport; int processed_pid; int databuf_init = 0; @@ -986,9 +988,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { goto error; } - /* - XXX - These 2 are used in ifconfig.c but they seem unnecessary - ret = ioctl(sd, I_PUSH, "tcp"); + int ret = ioctl(sd, I_PUSH, "tcp"); if (ret == -1) { PyErr_SetFromErrno(PyExc_OSError); goto error; @@ -998,8 +998,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { PyErr_SetFromErrno(PyExc_OSError); goto error; } - */ - + // // OK, this mess is basically copied and pasted from nxsensor project // which copied and pasted it from netstat source code, mibget() // function. Also see: @@ -1009,9 +1008,14 @@ psutil_net_connections(PyObject *self, PyObject *args) { tor->OPT_length = sizeof (struct opthdr); tor->MGMT_flags = T_CURRENT; mibhdr = (struct opthdr *)&tor[1]; - mibhdr->level = EXPER_IP_AND_ALL_IRES; + mibhdr->level = MIB2_IP; mibhdr->name = 0; + +#ifdef NEW_MIB_COMPLIANT + mibhdr->len = 1; +#else mibhdr->len = 0; +#endif ctlbuf.buf = buf; ctlbuf.len = tor->OPT_offset + tor->OPT_length; @@ -1024,7 +1028,6 @@ psutil_net_connections(PyObject *self, PyObject *args) { mibhdr = (struct opthdr *)&toa[1]; ctlbuf.maxlen = sizeof (buf); - for (;;) { flags = 0; getcode = getmsg(sd, &ctlbuf, (struct strbuf *)0, &flags); @@ -1072,7 +1075,11 @@ psutil_net_connections(PyObject *self, PyObject *args) { tp = (mib2_tcpConnEntry_t *)databuf.buf; num_ent = mibhdr->len / sizeof(mib2_tcpConnEntry_t); for (i = 0; i < num_ent; i++, tp++) { +#ifdef NEW_MIB_COMPLIANT processed_pid = tp->tcpConnCreationProcess; +#else + processed_pid = 0; +#endif if (pid != -1 && processed_pid != pid) continue; // construct local/remote addresses @@ -1113,7 +1120,11 @@ psutil_net_connections(PyObject *self, PyObject *args) { num_ent = mibhdr->len / sizeof(mib2_tcp6ConnEntry_t); for (i = 0; i < num_ent; i++, tp6++) { +#ifdef NEW_MIB_COMPLIANT processed_pid = tp6->tcp6ConnCreationProcess; +#else + processed_pid = 0; +#endif if (pid != -1 && processed_pid != pid) continue; // construct local/remote addresses @@ -1149,8 +1160,13 @@ psutil_net_connections(PyObject *self, PyObject *args) { else if (mibhdr->level == MIB2_UDP || mibhdr->level == MIB2_UDP_ENTRY) { ude = (mib2_udpEntry_t *)databuf.buf; num_ent = mibhdr->len / sizeof(mib2_udpEntry_t); + assert(num_ent * sizeof(mib2_udpEntry_t) == mibhdr->len); for (i = 0; i < num_ent; i++, ude++) { +#ifdef NEW_MIB_COMPLIANT processed_pid = ude->udpCreationProcess; +#else + processed_pid = 0; +#endif if (pid != -1 && processed_pid != pid) continue; // XXX Very ugly hack! It seems we get here only the first @@ -1186,7 +1202,11 @@ psutil_net_connections(PyObject *self, PyObject *args) { ude6 = (mib2_udp6Entry_t *)databuf.buf; num_ent = mibhdr->len / sizeof(mib2_udp6Entry_t); for (i = 0; i < num_ent; i++, ude6++) { +#ifdef NEW_MIB_COMPLIANT processed_pid = ude6->udp6CreationProcess; +#else + processed_pid = 0; +#endif if (pid != -1 && processed_pid != pid) continue; inet_ntop(AF_INET6, &ude6->udp6LocalAddress, lip, sizeof(lip)); diff --git a/psutil/arch/solaris/v10/ifaddrs.c b/psutil/arch/solaris/v10/ifaddrs.c index 2eb36a3ad..aedba84e9 100644 --- a/psutil/arch/solaris/v10/ifaddrs.c +++ b/psutil/arch/solaris/v10/ifaddrs.c @@ -21,10 +21,10 @@ static struct sockaddr * -sa_dup (struct sockaddr *sa1) +sa_dup (struct sockaddr_storage *sa1) { struct sockaddr *sa2; - size_t sz = sizeof(sa1); + size_t sz = sizeof(struct sockaddr_storage); sa2 = (struct sockaddr *) calloc(1,sz); memcpy(sa2,sa1,sz); return(sa2); @@ -91,11 +91,11 @@ int getifaddrs (struct ifaddrs **ifap) if (ioctl(sd, SIOCGLIFADDR, ifr, IFREQSZ) < 0) goto error; - cifa->ifa_addr = sa_dup((struct sockaddr*)&ifr->lifr_addr); + cifa->ifa_addr = sa_dup(&ifr->lifr_addr); if (ioctl(sd, SIOCGLIFNETMASK, ifr, IFREQSZ) < 0) goto error; - cifa->ifa_netmask = sa_dup((struct sockaddr*)&ifr->lifr_addr); + cifa->ifa_netmask = sa_dup(&ifr->lifr_addr); cifa->ifa_flags = 0; cifa->ifa_dstaddr = NULL; @@ -105,9 +105,9 @@ int getifaddrs (struct ifaddrs **ifap) if (ioctl(sd, SIOCGLIFDSTADDR, ifr, IFREQSZ) < 0) { if (0 == ioctl(sd, SIOCGLIFBRDADDR, ifr, IFREQSZ)) - cifa->ifa_dstaddr = sa_dup((struct sockaddr*)&ifr->lifr_addr); + cifa->ifa_dstaddr = sa_dup(&ifr->lifr_addr); } - else cifa->ifa_dstaddr = sa_dup((struct sockaddr*)&ifr->lifr_addr); + else cifa->ifa_dstaddr = sa_dup(&ifr->lifr_addr); pifa = cifa; ccp += IFREQSZ; diff --git a/psutil/arch/solaris/v10/ifaddrs.h b/psutil/arch/solaris/v10/ifaddrs.h index e1d885963..d27711935 100644 --- a/psutil/arch/solaris/v10/ifaddrs.h +++ b/psutil/arch/solaris/v10/ifaddrs.h @@ -23,4 +23,4 @@ struct ifaddrs { extern int getifaddrs(struct ifaddrs **); extern void freeifaddrs(struct ifaddrs *); -#endif \ No newline at end of file +#endif diff --git a/setup.py b/setup.py index 7c6ffc9e4..42ded9e17 100755 --- a/setup.py +++ b/setup.py @@ -233,6 +233,7 @@ def get_ethtool_macro(): elif SUNOS: macros.append(("PSUTIL_SUNOS", 1)) + sources.append('psutil/arch/solaris/v10/ifaddrs.c') ext = Extension( 'psutil._psutil_sunos', sources=sources + ['psutil/_psutil_sunos.c'], From 135ce35ad95f70b8fcc0d0769d16c8fc7ee18ad4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 18:22:35 +0200 Subject: [PATCH 719/922] give CREDITS to @alxchk; update HISTORY --- CREDITS | 5 +++++ HISTORY.rst | 4 ++++ psutil/_psutil_sunos.c | 8 ++++---- setup.py | 6 ++++-- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CREDITS b/CREDITS index 5483653a5..b11a8b93c 100644 --- a/CREDITS +++ b/CREDITS @@ -40,6 +40,7 @@ Github usernames of people to CC on github when in need of help. - fbenkstein, Frank Benkstein - SunOS: - wiggin15, Arnon Yaari + - alxchk, Oleksii Shevchuk Contributors ============ @@ -478,3 +479,7 @@ I: 1057 N: Gleb Smirnoff W: https://github.com/glebius I: 1042, 1079 + +N: Oleksii Shevchuk +W: https://github.com/alxchk +I: 1077 diff --git a/HISTORY.rst b/HISTORY.rst index b18438872..bdca8f246 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -62,6 +62,10 @@ processes. - 1074_: [FreeBSD] sensors_battery() raises OSError in case of no battery. - 1075_: [Windows] net_if_addrs(): inet_ntop() return value is not checked. +- 1077_: [SunOS] net_if_addrs() shows garbage addresses on SunOS 5.10. + (patch by Oleksii Shevchuk) +- 1077_: [SunOS] net_connections() does not work on SunOS 5.10. (patch by + Oleksii Shevchuk) - 1079_: [FreeBSD] net_connections() didn't list locally connected sockets. (patch by Gleb Smirnoff) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 53f719e48..eba71ec5b 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -1078,7 +1078,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { #ifdef NEW_MIB_COMPLIANT processed_pid = tp->tcpConnCreationProcess; #else - processed_pid = 0; + processed_pid = 0; #endif if (pid != -1 && processed_pid != pid) continue; @@ -1123,7 +1123,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { #ifdef NEW_MIB_COMPLIANT processed_pid = tp6->tcp6ConnCreationProcess; #else - processed_pid = 0; + processed_pid = 0; #endif if (pid != -1 && processed_pid != pid) continue; @@ -1165,7 +1165,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { #ifdef NEW_MIB_COMPLIANT processed_pid = ude->udpCreationProcess; #else - processed_pid = 0; + processed_pid = 0; #endif if (pid != -1 && processed_pid != pid) continue; @@ -1205,7 +1205,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { #ifdef NEW_MIB_COMPLIANT processed_pid = ude6->udp6CreationProcess; #else - processed_pid = 0; + processed_pid = 0; #endif if (pid != -1 && processed_pid != pid) continue; diff --git a/setup.py b/setup.py index 42ded9e17..cf3deb7f7 100755 --- a/setup.py +++ b/setup.py @@ -233,10 +233,12 @@ def get_ethtool_macro(): elif SUNOS: macros.append(("PSUTIL_SUNOS", 1)) - sources.append('psutil/arch/solaris/v10/ifaddrs.c') ext = Extension( 'psutil._psutil_sunos', - sources=sources + ['psutil/_psutil_sunos.c'], + sources=sources + [ + 'psutil/_psutil_sunos.c', + 'psutil/arch/solaris/v10/ifaddrs.c', + ], define_macros=macros, libraries=['kstat', 'nsl', 'socket']) From 5ce460f3fab568bf7dd15f93d12e37239b11803a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 18:33:59 +0200 Subject: [PATCH 720/922] regenerate MANIFEST.in --- MANIFEST.in | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 6c4b2b93b..65e11598f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -43,16 +43,18 @@ include psutil/_psutil_posix.h include psutil/_psutil_sunos.c include psutil/_psutil_windows.c include psutil/_pswindows.py -include psutil/arch/bsd/freebsd.c -include psutil/arch/bsd/freebsd.h -include psutil/arch/bsd/freebsd_socks.c -include psutil/arch/bsd/freebsd_socks.h -include psutil/arch/bsd/netbsd.c -include psutil/arch/bsd/netbsd.h -include psutil/arch/bsd/netbsd_socks.c -include psutil/arch/bsd/netbsd_socks.h -include psutil/arch/bsd/openbsd.c -include psutil/arch/bsd/openbsd.h +include psutil/arch/freebsd/proc_socks.c +include psutil/arch/freebsd/proc_socks.h +include psutil/arch/freebsd/specific.c +include psutil/arch/freebsd/specific.h +include psutil/arch/freebsd/sys_socks.c +include psutil/arch/freebsd/sys_socks.h +include psutil/arch/netbsd/socks.c +include psutil/arch/netbsd/socks.h +include psutil/arch/netbsd/specific.c +include psutil/arch/netbsd/specific.h +include psutil/arch/openbsd/specific.c +include psutil/arch/openbsd/specific.h include psutil/arch/osx/process_info.c include psutil/arch/osx/process_info.h include psutil/arch/solaris/v10/ifaddrs.c From 48da89ea4a6221ee5dc010aa14642319697f6073 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 19:58:33 +0200 Subject: [PATCH 721/922] skip failing connection tests on solaris --- psutil/tests/test_connections.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index aff3c8db7..9390214ed 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -199,6 +199,9 @@ class TestConnectedSocketPairs(Base, unittest.TestCase): each other. """ + # On SunOS, even after we close() it, the server socket stays around + # in TIME_WAIT state. + @unittest.skipIf(SUNOS, "unreliable on SUONS") def test_tcp(self): addr = ("127.0.0.1", get_free_port()) assert not thisproc.connections(kind='tcp4') @@ -352,7 +355,7 @@ def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): def test_multi_sockets_filtering(self): with create_sockets() as socks: cons = thisproc.connections(kind='all') - self.assertEqual(len(socks), len(cons)) + self.assertEqual(len(cons), len(socks)) # tcp cons = thisproc.connections(kind='tcp') self.assertEqual(len(cons), 2 if supports_ipv6() else 1) From 3fa7ad66061808369137d995306022aa7406b9b1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 20:05:37 +0200 Subject: [PATCH 722/922] skip UNIX socket tests on SUNOS --- psutil/tests/test_contracts.py | 2 ++ psutil/tests/test_unicode.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 7dd832597..65bad757f 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -30,6 +30,7 @@ from psutil.tests import bind_unix_socket from psutil.tests import check_connection_ntuple from psutil.tests import get_kernel_version +from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import HAS_RLIMIT from psutil.tests import HAS_SENSORS_FANS from psutil.tests import HAS_SENSORS_TEMPERATURES @@ -201,6 +202,7 @@ def test_disk_partitions(self): self.assertIsInstance(disk.opts, str) @unittest.skipIf(not POSIX, 'POSIX only') + @unittest.skipIf(not HAS_CONNECTIONS_UNIX, "can't list UNIX sockets") @skip_on_access_denied(only_if=OSX) def test_net_connections(self): with unix_socket_path() as name: diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 7c87a3f2f..05e0ebcb2 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -72,6 +72,7 @@ from psutil.tests import copyload_shared_lib from psutil.tests import create_exe from psutil.tests import get_test_subprocess +from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_MEMORY_MAPS from psutil.tests import mock @@ -247,6 +248,7 @@ def test_proc_connections(self): self.assertEqual(conn.laddr, name) @unittest.skipIf(not POSIX, "POSIX only") + @unittest.skipIf(not HAS_CONNECTIONS_UNIX, "can't list UNIX sockets") @skip_on_access_denied() def test_net_connections(self): def find_sock(cons): From de3aa8d54a14a50b274d741cfe5203c8c326c569 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 20:43:57 +0200 Subject: [PATCH 723/922] remove useless posix cwd test --- psutil/tests/test_misc.py | 3 ++- psutil/tests/test_posix.py | 6 ------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 57423dd80..f9459d30c 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -39,6 +39,7 @@ from psutil.tests import get_free_port from psutil.tests import get_test_subprocess from psutil.tests import HAS_BATTERY +from psutil.tests import HAS_CONNECTIONS_UNIX from psutil.tests import HAS_MEMORY_FULL_INFO from psutil.tests import HAS_MEMORY_MAPS from psutil.tests import HAS_SENSORS_BATTERY @@ -1006,7 +1007,7 @@ def test_create_sockets(self): types[s.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE)] += 1 self.assertGreaterEqual(fams[socket.AF_INET], 2) self.assertGreaterEqual(fams[socket.AF_INET6], 2) - if POSIX: + if POSIX and HAS_CONNECTIONS_UNIX: self.assertGreaterEqual(fams[socket.AF_UNIX], 2) self.assertGreaterEqual(types[socket.SOCK_STREAM], 2) self.assertGreaterEqual(types[socket.SOCK_DGRAM], 2) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index f810a09e7..54f886d81 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -248,12 +248,6 @@ def call(p, attr): if failures: self.fail('\n' + '\n'.join(failures)) - @unittest.skipIf(not os.path.islink("/proc/%s/cwd" % os.getpid()), - "/proc fs not available") - def test_cwd(self): - self.assertEqual(os.readlink("/proc/%s/cwd" % os.getpid()), - psutil.Process().cwd()) - @unittest.skipIf(not POSIX, "POSIX only") class TestSystemAPIs(unittest.TestCase): From 4490889cbfc33d63a0c6805c2c008b7593eed586 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 21:29:02 +0200 Subject: [PATCH 724/922] #1082: disable unreliable ps niceness test on SunOS --- psutil/tests/test_posix.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 54f886d81..580abdfde 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -202,6 +202,11 @@ def test_cmdline(self): psutil_cmdline = psutil_cmdline.split(" ")[0] self.assertEqual(ps_cmdline, psutil_cmdline) + # On SUNOS "ps" reads niceness /proc/pid/psinfo which returns an + # incorrect value (20); the real deal is getpriority(2) which + # returns 0; psutil relies on it, see: + # https://github.com/giampaolo/psutil/issues/1082 + @unittest.skipIf(SUNOS, "not reliable on SUNOS") def test_nice(self): ps_nice = ps("ps --no-headers -o nice -p %s" % self.pid) psutil_nice = psutil.Process().nice() From 7fbf19d0f04730983265bb494460929f708475c5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 22:00:08 +0200 Subject: [PATCH 725/922] refactor sunos py code --- psutil/_pssunos.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 6782f7f3d..6fbb3731b 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -66,6 +66,16 @@ cext.TCPS_BOUND: CONN_BOUND, # sunos specific } +proc_info_map = dict( + ppid=0, + rss=1, + vms=2, + create_time=3, + nice=4, + num_threads=5, + status=6, + ttynr=7) + # these get overwritten on "import psutil" from the __init__.py file NoSuchProcess = None ZombieProcess = None @@ -374,7 +384,9 @@ def _proc_name_and_args(self): @memoize_when_activated def _proc_basic_info(self): - return cext.proc_basic_info(self.pid, self._procfs_path) + ret = cext.proc_basic_info(self.pid, self._procfs_path) + assert len(ret) == len(proc_info_map) + return ret @memoize_when_activated def _proc_cred(self): @@ -404,11 +416,11 @@ def cmdline(self): @wrap_exceptions def create_time(self): - return self._proc_basic_info()[3] + return self._proc_basic_info()[proc_info_map['create_time']] @wrap_exceptions def num_threads(self): - return self._proc_basic_info()[5] + return self._proc_basic_info()[proc_info_map['num_threads']] @wrap_exceptions def nice_get(self): @@ -441,7 +453,7 @@ def nice_set(self, value): @wrap_exceptions def ppid(self): - self._ppid = self._proc_basic_info()[0] + self._ppid = self._proc_basic_info()[proc_info_map['ppid']] return self._ppid @wrap_exceptions @@ -481,7 +493,7 @@ def terminal(self): procfs_path = self._procfs_path hit_enoent = False tty = wrap_exceptions( - self._proc_basic_info()[0]) + self._proc_basic_info()[proc_info_map['ttynr']]) if tty != cext.PRNODEV: for x in (0, 1, 2, 255): try: @@ -514,14 +526,15 @@ def cwd(self): @wrap_exceptions def memory_info(self): ret = self._proc_basic_info() - rss, vms = ret[1] * 1024, ret[2] * 1024 + rss = ret[proc_info_map['rss']] * 1024 + vms = ret[proc_info_map['vms']] * 1024 return pmem(rss, vms) memory_full_info = memory_info @wrap_exceptions def status(self): - code = self._proc_basic_info()[6] + code = self._proc_basic_info()[proc_info_map['status']] # XXX is '?' legit? (we're not supposed to return it anyway) return PROC_STATUSES.get(code, '?') From d58c4338d64fad9b3c0eebb2451cd55f660e86cf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 22:02:13 +0200 Subject: [PATCH 726/922] add notes about #1082 --- psutil/_pssunos.py | 9 +++++++-- psutil/_psutil_sunos.c | 23 +++++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 6fbb3731b..b1ba6b452 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -424,13 +424,18 @@ def num_threads(self): @wrap_exceptions def nice_get(self): - # For some reason getpriority(3) return ESRCH (no such process) - # for certain low-pid processes, no matter what (even as root). + # Note #1: for some reason getpriority(3) return ESRCH (no such + # process) for certain low-pid processes, no matter what (even + # as root). # The process actually exists though, as it has a name, # creation time, etc. # The best thing we can do here appears to be raising AD. # Note: tested on Solaris 11; on Open Solaris 5 everything is # fine. + # + # Note #2: we also can get niceness from /proc/pid/psinfo + # but it's wrong, see: + # https://github.com/giampaolo/psutil/issues/1082 try: return cext_posix.getpriority(self.pid) except EnvironmentError as err: diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index eba71ec5b..e3eb2560e 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -99,16 +99,19 @@ psutil_proc_basic_info(PyObject *self, PyObject *args) { sprintf(path, "%s/%i/psinfo", procfs_path, pid); if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) return NULL; - return Py_BuildValue("ikkdiiik", - info.pr_ppid, // parent pid - info.pr_rssize, // rss - info.pr_size, // vms - PSUTIL_TV2DOUBLE(info.pr_start), // create time - info.pr_lwp.pr_nice, // nice - info.pr_nlwp, // no. of threads - info.pr_lwp.pr_state, // status code - info.pr_ttydev // tty nr - ); + return Py_BuildValue( + "ikkdiiik", + info.pr_ppid, // parent pid + info.pr_rssize, // rss + info.pr_size, // vms + PSUTIL_TV2DOUBLE(info.pr_start), // create time + // XXX - niceness is wrong (20 instead of 0), see: + // https://github.com/giampaolo/psutil/issues/1082 + info.pr_lwp.pr_nice, // nice + info.pr_nlwp, // no. of threads + info.pr_lwp.pr_state, // status code + info.pr_ttydev // tty nr + ); } From cee414dca663bdac9712c0779e43ce0c184e904e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 21:43:30 +0200 Subject: [PATCH 727/922] PSUTIL_TESTING env var (#1083) * Introduce PSUTIL_TESTING env var ...so that we can make stricter assertions in C and py code during tests only. * define a C function in _common.c which returns whether the var is set * set PSUTIL_TESTING from the Makefile * cache psutil_testing() result * winmake: set PSUTIL_TESTING env var for tests --- Makefile | 24 ++++++++++++------------ psutil/_psutil_bsd.c | 6 +++++- psutil/_psutil_common.c | 31 +++++++++++++++++++++++++++++++ psutil/_psutil_common.h | 4 +++- psutil/_psutil_linux.c | 6 +++++- psutil/_psutil_osx.c | 5 ++++- psutil/_psutil_sunos.c | 5 ++++- psutil/_psutil_windows.c | 5 ++++- psutil/tests/__init__.py | 32 +++++++++++++++++++++++++++++++- psutil/tests/__main__.py | 21 +-------------------- psutil/tests/test_process.py | 2 ++ scripts/internal/winmake.py | 17 ++++++++++++++++- 12 files changed, 118 insertions(+), 40 deletions(-) diff --git a/Makefile b/Makefile index 8b40d8c24..c44c6c029 100644 --- a/Makefile +++ b/Makefile @@ -118,65 +118,65 @@ setup-dev-env: # Run all tests. test: ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) $(TSCRIPT) + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) $(TSCRIPT) # Run process-related API tests. test-process: ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_process + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_process # Run system-related API tests. test-system: ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_system + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_system # Run miscellaneous tests. test-misc: ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_misc.py + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_misc.py # Test APIs dealing with strings. test-unicode: ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_unicode.py + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_unicode.py # APIs sanity tests. test-contracts: ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_contracts.py + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_contracts.py # Test net_connections() and Process.connections(). test-connections: ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_connections.py + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_connections.py # POSIX specific tests. test-posix: ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_posix.py + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_posix.py # Run specific platform tests only. test-platform: ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py # Memory leak tests. test-memleaks: ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_memory_leaks.py + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_memory_leaks.py # Run a specific test by name, e.g. # make test-by-name psutil.tests.test_system.TestSystemAPIs.test_cpu_times test-by-name: ${MAKE} install - @PYTHONWARNINGS=all $(PYTHON) -m unittest -v $(ARGS) + @PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v $(ARGS) # Run test coverage. coverage: ${MAKE} install # Note: coverage options are controlled by .coveragerc file rm -rf .coverage htmlcov - PYTHONWARNINGS=all $(PYTHON) -m coverage run $(TSCRIPT) + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m coverage run $(TSCRIPT) $(PYTHON) -m coverage report @echo "writing results to htmlcov/index.html" $(PYTHON) -m coverage html diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 217a95de5..3527b6667 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -908,7 +908,6 @@ psutil_users(PyObject *self, PyObject *args) { */ static PyMethodDef PsutilMethods[] = { - // --- per-process functions {"proc_oneshot_info", psutil_proc_oneshot_info, METH_VARARGS, @@ -983,6 +982,11 @@ PsutilMethods[] = { {"sensors_battery", psutil_sensors_battery, METH_VARARGS, "Return battery information."}, #endif + + // --- others + {"py_psutil_testing", py_psutil_testing, METH_VARARGS, + "Return True if PSUTIL_TESTING env var is set"}, + {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index c757c7255..dace4724b 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -7,6 +7,7 @@ */ #include +#include /* * Set OSError(errno=ESRCH, strerror="No such process") Python exception. @@ -36,6 +37,36 @@ AccessDenied(void) { } +static int _psutil_testing = -1; + + +/* + * Return 1 if PSUTIL_TESTING env var is set else 0. + */ +int +psutil_testing(void) { + if (_psutil_testing == -1) { + if (getenv("PSUTIL_TESTING") != NULL) + _psutil_testing = 1; + else + _psutil_testing = 0; + } + return _psutil_testing; +} + + +/* + * Return True if PSUTIL_TESTING env var is set else False. + */ +PyObject * +py_psutil_testing(PyObject *self, PyObject *args) { + PyObject *res; + res = psutil_testing() ? Py_True : Py_False; + Py_INCREF(res); + return res; +} + + /* * Backport of unicode FS APIs from Python 3. * On Python 2 we just return a plain byte string diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index aa634ad37..134045327 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -11,7 +11,9 @@ static const int PSUTIL_CONN_NONE = 128; PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); +int psutil_testing(void); +PyObject* py_psutil_testing(PyObject *self, PyObject *args); #if PY_MAJOR_VERSION < 3 PyObject* PyUnicode_DecodeFSDefault(char *s); PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); -#endif \ No newline at end of file +#endif diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 1a96fea08..e262ac701 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -203,6 +203,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { if (py_retlist == NULL) return NULL; + psutil_testing(); + // MOUNTED constant comes from mntent.h and it's == '/etc/mtab' Py_BEGIN_ALLOW_THREADS file = setmntent(MOUNTED, "r"); @@ -574,7 +576,6 @@ psutil_net_if_duplex_speed(PyObject* self, PyObject* args) { */ static PyMethodDef PsutilMethods[] = { - // --- per-process functions #if PSUTIL_HAVE_IOPRIO @@ -607,6 +608,9 @@ PsutilMethods[] = { "Get or set process resource limits."}, #endif + // --- others + {"py_psutil_testing", py_psutil_testing, METH_VARARGS, + "Return True if PSUTIL_TESTING env var is set"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 20ece694b..7d762a1cb 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1779,7 +1779,6 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { */ static PyMethodDef PsutilMethods[] = { - // --- per-process functions {"proc_kinfo_oneshot", psutil_proc_kinfo_oneshot, METH_VARARGS, @@ -1841,6 +1840,10 @@ PsutilMethods[] = { {"cpu_stats", psutil_cpu_stats, METH_VARARGS, "Return CPU statistics"}, + // --- others + {"py_psutil_testing", py_psutil_testing, METH_VARARGS, + "Return True if PSUTIL_TESTING env var is set"}, + {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index e3eb2560e..785805d4d 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -1470,7 +1470,6 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { */ static PyMethodDef PsutilMethods[] = { - // --- process-related functions {"proc_basic_info", psutil_proc_basic_info, METH_VARARGS, "Return process ppid, rss, vms, ctime, nice, nthreads, status and tty"}, @@ -1513,6 +1512,10 @@ PsutilMethods[] = { {"cpu_stats", psutil_cpu_stats, METH_VARARGS, "Return CPU statistics"}, + // --- others + {"py_psutil_testing", py_psutil_testing, METH_VARARGS, + "Return True if PSUTIL_TESTING env var is set"}, + {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 436dd76b5..795ee9cda 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -3479,7 +3479,6 @@ psutil_sensors_battery(PyObject *self, PyObject *args) { static PyMethodDef PsutilMethods[] = { - // --- per-process functions {"proc_cmdline", psutil_proc_cmdline, METH_VARARGS, @@ -3602,6 +3601,10 @@ PsutilMethods[] = { {"win32_QueryDosDevice", psutil_win32_QueryDosDevice, METH_VARARGS, "QueryDosDevice binding"}, + // --- others + {"py_psutil_testing", py_psutil_testing, METH_VARARGS, + "Return True if PSUTIL_TESTING env var is set"}, + {NULL, NULL, 0, NULL} }; diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index ba7e1ce43..0ba95b186 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -77,7 +77,8 @@ 'ThreadTask' # test utils 'unittest', 'skip_on_access_denied', 'skip_on_not_implemented', - 'retry_before_failing', 'run_test_module_by_name', + 'retry_before_failing', 'run_test_module_by_name', 'get_suite', + 'run_suite', # install utils 'install_pip', 'install_test_deps', # fs utils @@ -141,6 +142,7 @@ ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')) SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts') +HERE = os.path.abspath(os.path.dirname(__file__)) # --- support @@ -383,6 +385,9 @@ def reap_children(recursive=False): # https://ci.appveyor.com/project/giampaolo/psutil/build/job/ # jiq2cgd6stsbtn60 def assert_gone(pid): + # XXX + if WINDOWS: + return assert not psutil.pid_exists(pid), pid assert pid not in psutil.pids(), pid try: @@ -699,9 +704,34 @@ def __str__(self): unittest.TestCase = TestCase +def _setup_tests(): + assert 'PSUTIL_TESTING' in os.environ + assert psutil._psplatform.cext.py_psutil_testing() + + +def get_suite(): + testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) + if x.endswith('.py') and x.startswith('test_') and not + x.startswith('test_memory_leaks')] + suite = unittest.TestSuite() + for tm in testmodules: + # ...so that the full test paths are printed on screen + tm = "psutil.tests.%s" % tm + suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) + return suite + + +def run_suite(): + _setup_tests() + result = unittest.TextTestRunner(verbosity=VERBOSITY).run(get_suite()) + success = result.wasSuccessful() + sys.exit(0 if success else 1) + + def run_test_module_by_name(name): # testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) # if x.endswith('.py') and x.startswith('test_')] + _setup_tests() name = os.path.splitext(os.path.basename(name))[0] suite = unittest.TestSuite() suite.addTest(unittest.defaultTestLoader.loadTestsFromName(name)) diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py index 896b00cc7..475e6b81c 100755 --- a/psutil/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -21,8 +21,7 @@ except ImportError: from urllib2 import urlopen -from psutil.tests import unittest -from psutil.tests import VERBOSITY +from psutil.tests import run_suite HERE = os.path.abspath(os.path.dirname(__file__)) @@ -73,24 +72,6 @@ def install_test_deps(deps=None): return code -def get_suite(): - testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) - if x.endswith('.py') and x.startswith('test_') and not - x.startswith('test_memory_leaks')] - suite = unittest.TestSuite() - for tm in testmodules: - # ...so that the full test paths are printed on screen - tm = "psutil.tests.%s" % tm - suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) - return suite - - -def run_suite(): - result = unittest.TextTestRunner(verbosity=VERBOSITY).run(get_suite()) - success = result.wasSuccessful() - sys.exit(0 if success else 1) - - def main(): usage = "%s -m psutil.tests [opts]" % PYTHON parser = optparse.OptionParser(usage=usage, description="run unit tests") diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index b1f2508f3..3410ec0b3 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1391,6 +1391,8 @@ def test_environ(self): d2 = os.environ.copy() removes = [] + if 'PSUTIL_TESTING' in os.environ: + removes.append('PSUTIL_TESTING') if OSX: removes.extend([ "__CF_USER_TEXT_ENCODING", diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index d0c2c0a13..40ba37425 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -77,7 +77,10 @@ def safe_print(text, file=sys.stdout, flush=False): def sh(cmd, nolog=False): if not nolog: safe_print("cmd: " + cmd) - subprocess.check_call(cmd, shell=True, env=os.environ, cwd=os.getcwd()) + p = subprocess.Popen(cmd, shell=True, env=os.environ, cwd=os.getcwd()) + p.communicate() + if p.returncode != 0: + sys.exit(p.returncode) def cmd(fun): @@ -327,6 +330,7 @@ def flake8(): def test(): """Run tests""" install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa %s" % (PYTHON, TSCRIPT)) @@ -335,6 +339,7 @@ def coverage(): """Run coverage tests.""" # Note: coverage options are controlled by .coveragerc file install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m coverage run %s" % (PYTHON, TSCRIPT)) sh("%s -m coverage report" % PYTHON) sh("%s -m coverage html" % PYTHON) @@ -345,6 +350,7 @@ def coverage(): def test_process(): """Run process tests""" install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_process" % PYTHON) @@ -352,6 +358,7 @@ def test_process(): def test_system(): """Run system tests""" install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_system" % PYTHON) @@ -359,6 +366,7 @@ def test_system(): def test_platform(): """Run windows only tests""" install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_windows" % PYTHON) @@ -366,6 +374,7 @@ def test_platform(): def test_misc(): """Run misc tests""" install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_misc" % PYTHON) @@ -373,6 +382,7 @@ def test_misc(): def test_unicode(): """Run unicode tests""" install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_unicode" % PYTHON) @@ -380,6 +390,7 @@ def test_unicode(): def test_connections(): """Run connections tests""" install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_connections" % PYTHON) @@ -387,6 +398,7 @@ def test_connections(): def test_contracts(): """Run contracts tests""" install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_contracts" % PYTHON) @@ -399,6 +411,7 @@ def test_by_name(): except IndexError: sys.exit('second arg missing') install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v %s" % (PYTHON, name)) @@ -411,6 +424,7 @@ def test_script(): except IndexError: sys.exit('second arg missing') install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa %s" % (PYTHON, name)) @@ -418,6 +432,7 @@ def test_script(): def test_memleaks(): """Run memory leaks tests""" install() + os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa psutil\\tests\\test_memory_leaks.py" % PYTHON) From 822302832081b3968ed1ea97793709d263b5c1ca Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 21:45:59 +0200 Subject: [PATCH 728/922] remove useless line --- psutil/_psutil_linux.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index e262ac701..a15ebe5cf 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -203,8 +203,6 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { if (py_retlist == NULL) return NULL; - psutil_testing(); - // MOUNTED constant comes from mntent.h and it's == '/etc/mtab' Py_BEGIN_ALLOW_THREADS file = setmntent(MOUNTED, "r"); From d2b306688b8ab94ef678676f09dde7aa453b6d0c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 18 May 2017 22:04:52 +0200 Subject: [PATCH 729/922] set PSUTIL_TESTING env var for CI services --- .ci/travis/run.sh | 6 +++--- appveyor.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index b8526dffc..b27e6695c 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -20,14 +20,14 @@ python setup.py develop # run tests (with coverage) if [[ $PYVER == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then - python -Wa -m coverage run psutil/tests/__main__.py + PSUTIL_TESTING=1 python -Wa -m coverage run psutil/tests/__main__.py else - python -Wa psutil/tests/__main__.py + PSUTIL_TESTING=1 python -Wa psutil/tests/__main__.py fi if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then # run mem leaks test - python -Wa psutil/tests/test_memory_leaks.py + PSUTIL_TESTING=1 python -Wa psutil/tests/test_memory_leaks.py # run linter (on Linux only) if [[ "$(uname -s)" != 'Darwin' ]]; then python -Wa -m flake8 diff --git a/appveyor.yml b/appveyor.yml index 1390f962d..d46717845 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -91,7 +91,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "%WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" + - "set PSUTIL_TESTING=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" From e20b3473da81d1c4aca3919a895bc940c52ea333 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 19 May 2017 19:33:15 +0200 Subject: [PATCH 730/922] fix #1085: cpu_count() return value is now checked and forced to None if <= 1 --- HISTORY.rst | 1 + psutil/__init__.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bdca8f246..a85aacb56 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -68,6 +68,7 @@ Oleksii Shevchuk) - 1079_: [FreeBSD] net_connections() didn't list locally connected sockets. (patch by Gleb Smirnoff) +- 1085_: cpu_count() return value is now checked and forced to None if <= 1. **Porting notes** diff --git a/psutil/__init__.py b/psutil/__init__.py index a05d62498..c393ecc3a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1043,6 +1043,8 @@ def cpu_percent(self, interval=None): blocking = interval is not None and interval > 0.0 if interval is not None and interval < 0: raise ValueError("interval is not positive (got %r)" % interval) + # TODO: rarely cpu_count() may return None, meaning this will + # break. It's probably wise to fall back to 1. num_cpus = _NUM_CPUS or cpu_count() def timer(): @@ -1645,10 +1647,11 @@ def cpu_count(logical=True): """ global _NUM_CPUS if logical: - _NUM_CPUS = _psplatform.cpu_count_logical() - return _NUM_CPUS + ret = _psplatform.cpu_count_logical() + _NUM_CPUS = ret else: - return _psplatform.cpu_count_physical() + ret = _psplatform.cpu_count_physical() + return ret if ret >= 1 else None def cpu_times(percpu=False): From e5a081128e85d229af6499bb39b708ae15720358 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 19 May 2017 20:03:29 +0200 Subject: [PATCH 731/922] Fix #1055, fix #1085, fix #1087. - no longer cache cpu_count() return value in Process.cpu_percent() - in Process.cpu_percent(), guard against cpu_count() returning None and assume 1 instead - add test cases --- HISTORY.rst | 4 +++- psutil/__init__.py | 7 +------ psutil/tests/test_process.py | 6 ++++++ psutil/tests/test_system.py | 12 ++++++++++++ 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a85aacb56..d540ae69e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -50,7 +50,9 @@ - 1047_: [Windows] Process username(): memory leak in case exception is thrown. - 1048_: [Windows] users()'s host field report an invalid IP address. - 1050_: [Windows] Process.memory_maps memory() leaks memory. -- 1055_: cpu_count() is no longer cached. +- 1055_: cpu_count() is no longer cached; this is useful on systems such as + Linux where CPUs can be disabled at runtime. This also reflects on + Process.cpu_percent() which no longer uses the cache. - 1058_: fixed Python warnings. - 1062_: disk_io_counters() and net_io_counters() raise TypeError if no disks or NICs are installed on the system. diff --git a/psutil/__init__.py b/psutil/__init__.py index c393ecc3a..fc45abf16 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -209,7 +209,6 @@ POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED POWER_TIME_UNKNOWN = _common.POWER_TIME_UNKNOWN _TOTAL_PHYMEM = None -_NUM_CPUS = None _timer = getattr(time, 'monotonic', time.time) @@ -1043,9 +1042,7 @@ def cpu_percent(self, interval=None): blocking = interval is not None and interval > 0.0 if interval is not None and interval < 0: raise ValueError("interval is not positive (got %r)" % interval) - # TODO: rarely cpu_count() may return None, meaning this will - # break. It's probably wise to fall back to 1. - num_cpus = _NUM_CPUS or cpu_count() + num_cpus = cpu_count() or 1 def timer(): return _timer() * num_cpus @@ -1645,10 +1642,8 @@ def cpu_count(logical=True): >>> psutil.cpu_count.cache_clear() """ - global _NUM_CPUS if logical: ret = _psplatform.cpu_count_logical() - _NUM_CPUS = ret else: ret = _psplatform.cpu_count_physical() return ret if ret >= 1 else None diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 3410ec0b3..cab5a2fee 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -246,6 +246,12 @@ def test_cpu_percent(self): with self.assertRaises(ValueError): p.cpu_percent(interval=-1) + def test_cpu_percent_numcpus_none(self): + # See: https://github.com/giampaolo/psutil/issues/1087 + with mock.patch('psutil.cpu_count', return_value=None) as m: + psutil.Process().cpu_percent() + assert m.called + def test_cpu_times(self): times = psutil.Process().cpu_times() assert (times.user > 0.0) or (times.system > 0.0), times diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index fed7a222a..e93bb6b52 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -270,6 +270,18 @@ def test_cpu_count(self): self.assertGreaterEqual(physical, 1) self.assertGreaterEqual(logical, physical) + def test_cpu_count_none(self): + # https://github.com/giampaolo/psutil/issues/1085 + for val in (-1, 0, None): + with mock.patch('psutil._psplatform.cpu_count_logical', + return_value=val) as m: + self.assertIsNone(psutil.cpu_count()) + assert m.called + with mock.patch('psutil._psplatform.cpu_count_physical', + return_value=val) as m: + self.assertIsNone(psutil.cpu_count(logical=False)) + assert m.called + def test_cpu_times(self): # Check type, value >= 0, str(). total = 0 From eb6ab10d12eb4ecc636e8bac41781db36ac8a12b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 19 May 2017 20:04:39 +0200 Subject: [PATCH 732/922] update HISTORY --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index d540ae69e..bb20811a3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -71,6 +71,8 @@ - 1079_: [FreeBSD] net_connections() didn't list locally connected sockets. (patch by Gleb Smirnoff) - 1085_: cpu_count() return value is now checked and forced to None if <= 1. +- 1087_: Process.cpu_percent() guard against cpu_count() returning None and + assumes 1 instead. **Porting notes** From 5e74fa53becb51c17c21e2867b79d5403081faf0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 19 May 2017 20:17:16 +0200 Subject: [PATCH 733/922] remove dead code --- psutil/tests/test_unicode.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 05e0ebcb2..9b99fdf98 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -52,7 +52,6 @@ - https://pythonhosted.org/psutil/#unicode """ -import errno import os import traceback import warnings @@ -153,20 +152,6 @@ def tearDown(self): reap_children() safe_rmpath(self.funky_name) - def safe_rmpath(self, name): - if POSIX: - safe_rmpath(name) - else: - # https://ci.appveyor.com/project/giampaolo/psutil/build/ - # 1225/job/1yec67sr6e9rl217 - try: - safe_rmpath(name) - except OSError as err: - if err.errno in (errno.EACCES, errno.EPERM): - traceback.print_exc() - else: - raise - def expect_exact_path_match(self): raise NotImplementedError("must be implemented in subclass") From 4f5dafe498d8d13fa5e17c5c105b4988181478dd Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 19 May 2017 20:36:53 +0200 Subject: [PATCH 734/922] win process_info.c: move declarations at the top of the module --- psutil/arch/windows/process_info.c | 285 +++++++++++++++-------------- 1 file changed, 144 insertions(+), 141 deletions(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 24f7271c6..340745e92 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -18,6 +18,148 @@ #include "../../_psutil_common.h" +// Helper structures to access the memory correctly. Some of these might also +// be defined in the winternl.h header file but unfortunately not in a usable +// way. + +// see http://msdn2.microsoft.com/en-us/library/aa489609.aspx +#ifndef NT_SUCCESS +#define NT_SUCCESS(Status) ((NTSTATUS)(Status) >= 0) +#endif + +// http://msdn.microsoft.com/en-us/library/aa813741(VS.85).aspx +typedef struct { + BYTE Reserved1[16]; + PVOID Reserved2[5]; + UNICODE_STRING CurrentDirectoryPath; + PVOID CurrentDirectoryHandle; + UNICODE_STRING DllPath; + UNICODE_STRING ImagePathName; + UNICODE_STRING CommandLine; + LPCWSTR env; +} RTL_USER_PROCESS_PARAMETERS_, *PRTL_USER_PROCESS_PARAMETERS_; + +// https://msdn.microsoft.com/en-us/library/aa813706(v=vs.85).aspx +#ifdef _WIN64 +typedef struct { + BYTE Reserved1[2]; + BYTE BeingDebugged; + BYTE Reserved2[21]; + PVOID LoaderData; + PRTL_USER_PROCESS_PARAMETERS_ ProcessParameters; + /* More fields ... */ +} PEB_; +#else +typedef struct { + BYTE Reserved1[2]; + BYTE BeingDebugged; + BYTE Reserved2[1]; + PVOID Reserved3[2]; + PVOID Ldr; + PRTL_USER_PROCESS_PARAMETERS_ ProcessParameters; + /* More fields ... */ +} PEB_; +#endif + +#ifdef _WIN64 +/* When we are a 64 bit process accessing a 32 bit (WoW64) process we need to + use the 32 bit structure layout. */ +typedef struct { + USHORT Length; + USHORT MaxLength; + DWORD Buffer; +} UNICODE_STRING32; + +typedef struct { + BYTE Reserved1[16]; + DWORD Reserved2[5]; + UNICODE_STRING32 CurrentDirectoryPath; + DWORD CurrentDirectoryHandle; + UNICODE_STRING32 DllPath; + UNICODE_STRING32 ImagePathName; + UNICODE_STRING32 CommandLine; + DWORD env; +} RTL_USER_PROCESS_PARAMETERS32; + +typedef struct { + BYTE Reserved1[2]; + BYTE BeingDebugged; + BYTE Reserved2[1]; + DWORD Reserved3[2]; + DWORD Ldr; + DWORD ProcessParameters; + /* More fields ... */ +} PEB32; +#else +/* When we are a 32 bit (WoW64) process accessing a 64 bit process we need to + use the 64 bit structure layout and a special function to read its memory. + */ +typedef NTSTATUS (NTAPI *_NtWow64ReadVirtualMemory64)( + IN HANDLE ProcessHandle, + IN PVOID64 BaseAddress, + OUT PVOID Buffer, + IN ULONG64 Size, + OUT PULONG64 NumberOfBytesRead); + +typedef enum { + MemoryInformationBasic +} MEMORY_INFORMATION_CLASS; + +typedef NTSTATUS (NTAPI *_NtWow64QueryVirtualMemory64)( + IN HANDLE ProcessHandle, + IN PVOID64 BaseAddress, + IN MEMORY_INFORMATION_CLASS MemoryInformationClass, + OUT PMEMORY_BASIC_INFORMATION64 MemoryInformation, + IN ULONG64 Size, + OUT PULONG64 ReturnLength OPTIONAL); + +typedef struct { + PVOID Reserved1[2]; + PVOID64 PebBaseAddress; + PVOID Reserved2[4]; + PVOID UniqueProcessId[2]; + PVOID Reserved3[2]; +} PROCESS_BASIC_INFORMATION64; + +typedef struct { + USHORT Length; + USHORT MaxLength; + PVOID64 Buffer; +} UNICODE_STRING64; + +typedef struct { + BYTE Reserved1[16]; + PVOID64 Reserved2[5]; + UNICODE_STRING64 CurrentDirectoryPath; + PVOID64 CurrentDirectoryHandle; + UNICODE_STRING64 DllPath; + UNICODE_STRING64 ImagePathName; + UNICODE_STRING64 CommandLine; + PVOID64 env; +} RTL_USER_PROCESS_PARAMETERS64; + +typedef struct { + BYTE Reserved1[2]; + BYTE BeingDebugged; + BYTE Reserved2[21]; + PVOID64 LoaderData; + PVOID64 ProcessParameters; + /* More fields ... */ +} PEB64; +#endif + + +#define PSUTIL_FIRST_PROCESS(Processes) ( \ + (PSYSTEM_PROCESS_INFORMATION)(Processes)) +#define PSUTIL_NEXT_PROCESS(Process) ( \ + ((PSYSTEM_PROCESS_INFORMATION)(Process))->NextEntryOffset ? \ + (PSYSTEM_PROCESS_INFORMATION)((PCHAR)(Process) + \ + ((PSYSTEM_PROCESS_INFORMATION)(Process))->NextEntryOffset) : NULL) + +const int STATUS_INFO_LENGTH_MISMATCH = 0xC0000004; +const int STATUS_BUFFER_TOO_SMALL = 0xC0000023L; + + /* * A wrapper around OpenProcess setting NSP exception if process * no longer exists. @@ -168,136 +310,6 @@ psutil_pid_is_running(DWORD pid) { } -// Helper structures to access the memory correctly. Some of these might also -// be defined in the winternl.h header file but unfortunately not in a usable -// way. - -// see http://msdn2.microsoft.com/en-us/library/aa489609.aspx -#ifndef NT_SUCCESS -#define NT_SUCCESS(Status) ((NTSTATUS)(Status) >= 0) -#endif - -// http://msdn.microsoft.com/en-us/library/aa813741(VS.85).aspx -typedef struct { - BYTE Reserved1[16]; - PVOID Reserved2[5]; - UNICODE_STRING CurrentDirectoryPath; - PVOID CurrentDirectoryHandle; - UNICODE_STRING DllPath; - UNICODE_STRING ImagePathName; - UNICODE_STRING CommandLine; - LPCWSTR env; -} RTL_USER_PROCESS_PARAMETERS_, *PRTL_USER_PROCESS_PARAMETERS_; - -// https://msdn.microsoft.com/en-us/library/aa813706(v=vs.85).aspx -#ifdef _WIN64 -typedef struct { - BYTE Reserved1[2]; - BYTE BeingDebugged; - BYTE Reserved2[21]; - PVOID LoaderData; - PRTL_USER_PROCESS_PARAMETERS_ ProcessParameters; - /* More fields ... */ -} PEB_; -#else -typedef struct { - BYTE Reserved1[2]; - BYTE BeingDebugged; - BYTE Reserved2[1]; - PVOID Reserved3[2]; - PVOID Ldr; - PRTL_USER_PROCESS_PARAMETERS_ ProcessParameters; - /* More fields ... */ -} PEB_; -#endif - -#ifdef _WIN64 -/* When we are a 64 bit process accessing a 32 bit (WoW64) process we need to - use the 32 bit structure layout. */ -typedef struct { - USHORT Length; - USHORT MaxLength; - DWORD Buffer; -} UNICODE_STRING32; - -typedef struct { - BYTE Reserved1[16]; - DWORD Reserved2[5]; - UNICODE_STRING32 CurrentDirectoryPath; - DWORD CurrentDirectoryHandle; - UNICODE_STRING32 DllPath; - UNICODE_STRING32 ImagePathName; - UNICODE_STRING32 CommandLine; - DWORD env; -} RTL_USER_PROCESS_PARAMETERS32; - -typedef struct { - BYTE Reserved1[2]; - BYTE BeingDebugged; - BYTE Reserved2[1]; - DWORD Reserved3[2]; - DWORD Ldr; - DWORD ProcessParameters; - /* More fields ... */ -} PEB32; -#else -/* When we are a 32 bit (WoW64) process accessing a 64 bit process we need to - use the 64 bit structure layout and a special function to read its memory. - */ -typedef NTSTATUS (NTAPI *_NtWow64ReadVirtualMemory64)( - IN HANDLE ProcessHandle, - IN PVOID64 BaseAddress, - OUT PVOID Buffer, - IN ULONG64 Size, - OUT PULONG64 NumberOfBytesRead); - -typedef enum { - MemoryInformationBasic -} MEMORY_INFORMATION_CLASS; - -typedef NTSTATUS (NTAPI *_NtWow64QueryVirtualMemory64)( - IN HANDLE ProcessHandle, - IN PVOID64 BaseAddress, - IN MEMORY_INFORMATION_CLASS MemoryInformationClass, - OUT PMEMORY_BASIC_INFORMATION64 MemoryInformation, - IN ULONG64 Size, - OUT PULONG64 ReturnLength OPTIONAL); - -typedef struct { - PVOID Reserved1[2]; - PVOID64 PebBaseAddress; - PVOID Reserved2[4]; - PVOID UniqueProcessId[2]; - PVOID Reserved3[2]; -} PROCESS_BASIC_INFORMATION64; - -typedef struct { - USHORT Length; - USHORT MaxLength; - PVOID64 Buffer; -} UNICODE_STRING64; - -typedef struct { - BYTE Reserved1[16]; - PVOID64 Reserved2[5]; - UNICODE_STRING64 CurrentDirectoryPath; - PVOID64 CurrentDirectoryHandle; - UNICODE_STRING64 DllPath; - UNICODE_STRING64 ImagePathName; - UNICODE_STRING64 CommandLine; - PVOID64 env; -} RTL_USER_PROCESS_PARAMETERS64; - -typedef struct { - BYTE Reserved1[2]; - BYTE BeingDebugged; - BYTE Reserved2[21]; - PVOID64 LoaderData; - PVOID64 ProcessParameters; - /* More fields ... */ -} PEB64; -#endif - /* Given a pointer into a process's memory, figure out how much data can be * read from it. */ static int psutil_get_process_region_size(HANDLE hProcess, @@ -746,15 +758,6 @@ PyObject *psutil_get_environ(long pid) { return ret; } -#define PH_FIRST_PROCESS(Processes) ((PSYSTEM_PROCESS_INFORMATION)(Processes)) -#define PH_NEXT_PROCESS(Process) ( \ - ((PSYSTEM_PROCESS_INFORMATION)(Process))->NextEntryOffset ? \ - (PSYSTEM_PROCESS_INFORMATION)((PCHAR)(Process) + \ - ((PSYSTEM_PROCESS_INFORMATION)(Process))->NextEntryOffset) : \ - NULL) - -const int STATUS_INFO_LENGTH_MISMATCH = 0xC0000004; -const int STATUS_BUFFER_TOO_SMALL = 0xC0000023L; /* * Given a process PID and a PSYSTEM_PROCESS_INFORMATION structure @@ -816,14 +819,14 @@ psutil_get_proc_info(DWORD pid, PSYSTEM_PROCESS_INFORMATION *retProcess, if (bufferSize <= 0x20000) initialBufferSize = bufferSize; - process = PH_FIRST_PROCESS(buffer); + process = PSUTIL_FIRST_PROCESS(buffer); do { if (process->UniqueProcessId == (HANDLE)pid) { *retProcess = process; *retBuffer = buffer; return 1; } - } while ( (process = PH_NEXT_PROCESS(process)) ); + } while ( (process = PSUTIL_NEXT_PROCESS(process)) ); NoSuchProcess(); goto error; From 0d3e55a670887f865bb3c0ac9baaa86558f763e9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 20 May 2017 03:49:07 +0200 Subject: [PATCH 735/922] import psutil right after make build to make sure compilation worked --- Makefile | 5 +++-- scripts/internal/winmake.py | 7 ++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c44c6c029..f47d262cc 100644 --- a/Makefile +++ b/Makefile @@ -63,20 +63,21 @@ _: # Compile without installing. build: _ + # make sure setuptools is installed (needed for 'develop' / edit mode) + $(PYTHON) -c "import setuptools" PYTHONWARNINGS=all $(PYTHON) setup.py build @# copies compiled *.so files in ./psutil directory in order to allow @# "import psutil" when using the interactive interpreter from within @# this directory. PYTHONWARNINGS=all $(PYTHON) setup.py build_ext -i rm -rf tmp + $(PYTHON) -c "import psutil" # make sure it actually worked # Install this package + GIT hooks. Install is done: # - as the current user, in order to avoid permission issues # - in development / edit mode, so that source can be modified on the fly install: ${MAKE} build - # make sure setuptools is installed (needed for 'develop' / edit mode) - $(PYTHON) -c "import setuptools" PYTHONWARNINGS=all $(PYTHON) setup.py develop $(INSTALL_OPTS) rm -rf tmp diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 40ba37425..57546bd63 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -192,11 +192,16 @@ def help(): @cmd def build(): """Build / compile""" + # Make sure setuptools is installed (needed for 'develop' / + # edit mode). + sh("%s -c import setuptools" % PYTHON) sh("%s setup.py build" % PYTHON) - # copies compiled *.pyd files in ./psutil directory in order to + # Copies compiled *.pyd files in ./psutil directory in order to # allow "import psutil" when using the interactive interpreter # from within this directory. sh("%s setup.py build_ext -i" % PYTHON) + # Make sure it actually worked. + sh("%s -c 'import psutil'" % PYTHON) @cmd From 65cc00e0998cc0e645f0df386045365380be79ad Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 20 May 2017 05:00:00 +0200 Subject: [PATCH 736/922] fix TypeError --- psutil/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index fc45abf16..50ae2a337 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1646,7 +1646,9 @@ def cpu_count(logical=True): ret = _psplatform.cpu_count_logical() else: ret = _psplatform.cpu_count_physical() - return ret if ret >= 1 else None + if ret is not None and ret < 1: + ret = None + return ret def cpu_times(percpu=False): From d3fde86f8432f67caf360f90ab28016d7a1a430a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 20 May 2017 05:04:18 +0200 Subject: [PATCH 737/922] fix winmake --- scripts/internal/winmake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 57546bd63..138a0b0c7 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -194,14 +194,14 @@ def build(): """Build / compile""" # Make sure setuptools is installed (needed for 'develop' / # edit mode). - sh("%s -c import setuptools" % PYTHON) + sh('%s -c "import setuptools"' % PYTHON) sh("%s setup.py build" % PYTHON) # Copies compiled *.pyd files in ./psutil directory in order to # allow "import psutil" when using the interactive interpreter # from within this directory. sh("%s setup.py build_ext -i" % PYTHON) # Make sure it actually worked. - sh("%s -c 'import psutil'" % PYTHON) + sh('%s -c "import psutil"' % PYTHON) @cmd From a92cfcd574e876189b64449c8e1eea505768a83e Mon Sep 17 00:00:00 2001 From: Oleksii Shevchuk Date: Sat, 20 May 2017 23:50:46 +0400 Subject: [PATCH 738/922] SunOS: Fix .memory_maps(grouped=False) (#1093) --- CREDITS | 2 +- HISTORY.rst | 1 + psutil/_psutil_sunos.c | 12 ++++++------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CREDITS b/CREDITS index b11a8b93c..4dedb9f44 100644 --- a/CREDITS +++ b/CREDITS @@ -482,4 +482,4 @@ I: 1042, 1079 N: Oleksii Shevchuk W: https://github.com/alxchk -I: 1077 +I: 1077, 1093 diff --git a/HISTORY.rst b/HISTORY.rst index bb20811a3..95371d829 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -27,6 +27,7 @@ **Bug fixes** +- 1093_: [SunOS] memory_maps() shows wrong 64 bit addresses - 1007_: [Windows] boot_time() can have a 1 sec fluctuation between calls; the value of the first call is now cached so that boot_time() always returns the same value if fluctuation is <= 1 second. diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 785805d4d..fcfbd1acf 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -791,14 +791,14 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { if (! py_path) goto error; py_tuple = Py_BuildValue( - "iisOlll", - p->pr_vaddr, - pr_addr_sz, + "kksOkkk", + (unsigned long)p->pr_vaddr, + (unsigned long)pr_addr_sz, perms, py_path, - (long)p->pr_rss * p->pr_pagesize, - (long)p->pr_anon * p->pr_pagesize, - (long)p->pr_locked * p->pr_pagesize); + (unsigned long)p->pr_rss * p->pr_pagesize, + (unsigned long)p->pr_anon * p->pr_pagesize, + (unsigned long)p->pr_locked * p->pr_pagesize); if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) From 68e04def9b1f04462fa8f2eefcc5287b1b943cfe Mon Sep 17 00:00:00 2001 From: Oleksii Shevchuk Date: Sun, 21 May 2017 22:27:38 +0300 Subject: [PATCH 739/922] Add environment parsing (#1091) * Add common functions to extract information from SunOS process address space * SunOS feature: Add .environ() --- psutil/_pssunos.py | 4 + psutil/_psutil_sunos.c | 76 +++++ psutil/arch/solaris/process_as_utils.c | 386 +++++++++++++++++++++++++ psutil/arch/solaris/process_as_utils.h | 21 ++ setup.py | 1 + 5 files changed, 488 insertions(+) create mode 100644 psutil/arch/solaris/process_as_utils.c create mode 100644 psutil/arch/solaris/process_as_utils.h diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index b1ba6b452..53821829f 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -414,6 +414,10 @@ def exe(self): def cmdline(self): return self._proc_name_and_args()[1].split(' ') + @wrap_exceptions + def environ(self): + return cext.proc_environ(self.pid, self._procfs_path) + @wrap_exceptions def create_time(self): return self._proc_basic_info()[proc_info_map['create_time']] diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index fcfbd1acf..0af8bc471 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -50,6 +50,8 @@ #include "_psutil_common.h" #include "_psutil_posix.h" +#include "arch/solaris/process_as_utils.h" + #define PSUTIL_TV2DOUBLE(t) (((t).tv_nsec * 0.000000001) + (t).tv_sec) @@ -154,6 +156,78 @@ psutil_proc_name_and_args(PyObject *self, PyObject *args) { return NULL; } +/* + * Return process environ block + */ +static PyObject * +psutil_proc_environ(PyObject *self, PyObject *args) { + int pid; + char path[1000]; + psinfo_t info; + const char *procfs_path; + char **env = NULL; + ssize_t env_count = -1; + char *dm; + int i = 0; + PyObject *py_retdict = NULL; + PyObject *py_envname = NULL; + PyObject *py_envval = NULL; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + goto error; + + sprintf(path, "%s/%i/psinfo", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + goto error; + + env = psutil_read_raw_env(info, procfs_path, &env_count); + if (! env && env_count != 0) + goto error; + + py_retdict = PyDict_New(); + if (! py_retdict) { + PyErr_NoMemory(); + goto error; + } + + for (i=0; i= 0) + psutil_free_cstrings_array(env, env_count); + + Py_XDECREF(py_envname); + Py_XDECREF(py_envval); + Py_XDECREF(py_retdict); + return NULL; +} /* * Return process user and system CPU times as a Python tuple. @@ -1475,6 +1549,8 @@ PsutilMethods[] = { "Return process ppid, rss, vms, ctime, nice, nthreads, status and tty"}, {"proc_name_and_args", psutil_proc_name_and_args, METH_VARARGS, "Return process name and args."}, + {"proc_environ", psutil_proc_environ, METH_VARARGS, + "Return process environment."}, {"proc_cpu_times", psutil_proc_cpu_times, METH_VARARGS, "Return process user and system CPU times."}, {"proc_cred", psutil_proc_cred, METH_VARARGS, diff --git a/psutil/arch/solaris/process_as_utils.c b/psutil/arch/solaris/process_as_utils.c new file mode 100644 index 000000000..6af8f04a7 --- /dev/null +++ b/psutil/arch/solaris/process_as_utils.c @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2017, Oleksii Shevchuk. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * Functions specific to Sun OS Solaris platforms. + */ + +#define _STRUCTURED_PROC 1 + +#if !defined(_LP64) && _FILE_OFFSET_BITS == 64 +# undef _FILE_OFFSET_BITS +# undef _LARGEFILE64_SOURCE +#endif + +#include + +#include +#include +#include +#include + +#include "process_as_utils.h" + +/** Function opens address space of specified process and return file + * descriptor. + * @param pid a pid of process. + * @param procfs_path a path to mounted procfs filesystem. + * @return file descriptor or -1 in case of error. + */ +static int +open_address_space(pid_t pid, const char *procfs_path) { + int fd; + char proc_path[PATH_MAX]; + + snprintf(proc_path, PATH_MAX, "%s/%i/as", procfs_path, pid); + fd = open(proc_path, O_RDONLY); + if (fd < 0) + PyErr_SetFromErrno(PyExc_OSError); + + return fd; +} + +/** Function reads chunk of data by offset to specified + * buffer of the same size. + * @param fd a file descriptor. + * @param offset an required offset in file. + * @param buf a buffer where to store result. + * @param buf_size a size of buffer where data will be stored. + * @return amount of bytes stored to the buffer or -1 in case of + * error. + */ +static int +read_offt(int fd, off_t offset, char *buf, size_t buf_size) { + size_t to_read = buf_size; + size_t stored = 0; + + while (to_read) { + int r = pread(fd, buf + stored, to_read, offset + stored); + if (r < 0) + goto error; + else if (r == 0) + break; + + to_read -= r; + stored += r; + } + + return stored; + + error: + PyErr_SetFromErrno(PyExc_OSError); + return -1; +} + +#define STRING_SEARCH_BUF_SIZE 512 + +/** Function reads null-terminated string from file descriptor starting from + * specified offset. + * @param fd a file descriptor of opened address space. + * @param offset an offset in specified file descriptor. + * @return allocated null-terminated string or NULL in case of error. +*/ +static char * +read_cstring_offt(int fd, off_t offset) { + int r; + int i = 0; + off_t end = offset; + size_t len; + char buf[STRING_SEARCH_BUF_SIZE]; + char *result = NULL; + + if (lseek(fd, offset, SEEK_SET) == (off_t)-1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + // Search end of string + for (;;) { + r = read(fd, buf, sizeof(buf)); + if (r == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + else if (r == 0) + break; + else + for (i=0; i= 0 && count) + *count = env_count; + + if (env_count > 0) + result = read_cstrings_block( + as, info.pr_envp, ptr_size, env_count + ); + + close(as); + return result; +} + +/** Free array of cstrings. + * @param array an array of cstrings returned by psutil_read_raw_env, + * psutil_read_raw_args or any other function. + * @param count a count of strings in the passed array + */ +void +psutil_free_cstrings_array(char **array, size_t count) { + int i; + + if (!array) + return; + + for (i=0; i Date: Sun, 28 May 2017 21:45:36 +0200 Subject: [PATCH 740/922] Fix 1091 nitpicks (#1097) * rename C module * rename C file * fix compilation error * small refactoring * small refactoring * raise AccessDenied if info.pr_envp is empty, see https://github.com/giampaolo/psutil/pull/1091#issuecomment-304530771 * fix envs with no equal sign * style * update doc * style --- CREDITS | 2 +- HISTORY.rst | 1 + docs/index.rst | 3 +- psutil/_psutil_sunos.c | 27 ++- .../solaris/{process_as_utils.c => environ.c} | 217 ++++++++++-------- .../solaris/{process_as_utils.h => environ.h} | 8 +- setup.py | 2 +- 7 files changed, 141 insertions(+), 119 deletions(-) rename psutil/arch/solaris/{process_as_utils.c => environ.c} (54%) rename psutil/arch/solaris/{process_as_utils.h => environ.h} (59%) diff --git a/CREDITS b/CREDITS index 4dedb9f44..a9270c864 100644 --- a/CREDITS +++ b/CREDITS @@ -482,4 +482,4 @@ I: 1042, 1079 N: Oleksii Shevchuk W: https://github.com/alxchk -I: 1077, 1093 +I: 1077, 1093, 1091 diff --git a/HISTORY.rst b/HISTORY.rst index 95371d829..2e73df493 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -24,6 +24,7 @@ relevant files. - 1079_: [FreeBSD] net_connections()'s fd number is now being set for real (instead of -1). (patch by Gleb Smirnoff) +- 1091_: [SunOS] implemented Process.environ(). (patch by Oleksii Shevchuk) **Bug fixes** diff --git a/docs/index.rst b/docs/index.rst index 4e378ea44..309b2a68c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1130,9 +1130,10 @@ Process class >>> psutil.Process().environ() {'LC_NUMERIC': 'it_IT.UTF-8', 'QT_QPA_PLATFORMTHEME': 'appmenu-qt5', 'IM_CONFIG_PHASE': '1', 'XDG_GREETER_DATA_DIR': '/var/lib/lightdm-data/giampaolo', 'GNOME_DESKTOP_SESSION_ID': 'this-is-deprecated', 'XDG_CURRENT_DESKTOP': 'Unity', 'UPSTART_EVENTS': 'started starting', 'GNOME_KEYRING_PID': '', 'XDG_VTNR': '7', 'QT_IM_MODULE': 'ibus', 'LOGNAME': 'giampaolo', 'USER': 'giampaolo', 'PATH': '/home/giampaolo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/home/giampaolo/svn/sysconf/bin', 'LC_PAPER': 'it_IT.UTF-8', 'GNOME_KEYRING_CONTROL': '', 'GTK_IM_MODULE': 'ibus', 'DISPLAY': ':0', 'LANG': 'en_US.UTF-8', 'LESS_TERMCAP_se': '\x1b[0m', 'TERM': 'xterm-256color', 'SHELL': '/bin/bash', 'XDG_SESSION_PATH': '/org/freedesktop/DisplayManager/Session0', 'XAUTHORITY': '/home/giampaolo/.Xauthority', 'LANGUAGE': 'en_US', 'COMPIZ_CONFIG_PROFILE': 'ubuntu', 'LC_MONETARY': 'it_IT.UTF-8', 'QT_LINUX_ACCESSIBILITY_ALWAYS_ON': '1', 'LESS_TERMCAP_me': '\x1b[0m', 'LESS_TERMCAP_md': '\x1b[01;38;5;74m', 'LESS_TERMCAP_mb': '\x1b[01;31m', 'HISTSIZE': '100000', 'UPSTART_INSTANCE': '', 'CLUTTER_IM_MODULE': 'xim', 'WINDOWID': '58786407', 'EDITOR': 'vim', 'SESSIONTYPE': 'gnome-session', 'XMODIFIERS': '@im=ibus', 'GPG_AGENT_INFO': '/home/giampaolo/.gnupg/S.gpg-agent:0:1', 'HOME': '/home/giampaolo', 'HISTFILESIZE': '100000', 'QT4_IM_MODULE': 'xim', 'GTK2_MODULES': 'overlay-scrollbar', 'XDG_SESSION_DESKTOP': 'ubuntu', 'SHLVL': '1', 'XDG_RUNTIME_DIR': '/run/user/1000', 'INSTANCE': 'Unity', 'LC_ADDRESS': 'it_IT.UTF-8', 'SSH_AUTH_SOCK': '/run/user/1000/keyring/ssh', 'VTE_VERSION': '4205', 'GDMSESSION': 'ubuntu', 'MANDATORY_PATH': '/usr/share/gconf/ubuntu.mandatory.path', 'VISUAL': 'vim', 'DESKTOP_SESSION': 'ubuntu', 'QT_ACCESSIBILITY': '1', 'XDG_SEAT_PATH': '/org/freedesktop/DisplayManager/Seat0', 'LESSCLOSE': '/usr/bin/lesspipe %s %s', 'LESSOPEN': '| /usr/bin/lesspipe %s', 'XDG_SESSION_ID': 'c2', 'DBUS_SESSION_BUS_ADDRESS': 'unix:abstract=/tmp/dbus-9GAJpvnt8r', '_': '/usr/bin/python', 'DEFAULTS_PATH': '/usr/share/gconf/ubuntu.default.path', 'LC_IDENTIFICATION': 'it_IT.UTF-8', 'LESS_TERMCAP_ue': '\x1b[0m', 'UPSTART_SESSION': 'unix:abstract=/com/ubuntu/upstart-session/1000/1294', 'XDG_CONFIG_DIRS': '/etc/xdg/xdg-ubuntu:/usr/share/upstart/xdg:/etc/xdg', 'GTK_MODULES': 'gail:atk-bridge:unity-gtk-module', 'XDG_SESSION_TYPE': 'x11', 'PYTHONSTARTUP': '/home/giampaolo/.pythonstart', 'LC_NAME': 'it_IT.UTF-8', 'OLDPWD': '/home/giampaolo/svn/curio_giampaolo/tests', 'GDM_LANG': 'en_US', 'LC_TELEPHONE': 'it_IT.UTF-8', 'HISTCONTROL': 'ignoredups:erasedups', 'LC_MEASUREMENT': 'it_IT.UTF-8', 'PWD': '/home/giampaolo/svn/curio_giampaolo', 'JOB': 'gnome-session', 'LESS_TERMCAP_us': '\x1b[04;38;5;146m', 'UPSTART_JOB': 'unity-settings-daemon', 'LC_TIME': 'it_IT.UTF-8', 'LESS_TERMCAP_so': '\x1b[38;5;246m', 'PAGER': 'less', 'XDG_DATA_DIRS': '/usr/share/ubuntu:/usr/share/gnome:/usr/local/share/:/usr/share/:/var/lib/snapd/desktop', 'XDG_SEAT': 'seat0'} - Availability: Linux, OSX, Windows + Availability: Linux, OSX, Windows, SunOS .. versionadded:: 4.0.0 + .. versionchanged:: 5.3.0: added SunOS support .. method:: create_time() diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 0af8bc471..12caaec7e 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -50,7 +50,7 @@ #include "_psutil_common.h" #include "_psutil_posix.h" -#include "arch/solaris/process_as_utils.h" +#include "arch/solaris/environ.h" #define PSUTIL_TV2DOUBLE(t) (((t).tv_nsec * 0.000000001) + (t).tv_sec) @@ -156,8 +156,9 @@ psutil_proc_name_and_args(PyObject *self, PyObject *args) { return NULL; } + /* - * Return process environ block + * Return process environ block. */ static PyObject * psutil_proc_environ(PyObject *self, PyObject *args) { @@ -169,35 +170,36 @@ psutil_proc_environ(PyObject *self, PyObject *args) { ssize_t env_count = -1; char *dm; int i = 0; - PyObject *py_retdict = NULL; PyObject *py_envname = NULL; PyObject *py_envval = NULL; + PyObject *py_retdict = PyDict_New(); + + if (! py_retdict) + return PyErr_NoMemory(); if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) - goto error; + return NULL; sprintf(path, "%s/%i/psinfo", procfs_path, pid); if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) goto error; - env = psutil_read_raw_env(info, procfs_path, &env_count); - if (! env && env_count != 0) + if (! info.pr_envp) { + AccessDenied(); goto error; + } - py_retdict = PyDict_New(); - if (! py_retdict) { - PyErr_NoMemory(); + env = psutil_read_raw_env(info, procfs_path, &env_count); + if (! env && env_count != 0) goto error; - } for (i=0; i + #if !defined(_LP64) && _FILE_OFFSET_BITS == 64 # undef _FILE_OFFSET_BITS # undef _LARGEFILE64_SOURCE #endif -#include - #include #include #include #include -#include "process_as_utils.h" +#include "environ.h" + + +#define STRING_SEARCH_BUF_SIZE 512 + -/** Function opens address space of specified process and return file - * descriptor. +/* + * Open address space of specified process and return file descriptor. * @param pid a pid of process. * @param procfs_path a path to mounted procfs filesystem. * @return file descriptor or -1 in case of error. @@ -41,22 +46,24 @@ open_address_space(pid_t pid, const char *procfs_path) { return fd; } -/** Function reads chunk of data by offset to specified - * buffer of the same size. - * @param fd a file descriptor. - * @param offset an required offset in file. - * @param buf a buffer where to store result. - * @param buf_size a size of buffer where data will be stored. - * @return amount of bytes stored to the buffer or -1 in case of - * error. + +/* + * Read chunk of data by offset to specified buffer of the same size. + * @param fd a file descriptor. + * @param offset an required offset in file. + * @param buf a buffer where to store result. + * @param buf_size a size of buffer where data will be stored. + * @return amount of bytes stored to the buffer or -1 in case of + * error. */ static int read_offt(int fd, off_t offset, char *buf, size_t buf_size) { size_t to_read = buf_size; size_t stored = 0; + int r; while (to_read) { - int r = pread(fd, buf + stored, to_read, offset + stored); + r = pread(fd, buf + stored, to_read, offset + stored); if (r < 0) goto error; else if (r == 0) @@ -73,13 +80,13 @@ read_offt(int fd, off_t offset, char *buf, size_t buf_size) { return -1; } -#define STRING_SEARCH_BUF_SIZE 512 -/** Function reads null-terminated string from file descriptor starting from - * specified offset. - * @param fd a file descriptor of opened address space. - * @param offset an offset in specified file descriptor. - * @return allocated null-terminated string or NULL in case of error. +/* + * Read null-terminated string from file descriptor starting from + * specified offset. + * @param fd a file descriptor of opened address space. + * @param offset an offset in specified file descriptor. + * @return allocated null-terminated string or NULL in case of error. */ static char * read_cstring_offt(int fd, off_t offset) { @@ -102,17 +109,19 @@ read_cstring_offt(int fd, off_t offset) { PyErr_SetFromErrno(PyExc_OSError); goto error; } - else if (r == 0) + else if (r == 0) { break; - else + } + else { for (i=0; i 0) result = read_cstrings_block( - as, info.pr_envp, ptr_size, env_count - ); + as, info.pr_envp, ptr_size, env_count); close(as); return result; } -/** Free array of cstrings. - * @param array an array of cstrings returned by psutil_read_raw_env, - * psutil_read_raw_args or any other function. - * @param count a count of strings in the passed array + +/* + * Free array of cstrings. + * @param array an array of cstrings returned by psutil_read_raw_env, + * psutil_read_raw_args or any other function. + * @param count a count of strings in the passed array */ void psutil_free_cstrings_array(char **array, size_t count) { @@ -377,10 +396,10 @@ psutil_free_cstrings_array(char **array, size_t count) { if (!array) return; - - for (i=0; i Date: Sun, 28 May 2017 23:52:37 +0200 Subject: [PATCH 741/922] Windows: fix wrapper around OpenProcess (pid_exists() no longer lies) (#1094) * windows / C: add assert to make sure pid_is_running() is correct * set an error str * check if pid is actually gone in psutil_handle_from_pid_waccess * small refactoring * GetExitCodeProces() return code was not checked * define a reusable check_phandle() C function which checks whether the process handle is actually running * refactoring * re-enable windows test which now passes * check pid_is_running -1 return value in proc_connections(); also refactor some C code * fix memleak --- psutil/_psutil_windows.c | 8 +- psutil/arch/windows/process_info.c | 183 +++++++++++++++++++++++++---- psutil/arch/windows/process_info.h | 4 + psutil/tests/__init__.py | 3 - 4 files changed, 168 insertions(+), 30 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 795ee9cda..9c8782e36 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1510,6 +1510,7 @@ static PyObject * psutil_net_connections(PyObject *self, PyObject *args) { static long null_address[4] = { 0, 0, 0, 0 }; unsigned long pid; + int pid_return; typedef PSTR (NTAPI * _RtlIpv4AddressToStringA)(struct in_addr *, PSTR); _RtlIpv4AddressToStringA rtlIpv4AddressToStringA; typedef PSTR (NTAPI * _RtlIpv6AddressToStringA)(struct in6_addr *, PSTR); @@ -1551,10 +1552,15 @@ psutil_net_connections(PyObject *self, PyObject *args) { } if (pid != -1) { - if (psutil_pid_is_running(pid) == 0) { + pid_return = psutil_pid_is_running(pid); + if (pid_return == 0) { _psutil_conn_decref_objs(); return NoSuchProcess(); } + else if (pid_return == -1) { + _psutil_conn_decref_objs(); + return NULL; + } } // Import some functions. diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 340745e92..cc5669522 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -18,9 +18,11 @@ #include "../../_psutil_common.h" -// Helper structures to access the memory correctly. Some of these might also -// be defined in the winternl.h header file but unfortunately not in a usable -// way. +// ==================================================================== +// Helper structures to access the memory correctly. +// Some of these might also be defined in the winternl.h header file +// but unfortunately not in a usable way. +// ==================================================================== // see http://msdn2.microsoft.com/en-us/library/aa489609.aspx #ifndef NT_SUCCESS @@ -160,6 +162,108 @@ const int STATUS_INFO_LENGTH_MISMATCH = 0xC0000004; const int STATUS_BUFFER_TOO_SMALL = 0xC0000023L; +// ==================================================================== +// Process and PIDs utiilties. +// ==================================================================== + + +/* + * Return 1 if PID exists, 0 if not, -1 on error. + */ +int +psutil_pid_in_pids(DWORD pid) { + DWORD *proclist = NULL; + DWORD numberOfReturnedPIDs; + DWORD i; + + proclist = psutil_get_pids(&numberOfReturnedPIDs); + if (proclist == NULL) + return -1; + for (i = 0; i < numberOfReturnedPIDs; i++) { + if (proclist[i] == pid) { + free(proclist); + return 1; + } + } + free(proclist); + return 0; +} + + +/* + * Given a process HANDLE checks whether it's actually running. + * Returns: + * - 1: running + * - 0: not running + * - -1: WindowsError + * - -2: AssertionError + */ +int +psutil_is_phandle_running(HANDLE hProcess, DWORD pid) { + DWORD processExitCode = 0; + + if (hProcess == NULL) { + if (GetLastError() == ERROR_INVALID_PARAMETER) { + // Yeah, this is the actual error code in case of + // "no such process". + if (! psutil_assert_pid_not_exists( + pid, "iphr: OpenProcess() -> ERROR_INVALID_PARAMETER")) { + return -2; + } + return 0; + } + return -1; + } + + if (GetExitCodeProcess(hProcess, &processExitCode)) { + // XXX - maybe STILL_ACTIVE is not fully reliable as per: + // http://stackoverflow.com/questions/1591342/#comment47830782_1591379 + if (processExitCode == STILL_ACTIVE) { + if (! psutil_assert_pid_exists( + pid, "iphr: GetExitCodeProcess() -> STILL_ACTIVE")) { + return -2; + } + return 1; + } + else { + // We can't be sure so we look into pids. + if (psutil_pid_in_pids(pid) == 1) { + return 1; + } + else { + CloseHandle(hProcess); + return 0; + } + } + } + + CloseHandle(hProcess); + if (! psutil_assert_pid_not_exists( pid, "iphr: exit fun")) { + return -2; + } + return -1; +} + + +/* + * Given a process HANDLE checks whether it's actually running and if + * it does return it, else return NULL with the proper Python exception + * set. + */ +HANDLE +psutil_check_phandle(HANDLE hProcess, DWORD pid) { + int ret = psutil_is_phandle_running(hProcess, pid); + if (ret == 1) + return hProcess; + else if (ret == 0) + return NoSuchProcess(); + else if (ret == -1) + return PyErr_SetFromWindowsErr(0); + else if (ret == -2) + return NULL; +} + + /* * A wrapper around OpenProcess setting NSP exception if process * no longer exists. @@ -170,7 +274,6 @@ const int STATUS_BUFFER_TOO_SMALL = 0xC0000023L; HANDLE psutil_handle_from_pid_waccess(DWORD pid, DWORD dwDesiredAccess) { HANDLE hProcess; - DWORD processExitCode = 0; if (pid == 0) { // otherwise we'd get NoSuchProcess @@ -178,22 +281,7 @@ psutil_handle_from_pid_waccess(DWORD pid, DWORD dwDesiredAccess) { } hProcess = OpenProcess(dwDesiredAccess, FALSE, pid); - if (hProcess == NULL) { - if (GetLastError() == ERROR_INVALID_PARAMETER) - NoSuchProcess(); - else - PyErr_SetFromWindowsErr(0); - return NULL; - } - - // make sure the process is running - GetExitCodeProcess(hProcess, &processExitCode); - if (processExitCode == 0) { - NoSuchProcess(); - CloseHandle(hProcess); - return NULL; - } - return hProcess; + return psutil_check_phandle(hProcess, pid); } @@ -247,6 +335,30 @@ psutil_get_pids(DWORD *numberOfReturnedPIDs) { } +int +psutil_assert_pid_exists(DWORD pid, char *err) { + if (psutil_testing()) { + if (psutil_pid_in_pids(pid) == 0) { + PyErr_SetString(PyExc_AssertionError, err); + return 0; + } + } + return 1; +} + + +int +psutil_assert_pid_not_exists(DWORD pid, char *err) { + if (psutil_testing()) { + if (psutil_pid_in_pids(pid) == 1) { + PyErr_SetString(PyExc_AssertionError, err); + return 0; + } + } + return 1; +} + + /* /* Check for PID existance by using OpenProcess() + GetExitCodeProcess. /* Returns: @@ -271,10 +383,18 @@ psutil_pid_is_running(DWORD pid) { err = GetLastError(); // Yeah, this is the actual error code in case of "no such process". if (err == ERROR_INVALID_PARAMETER) { + if (! psutil_assert_pid_not_exists( + pid, "pir: OpenProcess() -> INVALID_PARAMETER")) { + return -1; + } return 0; } // Access denied obviously means there's a process to deny access to. else if (err == ERROR_ACCESS_DENIED) { + if (! psutil_assert_pid_exists( + pid, "pir: OpenProcess() ACCESS_DENIED")) { + return -1; + } return 1; } // Be strict and raise an exception; the caller is supposed @@ -289,23 +409,34 @@ psutil_pid_is_running(DWORD pid) { CloseHandle(hProcess); // XXX - maybe STILL_ACTIVE is not fully reliable as per: // http://stackoverflow.com/questions/1591342/#comment47830782_1591379 - if (exitCode == STILL_ACTIVE) + if (exitCode == STILL_ACTIVE) { + if (! psutil_assert_pid_exists( + pid, "pir: GetExitCodeProcess() -> STILL_ACTIVE")) { + return -1; + } return 1; - else - return 0; + } + // We can't be sure so we look into pids. + else { + return psutil_pid_in_pids(pid); + } } else { err = GetLastError(); CloseHandle(hProcess); // Same as for OpenProcess, assume access denied means there's // a process to deny access to. - if (err == ERROR_ACCESS_DENIED) + if (err == ERROR_ACCESS_DENIED) { + if (! psutil_assert_pid_exists( + pid, "pir: GetExitCodeProcess() -> ERROR_ACCESS_DENIED")) { + return -1; + } return 1; + } else { - PyErr_SetFromWindowsErr(err); + PyErr_SetFromWindowsErr(0); return -1; } - } } diff --git a/psutil/arch/windows/process_info.h b/psutil/arch/windows/process_info.h index a3b512e78..a2f70c2b9 100644 --- a/psutil/arch/windows/process_info.h +++ b/psutil/arch/windows/process_info.h @@ -23,6 +23,10 @@ int psutil_pid_is_running(DWORD pid); int psutil_get_proc_info(DWORD pid, PSYSTEM_PROCESS_INFORMATION *retProcess, PVOID *retBuffer); +int psutil_assert_pid_exists(DWORD pid, char *err); +int psutil_assert_pid_not_exists(DWORD pid, char *err); + + PyObject* psutil_get_cmdline(long pid); PyObject* psutil_get_cwd(long pid); PyObject* psutil_get_environ(long pid); diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 0ba95b186..24718edd5 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -385,9 +385,6 @@ def reap_children(recursive=False): # https://ci.appveyor.com/project/giampaolo/psutil/build/job/ # jiq2cgd6stsbtn60 def assert_gone(pid): - # XXX - if WINDOWS: - return assert not psutil.pid_exists(pid), pid assert pid not in psutil.pids(), pid try: From 70fff514f12c7f2789e5d8e26140a274dc64f841 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 30 May 2017 12:02:15 +0200 Subject: [PATCH 742/922] fix #1098: Windows: Process.wait() may return sooner, when the PID is still alive --- HISTORY.rst | 4 +++- psutil/_psutil_windows.c | 2 ++ psutil/_pswindows.py | 11 +++++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 2e73df493..99cb6f89c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,7 +28,6 @@ **Bug fixes** -- 1093_: [SunOS] memory_maps() shows wrong 64 bit addresses - 1007_: [Windows] boot_time() can have a 1 sec fluctuation between calls; the value of the first call is now cached so that boot_time() always returns the same value if fluctuation is <= 1 second. @@ -75,6 +74,9 @@ - 1085_: cpu_count() return value is now checked and forced to None if <= 1. - 1087_: Process.cpu_percent() guard against cpu_count() returning None and assumes 1 instead. +- 1093_: [SunOS] memory_maps() shows wrong 64 bit addresses. +- 1098_: [Windows] Process.wait() may erroneously return sooner, when the PID + is still alive. **Porting notes** diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 9c8782e36..9e16d2073 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -398,7 +398,9 @@ psutil_proc_wait(PyObject *self, PyObject *args) { CloseHandle(hProcess); return PyErr_SetFromWindowsErr(GetLastError()); } + CloseHandle(hProcess); + #if PY_MAJOR_VERSION >= 3 return PyLong_FromLong((long) ExitCode); #else diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 6d0679d63..ff868f2e7 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -791,10 +791,13 @@ def wait(self, timeout=None): else: # WaitForSingleObject() expects time in milliseconds cext_timeout = int(timeout * 1000) - ret = cext.proc_wait(self.pid, cext_timeout) - if ret == WAIT_TIMEOUT: - raise TimeoutExpired(timeout, self.pid, self._name) - return ret + while True: + ret = cext.proc_wait(self.pid, cext_timeout) + if ret == WAIT_TIMEOUT: + raise TimeoutExpired(timeout, self.pid, self._name) + if timeout is None and pid_exists(self.pid): + continue + return ret @wrap_exceptions def username(self): From 1cbd905ded0a8aab0fd842aeb1c6479f5aacc227 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 30 May 2017 12:06:17 +0200 Subject: [PATCH 743/922] update HISTORY --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 99cb6f89c..625f918d7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -75,6 +75,8 @@ - 1087_: Process.cpu_percent() guard against cpu_count() returning None and assumes 1 instead. - 1093_: [SunOS] memory_maps() shows wrong 64 bit addresses. +- 1094_: [Windows] psutil.pid_exists() may lie. Also, all process APIs relying + on OpenProcess Windows API now check whether the PID is actually running. - 1098_: [Windows] Process.wait() may erroneously return sooner, when the PID is still alive. From e492b4b2caa961c54c59191157343ef5c11c7b1f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 30 May 2017 12:11:40 +0200 Subject: [PATCH 744/922] windows / create_time: remove check using GetExitCodeProcess to make sure the process is not gone as it seems useless and may also be the cause of errors on appveyor --- psutil/_psutil_windows.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 9e16d2073..7c2479756 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -492,6 +492,7 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { } } +/* // Make sure the process is not gone as OpenProcess alone seems to be // unreliable in doing so (it seems a previous call to p.wait() makes // it unreliable). @@ -509,12 +510,10 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { if (GetLastError() != ERROR_ACCESS_DENIED) return PyErr_SetFromWindowsErr(0); } - - /* - Convert the FILETIME structure to a Unix time. - It's the best I could find by googling and borrowing code here and there. - The time returned has a precision of 1 second. - */ +*/ + // Convert the FILETIME structure to a Unix time. + // It's the best I could find by googling and borrowing code here + // and there. The time returned has a precision of 1 second. unix_time = ((LONGLONG)ftCreate.dwHighDateTime) << 32; unix_time += ftCreate.dwLowDateTime - 116444736000000000LL; unix_time /= 10000000; From 42bebc2ffdb874791b19b0362bad8c60120313ac Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 30 May 2017 12:18:53 +0200 Subject: [PATCH 745/922] fix C compiler warnings --- psutil/_psutil_windows.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 7c2479756..42c9e4e07 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -465,9 +465,7 @@ static PyObject * psutil_proc_create_time(PyObject *self, PyObject *args) { long pid; long long unix_time; - DWORD exitCode; HANDLE hProcess; - BOOL ret; FILETIME ftCreate, ftExit, ftKernel, ftUser; if (! PyArg_ParseTuple(args, "l", &pid)) @@ -492,7 +490,9 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { } } -/* + CloseHandle(hProcess); + + /* // Make sure the process is not gone as OpenProcess alone seems to be // unreliable in doing so (it seems a previous call to p.wait() makes // it unreliable). @@ -510,7 +510,8 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { if (GetLastError() != ERROR_ACCESS_DENIED) return PyErr_SetFromWindowsErr(0); } -*/ + */ + // Convert the FILETIME structure to a Unix time. // It's the best I could find by googling and borrowing code here // and there. The time returned has a precision of 1 second. From 1e49e9e4063bbcffb96092b39f3b8b233153aaf0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 30 May 2017 12:53:10 +0200 Subject: [PATCH 746/922] fix #1099 / windows: Process.terminate() may raise AccessDenied even if the process already dided --- HISTORY.rst | 2 ++ psutil/_psutil_windows.c | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 625f918d7..30658d689 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -79,6 +79,8 @@ on OpenProcess Windows API now check whether the PID is actually running. - 1098_: [Windows] Process.wait() may erroneously return sooner, when the PID is still alive. +- 1099_: [Windows] Process.terminate() may raise AccessDenied even if the + process already died. **Porting notes** diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 42c9e4e07..b8912bbb7 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -321,6 +321,7 @@ psutil_pids(PyObject *self, PyObject *args) { static PyObject * psutil_proc_kill(PyObject *self, PyObject *args) { HANDLE hProcess; + DWORD err; long pid; if (! PyArg_ParseTuple(args, "l", &pid)) @@ -342,9 +343,16 @@ psutil_proc_kill(PyObject *self, PyObject *args) { // kill the process if (! TerminateProcess(hProcess, 0)) { - PyErr_SetFromWindowsErr(0); + err = GetLastError(); CloseHandle(hProcess); - return NULL; + // See: https://github.com/giampaolo/psutil/issues/1099 + if (psutil_pid_is_running(pid) == 0) { + Py_RETURN_NONE; + } + else { + PyErr_SetFromWindowsErr(err); + return NULL; + } } CloseHandle(hProcess); From 0bc345be0bd6a61a6f3b5657057a07ca4f9622b1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 30 May 2017 17:34:22 +0200 Subject: [PATCH 747/922] #1099: look for ERROR_ACCESS_DENIED instead of using pid_is_running() --- psutil/_psutil_windows.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index b8912bbb7..90ef2d44e 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -344,12 +344,9 @@ psutil_proc_kill(PyObject *self, PyObject *args) { // kill the process if (! TerminateProcess(hProcess, 0)) { err = GetLastError(); - CloseHandle(hProcess); // See: https://github.com/giampaolo/psutil/issues/1099 - if (psutil_pid_is_running(pid) == 0) { - Py_RETURN_NONE; - } - else { + if (err != ERROR_ACCESS_DENIED) { + CloseHandle(hProcess); PyErr_SetFromWindowsErr(err); return NULL; } From 50a68a565252b47ce241651b33a2d35f1898ebf4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 30 May 2017 17:46:52 +0200 Subject: [PATCH 748/922] #1098: raise TimeoutExpired also if timeout param is passed --- psutil/_pswindows.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index ff868f2e7..52676183d 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -795,8 +795,11 @@ def wait(self, timeout=None): ret = cext.proc_wait(self.pid, cext_timeout) if ret == WAIT_TIMEOUT: raise TimeoutExpired(timeout, self.pid, self._name) - if timeout is None and pid_exists(self.pid): - continue + if pid_exists(self.pid): + if timeout is None: + continue + else: + raise TimeoutExpired(timeout, self.pid, self._name) return ret @wrap_exceptions From 279019043fd424eda265c5cefaa26a3f558b43d7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 30 May 2017 17:51:06 +0200 Subject: [PATCH 749/922] fix C compiler warning --- psutil/arch/windows/inet_ntop.h | 11 ++++++++--- psutil/arch/windows/process_info.c | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/psutil/arch/windows/inet_ntop.h b/psutil/arch/windows/inet_ntop.h index 0d97e28c8..70573a368 100644 --- a/psutil/arch/windows/inet_ntop.h +++ b/psutil/arch/windows/inet_ntop.h @@ -1,10 +1,15 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola', Jeff Tang. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + #include -PCSTR -WSAAPI +PCSTR WSAAPI inet_ntop( __in INT Family, __in PVOID pAddr, __out_ecount(StringBufSize) PSTR pStringBuf, __in size_t StringBufSize -); \ No newline at end of file +); diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index cc5669522..a9687f9cd 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -259,7 +259,7 @@ psutil_check_phandle(HANDLE hProcess, DWORD pid) { return NoSuchProcess(); else if (ret == -1) return PyErr_SetFromWindowsErr(0); - else if (ret == -2) + else // -2 return NULL; } From 8cd94a366c93a63aabfe69b5077b3471710c925d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 30 May 2017 18:53:11 +0200 Subject: [PATCH 750/922] disable failing test on travis --- psutil/tests/test_connections.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 9390214ed..203ddebba 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -40,6 +40,7 @@ from psutil.tests import skip_on_access_denied from psutil.tests import tcp_socketpair from psutil.tests import TESTFN +from psutil.tests import TRAVIS from psutil.tests import unittest from psutil.tests import unix_socket_path from psutil.tests import unix_socketpair @@ -451,6 +452,8 @@ def test_multi_socks(self): self.assertEqual(len(cons), len(socks)) @skip_on_access_denied() + # See: https://travis-ci.org/giampaolo/psutil/jobs/237566297 + @unittest.skipIf(OSX and TRAVIS, "unreliable on OSX + TRAVIS") def test_multi_sockets_procs(self): # Creates multiple sub processes, each creating different # sockets. For each process check that proc.connections() From 9da480c208db51b06cd2cc20b8edeeafc7f0b463 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 2 Jun 2017 11:14:59 +0200 Subject: [PATCH 751/922] small refactoring --- psutil/_pslinux.py | 10 ++++++---- psutil/_psutil_windows.c | 5 ++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index cf32bbd50..24ad43ac8 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1292,14 +1292,15 @@ def users(): def boot_time(): """Return the system boot time expressed in seconds since the epoch.""" global BOOT_TIME - with open_binary('%s/stat' % get_procfs_path()) as f: + path = '%s/stat' % get_procfs_path() + with open_binary(path) as f: for line in f: if line.startswith(b'btime'): ret = float(line.strip().split()[1]) BOOT_TIME = ret return ret raise RuntimeError( - "line 'btime' not found in %s/stat" % get_procfs_path()) + "line 'btime' not found in %s" % path) # ===================================================================== @@ -1331,14 +1332,15 @@ def pid_exists(pid): # Note: already checked that this is faster than using a # regular expr. Also (a lot) faster than doing # 'return pid in pids()' - with open_binary("%s/%s/status" % (get_procfs_path(), pid)) as f: + path = "%s/%s/status" % (get_procfs_path(), pid) + with open_binary(path) as f: for line in f: if line.startswith(b"Tgid:"): tgid = int(line.split()[1]) # If tgid and pid are the same then we're # dealing with a process PID. return tgid == pid - raise ValueError("'Tgid' line not found") + raise ValueError("'Tgid' line not found in %s" % path) except (EnvironmentError, ValueError): return pid in pids() diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 90ef2d44e..bec2d3aa9 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -65,8 +65,11 @@ typedef BOOL (WINAPI *LPFN_GLPI) (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); -// fix for mingw32, see +// Fix for mingw32, see: // https://github.com/giampaolo/psutil/issues/351#c2 +// This is actually a DISK_PERFORMANCE struct: +// https://msdn.microsoft.com/en-us/library/windows/desktop/ +// aa363991(v=vs.85).aspx typedef struct _DISK_PERFORMANCE_WIN_2008 { LARGE_INTEGER BytesRead; LARGE_INTEGER BytesWritten; From f435c2b6d308c12d0df33b10d828d97df303a614 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Jun 2017 13:05:59 +0200 Subject: [PATCH 752/922] 1044 osx zombies (#1100) * small refactoring * #1044: define a separate ctx manager which handles zombie processes * add create_zombie_proc utility function * disable test on windows * #1044: cmdline() was incorrectly raising AD instead of ZombieProcess * #1044: environ() was incorrectly raising AD instead of ZombieProcess * #1044: memory_maps() was incorrectly raising AD instead of ZombieProcess * #1044: threads() was incorrectly raising AD instead of ZombieProcess * enhance test * fix threads() --- psutil/_pslinux.py | 10 ++- psutil/_psosx.py | 77 ++++++++++++------ psutil/_psutil_osx.c | 17 ++-- psutil/_psutil_windows.c | 5 +- psutil/arch/osx/process_info.c | 26 +++--- psutil/tests/__init__.py | 41 +++++++++- psutil/tests/test_misc.py | 8 ++ psutil/tests/test_osx.py | 61 ++++++++++++++ psutil/tests/test_process.py | 143 ++++++++++++--------------------- 9 files changed, 246 insertions(+), 142 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index cf32bbd50..24ad43ac8 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1292,14 +1292,15 @@ def users(): def boot_time(): """Return the system boot time expressed in seconds since the epoch.""" global BOOT_TIME - with open_binary('%s/stat' % get_procfs_path()) as f: + path = '%s/stat' % get_procfs_path() + with open_binary(path) as f: for line in f: if line.startswith(b'btime'): ret = float(line.strip().split()[1]) BOOT_TIME = ret return ret raise RuntimeError( - "line 'btime' not found in %s/stat" % get_procfs_path()) + "line 'btime' not found in %s" % path) # ===================================================================== @@ -1331,14 +1332,15 @@ def pid_exists(pid): # Note: already checked that this is faster than using a # regular expr. Also (a lot) faster than doing # 'return pid in pids()' - with open_binary("%s/%s/status" % (get_procfs_path(), pid)) as f: + path = "%s/%s/status" % (get_procfs_path(), pid) + with open_binary(path) as f: for line in f: if line.startswith(b"Tgid:"): tgid = int(line.split()[1]) # If tgid and pid are the same then we're # dealing with a process PID. return tgid == pid - raise ValueError("'Tgid' line not found") + raise ValueError("'Tgid' line not found in %s" % path) except (EnvironmentError, ValueError): return pid in pids() diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 14cdb1e1f..a69f892c6 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -4,6 +4,7 @@ """OSX platform implementation.""" +import contextlib import errno import functools import os @@ -291,22 +292,40 @@ def wrapper(self, *args, **kwargs): try: return fun(self, *args, **kwargs) except OSError as err: - if self.pid == 0: - if 0 in pids(): - raise AccessDenied(self.pid, self._name) - else: - raise if err.errno == errno.ESRCH: - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - else: - raise ZombieProcess(self.pid, self._name, self._ppid) + raise NoSuchProcess(self.pid, self._name) if err.errno in (errno.EPERM, errno.EACCES): raise AccessDenied(self.pid, self._name) raise return wrapper +@contextlib.contextmanager +def catch_zombie(proc): + """There are some poor C APIs which incorrectly raise ESRCH when + the process is still alive or it's a zombie, or even RuntimeError + (those who don't set errno). This is here in order to solve: + https://github.com/giampaolo/psutil/issues/1044 + """ + try: + yield + except (OSError, RuntimeError) as err: + if isinstance(err, RuntimeError) or err.errno == errno.ESRCH: + try: + # status() is not supposed to lie and correctly detect + # zombies so if it raises ESRCH it's true. + status = proc.status() + except NoSuchProcess: + raise err + else: + if status == _common.STATUS_ZOMBIE: + raise ZombieProcess(proc.pid, proc._name, proc._ppid) + else: + raise AccessDenied(proc.pid, proc._name) + else: + raise + + class Process(object): """Wrapper class around underlying C implementation.""" @@ -327,7 +346,8 @@ def _get_kinfo_proc(self): @memoize_when_activated def _get_pidtaskinfo(self): # Note: should work for PIDs owned by user only. - ret = cext.proc_pidtaskinfo_oneshot(self.pid) + with catch_zombie(self): + ret = cext.proc_pidtaskinfo_oneshot(self.pid) assert len(ret) == len(pidtaskinfo_map) return ret @@ -346,19 +366,18 @@ def name(self): @wrap_exceptions def exe(self): - return cext.proc_exe(self.pid) + with catch_zombie(self): + return cext.proc_exe(self.pid) @wrap_exceptions def cmdline(self): - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - return cext.proc_cmdline(self.pid) + with catch_zombie(self): + return cext.proc_cmdline(self.pid) @wrap_exceptions def environ(self): - if not pid_exists(self.pid): - raise NoSuchProcess(self.pid, self._name) - return parse_environ_block(cext.proc_environ(self.pid)) + with catch_zombie(self): + return parse_environ_block(cext.proc_environ(self.pid)) @wrap_exceptions def ppid(self): @@ -367,7 +386,8 @@ def ppid(self): @wrap_exceptions def cwd(self): - return cext.proc_cwd(self.pid) + with catch_zombie(self): + return cext.proc_cwd(self.pid) @wrap_exceptions def uids(self): @@ -440,7 +460,8 @@ def open_files(self): if self.pid == 0: return [] files = [] - rawlist = cext.proc_open_files(self.pid) + with catch_zombie(self): + rawlist = cext.proc_open_files(self.pid) for path, fd in rawlist: if isfile_strict(path): ntuple = _common.popenfile(path, fd) @@ -453,7 +474,8 @@ def connections(self, kind='inet'): raise ValueError("invalid %r kind argument; choose between %s" % (kind, ', '.join([repr(x) for x in conn_tmap]))) families, types = conn_tmap[kind] - rawlist = cext.proc_connections(self.pid, families, types) + with catch_zombie(self): + rawlist = cext.proc_connections(self.pid, families, types) ret = [] for item in rawlist: fd, fam, type, laddr, raddr, status = item @@ -468,7 +490,8 @@ def connections(self, kind='inet'): def num_fds(self): if self.pid == 0: return 0 - return cext.proc_num_fds(self.pid) + with catch_zombie(self): + return cext.proc_num_fds(self.pid) @wrap_exceptions def wait(self, timeout=None): @@ -479,11 +502,13 @@ def wait(self, timeout=None): @wrap_exceptions def nice_get(self): - return cext_posix.getpriority(self.pid) + with catch_zombie(self): + return cext_posix.getpriority(self.pid) @wrap_exceptions def nice_set(self, value): - return cext_posix.setpriority(self.pid, value) + with catch_zombie(self): + return cext_posix.setpriority(self.pid, value) @wrap_exceptions def status(self): @@ -493,7 +518,8 @@ def status(self): @wrap_exceptions def threads(self): - rawlist = cext.proc_threads(self.pid) + with catch_zombie(self): + rawlist = cext.proc_threads(self.pid) retlist = [] for thread_id, utime, stime in rawlist: ntuple = _common.pthread(thread_id, utime, stime) @@ -502,4 +528,5 @@ def threads(self): @wrap_exceptions def memory_maps(self): - return cext.proc_memory_maps(self.pid) + with catch_zombie(self): + return cext.proc_memory_maps(self.pid) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 7d762a1cb..450f1db08 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -125,6 +125,8 @@ psutil_pids(PyObject *self, PyObject *args) { * using sysctl() and filling up a kinfo_proc struct. * It should be possible to do this for all processes without * incurring into permission (EPERM) errors. + * This will also succeed for zombie processes returning correct + * information. */ static PyObject * psutil_proc_kinfo_oneshot(PyObject *self, PyObject *args) { @@ -173,8 +175,9 @@ psutil_proc_kinfo_oneshot(PyObject *self, PyObject *args) { * Return multiple process info as a Python tuple in one shot by * using proc_pidinfo(PROC_PIDTASKINFO) and filling a proc_taskinfo * struct. - * Contrarily from proc_kinfo above this function will return EACCES - * for PIDs owned by another user. + * Contrarily from proc_kinfo above this function will fail with + * EACCES for PIDs owned by another user and with ESRCH for zombie + * processes. */ static PyObject * psutil_proc_pidtaskinfo_oneshot(PyObject *self, PyObject *args) { @@ -226,6 +229,7 @@ psutil_proc_name(PyObject *self, PyObject *args) { /* * Return process current working directory. + * Raises NSP in case of zombie process. */ static PyObject * psutil_proc_cwd(PyObject *self, PyObject *args) { @@ -332,10 +336,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { err = task_for_pid(mach_task_self(), (pid_t)pid, &task); if (err != KERN_SUCCESS) { - if (psutil_pid_exists(pid) == 0) - NoSuchProcess(); - else - AccessDenied(); + psutil_raise_for_pid(pid, "task_for_pid() failed"); goto error; } @@ -1002,7 +1003,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) goto error; - // task_for_pid() requires special privileges + // task_for_pid() requires root privileges err = task_for_pid(mach_task_self(), (pid_t)pid, &task); if (err != KERN_SUCCESS) { if (psutil_pid_exists(pid) == 0) @@ -1186,6 +1187,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { /* * Return process TCP and UDP connections as a list of tuples. + * Raises NSP in case of zombie process. * References: * - lsof source code: http://goo.gl/SYW79 and http://goo.gl/wNrC0 * - /usr/include/sys/proc_info.h @@ -1389,6 +1391,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { /* * Return number of file descriptors opened by process. + * Raises NSP in case of zombie process. */ static PyObject * psutil_proc_num_fds(PyObject *self, PyObject *args) { diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 90ef2d44e..bec2d3aa9 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -65,8 +65,11 @@ typedef BOOL (WINAPI *LPFN_GLPI) (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); -// fix for mingw32, see +// Fix for mingw32, see: // https://github.com/giampaolo/psutil/issues/351#c2 +// This is actually a DISK_PERFORMANCE struct: +// https://msdn.microsoft.com/en-us/library/windows/desktop/ +// aa363991(v=vs.85).aspx typedef struct _DISK_PERFORMANCE_WIN_2008 { LARGE_INTEGER BytesRead; LARGE_INTEGER BytesWritten; diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/osx/process_info.c index 7d6861a52..7c715be81 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -141,13 +141,12 @@ psutil_get_cmdline(long pid) { mib[1] = KERN_PROCARGS2; mib[2] = (pid_t)pid; if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { - if (EINVAL == errno) { - // EINVAL == access denied OR nonexistent PID - if (psutil_pid_exists(pid)) - AccessDenied(); - else - NoSuchProcess(); - } + // In case of zombie process we'll get EINVAL. We translate it + // to NSP and _psosx.py will translate it to ZP. + if ((errno == EINVAL) && (psutil_pid_exists(pid))) + NoSuchProcess(); + else + PyErr_SetFromErrno(PyExc_OSError); goto error; } @@ -236,13 +235,12 @@ psutil_get_environ(long pid) { mib[1] = KERN_PROCARGS2; mib[2] = (pid_t)pid; if (sysctl(mib, 3, procargs, &argmax, NULL, 0) < 0) { - if (EINVAL == errno) { - // EINVAL == access denied OR nonexistent PID - if (psutil_pid_exists(pid)) - AccessDenied(); - else - NoSuchProcess(); - } + // In case of zombie process we'll get EINVAL. We translate it + // to NSP and _psosx.py will translate it to ZP. + if ((errno == EINVAL) && (psutil_pid_exists(pid))) + NoSuchProcess(); + else + PyErr_SetFromErrno(PyExc_OSError); goto error; } diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 24718edd5..9eedb40c9 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -18,6 +18,7 @@ import os import random import re +import select import shutil import socket import stat @@ -35,6 +36,7 @@ from socket import SOCK_STREAM import psutil +from psutil import OSX from psutil import POSIX from psutil import SUNOS from psutil import WINDOWS @@ -71,7 +73,7 @@ "HAS_SENSORS_BATTERY", "HAS_BATTERY""HAS_SENSORS_FANS", "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", # subprocesses - 'pyrun', 'reap_children', 'get_test_subprocess', + 'pyrun', 'reap_children', 'get_test_subprocess', 'create_zombie_proc', 'create_proc_children_pair', # threads 'ThreadTask' @@ -330,6 +332,43 @@ def create_proc_children_pair(): return (child1, child2) +def create_zombie_proc(): + """Create a zombie process and return its PID.""" + assert psutil.POSIX + unix_file = tempfile.mktemp(prefix=TESTFILE_PREFIX) if OSX else TESTFN + src = textwrap.dedent("""\ + import os, sys, time, socket, contextlib + child_pid = os.fork() + if child_pid > 0: + time.sleep(3000) + else: + # this is the zombie process + s = socket.socket(socket.AF_UNIX) + with contextlib.closing(s): + s.connect('%s') + if sys.version_info < (3, ): + pid = str(os.getpid()) + else: + pid = bytes(str(os.getpid()), 'ascii') + s.sendall(pid) + """ % unix_file) + with contextlib.closing(socket.socket(socket.AF_UNIX)) as sock: + sock.settimeout(GLOBAL_TIMEOUT) + sock.bind(unix_file) + sock.listen(1) + pyrun(src) + conn, _ = sock.accept() + try: + select.select([conn.fileno()], [], [], GLOBAL_TIMEOUT) + zpid = int(conn.recv(1024)) + _pids_started.add(zpid) + zproc = psutil.Process(zpid) + call_until(lambda: zproc.status(), "ret == psutil.STATUS_ZOMBIE") + return zpid + finally: + conn.close() + + @_cleanup_on_err def pyrun(src, **kwds): """Run python 'src' code string in a separate interpreter. diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index f9459d30c..85bab84c7 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -35,6 +35,7 @@ from psutil.tests import chdir from psutil.tests import create_proc_children_pair from psutil.tests import create_sockets +from psutil.tests import create_zombie_proc from psutil.tests import DEVNULL from psutil.tests import get_free_port from psutil.tests import get_test_subprocess @@ -944,6 +945,13 @@ def test_create_proc_children_pair(self): assert not psutil.tests._pids_started assert not psutil.tests._subprocesses_started + @unittest.skipIf(not POSIX, "POSIX only") + def test_create_zombie_proc(self): + zpid = create_zombie_proc() + self.addCleanup(reap_children, recursive=True) + p = psutil.Process(zpid) + self.assertEqual(p.status(), psutil.STATUS_ZOMBIE) + class TestNetUtils(unittest.TestCase): diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 225f95db2..5658f9885 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -12,6 +12,7 @@ import psutil from psutil import OSX +from psutil.tests import create_zombie_proc from psutil.tests import get_test_subprocess from psutil.tests import MEMORY_TOLERANCE from psutil.tests import reap_children @@ -99,6 +100,66 @@ def test_process_create_time(self): time.strftime("%Y", time.localtime(start_psutil))) +@unittest.skipIf(not OSX, "OSX only") +class TestZombieProcessAPIs(unittest.TestCase): + + @classmethod + def setUpClass(cls): + zpid = create_zombie_proc() + cls.p = psutil.Process(zpid) + + @classmethod + def tearDownClass(cls): + reap_children(recursive=True) + + def test_pidtask_info(self): + self.assertEqual(self.p.status(), psutil.STATUS_ZOMBIE) + self.p.ppid() + self.p.uids() + self.p.gids() + self.p.terminal() + self.p.create_time() + + def test_exe(self): + self.assertRaises(psutil.ZombieProcess, self.p.exe) + + def test_cmdline(self): + self.assertRaises(psutil.ZombieProcess, self.p.cmdline) + + def test_environ(self): + self.assertRaises(psutil.ZombieProcess, self.p.environ) + + def test_cwd(self): + self.assertRaises(psutil.ZombieProcess, self.p.cwd) + + def test_memory_full_info(self): + self.assertRaises(psutil.ZombieProcess, self.p.memory_full_info) + + def test_cpu_times(self): + self.assertRaises(psutil.ZombieProcess, self.p.cpu_times) + + def test_num_ctx_switches(self): + self.assertRaises(psutil.ZombieProcess, self.p.num_ctx_switches) + + def test_num_threads(self): + self.assertRaises(psutil.ZombieProcess, self.p.num_threads) + + def test_open_files(self): + self.assertRaises(psutil.ZombieProcess, self.p.open_files) + + def test_connections(self): + self.assertRaises(psutil.ZombieProcess, self.p.connections) + + def test_num_fds(self): + self.assertRaises(psutil.ZombieProcess, self.p.num_fds) + + def test_threads(self): + self.assertRaises(psutil.ZombieProcess, self.p.threads) + + def test_memory_maps(self): + self.assertRaises(psutil.ZombieProcess, self.p.memory_maps) + + @unittest.skipIf(not OSX, "OSX only") class TestSystemAPIs(unittest.TestCase): diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index cab5a2fee..dd0c507ec 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -7,11 +7,9 @@ """Tests for psutil.Process class.""" import collections -import contextlib import errno import getpass import os -import select import signal import socket import subprocess @@ -38,10 +36,10 @@ from psutil.tests import copyload_shared_lib from psutil.tests import create_exe from psutil.tests import create_proc_children_pair +from psutil.tests import create_zombie_proc from psutil.tests import enum from psutil.tests import get_test_subprocess from psutil.tests import get_winver -from psutil.tests import GLOBAL_TIMEOUT from psutil.tests import HAS_CPU_AFFINITY from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_IONICE @@ -51,7 +49,6 @@ from psutil.tests import HAS_RLIMIT from psutil.tests import mock from psutil.tests import PYPY -from psutil.tests import pyrun from psutil.tests import PYTHON from psutil.tests import reap_children from psutil.tests import retry_before_failing @@ -1243,92 +1240,58 @@ def succeed_or_zombie_p_exc(fun, *args, **kwargs): except (psutil.ZombieProcess, psutil.AccessDenied): pass - # Note: in this test we'll be creating two sub processes. - # Both of them are supposed to be freed / killed by - # reap_children() as they are attributable to 'us' - # (os.getpid()) via children(recursive=True). - unix_file = tempfile.mktemp(prefix=TESTFILE_PREFIX) if OSX else TESTFN - src = textwrap.dedent("""\ - import os, sys, time, socket, contextlib - child_pid = os.fork() - if child_pid > 0: - time.sleep(3000) - else: - # this is the zombie process - s = socket.socket(socket.AF_UNIX) - with contextlib.closing(s): - s.connect('%s') - if sys.version_info < (3, ): - pid = str(os.getpid()) - else: - pid = bytes(str(os.getpid()), 'ascii') - s.sendall(pid) - """ % unix_file) - with contextlib.closing(socket.socket(socket.AF_UNIX)) as sock: - try: - sock.settimeout(GLOBAL_TIMEOUT) - sock.bind(unix_file) - sock.listen(1) - pyrun(src) - conn, _ = sock.accept() - self.addCleanup(conn.close) - select.select([conn.fileno()], [], [], GLOBAL_TIMEOUT) - zpid = int(conn.recv(1024)) - zproc = psutil.Process(zpid) - call_until(lambda: zproc.status(), - "ret == psutil.STATUS_ZOMBIE") - # A zombie process should always be instantiable - zproc = psutil.Process(zpid) - # ...and at least its status always be querable - self.assertEqual(zproc.status(), psutil.STATUS_ZOMBIE) - # ...and it should be considered 'running' - self.assertTrue(zproc.is_running()) - # ...and as_dict() shouldn't crash - zproc.as_dict() - # if cmdline succeeds it should be an empty list - ret = succeed_or_zombie_p_exc(zproc.suspend) - if ret is not None: - self.assertEqual(ret, []) - - if hasattr(zproc, "rlimit"): - succeed_or_zombie_p_exc(zproc.rlimit, psutil.RLIMIT_NOFILE) - succeed_or_zombie_p_exc(zproc.rlimit, psutil.RLIMIT_NOFILE, - (5, 5)) - # set methods - succeed_or_zombie_p_exc(zproc.parent) - if hasattr(zproc, 'cpu_affinity'): - succeed_or_zombie_p_exc(zproc.cpu_affinity, [0]) - succeed_or_zombie_p_exc(zproc.nice, 0) - if hasattr(zproc, 'ionice'): - if LINUX: - succeed_or_zombie_p_exc(zproc.ionice, 2, 0) - else: - succeed_or_zombie_p_exc(zproc.ionice, 0) # Windows - if hasattr(zproc, 'rlimit'): - succeed_or_zombie_p_exc(zproc.rlimit, - psutil.RLIMIT_NOFILE, (5, 5)) - succeed_or_zombie_p_exc(zproc.suspend) - succeed_or_zombie_p_exc(zproc.resume) - succeed_or_zombie_p_exc(zproc.terminate) - succeed_or_zombie_p_exc(zproc.kill) - - # ...its parent should 'see' it - # edit: not true on BSD and OSX - # descendants = [x.pid for x in psutil.Process().children( - # recursive=True)] - # self.assertIn(zpid, descendants) - # XXX should we also assume ppid be usable? Note: this - # would be an important use case as the only way to get - # rid of a zombie is to kill its parent. - # self.assertEqual(zpid.ppid(), os.getpid()) - # ...and all other APIs should be able to deal with it - self.assertTrue(psutil.pid_exists(zpid)) - self.assertIn(zpid, psutil.pids()) - self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) - psutil._pmap = {} - self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) - finally: - reap_children(recursive=True) + zpid = create_zombie_proc() + self.addCleanup(reap_children, recursive=True) + # A zombie process should always be instantiable + zproc = psutil.Process(zpid) + # ...and at least its status always be querable + self.assertEqual(zproc.status(), psutil.STATUS_ZOMBIE) + # ...and it should be considered 'running' + self.assertTrue(zproc.is_running()) + # ...and as_dict() shouldn't crash + zproc.as_dict() + # if cmdline succeeds it should be an empty list + ret = succeed_or_zombie_p_exc(zproc.suspend) + if ret is not None: + self.assertEqual(ret, []) + + if hasattr(zproc, "rlimit"): + succeed_or_zombie_p_exc(zproc.rlimit, psutil.RLIMIT_NOFILE) + succeed_or_zombie_p_exc(zproc.rlimit, psutil.RLIMIT_NOFILE, + (5, 5)) + # set methods + succeed_or_zombie_p_exc(zproc.parent) + if hasattr(zproc, 'cpu_affinity'): + succeed_or_zombie_p_exc(zproc.cpu_affinity, [0]) + succeed_or_zombie_p_exc(zproc.nice, 0) + if hasattr(zproc, 'ionice'): + if LINUX: + succeed_or_zombie_p_exc(zproc.ionice, 2, 0) + else: + succeed_or_zombie_p_exc(zproc.ionice, 0) # Windows + if hasattr(zproc, 'rlimit'): + succeed_or_zombie_p_exc(zproc.rlimit, + psutil.RLIMIT_NOFILE, (5, 5)) + succeed_or_zombie_p_exc(zproc.suspend) + succeed_or_zombie_p_exc(zproc.resume) + succeed_or_zombie_p_exc(zproc.terminate) + succeed_or_zombie_p_exc(zproc.kill) + + # ...its parent should 'see' it + # edit: not true on BSD and OSX + # descendants = [x.pid for x in psutil.Process().children( + # recursive=True)] + # self.assertIn(zpid, descendants) + # XXX should we also assume ppid be usable? Note: this + # would be an important use case as the only way to get + # rid of a zombie is to kill its parent. + # self.assertEqual(zpid.ppid(), os.getpid()) + # ...and all other APIs should be able to deal with it + self.assertTrue(psutil.pid_exists(zpid)) + self.assertIn(zpid, psutil.pids()) + self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) + psutil._pmap = {} + self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) @unittest.skipIf(not POSIX, 'POSIX only') def test_zombie_process_is_running_w_exc(self): From d17080ca96dd47eddac7c7566458b2ea3e786f03 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Jun 2017 13:06:47 +0200 Subject: [PATCH 753/922] update HISTORY --- HISTORY.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 30658d689..fadbc5f91 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -47,6 +47,8 @@ - 1040_: fixed many unicode related issues such as UnicodeDecodeError on Python 3 + UNIX and invalid encoded data on Windows. - 1042_: [FreeBSD] psutil won't compile on FreeBSD 12. +- 1044_: [OSX] different Process methods incorrectly raise AccessDenied for + zombie processes. - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. - 1048_: [Windows] users()'s host field report an invalid IP address. From 0a6953cfd59009b422b808b2c59e37077c0bdcb1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Jun 2017 13:25:12 +0200 Subject: [PATCH 754/922] fix #1071: provide fallback for cpu_freq() in case main current freq file is not available on old RedHat versions --- HISTORY.rst | 1 + psutil/_pslinux.py | 15 ++++++++++++--- psutil/tests/test_linux.py | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fadbc5f91..c37981426 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -65,6 +65,7 @@ - 1067_: [NetBSD] Process.cmdline() leaks memory if proces has terminated. - 1069_: [FreeBSD] Process.cpu_num() may return 255 for certain kernel processes. +- 1071_: [Linux] cpu_freq() may raise IOError on old RedHat distros. - 1074_: [FreeBSD] sensors_battery() raises OSError in case of no battery. - 1075_: [Windows] net_if_addrs(): inet_ntop() return value is not checked. - 1077_: [SunOS] net_if_addrs() shows garbage addresses on SunOS 5.10. diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 24ad43ac8..e37139b8e 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -678,10 +678,19 @@ def cpu_freq(): ls = glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq") ls.sort(key=lambda x: int(re.search('[0-9]+', x).group(0))) + pjoin = os.path.join for path in ls: - curr = int(cat(os.path.join(path, "scaling_cur_freq"))) / 1000 - max_ = int(cat(os.path.join(path, "scaling_max_freq"))) / 1000 - min_ = int(cat(os.path.join(path, "scaling_min_freq"))) / 1000 + curr = cat(pjoin(path, "scaling_cur_freq"), fallback=None) + if curr is None: + # Likely an old RedHat, see: + # https://github.com/giampaolo/psutil/issues/1071 + curr = cat(pjoin(path, "cpuinfo_cur_freq"), fallback=None) + if curr is None: + raise NotImplementedError( + "can't find current frequency file") + curr = int(curr) / 1000 + max_ = int(cat(pjoin(path, "scaling_max_freq"))) / 1000 + min_ = int(cat(pjoin(path, "scaling_min_freq"))) / 1000 ret.append(_common.scpufreq(curr, min_, max_)) return ret diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 2054da8b2..162920bba 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -674,6 +674,38 @@ def open_mock(name, *args, **kwargs): self.assertEqual(freq.min, 200.0) self.assertEqual(freq.max, 300.0) + def test_cpu_frequ_no_scaling_cur_freq_file(self): + # See: https://github.com/giampaolo/psutil/issues/1071 + def open_mock(name, *args, **kwargs): + if name.endswith('/scaling_cur_freq'): + raise IOError(errno.ENOENT, "") + elif name.endswith('/cpuinfo_cur_freq'): + return io.BytesIO(b"200000") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock): + ret = psutil.cpu_freq() + self.assertEqual(ret.current, 200) + + # Also test that NotImplementedError is raised in case no + # current freq file is present. + + def open_mock(name, *args, **kwargs): + if name.endswith('/scaling_cur_freq'): + raise IOError(errno.ENOENT, "") + elif name.endswith('/cpuinfo_cur_freq'): + raise IOError(errno.ENOENT, "") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock): + self.assertRaises(NotImplementedError, psutil.cpu_freq) + # ===================================================================== # --- system CPU stats From 4a20d974515e408469f6018d5336bb7422f51d1c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Jun 2017 13:26:21 +0200 Subject: [PATCH 755/922] fix OSX failure --- psutil/tests/test_osx.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index 5658f9885..c8214f14c 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -154,7 +154,8 @@ def test_num_fds(self): self.assertRaises(psutil.ZombieProcess, self.p.num_fds) def test_threads(self): - self.assertRaises(psutil.ZombieProcess, self.p.threads) + self.assertRaises((psutil.ZombieProcess, psutil.AccessDenied), + self.p.threads) def test_memory_maps(self): self.assertRaises(psutil.ZombieProcess, self.p.memory_maps) From 7eb1581e41d0453caa625802556d209dbbf8b415 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Jun 2017 13:40:10 +0200 Subject: [PATCH 756/922] fix test and update doc --- HISTORY.rst | 3 ++- docs/index.rst | 24 +++++++----------------- psutil/tests/test_linux.py | 12 +++++++++--- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index c37981426..bc535643b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -94,7 +94,8 @@ - OpenBSD: connections('unix'): laddr and raddr are now set to "" instead of None - 1040_: all strings are encoded by using OS fs encoding. -- 1040_: the following Windows APIs returned unicode and now they return str: +- 1040_: the following Windows APIs on Python 2 now return a string instead of + unicode: - Process.memory_maps().path - WindosService.bin_path() - WindosService.description() diff --git a/docs/index.rst b/docs/index.rst index 309b2a68c..d56425c4e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2342,20 +2342,12 @@ A bit more advanced, check string against :meth:`Process.name()`, def find_procs_by_name(name): "Return a list of processes matching 'name'." - assert name, name ls = [] - for p in psutil.process_iter(): - name_, exe, cmdline = "", "", [] - try: - name_ = p.name() - cmdline = p.cmdline() - exe = p.exe() - except (psutil.AccessDenied, psutil.ZombieProcess): - pass - except psutil.NoSuchProcess: - continue - if name == name_ or cmdline[0] == name or os.path.basename(exe) == name: - ls.append(name) + for p in psutil.process_iter(attrs=["name", "exe", "cmdline"]): + if name == p.info['name'] or \ + p.info['exe'] and os.path.basename(p.info['exe']) == name or \ + p.info['cmdline'] and p.info['cmdline'][0] == name: + ls.append(p) return ls Kill process tree @@ -2371,8 +2363,8 @@ Kill process tree timeout=None, on_terminate=None): """Kill a process tree (including grandchildren) with signal "sig" and return a (gone, still_alive) tuple. - "on_terminate", if specified, is a callabck as soon as a child - terminates. + "on_terminate", if specified, is a callabck function which is + called as soon as a child terminates. """ if pid == os.getpid(): raise RuntimeError("I refuse to kill myself") @@ -2418,8 +2410,6 @@ resources. for p in alive: print("process {} survived SIGKILL; giving up" % p) - reap_children() - Filtering and sorting processes ------------------------------- diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 162920bba..b16e90e0c 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -686,9 +686,14 @@ def open_mock(name, *args, **kwargs): orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' + policies = ['/sys/devices/system/cpu/cpufreq/policy0', + '/sys/devices/system/cpu/cpufreq/policy1', + '/sys/devices/system/cpu/cpufreq/policy2'] + with mock.patch(patch_point, side_effect=open_mock): - ret = psutil.cpu_freq() - self.assertEqual(ret.current, 200) + with mock.patch('glob.glob', return_value=policies): + freq = psutil.cpu_freq() + self.assertEqual(freq.current, 200) # Also test that NotImplementedError is raised in case no # current freq file is present. @@ -704,7 +709,8 @@ def open_mock(name, *args, **kwargs): orig_open = open patch_point = 'builtins.open' if PY3 else '__builtin__.open' with mock.patch(patch_point, side_effect=open_mock): - self.assertRaises(NotImplementedError, psutil.cpu_freq) + with mock.patch('glob.glob', return_value=policies): + self.assertRaises(NotImplementedError, psutil.cpu_freq) # ===================================================================== From 612e5719299d3852e4fd02ecf177480fa6db4437 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Jun 2017 13:54:09 +0200 Subject: [PATCH 757/922] skip test on travis; update MANIFEST --- MANIFEST.in | 2 ++ psutil/tests/test_linux.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 65e11598f..89c422160 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -57,6 +57,8 @@ include psutil/arch/openbsd/specific.c include psutil/arch/openbsd/specific.h include psutil/arch/osx/process_info.c include psutil/arch/osx/process_info.h +include psutil/arch/solaris/environ.c +include psutil/arch/solaris/environ.h include psutil/arch/solaris/v10/ifaddrs.c include psutil/arch/solaris/v10/ifaddrs.h include psutil/arch/windows/glpi.h diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index b16e90e0c..1fe72a01a 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -674,7 +674,8 @@ def open_mock(name, *args, **kwargs): self.assertEqual(freq.min, 200.0) self.assertEqual(freq.max, 300.0) - def test_cpu_frequ_no_scaling_cur_freq_file(self): + @unittest.skipIf(TRAVIS, "fails on Travis") + def test_cpu_freq_no_scaling_cur_freq_file(self): # See: https://github.com/giampaolo/psutil/issues/1071 def open_mock(name, *args, **kwargs): if name.endswith('/scaling_cur_freq'): From ec1d35e41c288248818388830f0e4f98536b93e4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Jun 2017 16:29:13 +0200 Subject: [PATCH 758/922] #989 / windows / boot_time(): try to return the correct C type in order to avoid negative values --- HISTORY.rst | 1 + psutil/_psutil_windows.c | 12 ++++++++---- psutil/_pswindows.py | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bc535643b..c61181a29 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -28,6 +28,7 @@ **Bug fixes** +- 989_: [Windows] boot_time() may return a negative value. - 1007_: [Windows] boot_time() can have a 1 sec fluctuation between calls; the value of the first call is now cached so that boot_time() always returns the same value if fluctuation is <= 1 second. diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index bec2d3aa9..74e7cfef0 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -235,8 +235,12 @@ psutil_boot_time(PyObject *self, PyObject *args) { and 01-01-1601, from time_t the divide by 1e+7 to get to the same base granularity. */ - ll = (((LONGLONG)(fileTime.dwHighDateTime)) << 32) \ - + fileTime.dwLowDateTime; +#if (_WIN32_WINNT >= 0x0600) // Windows Vista + ll = (((ULONGLONG) +#else + ll = (((LONGLONG) +#endif + (fileTime.dwHighDateTime)) << 32) + fileTime.dwLowDateTime; pt = (time_t)((ll - 116444736000000000ull) / 10000000ull); // GetTickCount64() is Windows Vista+ only. Dinamically load @@ -249,15 +253,15 @@ psutil_boot_time(PyObject *self, PyObject *args) { if (psutil_GetTickCount64 != NULL) { // Windows >= Vista uptime = psutil_GetTickCount64() / (ULONGLONG)1000.00f; + return Py_BuildValue("K", pt - uptime); } else { // Windows XP. // GetTickCount() time will wrap around to zero if the // system is run continuously for 49.7 days. uptime = GetTickCount() / (LONGLONG)1000.00f; + return Py_BuildValue("L", pt - uptime); } - - return Py_BuildValue("d", (double)pt - (double)uptime); } diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 52676183d..80225ee9e 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -410,7 +410,7 @@ def boot_time(): # value which may have a 1 second fluctuation, see: # https://github.com/giampaolo/psutil/issues/1007 global _last_btime - ret = cext.boot_time() + ret = float(cext.boot_time()) if abs(ret - _last_btime) <= 1: return _last_btime else: From 938e6d799a6345bbb2a0eefe4e2d0a1a63d9f0db Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 7 Jun 2017 19:25:18 +0200 Subject: [PATCH 759/922] update doc --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index ed55ea13d..ebd427872 100644 --- a/README.rst +++ b/README.rst @@ -66,15 +66,15 @@ Example applications | :target: https://github.com/giampaolo/psutil/blob/master/docs/_static/procsmem.png | :target: https://github.com/giampaolo/psutil/blob/master/docs/_static/pmap.png | +------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------+ -Also see https://github.com/giampaolo/psutil/tree/master/scripts and -`doc recipes `__. +Also see `scripts directory `__ +and `doc recipes `__. ===================== Projects using psutil ===================== At the time of writing there are over -`5200 open source projects `__ +`5400 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: From 909c387cfcef1801245ffef35c8701099604e48b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 13 Jun 2017 21:01:53 +0200 Subject: [PATCH 760/922] fix #1101: [Linux] sensors_temperatures() may raise ENODEV. --- HISTORY.rst | 1 + psutil/_pslinux.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index bc535643b..26dd82ed6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -84,6 +84,7 @@ is still alive. - 1099_: [Windows] Process.terminate() may raise AccessDenied even if the process already died. +- 1101_: [Linux] sensors_temperatures() may raise ENODEV. **Porting notes** diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index e37139b8e..10f1f6186 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1151,7 +1151,8 @@ def sensors_temperatures(): current = float(cat(base + '_input')) / 1000.0 except OSError as err: # https://github.com/giampaolo/psutil/issues/1009 - if err.errno == errno.EIO: + # https://github.com/giampaolo/psutil/issues/1101 + if err.errno in (errno.EIO, errno.ENODEV): warnings.warn("ignoring %r" % err, RuntimeWarning) continue else: From 8b8da39e0c62432504fb5f67c418715aad35b291 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 23 Jun 2017 14:36:34 +0200 Subject: [PATCH 761/922] fix #928: turn connections()' 'laddr' and 'raddr' into named tuples --- HISTORY.rst | 2 ++ README.rst | 16 ++++++++-------- docs/index.rst | 32 ++++++++++++++++++-------------- psutil/_common.py | 3 +++ psutil/_psbsd.py | 10 ++++++++++ psutil/_pslinux.py | 2 +- psutil/_psosx.py | 5 +++++ psutil/_pssunos.py | 5 +++++ psutil/_pswindows.py | 2 ++ psutil/tests/__init__.py | 7 +++---- 10 files changed, 57 insertions(+), 27 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8aef81b6c..aa9628b2b 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ - 802_: disk_io_counters() and net_io_counters() numbers no longer wrap (restart from 0). Introduced a new "nowrap" argument. +- 928_: psutil.net_connections() and psutil.Process.connections() "laddr" and + "raddr" are now named tuples. - 1015_: swap_memory() now relies on /proc/meminfo instead of sysinfo() syscall so that it can be used in conjunction with PROCFS_PATH in order to retrieve memory info about Linux containers such as Docker and Heroku. diff --git a/README.rst b/README.rst index ebd427872..11898c47f 100644 --- a/README.rst +++ b/README.rst @@ -182,10 +182,10 @@ Network 'lo': netio(bytes_sent=2838627, bytes_recv=2838627, packets_sent=30567, packets_recv=30567, errin=0, errout=0, dropin=0, dropout=0)} >>> >>> psutil.net_connections() - [pconn(fd=115, family=, type=, laddr=('10.0.0.1', 48776), raddr=('93.186.135.91', 80), status='ESTABLISHED', pid=1254), - pconn(fd=117, family=, type=, laddr=('10.0.0.1', 43761), raddr=('72.14.234.100', 80), status='CLOSING', pid=2987), - pconn(fd=-1, family=, type=, laddr=('10.0.0.1', 60759), raddr=('72.14.234.104', 80), status='ESTABLISHED', pid=None), - pconn(fd=-1, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT', pid=None) + [sconn(fd=115, family=, type=, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED', pid=1254), + sconn(fd=117, family=, type=, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status='CLOSING', pid=2987), + sconn(fd=-1, family=, type=, laddr=addr(ip='10.0.0.1', port=60759), raddr=addr(ip='72.14.234.104', port=80), status='ESTABLISHED', pid=None), + sconn(fd=-1, family=, type=, laddr=addr(ip='10.0.0.1', port=51314), raddr=addr(ip='72.14.234.83', port=443), status='SYN_SENT', pid=None) ...] >>> >>> psutil.net_if_addrs() @@ -315,10 +315,10 @@ Process management popenfile(path='/var/log/monitd', fd=4, position=235542, mode='a', flags=33793)] >>> >>> p.connections() - [pconn(fd=115, family=, type=, laddr=('10.0.0.1', 48776), raddr=('93.186.135.91', 80), status='ESTABLISHED'), - pconn(fd=117, family=, type=, laddr=('10.0.0.1', 43761), raddr=('72.14.234.100', 80), status='CLOSING'), - pconn(fd=119, family=, type=, laddr=('10.0.0.1', 60759), raddr=('72.14.234.104', 80), status='ESTABLISHED'), - pconn(fd=123, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT')] + [pconn(fd=115, family=, type=, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED'), + pconn(fd=117, family=, type=, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status='CLOSING'), + pconn(fd=119, family=, type=, laddr=addr(ip='10.0.0.1', port=60759), raddr=addr(ip='72.14.234.104', port=80), status='ESTABLISHED'), + pconn(fd=123, family=, type=, laddr=addr(ip='10.0.0.1', port=51314), raddr=addr(ip='72.14.234.83', port=443), status='SYN_SENT')] >>> >>> p.num_threads() 4 diff --git a/docs/index.rst b/docs/index.rst index d56425c4e..6ccbedc27 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -489,10 +489,10 @@ Network `__ or `SOCK_DGRAM `__. - - **laddr**: the local address as a ``(ip, port)`` tuple or a ``path`` + - **laddr**: the local address as a ``(ip, port)`` named tuple or a ``path`` in case of AF_UNIX sockets. For UNIX sockets see notes below. - - **raddr**: the remote address as a ``(ip, port)`` tuple or an absolute - ``path`` in case of UNIX sockets. + - **raddr**: the remote address as a ``(ip, port)`` named tuple or an + absolute ``path`` in case of UNIX sockets. When the remote endpoint is not connected you'll get an empty tuple (AF_INET*) or ``""`` (AF_UNIX). For UNIX sockets see notes below. - **status**: represents the status of a TCP connection. The return value @@ -543,10 +543,10 @@ Network >>> import psutil >>> psutil.net_connections() - [pconn(fd=115, family=, type=, laddr=('10.0.0.1', 48776), raddr=('93.186.135.91', 80), status='ESTABLISHED', pid=1254), - pconn(fd=117, family=, type=, laddr=('10.0.0.1', 43761), raddr=('72.14.234.100', 80), status='CLOSING', pid=2987), - pconn(fd=-1, family=, type=, laddr=('10.0.0.1', 60759), raddr=('72.14.234.104', 80), status='ESTABLISHED', pid=None), - pconn(fd=-1, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT', pid=None) + [pconn(fd=115, family=, type=, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED', pid=1254), + pconn(fd=117, family=, type=, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status='CLOSING', pid=2987), + pconn(fd=-1, family=, type=, laddr=addr(ip='10.0.0.1', port=60759), raddr=addr(ip='72.14.234.104', port=80), status='ESTABLISHED', pid=None), + pconn(fd=-1, family=, type=, laddr=addr(ip='10.0.0.1', port=51314), raddr=addr(ip='72.14.234.83', port=443), status='SYN_SENT', pid=None) ...] .. note:: @@ -569,6 +569,8 @@ Network .. versionchanged:: 5.3.0 : socket "fd" is now set for real instead of being ``-1``. + .. versionchanged:: 5.3.0 : "laddr" and "raddr" are named tuples. + .. function:: net_if_addrs() Return the addresses associated to each NIC (network interface card) @@ -1788,10 +1790,10 @@ Process class - **type**: the address type, either `SOCK_STREAM `__ or `SOCK_DGRAM `__. - - **laddr**: the local address as a ``(ip, port)`` tuple or a ``path`` + - **laddr**: the local address as a ``(ip, port)`` named tuple or a ``path`` in case of AF_UNIX sockets. For UNIX sockets see notes below. - - **raddr**: the remote address as a ``(ip, port)`` tuple or an absolute - ``path`` in case of UNIX sockets. + - **raddr**: the remote address as a ``(ip, port)`` named tuple or an + absolute ``path`` in case of UNIX sockets. When the remote endpoint is not connected you'll get an empty tuple (AF_INET*) or ``""`` (AF_UNIX). For UNIX sockets see notes below. - **status**: represents the status of a TCP connection. The return value @@ -1835,10 +1837,10 @@ Process class >>> p.name() 'firefox' >>> p.connections() - [pconn(fd=115, family=, type=, laddr=('10.0.0.1', 48776), raddr=('93.186.135.91', 80), status='ESTABLISHED'), - pconn(fd=117, family=, type=, laddr=('10.0.0.1', 43761), raddr=('72.14.234.100', 80), status='CLOSING'), - pconn(fd=119, family=, type=, laddr=('10.0.0.1', 60759), raddr=('72.14.234.104', 80), status='ESTABLISHED'), - pconn(fd=123, family=, type=, laddr=('10.0.0.1', 51314), raddr=('72.14.234.83', 443), status='SYN_SENT')] + [pconn(fd=115, family=, type=, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED'), + pconn(fd=117, family=, type=, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status='CLOSING'), + pconn(fd=119, family=, type=, laddr=addr(ip='10.0.0.1', port=60759), raddr=addr(ip='72.14.234.104', port=80), status='ESTABLISHED'), + pconn(fd=123, family=, type=, laddr=addr(ip='10.0.0.1', port=51314), raddr=addr(ip='72.14.234.83', port=443), status='SYN_SENT')] .. note:: (Solaris) UNIX sockets are not supported. @@ -1851,6 +1853,8 @@ Process class (OpenBSD) "laddr" and "raddr" fields for UNIX sockets are always set to "". This is a limitation of the OS. + .. versionchanged:: 5.3.0 : "laddr" and "raddr" are named tuples. + .. method:: is_running() Return whether the current process is running in the current process list. diff --git a/psutil/_common.py b/psutil/_common.py index c08c60c13..7b1d3cde2 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -221,6 +221,9 @@ class BatteryTime(enum.IntEnum): pconn = namedtuple('pconn', ['fd', 'family', 'type', 'laddr', 'raddr', 'status']) +# psutil.connections() and psutil.Process.connections() +addr = namedtuple('addr', ['ip', 'port']) + # =================================================================== # --- Process.connections() 'kind' parameter mapping diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 407d5a957..fe55f92f2 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -10,11 +10,13 @@ import os import xml.etree.ElementTree as ET from collections import namedtuple +from socket import AF_INET from . import _common from . import _psposix from . import _psutil_bsd as cext from . import _psutil_posix as cext_posix +from ._common import AF_INET6 from ._common import conn_tmap from ._common import FREEBSD from ._common import memoize @@ -393,6 +395,8 @@ def net_connections(kind): # can't initialize their status? status = TCP_STATUSES[cext.PSUTIL_CONN_NONE] fam = sockfam_to_enum(fam) + laddr = _common.addr(*laddr) + raddr = _common.addr(*raddr) type = socktype_to_enum(type) nt = _common.sconn(fd, fam, type, laddr, raddr, status, pid) ret.add(nt) @@ -714,6 +718,9 @@ def connections(self, kind='inet'): status = TCP_STATUSES[status] except KeyError: status = TCP_STATUSES[cext.PSUTIL_CONN_NONE] + if fam in (AF_INET, AF_INET6): + laddr = _common.addr(*laddr) + raddr = _common.addr(*raddr) fam = sockfam_to_enum(fam) type = socktype_to_enum(type) nt = _common.pconn(fd, fam, type, laddr, raddr, status) @@ -729,6 +736,9 @@ def connections(self, kind='inet'): ret = [] for item in rawlist: fd, fam, type, laddr, raddr, status = item + if fam in (AF_INET, AF_INET6): + laddr = _common.addr(*laddr) + raddr = _common.addr(*raddr) fam = sockfam_to_enum(fam) type = socktype_to_enum(type) status = TCP_STATUSES[status] diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 10f1f6186..d20b12730 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -834,7 +834,7 @@ def decode_address(addr, family): raise _Ipv6UnsupportedError else: raise - return (ip, port) + return _common.addr(ip, port) @staticmethod def process_inet(file, family, type_, inodes, filter_pid=None): diff --git a/psutil/_psosx.py b/psutil/_psosx.py index a69f892c6..282bdeb44 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -8,12 +8,14 @@ import errno import functools import os +from socket import AF_INET from collections import namedtuple from . import _common from . import _psposix from . import _psutil_osx as cext from . import _psutil_posix as cext_posix +from ._common import AF_INET6 from ._common import conn_tmap from ._common import isfile_strict from ._common import memoize_when_activated @@ -482,6 +484,9 @@ def connections(self, kind='inet'): status = TCP_STATUSES[status] fam = sockfam_to_enum(fam) type = socktype_to_enum(type) + if fam in (AF_INET, AF_INET6): + laddr = _common.addr(*laddr) + raddr = _common.addr(*raddr) nt = _common.pconn(fd, fam, type, laddr, raddr, status) ret.append(nt) return ret diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 53821829f..9931d885f 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -10,11 +10,13 @@ import subprocess import sys from collections import namedtuple +from socket import AF_INET from . import _common from . import _psposix from . import _psutil_posix as cext_posix from . import _psutil_sunos as cext +from ._common import AF_INET6 from ._common import isfile_strict from ._common import memoize_when_activated from ._common import sockfam_to_enum @@ -263,6 +265,9 @@ def net_connections(kind, _pid=-1): continue if type_ not in types: continue + if fam in (AF_INET, AF_INET6): + laddr = _common.addr(*laddr) + raddr = _common.addr(*raddr) status = TCP_STATUSES[status] fam = sockfam_to_enum(fam) type_ = socktype_to_enum(type_) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 80225ee9e..aedb79306 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -327,6 +327,8 @@ def net_connections(kind, _pid=-1): ret = set() for item in rawlist: fd, fam, type, laddr, raddr, status, pid = item + laddr = _common.addr(*laddr) + raddr = _common.addr(*raddr) status = TCP_STATUSES[status] fam = sockfam_to_enum(fam) type = socktype_to_enum(type) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 9eedb40c9..c9cd5006b 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -1038,10 +1038,9 @@ def check_connection_ntuple(conn): assert isinstance(addr, tuple), addr if not addr: continue - ip, port = addr - assert isinstance(port, int), port - assert 0 <= port <= 65535, port - check_net_address(ip, conn.family) + assert isinstance(addr.port, int), addr.port + assert 0 <= addr.port <= 65535, addr.port + check_net_address(addr.ip, conn.family) elif conn.family == AF_UNIX: assert isinstance(addr, str), addr From 7cb0de84aae64301396ea1f8735539bf0b46bb0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Ku=C5=BAmi=C5=84ski?= Date: Fri, 30 Jun 2017 20:03:16 +0200 Subject: [PATCH 762/922] Update index.rst (#1106) Fix reference to undefined variable still_alive --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 6ccbedc27..44c79110e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -918,7 +918,7 @@ Functions for p in procs: p.terminate() gone, alive = psutil.wait_procs(procs, timeout=3, callback=on_terminate) - for p in still_alive: + for p in alive: p.kill() Exceptions From 625030ba4862adac87a75df6b6413e671a183e1d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 18 Jul 2017 17:10:43 +0200 Subject: [PATCH 763/922] variables rewording --- psutil/_pslinux.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index d20b12730..10c4aebe1 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -83,7 +83,7 @@ # Used when reading "big" files, namely /proc/{pid}/smaps and /proc/net/*. # On Python 2, using a buffer with open() for such files may result in a # speedup, see: https://github.com/giampaolo/psutil/issues/708 -BIGGER_FILE_BUFFERING = -1 if PY3 else 8192 +BIGFILE_BUFFERING = -1 if PY3 else 8192 LITTLE_ENDIAN = sys.byteorder == 'little' SECTOR_SIZE_FALLBACK = 512 if enum is None: @@ -842,7 +842,7 @@ def process_inet(file, family, type_, inodes, filter_pid=None): if file.endswith('6') and not os.path.exists(file): # IPv6 not supported return - with open_text(file, buffering=BIGGER_FILE_BUFFERING) as f: + with open_text(file, buffering=BIGFILE_BUFFERING) as f: f.readline() # skip the first line for lineno, line in enumerate(f, 1): try: @@ -879,7 +879,7 @@ def process_inet(file, family, type_, inodes, filter_pid=None): @staticmethod def process_unix(file, family, inodes, filter_pid=None): """Parse /proc/net/unix files.""" - with open_text(file, buffering=BIGGER_FILE_BUFFERING) as f: + with open_text(file, buffering=BIGFILE_BUFFERING) as f: f.readline() # skip the first line for line in f: tokens = line.split() @@ -1398,7 +1398,6 @@ def _parse_stat_file(self): Using "man proc" as a reference: where "man proc" refers to position N, always substract 2 (e.g starttime pos 22 in 'man proc' == pos 20 in the list returned here). - The return value is cached in case oneshot() ctx manager is in use. """ @@ -1409,13 +1408,12 @@ def _parse_stat_file(self): # the first occurrence of "(" and the last occurence of ")". rpar = data.rfind(b')') name = data[data.find(b'(') + 1:rpar] - fields_after_name = data[rpar + 2:].split() - return [name] + fields_after_name + others = data[rpar + 2:].split() + return [name] + others @memoize_when_activated def _read_status_file(self): """Read /proc/{pid}/stat file and return its content. - The return value is cached in case oneshot() ctx manager is in use. """ @@ -1425,7 +1423,7 @@ def _read_status_file(self): @memoize_when_activated def _read_smaps_file(self): with open_binary("%s/%s/smaps" % (self._procfs_path, self.pid), - buffering=BIGGER_FILE_BUFFERING) as f: + buffering=BIGFILE_BUFFERING) as f: return f.read().strip() def oneshot_enter(self): From 3b3f97cc06e54f0dcfc4794a97808d8f534f3e6a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 29 Jul 2017 19:37:55 +0200 Subject: [PATCH 764/922] update doc --- docs/index.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 6ccbedc27..e0f5389bd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -156,6 +156,9 @@ CPU Return the number of logical CPUs in the system (same as `os.cpu_count() `__ in Python 3.4). + This number is not equivalent to the number of CPUs the current process can + use. The number of usable CPUs can be obtained with + ``len(psutil.Process().cpu_affinity())``. If *logical* is ``False`` return the number of physical cores only (hyper thread CPUs are excluded). Return ``None`` if undetermined. On OpenBSD and NetBSD ``psutil.cpu_count(logical=False)`` always return @@ -1445,14 +1448,13 @@ Process class Get or set process current `CPU affinity `__. CPU affinity consists in telling the OS to run a process on a limited set - of CPUs only. - On Linux this is done via the ``taskset`` command. + of CPUs only (on Linux cmdline, ``taskset`` command is typically used). If no argument is passed it returns the current CPU affinity as a list of integers. If passed it must be a list of integers specifying the new CPUs affinity. - If an empty list is passed all eligible CPUs are assumed (and set); - on Linux this may not necessarily mean all available CPUs as in - ``list(range(psutil.cpu_count()))``). + If an empty list is passed all eligible CPUs are assumed (and set). + On some systems such as Linux this may not necessarily mean all available + logical CPUs as in ``list(range(psutil.cpu_count()))``). >>> import psutil >>> psutil.cpu_count() From 28e0aef2d2a40b61dad7081ddaa88917abba52e7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 31 Jul 2017 21:31:23 +0200 Subject: [PATCH 765/922] #1110: add test for parsing of /stat file on Linux --- psutil/tests/test_linux.py | 66 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 1fe72a01a..fb023ad8b 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1673,6 +1673,72 @@ def test_cwd_zombie(self): self.assertEqual(exc.exception.pid, p.pid) self.assertEqual(exc.exception.name, p.name()) + def test_stat_file_parsing(self): + from psutil._pslinux import CLOCK_TICKS + + def open_mock(name, *args, **kwargs): + if name.startswith('/proc/%s/stat' % os.getpid()): + args = [ + "0", # pid + "(cat)", # name + "Z", # status + "1", # ppid + "0", # pgrp + "0", # session + "0", # tty + "0", # tpgid + "0", # flags + "0", # minflt + "0", # cminflt + "0", # majflt + "0", # cmajflt + "2", # utime + "3", # stime + "4", # cutime + "5", # cstime + "0", # priority + "0", # nice + "0", # num_threads + "0", # itrealvalue + "6", # starttime + "0", # vsize + "0", # rss + "0", # rsslim + "0", # startcode + "0", # endcode + "0", # startstack + "0", # kstkesp + "0", # kstkeip + "0", # signal + "0", # blocked + "0", # sigignore + "0", # sigcatch + "0", # wchan + "0", # nswap + "0", # cnswap + "0", # exit_signal + "6", # processor + ] + return io.BytesIO(" ".join(args).encode()) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock): + p = psutil.Process() + self.assertEqual(p.name(), 'cat') + self.assertEqual(p.status(), psutil.STATUS_ZOMBIE) + self.assertEqual(p.ppid(), 1) + self.assertEqual( + p.create_time(), 6 / CLOCK_TICKS + psutil.boot_time()) + cpu = p.cpu_times() + self.assertEqual(cpu.user, 2 / CLOCK_TICKS) + self.assertEqual(cpu.system, 3 / CLOCK_TICKS) + self.assertEqual(cpu.children_user, 4 / CLOCK_TICKS) + self.assertEqual(cpu.children_system, 5 / CLOCK_TICKS) + self.assertEqual(p.cpu_num(), 6) + @unittest.skipIf(not LINUX, "LINUX only") class TestProcessAgainstStatus(unittest.TestCase): From c895a4aa1a08e7dac7213ab1c2458b0a28cea343 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 31 Jul 2017 21:46:57 +0200 Subject: [PATCH 766/922] add test for parsing of /status file on Linux --- psutil/_pslinux.py | 2 +- psutil/tests/test_linux.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 10c4aebe1..e9ee7df04 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1697,7 +1697,7 @@ def num_ctx_switches(self, "'voluntary_ctxt_switches' and 'nonvoluntary_ctxt_switches'" "lines were not found in %s/%s/status; the kernel is " "probably older than 2.6.23" % ( - self._procfs_path, self.self.pid)) + self._procfs_path, self.pid)) else: return _common.pctxsw(int(ctxsw[0]), int(ctxsw[1])) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index fb023ad8b..fa2ba0234 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1739,6 +1739,37 @@ def open_mock(name, *args, **kwargs): self.assertEqual(cpu.children_system, 5 / CLOCK_TICKS) self.assertEqual(p.cpu_num(), 6) + def test_status_file_parsing(self): + def open_mock(name, *args, **kwargs): + if name.startswith('/proc/%s/status' % os.getpid()): + return io.BytesIO(textwrap.dedent("""\ + Uid:\t1000\t1001\t1002\t1003 + Gid:\t1004\t1005\t1006\t1007 + Threads:\t66 + Cpus_allowed:\tf + Cpus_allowed_list:\t0-7 + voluntary_ctxt_switches:\t12 + nonvoluntary_ctxt_switches:\t13""")) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + patch_point = 'builtins.open' if PY3 else '__builtin__.open' + with mock.patch(patch_point, side_effect=open_mock): + p = psutil.Process() + self.assertEqual(p.num_ctx_switches().voluntary, 12) + self.assertEqual(p.num_ctx_switches().involuntary, 13) + self.assertEqual(p.num_threads(), 66) + uids = p.uids() + self.assertEqual(uids.real, 1000) + self.assertEqual(uids.effective, 1001) + self.assertEqual(uids.saved, 1002) + gids = p.gids() + self.assertEqual(gids.real, 1004) + self.assertEqual(gids.effective, 1005) + self.assertEqual(gids.saved, 1006) + self.assertEqual(p._proc._get_eligible_cpus(), list(range(0, 8))) + @unittest.skipIf(not LINUX, "LINUX only") class TestProcessAgainstStatus(unittest.TestCase): From ab233784640d18d8dd0840b584f90eed0cc470c7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 31 Jul 2017 21:50:54 +0200 Subject: [PATCH 767/922] fix #1112 fix NameError on OSX --- psutil/_common.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 7b1d3cde2..7c4af3d85 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -44,7 +44,7 @@ # constants 'FREEBSD', 'BSD', 'LINUX', 'NETBSD', 'OPENBSD', 'OSX', 'POSIX', 'SUNOS', 'WINDOWS', - 'ENCODING', 'ENCODING_ERRS', + 'ENCODING', 'ENCODING_ERRS', 'AF_INET6', # connection constants 'CONN_CLOSE', 'CONN_CLOSE_WAIT', 'CONN_CLOSING', 'CONN_ESTABLISHED', 'CONN_FIN_WAIT1', 'CONN_FIN_WAIT2', 'CONN_LAST_ACK', 'CONN_LISTEN', @@ -252,7 +252,7 @@ class BatteryTime(enum.IntEnum): "unix": ([AF_UNIX], [SOCK_STREAM, SOCK_DGRAM]), }) -del AF_INET, AF_INET6, AF_UNIX, SOCK_STREAM, SOCK_DGRAM +del AF_INET, AF_UNIX, SOCK_STREAM, SOCK_DGRAM # =================================================================== @@ -390,10 +390,10 @@ def path_exists_strict(path): @memoize def supports_ipv6(): """Return True if IPv6 is supported on this platform.""" - if not socket.has_ipv6 or not hasattr(socket, "AF_INET6"): + if not socket.has_ipv6 or AF_INET6 is None: return False try: - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock = socket.socket(AF_INET6, socket.SOCK_STREAM) with contextlib.closing(sock): sock.bind(("::1", 0)) return True From 3073311c32b31e4f2b7d1ca2c942bdf476d3b550 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 31 Jul 2017 21:57:26 +0200 Subject: [PATCH 768/922] signal @sunpoet and @kostikbel in CREDITS as possible helpers for FreeBSD related issues --- CREDITS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index a9270c864..77bdfcafa 100644 --- a/CREDITS +++ b/CREDITS @@ -30,8 +30,10 @@ Github usernames of people to CC on github when in need of help. - ryoqun, Ryo Onodera - OpenBSD: - landryb, Landry Breuil -- FreeNBSD: +- FreeBSD: - glebius, Gleb Smirnoff (#1013) + - sunpoet, Po-Chuan Hsieh (pkg maintainer, #1105) + - kostikbel, Konstantin Belousov (#1105) - OSX: - whitlockjc, Jeremy Whitlock - Windows: From 32edf35e95ff1c99ba2f4c94eb9913e65f33bd45 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 1 Aug 2017 11:59:03 +0200 Subject: [PATCH 769/922] fix TypeError on OSX --- psutil/_psosx.py | 6 ++++-- psutil/tests/test_linux.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 282bdeb44..8093e8149 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -485,8 +485,10 @@ def connections(self, kind='inet'): fam = sockfam_to_enum(fam) type = socktype_to_enum(type) if fam in (AF_INET, AF_INET6): - laddr = _common.addr(*laddr) - raddr = _common.addr(*raddr) + if laddr: + laddr = _common.addr(*laddr) + if raddr: + raddr = _common.addr(*raddr) nt = _common.pconn(fd, fam, type, laddr, raddr, status) ret.append(nt) return ret diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index fa2ba0234..468b3c663 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1749,7 +1749,7 @@ def open_mock(name, *args, **kwargs): Cpus_allowed:\tf Cpus_allowed_list:\t0-7 voluntary_ctxt_switches:\t12 - nonvoluntary_ctxt_switches:\t13""")) + nonvoluntary_ctxt_switches:\t13""").encode()) else: return orig_open(name, *args, **kwargs) From 46723905330c4c37001e8c3a5cec22132eeded61 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 1 Aug 2017 16:39:59 +0200 Subject: [PATCH 770/922] fix TypeError --- psutil/_pswindows.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index aedb79306..8903a3072 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -327,8 +327,10 @@ def net_connections(kind, _pid=-1): ret = set() for item in rawlist: fd, fam, type, laddr, raddr, status, pid = item - laddr = _common.addr(*laddr) - raddr = _common.addr(*raddr) + if laddr: + laddr = _common.addr(*laddr) + if raddr: + raddr = _common.addr(*raddr) status = TCP_STATUSES[status] fam = sockfam_to_enum(fam) type = socktype_to_enum(type) From 5ba055a8e514698058589d3b615d408767a6e330 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 1 Aug 2017 16:45:25 +0200 Subject: [PATCH 771/922] #928: fix possible TypeError --- psutil/_psbsd.py | 12 ++++++++---- psutil/_pssunos.py | 6 ++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index fe55f92f2..ba2414cd4 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -719,8 +719,10 @@ def connections(self, kind='inet'): except KeyError: status = TCP_STATUSES[cext.PSUTIL_CONN_NONE] if fam in (AF_INET, AF_INET6): - laddr = _common.addr(*laddr) - raddr = _common.addr(*raddr) + if laddr: + laddr = _common.addr(*laddr) + if raddr: + raddr = _common.addr(*raddr) fam = sockfam_to_enum(fam) type = socktype_to_enum(type) nt = _common.pconn(fd, fam, type, laddr, raddr, status) @@ -737,8 +739,10 @@ def connections(self, kind='inet'): for item in rawlist: fd, fam, type, laddr, raddr, status = item if fam in (AF_INET, AF_INET6): - laddr = _common.addr(*laddr) - raddr = _common.addr(*raddr) + if laddr: + laddr = _common.addr(*laddr) + if raddr: + raddr = _common.addr(*raddr) fam = sockfam_to_enum(fam) type = socktype_to_enum(type) status = TCP_STATUSES[status] diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 9931d885f..06e8bbbaa 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -266,8 +266,10 @@ def net_connections(kind, _pid=-1): if type_ not in types: continue if fam in (AF_INET, AF_INET6): - laddr = _common.addr(*laddr) - raddr = _common.addr(*raddr) + if laddr: + laddr = _common.addr(*laddr) + if raddr: + raddr = _common.addr(*raddr) status = TCP_STATUSES[status] fam = sockfam_to_enum(fam) type_ = socktype_to_enum(type_) From 484fc3d4efe9be6f626a6e94db17d8fa5be48b66 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 1 Sep 2017 17:56:46 +0800 Subject: [PATCH 772/922] update HISTORY --- HISTORY.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index aa9628b2b..b4fed170d 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,6 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -*XXXX-XX-XX* +*2017-09-01* 5.3.0 ===== @@ -65,7 +65,7 @@ - 1063_: [NetBSD] net_connections() may list incorrect sockets. - 1064_: [NetBSD] swap_memory() may segfault in case of error. - 1065_: [OpenBSD] Process.cmdline() may raise SystemError. -- 1067_: [NetBSD] Process.cmdline() leaks memory if proces has terminated. +- 1067_: [NetBSD] Process.cmdline() leaks memory if process has terminated. - 1069_: [FreeBSD] Process.cpu_num() may return 255 for certain kernel processes. - 1071_: [Linux] cpu_freq() may raise IOError on old RedHat distros. From fe0799f98e04b980c3f9aee0dd577567eb932e0b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 1 Sep 2017 18:27:38 +0800 Subject: [PATCH 773/922] pre-release --- docs/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index ef597f28d..cde425ed5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2556,6 +2556,10 @@ take a look at the Timeline ======== +- 2017-09-01: + `5.3.0 `__ - + `what's new `__ - + `diff `__ - 2017-04-10: `5.2.2 `__ - `what's new `__ - From a893f767794cd2fab75a342861e841831e8402f8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 5 Sep 2017 15:34:54 +0800 Subject: [PATCH 774/922] update doc --- HISTORY.rst | 8 ++++---- README.rst | 4 ++-- setup.py | 7 ++----- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b4fed170d..3fe6858b2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -101,10 +101,10 @@ - 1040_: the following Windows APIs on Python 2 now return a string instead of unicode: - Process.memory_maps().path - - WindosService.bin_path() - - WindosService.description() - - WindosService.display_name() - - WindosService.username() + - WindowsService.bin_path() + - WindowsService.description() + - WindowsService.display_name() + - WindowsService.username() *2017-04-10* diff --git a/README.rst b/README.rst index 11898c47f..57dafe05b 100644 --- a/README.rst +++ b/README.rst @@ -74,7 +74,7 @@ Projects using psutil ===================== At the time of writing there are over -`5400 open source projects `__ +`6000 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: @@ -83,7 +83,7 @@ Here's some I find particularly interesting: - https://github.com/google/grr - https://github.com/Jahaja/psdash - https://github.com/ajenti/ajenti - +- https://github.com/home-assistant/home-assistant/ ======== Portings diff --git a/setup.py b/setup.py index a0ce34d63..5f2683497 100755 --- a/setup.py +++ b/setup.py @@ -4,10 +4,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -"""psutil is a cross-platform library for retrieving information on -running processes and system utilization (CPU, memory, disks, network) -in Python. -""" +"""Cross-platform lib for process and system monitoring in Python.""" import contextlib import io @@ -267,7 +264,7 @@ def main(): kwargs = dict( name='psutil', version=VERSION, - description=__doc__ .replace('\n', '').strip() if __doc__ else '', + description=__doc__ .replace('\n', ' ').strip() if __doc__ else '', long_description=get_description(), keywords=[ 'ps', 'top', 'kill', 'free', 'lsof', 'netstat', 'nice', 'tty', From 0abd8c687cbe9419668bb553e2d154e2f7fa62f8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 5 Sep 2017 15:48:01 +0800 Subject: [PATCH 775/922] update doc --- docs/index.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index cde425ed5..f9808d6c3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -155,12 +155,14 @@ CPU Return the number of logical CPUs in the system (same as `os.cpu_count() `__ - in Python 3.4). - This number is not equivalent to the number of CPUs the current process can - use. The number of usable CPUs can be obtained with + in Python 3.4) or ``None`` if undetermined. + This number may not be equivalent to the number of CPUs the current process + can actually use in case process CPU affinity has been changed or Linux + cgroups are being used. + The number of usable CPUs can be obtained with ``len(psutil.Process().cpu_affinity())``. If *logical* is ``False`` return the number of physical cores only (hyper - thread CPUs are excluded). Return ``None`` if undetermined. + thread CPUs are excluded). On OpenBSD and NetBSD ``psutil.cpu_count(logical=False)`` always return ``None``. Example on a system having 2 physical hyper-thread CPU cores: @@ -170,6 +172,11 @@ CPU >>> psutil.cpu_count(logical=False) 2 + Example returning the number of CPUs usable by the current process: + + >>> len(psutil.Process().cpu_affinity()) + 1 + .. function:: cpu_stats() Return various CPU statistics as a named tuple: From 274d6f63dbab0603d4cc714fa0ea4e5f7f5de883 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 7 Sep 2017 16:02:13 +0800 Subject: [PATCH 776/922] migration towards RTD doc theme --- docs/Makefile | 57 +- docs/_static/css/custom.css | 518 ++++++++++++++++++ docs/_template/globaltoc.html | 12 - docs/_template/indexcontent.html | 4 - docs/_template/indexsidebar.html | 8 - docs/_template/page.html | 66 --- docs/_themes/pydoctheme/static/pydoctheme.css | 197 ------- docs/_themes/pydoctheme/theme.conf | 23 - docs/conf.py | 235 ++++++-- 9 files changed, 754 insertions(+), 366 deletions(-) create mode 100644 docs/_static/css/custom.css delete mode 100644 docs/_template/globaltoc.html delete mode 100644 docs/_template/indexcontent.html delete mode 100644 docs/_template/indexsidebar.html delete mode 100644 docs/_template/page.html delete mode 100644 docs/_themes/pydoctheme/static/pydoctheme.css delete mode 100644 docs/_themes/pydoctheme/theme.conf diff --git a/docs/Makefile b/docs/Makefile index a69fc329e..0c4bdf48a 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -15,8 +15,7 @@ ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - +.PHONY: help help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @@ -26,8 +25,10 @@ help: @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" @echo " devhelp to make HTML files and a Devhelp project" @echo " epub to make an epub" + @echo " epub3 to make an epub3" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " latexpdf to make LaTeX files and run them through pdflatex" @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" @@ -41,41 +42,51 @@ help: @echo " pseudoxml to make pseudoxml-XML files for display purposes" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + @echo " dummy to check syntax errors of document sources" +.PHONY: clean clean: rm -rf $(BUILDDIR) +.PHONY: html html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." +.PHONY: dirhtml dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." +.PHONY: singlehtml singlehtml: $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." +.PHONY: pickle pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." +.PHONY: json json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." +.PHONY: htmlhelp htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." +.PHONY: qthelp qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @@ -85,6 +96,16 @@ qthelp: @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/psutil.qhc" +.PHONY: applehelp +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +.PHONY: devhelp devhelp: $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp @echo @@ -94,11 +115,19 @@ devhelp: @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/psutil" @echo "# devhelp" +.PHONY: epub epub: $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." +.PHONY: epub3 +epub3: + $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 + @echo + @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." + +.PHONY: latex latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @@ -106,28 +135,33 @@ latex: @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." +.PHONY: latexpdf latexpdf: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." +.PHONY: latexpdfja latexpdfja: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." +.PHONY: text text: $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." +.PHONY: man man: $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." +.PHONY: texinfo texinfo: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo @@ -135,39 +169,58 @@ texinfo: @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." +.PHONY: info info: $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." +.PHONY: gettext gettext: $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." +.PHONY: changes changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." +.PHONY: linkcheck linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." +.PHONY: doctest doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." +.PHONY: coverage +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +.PHONY: xml xml: $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." +.PHONY: pseudoxml pseudoxml: $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +.PHONY: dummy +dummy: + $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy + @echo + @echo "Build finished. Dummy builder generates no files." diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css new file mode 100644 index 000000000..b76f442af --- /dev/null +++ b/docs/_static/css/custom.css @@ -0,0 +1,518 @@ +.wy-nav-content { + max-width: 100% !important; + padding: 15px !important; +} + +.rst-content dl:not(.docutils) { + margin: 0px 0px 0px 0px; +} + +.data dd { + margin-bottom: 0px !important; +} + +.data .descname { + border-right:10px !important; +} + +.local-toc li ul li{ + padding-left: 20px !important; +} + +.function .descclassname { + font-weight: normal !important; +} + +.class .descclassname { + font-weight: normal !important; +} + +.admonition.warning { + padding-top: 2px !important; + padding-bottom: 2px !important; +} + +.admonition.note { + padding-top: 2px !important; + padding-bottom: 2px !important; +} + +.rst-content dl:not(.docutils) dt { + color: #555; +} + +.sig-paren { + padding-left: 2px; + padding-right: 2px; +} + +h1, h2, h3 { + background: #eee; + padding: 5px; + border-bottom: 1px solid #ccc; +} + +h1 { + font-size: 35px; +} + +.admonition.warning { + padding-top: 5px !important; + padding-bottom: 5px !important; +} + +.admonition.warning p { + margin-bottom: 5px !important; +} + +.admonition.note { + padding-top: 5px !important; + padding-bottom: 5px !important; +} + +.admonition.note p { + margin-bottom: 5px !important; + backround-color: rgb(238, 255, 204) !important; +} + +.codeblock div[class^='highlight'], pre.literal-block div[class^='highlight'], .rst-content .literal-block div[class^='highlight'], div[class^='highlight'] div[class^='highlight'] { + background-color: #eeffcc !important; +} + +.highlight .hll { + background-color: #ffffcc +} + +.highlight { + background: #eeffcc; +} + +.highlight-default, .highlight-python { + border-radius: 3px !important; + border: 1px solid #ac9 !important; +} + +.highlight .c { + color: #408090; + font-style: italic +} + +.wy-side-nav-search { + background-color: grey !important +} + +.highlight { + border-radius: 3px !important; + +} + +div.highlight-default { + margin-bottom: 10px !important; +} + +pre { + padding: 5px !important; +} + +/* ================================================================== */ +/* Warnings and info boxes like python doc */ +/* ================================================================== */ + +div.admonition { + margin-top: 10px !important; + margin-bottom: 10px !important; +} + +div.warning { + background-color: #ffe4e4 !important; + border: 1px solid #f66 !important; + border-radius: 3px !important; +} + +div.note { + background-color: #eee !important; + border: 1px solid #ccc !important; + border-radius: 3px !important; +} + +div.admonition p.admonition-title + p { + display: inline !important; +} + +p.admonition-title { + display: inline !important; + background: none !important; + color: black !important; +} + +p.admonition-title:after { + content: ":" !important; +} + +div.body div.admonition, div.body div.impl-detail { +} + +.fa-exclamation-circle:before, .wy-inline-validate.wy-inline-validate-warning .wy-input-context:before, .wy-inline-validate.wy-inline-validate-info .wy-input-context:before, .rst-content .admonition-title:before { + display: none !important; +} + +.note code { + background: #d6d6d6 !important; +} + +/* ================================================================== */ +/* Syntax highlight like Python doc. +/* ================================================================== */ + +/* Comment */ +.highlight .err { + border: 1px solid #FF0000 +} + +/* Error */ +.highlight .k { + color: #007020; + font-weight: bold +} + +/* Keyword */ +.highlight .o { + color: #666666 +} + +/* Operator */ +.highlight .ch { + color: #408090; + font-style: italic +} + +/* Comment.Hashbang */ +.highlight .cm { + color: #408090; + font-style: italic +} + +/* Comment.Multiline */ +.highlight .cp { + color: #007020 +} + +/* Comment.Preproc */ +.highlight .cpf { + color: #408090; + font-style: italic +} + +/* Comment.PreprocFile */ +.highlight .c1 { + color: #408090; + font-style: italic +} + +/* Comment.Single */ +.highlight .cs { + color: #408090; + background-color: #fff0f0 +} + +/* Comment.Special */ +.highlight .gd { + color: #A00000 +} + +/* Generic.Deleted */ +.highlight .ge { + font-style: italic +} + +/* Generic.Emph */ +.highlight .gr { + color: #FF0000 +} + +/* Generic.Error */ +.highlight .gh { + color: #000080; + font-weight: bold +} + +/* Generic.Heading */ +.highlight .gi { + color: #00A000 +} + +/* Generic.Inserted */ +.highlight .go { + color: #333333 +} + +/* Generic.Output */ +.highlight .gp { + color: #c65d09; + font-weight: bold +} + +/* Generic.Prompt */ +.highlight .gs { + font-weight: bold +} + +/* Generic.Strong */ +.highlight .gu { + color: #800080; + font-weight: bold +} + +/* Generic.Subheading */ +.highlight .gt { + color: #0044DD +} + +/* Generic.Traceback */ +.highlight .kc { + color: #007020; + font-weight: bold +} + +/* Keyword.Constant */ +.highlight .kd { + color: #007020; + font-weight: bold +} + +/* Keyword.Declaration */ +.highlight .kn { + color: #007020; + font-weight: bold +} + +/* Keyword.Namespace */ +.highlight .kp { + color: #007020 +} + +/* Keyword.Pseudo */ +.highlight .kr { + color: #007020; + font-weight: bold +} + +/* Keyword.Reserved */ +.highlight .kt { + color: #902000 +} + +/* Keyword.Type */ +.highlight .m { + color: #208050 +} + +/* Literal.Number */ +.highlight .s { + color: #4070a0 +} + +/* Literal.String */ +.highlight .na { + color: #4070a0 +} + +/* Name.Attribute */ +.highlight .nb { + color: #007020 +} + +/* Name.Builtin */ +.highlight .nc { + color: #0e84b5; + font-weight: bold +} + +/* Name.Class */ +.highlight .no { + color: #60add5 +} + +/* Name.Constant */ +.highlight .nd { + color: #555555; + font-weight: bold +} + +/* Name.Decorator */ +.highlight .ni { + color: #d55537; + font-weight: bold +} + +/* Name.Entity */ +.highlight .ne { + color: #007020 +} + +/* Name.Exception */ +.highlight .nf { + color: #06287e +} + +/* Name.Function */ +.highlight .nl { + color: #002070; + font-weight: bold +} + +/* Name.Label */ +.highlight .nn { + color: #0e84b5; + font-weight: bold +} + +/* Name.Namespace */ +.highlight .nt { + color: #062873; + font-weight: bold +} + +/* Name.Tag */ +.highlight .nv { + color: #bb60d5 +} + +/* Name.Variable */ +.highlight .ow { + color: #007020; + font-weight: bold +} + +/* Operator.Word */ +.highlight .w { + color: #bbbbbb +} + +/* Text.Whitespace */ +.highlight .mb { + color: #208050 +} + +/* Literal.Number.Bin */ +.highlight .mf { + color: #208050 +} + +/* Literal.Number.Float */ +.highlight .mh { + color: #208050 +} + +/* Literal.Number.Hex */ +.highlight .mi { + color: #208050 +} + +/* Literal.Number.Integer */ +.highlight .mo { + color: #208050 +} + +/* Literal.Number.Oct */ +.highlight .sa { + color: #4070a0 +} + +/* Literal.String.Affix */ +.highlight .sb { + color: #4070a0 +} + +/* Literal.String.Backtick */ +.highlight .sc { + color: #4070a0 +} + +/* Literal.String.Char */ +.highlight .dl { + color: #4070a0 +} + +/* Literal.String.Delimiter */ +.highlight .sd { + color: #4070a0; + font-style: italic +} + +/* Literal.String.Doc */ +.highlight .s2 { + color: #4070a0 +} + +/* Literal.String.Double */ +.highlight .se { + color: #4070a0; + font-weight: bold +} + +/* Literal.String.Escape */ +.highlight .sh { + color: #4070a0 +} + +/* Literal.String.Heredoc */ +.highlight .si { + color: #70a0d0; + font-style: italic +} + +/* Literal.String.Interpol */ +.highlight .sx { + color: #c65d09 +} + +/* Literal.String.Other */ +.highlight .sr { + color: #235388 +} + +/* Literal.String.Regex */ +.highlight .s1 { + color: #4070a0 +} + +/* Literal.String.Single */ +.highlight .ss { + color: #517918 +} + +/* Literal.String.Symbol */ +.highlight .bp { + color: #007020 +} + +/* Name.Builtin.Pseudo */ +.highlight .fm { + color: #06287e +} + +/* Name.Function.Magic */ +.highlight .vc { + color: #bb60d5 +} + +/* Name.Variable.Class */ +.highlight .vg { + color: #bb60d5 +} + +/* Name.Variable.Global */ +.highlight .vi { + color: #bb60d5 +} + +/* Name.Variable.Instance */ +.highlight .vm { + color: #bb60d5 +} + +/* Name.Variable.Magic */ +.highlight .il { + color: #208050 +} diff --git a/docs/_template/globaltoc.html b/docs/_template/globaltoc.html deleted file mode 100644 index f5fbb406c..000000000 --- a/docs/_template/globaltoc.html +++ /dev/null @@ -1,12 +0,0 @@ -{# - basic/globaltoc.html - ~~~~~~~~~~~~~~~~~~~~ - - Sphinx sidebar template: global table of contents. - - :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS. - :license: BSD, see LICENSE for details. -#} -

{{ _('Manual') }}

-{{ toctree() }} -Back to Welcome diff --git a/docs/_template/indexcontent.html b/docs/_template/indexcontent.html deleted file mode 100644 index dd5e7249a..000000000 --- a/docs/_template/indexcontent.html +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "defindex.html" %} -{% block tables %} - -{% endblock %} diff --git a/docs/_template/indexsidebar.html b/docs/_template/indexsidebar.html deleted file mode 100644 index 903675d10..000000000 --- a/docs/_template/indexsidebar.html +++ /dev/null @@ -1,8 +0,0 @@ -

Useful links

- diff --git a/docs/_template/page.html b/docs/_template/page.html deleted file mode 100644 index 04b47b415..000000000 --- a/docs/_template/page.html +++ /dev/null @@ -1,66 +0,0 @@ -{% extends "!page.html" %} -{% block extrahead %} -{{ super() }} -{% if not embedded %}{% endif %} - - -{% endblock %} - -{% block rootrellink %} -
  • Project Homepage{{ reldelim1 }}
    • -
  • {{ shorttitle }}{{ reldelim1 }}
    • -{% endblock %} - - -{% block footer %} -
    - © Copyright {{ copyright|e }}. -
    - Last updated on {{ last_updated|e }}. -
    - Created using Sphinx {{ sphinx_version|e }}. -
    -{% endblock %} \ No newline at end of file diff --git a/docs/_themes/pydoctheme/static/pydoctheme.css b/docs/_themes/pydoctheme/static/pydoctheme.css deleted file mode 100644 index c6f19ab79..000000000 --- a/docs/_themes/pydoctheme/static/pydoctheme.css +++ /dev/null @@ -1,197 +0,0 @@ -@import url("default.css"); - -body { - background-color: white; - margin-left: 1em; - margin-right: 1em; -} - -div.related { - margin-bottom: 1.2em; - padding: 0.5em 0; - border-top: 1px solid #ccc; - margin-top: 0.5em; -} - -div.related a:hover { - color: #0095C4; -} - -div.related:first-child { - border-top: 0; - padding-top: 0; - border-bottom: 1px solid #ccc; -} - -div.sphinxsidebar { - background-color: #eeeeee; - border-radius: 5px; - line-height: 130%; - font-size: smaller; -} - -div.sphinxsidebar h3, div.sphinxsidebar h4 { - margin-top: 1.5em; -} - -div.sphinxsidebarwrapper > h3:first-child { - margin-top: 0.2em; -} - -div.sphinxsidebarwrapper > ul > li > ul > li { - margin-bottom: 0.4em; -} - -div.sphinxsidebar a:hover { - color: #0095C4; -} - -div.sphinxsidebar input { - font-family: 'Lucida Grande','Lucida Sans','DejaVu Sans',Arial,sans-serif; - border: 1px solid #999999; - font-size: smaller; - border-radius: 3px; -} - -div.sphinxsidebar input[type=text] { - max-width: 150px; -} - -div.body { - padding: 0 0 0 1.2em; -} - -div.body p { - line-height: 140%; -} - -div.body h1, div.body h2, div.body h3, div.body h4, div.body h5, div.body h6 { - margin: 0; - border: 0; - padding: 0.3em 0; -} - -div.body hr { - border: 0; - background-color: #ccc; - height: 1px; -} - -div.body pre { - border-radius: 3px; - border: 1px solid #ac9; -} - -div.body div.admonition, div.body div.impl-detail { - border-radius: 3px; -} - -div.body div.impl-detail > p { - margin: 0; -} - -div.body div.seealso { - border: 1px solid #dddd66; -} - -div.body a { - color: #00608f; -} - -div.body a:visited { - color: #30306f; -} - -div.body a:hover { - color: #00B0E4; -} - -tt, pre { - font-family: monospace, sans-serif; - font-size: 96.5%; -} - -div.body tt { - border-radius: 3px; -} - -div.body tt.descname { - font-size: 120%; -} - -div.body tt.xref, div.body a tt { - font-weight: normal; -} - -p.deprecated { - border-radius: 3px; -} - -table.docutils { - border: 1px solid #ddd; - min-width: 20%; - border-radius: 3px; - margin-top: 10px; - margin-bottom: 10px; -} - -table.docutils td, table.docutils th { - border: 1px solid #ddd !important; - border-radius: 3px; -} - -table p, table li { - text-align: left !important; -} - -table.docutils th { - background-color: #eee; - padding: 0.3em 0.5em; -} - -table.docutils td { - background-color: white; - padding: 0.3em 0.5em; -} - -table.footnote, table.footnote td { - border: 0 !important; -} - -div.footer { - line-height: 150%; - margin-top: -2em; - text-align: right; - width: auto; - margin-right: 10px; -} - -div.footer a:hover { - color: #0095C4; -} - -div.body h1, -div.body h2, -div.body h3 { - background-color: #EAEAEA; - border-bottom: 1px solid #CCC; - padding-top: 2px; - padding-bottom: 2px; - padding-left: 5px; - margin-top: 5px; - margin-bottom: 5px; -} - -div.body h2 { - padding-left:10px; -} - -.data { - margin-top: 4px !important; - margin-bottom: 4px !important; -} - -.data dd { - margin-top: 0px !important; - margin-bottom: 0px !important; -} diff --git a/docs/_themes/pydoctheme/theme.conf b/docs/_themes/pydoctheme/theme.conf deleted file mode 100644 index 95b97e536..000000000 --- a/docs/_themes/pydoctheme/theme.conf +++ /dev/null @@ -1,23 +0,0 @@ -[theme] -inherit = default -stylesheet = pydoctheme.css -pygments_style = sphinx - -[options] -bodyfont = 'Lucida Grande', 'Lucida Sans', 'DejaVu Sans', Arial, sans-serif -headfont = 'Lucida Grande', 'Lucida Sans', 'DejaVu Sans', Arial, sans-serif -footerbgcolor = white -footertextcolor = #555555 -relbarbgcolor = white -relbartextcolor = #666666 -relbarlinkcolor = #444444 -sidebarbgcolor = white -sidebartextcolor = #444444 -sidebarlinkcolor = #444444 -bgcolor = white -textcolor = #222222 -linkcolor = #0090c0 -visitedlinkcolor = #00608f -headtextcolor = #1a1a1a -headbgcolor = white -headlinkcolor = #aaaaaa diff --git a/docs/conf.py b/docs/conf.py index f0a206db7..df825cbd5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # psutil documentation build configuration file, created by -# sphinx-quickstart. +# sphinx-quickstart on Wed Oct 19 21:54:30 2016. # # This file is execfile()d with the current directory set to its # containing dir. @@ -12,12 +12,22 @@ # All configuration values have a default; values that are commented out # serve to show the default. +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + import datetime import os PROJECT_NAME = "psutil" -AUTHOR = "Giampaolo Rodola'" +AUTHOR = u"Giampaolo Rodola" THIS_YEAR = str(datetime.datetime.now().year) HERE = os.path.abspath(os.path.dirname(__file__)) @@ -39,7 +49,8 @@ def get_version(): VERSION = get_version() # If your documentation needs a minimal Sphinx version, state it here. -needs_sphinx = '1.0' +# +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom @@ -51,12 +62,16 @@ def get_version(): 'sphinx.ext.intersphinx'] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_template'] +templates_path = ['_templates'] -# The suffix of source filenames. +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] source_suffix = '.rst' # The encoding of source files. +# # source_encoding = 'utf-8-sig' # The master toctree document. @@ -65,6 +80,7 @@ def get_version(): # General information about the project. project = PROJECT_NAME copyright = '2009-%s, %s' % (THIS_YEAR, AUTHOR) +author = AUTHOR # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -72,35 +88,47 @@ def get_version(): # # The short X.Y version. version = VERSION +# The full version, including alpha/beta/rc tags. +release = VERSION # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -# language = None +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: +# # today = '' +# # Else, today_fmt is used as the format for a strftime call. +# # today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] # The reST default role (used for this markup: `text`) to use for all # documents. +# # default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -add_function_parentheses = True +# +# add_function_parentheses = True + # If true, the current module name will be prepended to all description # unit titles (such as .. function::). +# # add_module_names = True -autodoc_docstring_signature = True - # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. +# # show_authors = False # The name of the Pygments (syntax highlighting) style to use. @@ -109,141 +137,240 @@ def get_version(): # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False -# -- Options for HTML output ------------------------------------------------- + +# -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -html_theme = 'pydoctheme' -html_theme_options = {'collapsiblesidebar': True} +# +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -html_theme_path = ["_themes"] +# html_theme_path = [] -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -html_title = "{project} {version} documentation".format(**locals()) +# The name for this set of Sphinx documents. +# " v documentation" by default. +# +# html_title = u'psutil v1.0' # A shorter title for the navigation bar. Default is the same as html_title. +# # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -# html_logo = 'logo.png' +# +# html_logo = None + +# The name of an image file (relative to this directory) to use as a favicon of +# the docs. This file should be a Windows icon file (.ico) being 16x16 or +# 32x32 pixels large. -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -html_favicon = '_static/favicon.ico' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# +# html_extra_path = [] + +# If not None, a 'Last updated on:' timestamp is inserted at every page +# bottom, using the given strftime format. +# The empty string is equivalent to '%b %d, %Y'. +# +# html_last_updated_fmt = None # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -html_use_smartypants = True +# +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -html_sidebars = { - 'index': 'indexsidebar.html', - '**': ['globaltoc.html', - 'relations.html', - 'sourcelink.html', - 'searchbox.html'] -} +# +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -# html_additional_pages = { -# 'index': 'indexcontent.html', -# } +# +# html_additional_pages = {} # If false, no module index is generated. -html_domain_indices = False +# +# html_domain_indices = True # If false, no index is generated. -html_use_index = True +# +# html_use_index = True # If true, the index is split into individual pages for each letter. +# # html_split_index = False # If true, links to the reST sources are added to the pages. +# # html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# # html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# # html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. +# # html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = None +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' +# +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# 'ja' uses this config value. +# 'zh' user can custom change `jieba` dictionary path. +# +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# +# html_search_scorer = 'scorer.js' + # Output file base name for HTML help builder. htmlhelp_basename = '%s-doc' % PROJECT_NAME -# -- Options for LaTeX output ------------------------------------------------ +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', -# The paper size ('letter' or 'a4'). -# latex_paper_size = 'letter' + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', -# The font size ('10pt', '11pt' or '12pt'). -# latex_font_size = '10pt' + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} # Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass -# [howto/manual]). +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', '%s.tex' % PROJECT_NAME, - '%s documentation' % PROJECT_NAME, AUTHOR), + (master_doc, 'psutil.tex', u'psutil Documentation', + AUTHOR, 'manual'), ] -# The name of an image file (relative to this directory) to place at -# the top of the title page. +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. +# # latex_use_parts = False # If true, show page references after internal links. +# # latex_show_pagerefs = False # If true, show URL addresses after external links. +# # latex_show_urls = False -# Additional stuff for the LaTeX preamble. -# latex_preamble = '' - # Documents to append as an appendix to all manuals. +# # latex_appendices = [] +# It false, will not define \strong, \code, itleref, \crossref ... but only +# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added +# packages. +# +# latex_keep_old_macro_names = True + # If false, no module index is generated. +# # latex_domain_indices = True -# -- Options for manual page output ------------------------------------------ +# -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', PROJECT_NAME, '%s documentation' % PROJECT_NAME, [AUTHOR], 1) + (master_doc, 'psutil', u'psutil Documentation', + [author], 1) ] # If true, show URL addresses after external links. +# # man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'psutil', u'psutil Documentation', + author, 'psutil', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# +# texinfo_appendices = [] + +# If false, no module index is generated. +# +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# +# texinfo_no_detailmenu = False + + +html_context = { + 'css_files': [ + 'https://media.readthedocs.org/css/sphinx_rtd_theme.css', + 'https://media.readthedocs.org/css/readthedocs-doc-embed.css', + '_static/css/custom.css', + ], +} From 6b7f8fbad032e88b96e700e0e27b214a1e7a0b95 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 7 Sep 2017 16:11:39 +0800 Subject: [PATCH 777/922] update doc url --- DEVGUIDE.rst | 2 +- README.rst | 6 +++--- docs/README | 4 ++-- psutil/tests/test_unicode.py | 2 +- scripts/internal/print_announce.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index 6a0f08fc8..da5c9d799 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -164,7 +164,7 @@ Documentation - it uses `RsT syntax `_ and it's built with `sphinx `_. - doc can be built with ``make setup-dev-env; cd docs; make html``. -- public doc is hosted on http://pythonhosted.org/psutil/ +- public doc is hosted on http://psutil.readthedocs.io/ ======================= Releasing a new version diff --git a/README.rst b/README.rst index 57dafe05b..acf400dee 100644 --- a/README.rst +++ b/README.rst @@ -28,7 +28,7 @@ Quick links - `Home page `_ - `Install `_ -- `Documentation `_ +- `Documentation `_ - `Download `_ - `Forum `_ - `Blog `_ @@ -67,7 +67,7 @@ Example applications +------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------+ Also see `scripts directory `__ -and `doc recipes `__. +and `doc recipes `__. ===================== Projects using psutil @@ -438,7 +438,7 @@ Windows services Other samples ============= -See `doc recipes `__. +See `doc recipes `__. ====== Author diff --git a/docs/README b/docs/README index 3aaea8a5b..8ceb5f21f 100644 --- a/docs/README +++ b/docs/README @@ -3,7 +3,7 @@ About This directory contains the reStructuredText (reST) sources to the psutil documentation. You don't need to build them yourself, prebuilt versions are -available at https://pythonhosted.org/psutil/. +available at http://psutil.readthedocs.io. In case you want, you need to install sphinx first: $ pip install sphinx @@ -12,4 +12,4 @@ Then run: $ make html -You'll then have an HTML version of the doc at _build/html/index.html. \ No newline at end of file +You'll then have an HTML version of the doc at _build/html/index.html. diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 9b99fdf98..2aa752216 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -49,7 +49,7 @@ For a detailed explanation of how psutil handles unicode see: - https://github.com/giampaolo/psutil/issues/1040 -- https://pythonhosted.org/psutil/#unicode +- http://psutil.readthedocs.io/#unicode """ import os diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py index e47911c24..9d2cbb62c 100755 --- a/scripts/internal/print_announce.py +++ b/scripts/internal/print_announce.py @@ -19,7 +19,7 @@ PRJ_NAME = 'psutil' PRJ_URL_HOME = 'https://github.com/giampaolo/psutil' -PRJ_URL_DOC = 'http://pythonhosted.org/psutil' +PRJ_URL_DOC = 'http://psutil.readthedocs.io' PRJ_URL_DOWNLOAD = 'https://pypi.python.org/pypi/psutil' PRJ_URL_WHATSNEW = \ 'https://github.com/giampaolo/psutil/blob/master/HISTORY.rst' From 5c0dfc851950b820aa13ef66ce3ab25424aedcec Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 7 Sep 2017 17:03:46 +0800 Subject: [PATCH 778/922] add doc badge --- README.rst | 4 ++++ docs/index.rst | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index acf400dee..bd77387b9 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,10 @@ :target: https://coveralls.io/github/giampaolo/psutil?branch=master :alt: Test coverage (coverall.io) +.. image:: https://readthedocs.org/projects/psutil/badge/?version=latest + :target: http://psutil.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + .. image:: https://img.shields.io/pypi/v/psutil.svg?label=pypi :target: https://pypi.python.org/pypi/psutil/ :alt: Latest version diff --git a/docs/index.rst b/docs/index.rst index f9808d6c3..e7081f7bd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -837,9 +837,9 @@ Functions Sorting order in which processes are returned is based on their PID. *attrs* and *ad_value* have the same meaning as in :meth:`Process.as_dict()`. - If *attrs* is specified :meth:`Process.as_dict()` is called and the resulting - dict is stored as a ``info`` attribute which is attached to the returned - :class:`Process` instance. + If *attrs* is specified :meth:`Process.as_dict()` is called interanally and + the resulting dict is stored as a ``info`` attribute which is attached to the + returned :class:`Process` instances. If *attrs* is an empty list it will retrieve all process info (slow). Example usage:: From e950259b745317ceac9870d2acfcbfc7a946cbcb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 8 Sep 2017 12:24:49 +0800 Subject: [PATCH 779/922] update doc, bump up ver --- HISTORY.rst | 9 +++++++++ docs/index.rst | 26 +++++++++++++------------- psutil/__init__.py | 2 +- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3fe6858b2..7fe754901 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,14 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +XXXX-XX-XX + +5.3.1 +===== + +**Enhancements** + +- 1124_: documentation moved to http://psutil.readthedocs.io + *2017-09-01* 5.3.0 diff --git a/docs/index.rst b/docs/index.rst index e7081f7bd..da730a7b0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -694,7 +694,7 @@ Sensors .. warning:: - This API is experimental. Backward incompatible changes may occur if + this API is experimental. Backward incompatible changes may occur if deemed necessary. .. function:: sensors_fans() @@ -718,7 +718,7 @@ Sensors .. warning:: - This API is experimental. Backward incompatible changes may occur if + this API is experimental. Backward incompatible changes may occur if deemed necessary. .. function:: sensors_battery() @@ -760,7 +760,7 @@ Sensors .. warning:: - This API is experimental. Backward incompatible changes may occur if + this API is experimental. Backward incompatible changes may occur if deemed necessary. Other system info @@ -993,14 +993,13 @@ Process class at the same time, make sure to use either :meth:`as_dict` or :meth:`oneshot` context manager. - .. warning:: + .. note:: - the way this class is bound to a process is via its **PID**. - That means that if the :class:`Process` instance is old enough and - the PID has been reused in the meantime you might end up interacting - with another process. + the way this class is bound to a process is uniquely via its **PID**. + That means that if the process terminates and the OS reuses its PID you may + end up interacting with another process. The only exceptions for which process identity is preemptively checked - (via PID + creation time) and guaranteed are for + (via PID + creation time) is for the following methods: :meth:`nice` (set), :meth:`ionice` (set), :meth:`cpu_affinity` (set), @@ -1010,12 +1009,13 @@ Process class :meth:`suspend` :meth:`resume`, :meth:`send_signal`, - :meth:`terminate`, and - :meth:`kill` - methods. + :meth:`terminate` + :meth:`kill`. To prevent this problem for all other methods you can use - :meth:`is_running()` before querying the process or use + :meth:`is_running()` before querying the process or :func:`process_iter()` in case you're iterating over all processes. + It must be noted though that unless you deal with very "old" (inactive) + :class:`Process` instances this will hardly represent a problem. .. method:: oneshot() diff --git a/psutil/__init__.py b/psutil/__init__.py index 50ae2a337..70f9bc060 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -203,7 +203,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.3.0" +__version__ = "5.3.1" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From de5f2f80fe54194663eef3c9cdd709efd91576ec Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 8 Sep 2017 12:47:16 +0800 Subject: [PATCH 780/922] re: #1120 / PEP527: no longer provide .exe files for Windows --- HISTORY.rst | 5 +++++ appveyor.yml | 1 - scripts/internal/download_exes.py | 4 ++-- scripts/internal/winmake.py | 7 ------- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7fe754901..6379930e7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,11 @@ XXXX-XX-XX - 1124_: documentation moved to http://psutil.readthedocs.io +**Compatibility notes** + +- 1120_: .exe files for Windows are no longer uploaded on PYPI as per PEP-527; + only wheels are provided. + *2017-09-01* 5.3.0 diff --git a/appveyor.yml b/appveyor.yml index d46717845..b18677242 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -95,7 +95,6 @@ test_script: after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" - - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wininst" artifacts: - path: dist\* diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_exes.py index 9688919bf..5c2d70acd 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_exes.py @@ -170,8 +170,8 @@ def main(options): completed += 1 print("downloaded %-45s %s" % ( local_fname, bytes2human(os.path.getsize(local_fname)))) - # 2 exes (32 and 64 bit) and 2 wheels (32 and 64 bit) for each ver. - expected = len(PY_VERSIONS) * 4 + # 2 wheels (32 and 64 bit) per supported python version + expected = len(PY_VERSIONS) * 2 if expected != completed: return exit("expected %s files, got %s" % (expected, completed)) if exc: diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 138a0b0c7..c2ee2ab04 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -204,13 +204,6 @@ def build(): sh('%s -c "import psutil"' % PYTHON) -@cmd -def build_exe(): - """Create exe file.""" - build() - sh("%s setup.py bdist_wininst" % PYTHON) - - @cmd def build_wheel(): """Create wheel file.""" From 9e4b231df96138e7dc0425978f64b52f194c3b3e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 8 Sep 2017 15:29:33 +0800 Subject: [PATCH 781/922] generate-manifest on pre-release --- MANIFEST.in | 7 +------ Makefile | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 89c422160..9f84c4c20 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,14 +13,9 @@ include README.rst include docs/Makefile include docs/README include docs/_static/copybutton.js +include docs/_static/css/custom.css include docs/_static/favicon.ico include docs/_static/sidebar.js -include docs/_template/globaltoc.html -include docs/_template/indexcontent.html -include docs/_template/indexsidebar.html -include docs/_template/page.html -include docs/_themes/pydoctheme/static/pydoctheme.css -include docs/_themes/pydoctheme/theme.conf include docs/conf.py include docs/index.rst include docs/make.bat diff --git a/Makefile b/Makefile index f47d262cc..62b79c8be 100644 --- a/Makefile +++ b/Makefile @@ -237,6 +237,7 @@ win-upload-exes: # All the necessary steps before making a release. pre-release: + ${MAKE} generate-manifest @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes') if out else sys.exit(0);" ${MAKE} sdist ${MAKE} install From c15fb0d09f6dd89238fc341e15b839815d3bc078 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 8 Sep 2017 11:04:27 +0000 Subject: [PATCH 782/922] fix #1105: can't compile psutil on freebsd 12 --- HISTORY.rst | 4 ++++ psutil/arch/freebsd/proc_socks.c | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index 6379930e7..1213db2c3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,10 @@ XXXX-XX-XX - 1124_: documentation moved to http://psutil.readthedocs.io +**Big fixes** + +- 1105_: [FreeBSD] psutil does not compile on FreeBSD 12. + **Compatibility notes** - 1120_: .exe files for Windows are no longer uploaded on PYPI as per PEP-527; diff --git a/psutil/arch/freebsd/proc_socks.c b/psutil/arch/freebsd/proc_socks.c index 9b03e0591..c5b19a0de 100644 --- a/psutil/arch/freebsd/proc_socks.c +++ b/psutil/arch/freebsd/proc_socks.c @@ -136,20 +136,36 @@ psutil_search_tcplist(char *buf, struct kinfo_file *kif) { if (kif->kf_sock_domain == AF_INET) { if (!psutil_sockaddr_matches( AF_INET, inp->inp_lport, &inp->inp_laddr, +#if __FreeBSD_version < 1200031 &kif->kf_sa_local)) +#else + &kif->kf_un.kf_sock.kf_sa_local)) +#endif continue; if (!psutil_sockaddr_matches( AF_INET, inp->inp_fport, &inp->inp_faddr, +#if __FreeBSD_version < 1200031 &kif->kf_sa_peer)) +#else + &kif->kf_un.kf_sock.kf_sa_peer)) +#endif continue; } else { if (!psutil_sockaddr_matches( AF_INET6, inp->inp_lport, &inp->in6p_laddr, +#if __FreeBSD_version < 1200031 &kif->kf_sa_local)) +#else + &kif->kf_un.kf_sock.kf_sa_local)) +#endif continue; if (!psutil_sockaddr_matches( AF_INET6, inp->inp_fport, &inp->in6p_faddr, +#if __FreeBSD_version < 1200031 &kif->kf_sa_peer)) +#else + &kif->kf_un.kf_sock.kf_sa_peer)) +#endif continue; } @@ -243,19 +259,35 @@ psutil_proc_connections(PyObject *self, PyObject *args) { inet_ntop( kif->kf_sock_domain, psutil_sockaddr_addr(kif->kf_sock_domain, +#if __FreeBSD_version < 1200031 &kif->kf_sa_local), +#else + &kif->kf_un.kf_sock.kf_sa_local), +#endif lip, sizeof(lip)); inet_ntop( kif->kf_sock_domain, psutil_sockaddr_addr(kif->kf_sock_domain, +#if __FreeBSD_version < 1200031 &kif->kf_sa_peer), +#else + &kif->kf_un.kf_sock.kf_sa_peer), +#endif rip, sizeof(rip)); lport = htons(psutil_sockaddr_port(kif->kf_sock_domain, +#if __FreeBSD_version < 1200031 &kif->kf_sa_local)); +#else + &kif->kf_un.kf_sock.kf_sa_local)); +#endif rport = htons(psutil_sockaddr_port(kif->kf_sock_domain, +#if __FreeBSD_version < 1200031 &kif->kf_sa_peer)); +#else + &kif->kf_un.kf_sock.kf_sa_peer)); +#endif // construct python tuple/list py_laddr = Py_BuildValue("(si)", lip, lport); @@ -287,7 +319,11 @@ psutil_proc_connections(PyObject *self, PyObject *args) { else if (kif->kf_sock_domain == AF_UNIX) { struct sockaddr_un *sun; +#if __FreeBSD_version < 1200031 sun = (struct sockaddr_un *)&kif->kf_sa_local; +#else + sun = (struct sockaddr_un *)&kif->kf_un.kf_sock.kf_sa_local; +#endif snprintf( path, sizeof(path), "%.*s", (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), From 56e82a63a5fa4b7ab77d13b08179f3d035523849 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 8 Sep 2017 20:49:41 +0800 Subject: [PATCH 783/922] fix #1125: [BSD] net_connections() raises TypeError. --- HISTORY.rst | 1 + psutil/_psbsd.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 1213db2c3..17e154a67 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,7 @@ XXXX-XX-XX **Big fixes** - 1105_: [FreeBSD] psutil does not compile on FreeBSD 12. +- 1125_: [BSD] net_connections() raises TypeError. **Compatibility notes** diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index ba2414cd4..6517f2446 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -394,9 +394,12 @@ def net_connections(kind): # have a very short lifetime so maybe the kernel # can't initialize their status? status = TCP_STATUSES[cext.PSUTIL_CONN_NONE] + if fam in (AF_INET, AF_INET6): + if laddr: + laddr = _common.addr(*laddr) + if raddr: + raddr = _common.addr(*raddr) fam = sockfam_to_enum(fam) - laddr = _common.addr(*laddr) - raddr = _common.addr(*raddr) type = socktype_to_enum(type) nt = _common.sconn(fd, fam, type, laddr, raddr, status, pid) ret.add(nt) From efb85df904b693eb8ab540fc3959cfdc439ef396 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 10 Sep 2017 13:01:48 +0800 Subject: [PATCH 784/922] pre release --- HISTORY.rst | 2 +- docs/index.rst | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 17e154a67..3649eafdb 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,6 +1,6 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -XXXX-XX-XX +2017-09-10 5.3.1 ===== diff --git a/docs/index.rst b/docs/index.rst index da730a7b0..80360046a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2563,6 +2563,10 @@ take a look at the Timeline ======== +- 2017-09-10: + `5.3.1 `__ - + `what's new `__ - + `diff `__ - 2017-09-01: `5.3.0 `__ - `what's new `__ - From c34dac19561501a2b37262f139248552e46c338c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 10 Sep 2017 13:26:08 +0800 Subject: [PATCH 785/922] pre release --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 62b79c8be..f47d262cc 100644 --- a/Makefile +++ b/Makefile @@ -237,7 +237,6 @@ win-upload-exes: # All the necessary steps before making a release. pre-release: - ${MAKE} generate-manifest @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes') if out else sys.exit(0);" ${MAKE} sdist ${MAKE} install From b42b9bba805b93b7adda7f983dba11620206d04f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 10 Sep 2017 13:44:13 +0800 Subject: [PATCH 786/922] bump up ver --- Makefile | 8 +++++--- psutil/__init__.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f47d262cc..1ed62c393 100644 --- a/Makefile +++ b/Makefile @@ -237,8 +237,9 @@ win-upload-exes: # All the necessary steps before making a release. pre-release: - @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes') if out else sys.exit(0);" - ${MAKE} sdist + ${MAKE} generate-manifest + git diff MANIFEST.in > /dev/null # ...otherwise 'git diff-index HEAD' will complain + @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else sys.exit(0);" ${MAKE} install @PYTHONWARNINGS=all $(PYTHON) -c \ "from psutil import __version__ as ver; \ @@ -248,12 +249,13 @@ pre-release: assert ver in history, '%r not in HISTORY.rst' % ver; \ assert 'XXXX' not in history, 'XXXX in HISTORY.rst';" ${MAKE} win-download-exes + ${MAKE} sdist # Create a release: creates tar.gz and exes/wheels, uploads them, # upload doc, git tag release. release: ${MAKE} pre-release - PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/* # upload tar.gz, exes, wheels on PYPI + PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/* # upload tar.gz and Windows wheels on PYPI ${MAKE} git-tag-release # Print announce of new release. diff --git a/psutil/__init__.py b/psutil/__init__.py index 70f9bc060..ca9bc2390 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -203,7 +203,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.3.1" +__version__ = "5.3.2" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From c4c6e22ca4dad687d2a8505f9e0207d8ecd64358 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 10 Sep 2017 22:28:10 +0800 Subject: [PATCH 787/922] #1126: temporarily disable failing test --- psutil/tests/test_process.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index dd0c507ec..1f695a064 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -895,6 +895,8 @@ def test_cpu_affinity(self): p.cpu_affinity(set(all_cpus)) p.cpu_affinity(tuple(all_cpus)) + # TODO: temporary, see: https://github.com/MacPython/psutil/issues/1 + @unittest.skipIf(LINUX, "temporary") @unittest.skipIf(not HAS_CPU_AFFINITY, 'not supported') def test_cpu_affinity_errs(self): sproc = get_test_subprocess() From 8aa64713c634a5d31774cac3b224308a0edeb083 Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Mon, 25 Sep 2017 18:53:46 +0300 Subject: [PATCH 788/922] Solaris: fix compilation warnings (#1131) --- psutil/_psutil_sunos.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 12caaec7e..56ec002a8 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -9,13 +9,13 @@ * this in Cython which I later on translated in C. */ -// fix compilation issue on SunOS 5.10, see: -// https://github.com/giampaolo/psutil/issues/421 -// https://github.com/giampaolo/psutil/issues/1077 -// http://us-east.manta.joyent.com/jmc/public/opensolaris/ARChive/PSARC/2010/111/materials/s10ceval.txt -// -// Because LEGACY_MIB_SIZE defined in the same file there is no way to make autoconfiguration =\ -// +/* fix compilation issue on SunOS 5.10, see: + * https://github.com/giampaolo/psutil/issues/421 + * https://github.com/giampaolo/psutil/issues/1077 + * http://us-east.manta.joyent.com/jmc/public/opensolaris/ARChive/PSARC/2010/111/materials/s10ceval.txt + * + * Because LEGACY_MIB_SIZE defined in the same file there is no way to make autoconfiguration =\ +*/ #define NEW_MIB_COMPLIANT 1 #define _STRUCTURED_PROC 1 @@ -126,9 +126,9 @@ psutil_proc_name_and_args(PyObject *self, PyObject *args) { char path[1000]; psinfo_t info; const char *procfs_path; - PyObject *py_name; - PyObject *py_args; - PyObject *py_retlist; + PyObject *py_name = NULL; + PyObject *py_args = NULL; + PyObject *py_retlist = NULL; if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) return NULL; @@ -328,7 +328,7 @@ psutil_proc_cpu_num(PyObject *self, PyObject *args) { return Py_BuildValue("i", proc_num); error: - if (fd != NULL) + if (fd != -1) close(fd); if (ptr != NULL) free(ptr); @@ -832,7 +832,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { pr_addr_sz = p->pr_vaddr + p->pr_size; // perms - sprintf(perms, "%c%c%c%c%c%c", p->pr_mflags & MA_READ ? 'r' : '-', + sprintf(perms, "%c%c%c%c", p->pr_mflags & MA_READ ? 'r' : '-', p->pr_mflags & MA_WRITE ? 'w' : '-', p->pr_mflags & MA_EXEC ? 'x' : '-', p->pr_mflags & MA_SHARED ? 's' : '-'); From b9aeea4eed3bf805012304133962efdcfdb48c53 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 26 Sep 2017 00:36:07 +0800 Subject: [PATCH 789/922] #1131: give CREDITS to @wiggin15 --- CREDITS | 4 ++++ HISTORY.rst | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/CREDITS b/CREDITS index 77bdfcafa..ffca763ee 100644 --- a/CREDITS +++ b/CREDITS @@ -485,3 +485,7 @@ I: 1042, 1079 N: Oleksii Shevchuk W: https://github.com/alxchk I: 1077, 1093, 1091 + +N: Arnon Yaari +W: https://github.com/wiggin15 +I: 1131, 1123, 1130 diff --git a/HISTORY.rst b/HISTORY.rst index 3649eafdb..8096e82db 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +XXXX-XX-XX + +5.3.2 +===== + +*Bug fixes* + +- 1131_: [SunOS] fix compilation warnings. (patch by Arnon Yaari) + + 2017-09-10 5.3.1 From 452949b36c39d22e7702f5e099e18f29ef275ec4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 26 Sep 2017 17:57:14 +0800 Subject: [PATCH 790/922] #1131: merge wiggin15 entries in CREDITS --- CREDITS | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/CREDITS b/CREDITS index ffca763ee..3948189d2 100644 --- a/CREDITS +++ b/CREDITS @@ -363,9 +363,9 @@ N: maozguttman W: https://github.com/maozguttman I: 659 -N: wiggin15 +N: Arnon Yaari (wiggin15) W: https://github.com/wiggin15 -I: 517, 607, 610 +I: 517, 607, 610, 1131, 1123, 1130 N: dasumin W: https://github.com/dasumin @@ -485,7 +485,3 @@ I: 1042, 1079 N: Oleksii Shevchuk W: https://github.com/alxchk I: 1077, 1093, 1091 - -N: Arnon Yaari -W: https://github.com/wiggin15 -I: 1131, 1123, 1130 From 7a111cfe490f9c81722b2d896d87b9fb77228141 Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Tue, 26 Sep 2017 12:58:07 +0300 Subject: [PATCH 791/922] Solaris: Use the correct set/get/end functions for utmp (#1130) boot_time() and users() use utmp (getutxent), but don't call "set" and don't use the correct "end" variant (for utmpx) --- psutil/_psutil_sunos.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 56ec002a8..8f9773424 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -548,6 +548,7 @@ psutil_users(PyObject *self, PyObject *args) { if (py_retlist == NULL) return NULL; + setutxent(); while (NULL != (ut = getutxent())) { if (ut->ut_type == USER_PROCESS) py_user_proc = Py_True; @@ -580,7 +581,7 @@ psutil_users(PyObject *self, PyObject *args) { Py_DECREF(py_hostname); Py_DECREF(py_tuple); } - endutent(); + endutxent(); return py_retlist; @@ -590,8 +591,7 @@ psutil_users(PyObject *self, PyObject *args) { Py_XDECREF(py_hostname); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); - if (ut != NULL) - endutent(); + endutxent(); return NULL; } @@ -1332,20 +1332,20 @@ psutil_boot_time(PyObject *self, PyObject *args) { float boot_time = 0.0; struct utmpx *ut; + setutxent(); while (NULL != (ut = getutxent())) { if (ut->ut_type == BOOT_TIME) { boot_time = (float)ut->ut_tv.tv_sec; break; } } - endutent(); - if (boot_time != 0.0) { - return Py_BuildValue("f", boot_time); - } - else { + endutxent(); + if (boot_time == 0.0) { + /* could not find BOOT_TIME in getutxent loop */ PyErr_SetString(PyExc_RuntimeError, "can't determine boot time"); return NULL; } + return Py_BuildValue("f", boot_time); } From 730e0fbba6a2ef4a01a5a67759e15dad5613d3a9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 26 Sep 2017 19:19:42 +0800 Subject: [PATCH 792/922] try to fix travis failure --- psutil/tests/test_process.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 1f695a064..0686ba25b 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -868,7 +868,14 @@ def test_cpu_affinity(self): # CPUs on get): # AssertionError: Lists differ: [0, 1, 2, 3, 4, 5, 6, ... != [0] for n in all_cpus: - p.cpu_affinity([n]) + try: + p.cpu_affinity([n]) + except ValueError as err: + if TRAVIS and LINUX and "not eligible" in str(err): + # https://travis-ci.org/giampaolo/psutil/jobs/279890461 + continue + else: + raise self.assertEqual(p.cpu_affinity(), [n]) if hasattr(os, "sched_getaffinity"): self.assertEqual(p.cpu_affinity(), @@ -1264,7 +1271,15 @@ def succeed_or_zombie_p_exc(fun, *args, **kwargs): # set methods succeed_or_zombie_p_exc(zproc.parent) if hasattr(zproc, 'cpu_affinity'): - succeed_or_zombie_p_exc(zproc.cpu_affinity, [0]) + try: + succeed_or_zombie_p_exc(zproc.cpu_affinity, [0]) + except ValueError as err: + if TRAVIS and LINUX and "not eligible" in str(err): + # https://travis-ci.org/giampaolo/psutil/jobs/279890461 + pass + else: + raise + succeed_or_zombie_p_exc(zproc.nice, 0) if hasattr(zproc, 'ionice'): if LINUX: From 1ebe625e5aa21b33e9de5652c305d1d0a2147059 Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Tue, 26 Sep 2017 14:52:42 +0300 Subject: [PATCH 793/922] AIX support (#1123) * AIX support * AIX support * AIX support * AIX support - use get_procfs_path() instead of /proc * AIX support - group sections like in other modules * AIX support * AIX support * AIX support * AIX support - remove unnecessary dict copy --- MANIFEST.in | 1 + docs/index.rst | 74 ++- psutil/TODO.aix | 16 + psutil/__init__.py | 25 +- psutil/_common.py | 1 + psutil/_psaix.py | 590 +++++++++++++++++ psutil/_psutil_aix.c | 930 +++++++++++++++++++++++++++ psutil/_psutil_posix.c | 6 +- psutil/arch/aix/ifaddrs.c | 149 +++++ psutil/arch/aix/ifaddrs.h | 35 + psutil/arch/aix/net_connections.c | 346 ++++++++++ psutil/arch/aix/net_connections.h | 10 + psutil/arch/aix/net_kernel_structs.h | 110 ++++ psutil/tests/__init__.py | 2 + psutil/tests/test_aix.py | 129 ++++ psutil/tests/test_contracts.py | 18 +- psutil/tests/test_memory_leaks.py | 3 + psutil/tests/test_posix.py | 9 +- psutil/tests/test_process.py | 5 +- psutil/tests/test_system.py | 4 +- scripts/procinfo.py | 3 +- setup.py | 15 +- 22 files changed, 2429 insertions(+), 52 deletions(-) create mode 100644 psutil/TODO.aix create mode 100644 psutil/_psaix.py create mode 100644 psutil/_psutil_aix.c create mode 100644 psutil/arch/aix/ifaddrs.c create mode 100644 psutil/arch/aix/ifaddrs.h create mode 100644 psutil/arch/aix/net_connections.c create mode 100644 psutil/arch/aix/net_connections.h create mode 100644 psutil/arch/aix/net_kernel_structs.h create mode 100644 psutil/tests/test_aix.py diff --git a/MANIFEST.in b/MANIFEST.in index 9f84c4c20..e2f8a3192 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -71,6 +71,7 @@ include psutil/arch/windows/services.h include psutil/tests/README.rst include psutil/tests/__init__.py include psutil/tests/__main__.py +include psutil/tests/test_aix.py include psutil/tests/test_bsd.py include psutil/tests/test_connections.py include psutil/tests/test_contracts.py diff --git a/docs/index.rst b/docs/index.rst index 80360046a..3ab444617 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -545,7 +545,7 @@ Network | ``"all"`` | the sum of all the possible families and protocols | +----------------+-----------------------------------------------------+ - On OSX this function requires root privileges. + On OSX and AIX this function requires root privileges. To get per-process connections use :meth:`Process.connections`. Also, see `netstat.py sample script `__. @@ -563,6 +563,10 @@ Network (OSX) :class:`psutil.AccessDenied` is always raised unless running as root. This is a limitation of the OS and ``lsof`` does the same. + .. note:: + (AIX) :class:`psutil.AccessDenied` is always raised unless running as root + (lsof does the same). + .. note:: (Solaris) UNIX sockets are not supported. @@ -1358,7 +1362,7 @@ Process class >>> p.io_counters() pio(read_count=454556, write_count=3456, read_bytes=110592, write_bytes=0, read_chars=769931, write_chars=203) - Availability: all platforms except OSX and Solaris + Availability: Linux, BSD, Windows, AIX .. versionchanged:: 5.2.0 added *read_chars* and *write_chars* on Linux; added *other_count* and *other_bytes* on Windows. @@ -1368,6 +1372,8 @@ Process class The number voluntary and involuntary context switches performed by this process (cumulative). + Availability: all platforms except AIX + .. method:: num_fds() The number of file descriptors currently opened by this process @@ -1503,33 +1509,33 @@ Process class The "portable" fields available on all plaforms are `rss` and `vms`. All numbers are expressed in bytes. - +---------+---------+-------+---------+------------------------------+ - | Linux | OSX | BSD | Solaris | Windows | - +=========+=========+=======+=========+==============================+ - | rss | rss | rss | rss | rss (alias for ``wset``) | - +---------+---------+-------+---------+------------------------------+ - | vms | vms | vms | vms | vms (alias for ``pagefile``) | - +---------+---------+-------+---------+------------------------------+ - | shared | pfaults | text | | num_page_faults | - +---------+---------+-------+---------+------------------------------+ - | text | pageins | data | | peak_wset | - +---------+---------+-------+---------+------------------------------+ - | lib | | stack | | wset | - +---------+---------+-------+---------+------------------------------+ - | data | | | | peak_paged_pool | - +---------+---------+-------+---------+------------------------------+ - | dirty | | | | paged_pool | - +---------+---------+-------+---------+------------------------------+ - | | | | | peak_nonpaged_pool | - +---------+---------+-------+---------+------------------------------+ - | | | | | nonpaged_pool | - +---------+---------+-------+---------+------------------------------+ - | | | | | pagefile | - +---------+---------+-------+---------+------------------------------+ - | | | | | peak_pagefile | - +---------+---------+-------+---------+------------------------------+ - | | | | | private | - +---------+---------+-------+---------+------------------------------+ + +---------+---------+-------+---------+-----+------------------------------+ + | Linux | OSX | BSD | Solaris | AIX | Windows | + +=========+=========+=======+=========+=====+==============================+ + | rss | rss | rss | rss | rss | rss (alias for ``wset``) | + +---------+---------+-------+---------+-----+------------------------------+ + | vms | vms | vms | vms | vms | vms (alias for ``pagefile``) | + +---------+---------+-------+---------+-----+------------------------------+ + | shared | pfaults | text | | | num_page_faults | + +---------+---------+-------+---------+-----+------------------------------+ + | text | pageins | data | | | peak_wset | + +---------+---------+-------+---------+-----+------------------------------+ + | lib | | stack | | | wset | + +---------+---------+-------+---------+-----+------------------------------+ + | data | | | | | peak_paged_pool | + +---------+---------+-------+---------+-----+------------------------------+ + | dirty | | | | | paged_pool | + +---------+---------+-------+---------+-----+------------------------------+ + | | | | | | peak_nonpaged_pool | + +---------+---------+-------+---------+-----+------------------------------+ + | | | | | | nonpaged_pool | + +---------+---------+-------+---------+-----+------------------------------+ + | | | | | | pagefile | + +---------+---------+-------+---------+-----+------------------------------+ + | | | | | | peak_pagefile | + +---------+---------+-------+---------+-----+------------------------------+ + | | | | | | private | + +---------+---------+-------+---------+-----+------------------------------+ - **rss**: aka "Resident Set Size", this is the non-swapped physical memory a process has used. @@ -1700,7 +1706,7 @@ Process class pmmap_ext(addr='02829000-02ccf000', perms='rw-p', path='[heap]', rss=4743168, size=4874240, pss=4743168, shared_clean=0, shared_dirty=0, private_clean=0, private_dirty=4743168, referenced=4718592, anonymous=4743168, swap=0), ...] - Availability: All platforms except OpenBSD and NetBSD. + Availability: All platforms except OpenBSD, NetBSD and AIX. .. method:: children(recursive=False) @@ -1864,6 +1870,10 @@ Process class .. versionchanged:: 5.3.0 : "laddr" and "raddr" are named tuples. + .. note:: + (AIX) :class:`psutil.AccessDenied` is always raised unless running + as root (lsof does the same). + .. method:: is_running() Return whether the current process is running in the current process list. @@ -2101,16 +2111,18 @@ Constants .. data:: OPENBSD .. data:: BSD .. data:: SUNOS +.. data:: AIX ``bool`` constants which define what platform you're on. E.g. if on Windows, :const:`WINDOWS` constant will be ``True``, all others will be ``False``. .. versionadded:: 4.0.0 + .. versionchanged:: 5.4.0 added AIX .. _const-procfs_path: .. data:: PROCFS_PATH - The path of the /proc filesystem on Linux and Solaris (defaults to + The path of the /proc filesystem on Linux, Solaris and AIX (defaults to ``"/proc"``). You may want to re-set this constant right after importing psutil in case your /proc filesystem is mounted elsewhere or if you want to retrieve diff --git a/psutil/TODO.aix b/psutil/TODO.aix new file mode 100644 index 000000000..495f4963d --- /dev/null +++ b/psutil/TODO.aix @@ -0,0 +1,16 @@ +AIX support is experimental at this time. +The following functions and methods are unsupported on the AIX platform: + + psutil.Process.memory_maps + psutil.Process.num_ctx_switches + +Known limitations: + psutil.Process.io_counters read count is always 0 + reading basic process info may fail or return incorrect values when process is starting + (see IBM APAR IV58499 - fixed in newer AIX versions) + sockets and pipes may not be counted in num_fds (fixed in newer AIX versions) + +The following unit tests may fail: + test_prog_w_funky_name funky name tests don't work, name is truncated + test_cmdline long args are cut from cmdline in /proc/pid/psinfo and getargs + test_pid_exists_2 there are pids in /proc that don't really exist diff --git a/psutil/__init__.py b/psutil/__init__.py index ca9bc2390..9c6451e46 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -73,6 +73,7 @@ from ._common import NIC_DUPLEX_HALF from ._common import NIC_DUPLEX_UNKNOWN +from ._common import AIX from ._common import BSD from ._common import FREEBSD # NOQA from ._common import LINUX @@ -158,6 +159,13 @@ # _pssunos.py via sys.modules. PROCFS_PATH = "/proc" +elif AIX: + from . import _psaix as _psplatform + + # This is public API and it will be retrieved from _pslinux.py + # via sys.modules. + PROCFS_PATH = "/proc" + else: # pragma: no cover raise NotImplementedError('platform %s is not supported' % sys.platform) @@ -185,7 +193,7 @@ "POWER_TIME_UNKNOWN", "POWER_TIME_UNLIMITED", "BSD", "FREEBSD", "LINUX", "NETBSD", "OPENBSD", "OSX", "POSIX", "SUNOS", - "WINDOWS", + "WINDOWS", "AIX", # classes "Process", "Popen", @@ -785,7 +793,7 @@ def num_fds(self): """ return self._proc.num_fds() - # Linux, BSD and Windows only + # Linux, BSD, AIX and Windows only if hasattr(_psplatform.Process, "io_counters"): def io_counters(self): @@ -890,11 +898,13 @@ def num_handles(self): """ return self._proc.num_handles() - def num_ctx_switches(self): - """Return the number of voluntary and involuntary context - switches performed by this process. - """ - return self._proc.num_ctx_switches() + if hasattr(_psplatform.Process, "num_ctx_switches"): + + def num_ctx_switches(self): + """Return the number of voluntary and involuntary context + switches performed by this process. + """ + return self._proc.num_ctx_switches() def num_threads(self): """Return the number of threads used by this process.""" @@ -1171,7 +1181,6 @@ def memory_percent(self, memtype="rss"): if hasattr(_psplatform.Process, "memory_maps"): # Available everywhere except OpenBSD and NetBSD. - def memory_maps(self, grouped=True): """Return process' mapped memory regions as a list of namedtuples whose fields are variable depending on the platform. diff --git a/psutil/_common.py b/psutil/_common.py index 7c4af3d85..2d562f93e 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -81,6 +81,7 @@ NETBSD = sys.platform.startswith("netbsd") BSD = FREEBSD or OPENBSD or NETBSD SUNOS = sys.platform.startswith("sunos") or sys.platform.startswith("solaris") +AIX = sys.platform.startswith("aix") # =================================================================== diff --git a/psutil/_psaix.py b/psutil/_psaix.py new file mode 100644 index 000000000..102e0f5f6 --- /dev/null +++ b/psutil/_psaix.py @@ -0,0 +1,590 @@ +# Copyright (c) 2009, Giampaolo Rodola' +# Copyright (c) 2017, Arnon Yaari +# All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""AIX platform implementation.""" + +import errno +import glob +import os +import re +import subprocess +import sys +from collections import namedtuple +from socket import AF_INET + +from . import _common +from . import _psposix +from . import _psutil_aix as cext +from . import _psutil_posix as cext_posix +from ._common import AF_INET6 +from ._common import memoize_when_activated +from ._common import NIC_DUPLEX_FULL +from ._common import NIC_DUPLEX_HALF +from ._common import NIC_DUPLEX_UNKNOWN +from ._common import sockfam_to_enum +from ._common import socktype_to_enum +from ._common import usage_percent +from ._compat import PY3 + + +__extra__all__ = ["PROCFS_PATH"] + + +# ===================================================================== +# --- globals +# ===================================================================== + + +PAGE_SIZE = os.sysconf('SC_PAGE_SIZE') +AF_LINK = cext_posix.AF_LINK + +PROC_STATUSES = { + + cext.SIDL: _common.STATUS_IDLE, + cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SACTIVE: _common.STATUS_RUNNING, + cext.SSWAP: _common.STATUS_RUNNING, # TODO what status is this? + cext.SSTOP: _common.STATUS_STOPPED, +} + +TCP_STATUSES = { + cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, + cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, + cext.TCPS_SYN_RCVD: _common.CONN_SYN_RECV, + cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1, + cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2, + cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.TCPS_CLOSED: _common.CONN_CLOSE, + cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK, + cext.TCPS_LISTEN: _common.CONN_LISTEN, + cext.TCPS_CLOSING: _common.CONN_CLOSING, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, +} + +proc_info_map = dict( + ppid=0, + rss=1, + vms=2, + create_time=3, + nice=4, + num_threads=5, + status=6, + ttynr=7) + +# these get overwritten on "import psutil" from the __init__.py file +NoSuchProcess = None +ZombieProcess = None +AccessDenied = None +TimeoutExpired = None + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# psutil.Process.memory_info() +pmem = namedtuple('pmem', ['rss', 'vms']) +# psutil.Process.memory_full_info() +pfullmem = pmem +# psutil.Process.cpu_times() +scputimes = namedtuple('scputimes', ['user', 'system', 'idle', 'iowait']) +# psutil.virtual_memory() +svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) +# psutil.Process.memory_maps(grouped=True) +pmmap_grouped = namedtuple('pmmap_grouped', ['path', 'rss', 'anon', 'locked']) +# psutil.Process.memory_maps(grouped=False) +pmmap_ext = namedtuple( + 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) + + +# ===================================================================== +# --- utils +# ===================================================================== + + +def get_procfs_path(): + """Return updated psutil.PROCFS_PATH constant.""" + return sys.modules['psutil'].PROCFS_PATH + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + total, avail, free, pinned, inuse = cext.virtual_mem() + percent = usage_percent((total - avail), total, _round=1) + return svmem(total, avail, percent, inuse, free) + + +def swap_memory(): + """Swap system memory as a (total, used, free, sin, sout) tuple.""" + total, free, sin, sout = cext.swap_mem() + used = total - free + percent = usage_percent(used, total, _round=1) + return _common.sswap(total, used, free, percent, sin, sout) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system-wide CPU times as a named tuple""" + ret = cext.per_cpu_times() + return scputimes(*[sum(x) for x in zip(*ret)]) + + +def per_cpu_times(): + """Return system per-CPU times as a list of named tuples""" + ret = cext.per_cpu_times() + return [scputimes(*x) for x in ret] + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + try: + return os.sysconf("SC_NPROCESSORS_ONLN") + except ValueError: + # mimic os.cpu_count() behavior + return None + + +def cpu_count_physical(): + cmd = "lsdev -Cc processor" + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if PY3: + stdout, stderr = [x.decode(sys.stdout.encoding) + for x in (stdout, stderr)] + if p.returncode != 0: + raise RuntimeError("%r command error\n%s" % (cmd, stderr)) + processors = stdout.strip().splitlines() + return len(processors) or None + + +def cpu_stats(): + """Return various CPU stats as a named tuple.""" + ctx_switches, interrupts, soft_interrupts, syscalls = cext.cpu_stats() + return _common.scpustats( + ctx_switches, interrupts, soft_interrupts, syscalls) + + +# ===================================================================== +# --- disks +# ===================================================================== + + +disk_io_counters = cext.disk_io_counters +disk_usage = _psposix.disk_usage + + +def disk_partitions(all=False): + """Return system disk partitions.""" + # TODO - the filtering logic should be better checked so that + # it tries to reflect 'df' as much as possible + retlist = [] + partitions = cext.disk_partitions() + for partition in partitions: + device, mountpoint, fstype, opts = partition + if device == 'none': + device = '' + if not all: + # Differently from, say, Linux, we don't have a list of + # common fs types so the best we can do, AFAIK, is to + # filter by filesystem having a total size > 0. + if not disk_usage(mountpoint).total: + continue + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + return retlist + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_if_addrs = cext_posix.net_if_addrs +net_io_counters = cext.net_io_counters + + +def net_connections(kind, _pid=-1): + """Return socket connections. If pid == -1 return system-wide + connections (as opposed to connections opened by one process only). + """ + cmap = _common.conn_tmap + if kind not in cmap: + raise ValueError("invalid %r kind argument; choose between %s" + % (kind, ', '.join([repr(x) for x in cmap]))) + families, types = _common.conn_tmap[kind] + rawlist = cext.net_connections(_pid) + ret = set() + for item in rawlist: + fd, fam, type_, laddr, raddr, status, pid = item + if fam not in families: + continue + if type_ not in types: + continue + status = TCP_STATUSES[status] + if fam in (AF_INET, AF_INET6): + if laddr: + laddr = _common.addr(*laddr) + if raddr: + raddr = _common.addr(*raddr) + fam = sockfam_to_enum(fam) + type_ = socktype_to_enum(type_) + if _pid == -1: + nt = _common.sconn(fd, fam, type_, laddr, raddr, status, pid) + else: + nt = _common.pconn(fd, fam, type_, laddr, raddr, status) + ret.add(nt) + return list(ret) + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + duplex_map = {"Full": NIC_DUPLEX_FULL, + "Half": NIC_DUPLEX_HALF} + names = set([x[0] for x in net_if_addrs()]) + ret = {} + for name in names: + isup, mtu = cext.net_if_stats(name) + + # try to get speed and duplex + # TODO: rewrite this in C (entstat forks, so use truss -f to follow. + # looks like it is using an undocumented ioctl?) + duplex = "" + speed = 0 + p = subprocess.Popen(["/usr/bin/entstat", "-d", name], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if PY3: + stdout, stderr = [x.decode(sys.stdout.encoding) + for x in (stdout, stderr)] + if p.returncode == 0: + re_result = re.search("Running: (\d+) Mbps.*?(\w+) Duplex", stdout) + if re_result is not None: + speed = int(re_result.group(1)) + duplex = re_result.group(2) + + duplex = duplex_map.get(duplex, NIC_DUPLEX_UNKNOWN) + ret[name] = _common.snicstats(isup, duplex, speed, mtu) + return ret + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + return cext.boot_time() + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + localhost = (':0.0', ':0') + for item in rawlist: + user, tty, hostname, tstamp, user_process, pid = item + # note: the underlying C function includes entries about + # system boot, run level and others. We might want + # to use them in the future. + if not user_process: + continue + if hostname in localhost: + hostname = 'localhost' + nt = _common.suser(user, tty, hostname, tstamp, pid) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- processes +# ===================================================================== + + +def pids(): + """Returns a list of PIDs currently running on the system.""" + return [int(x) for x in os.listdir(get_procfs_path()) if x.isdigit()] + + +def pid_exists(pid): + """Check for the existence of a unix pid.""" + return _psposix.pid_exists(pid) + + +def wrap_exceptions(fun): + """Call callable into a try/except clause and translate ENOENT, + EACCES and EPERM in NoSuchProcess or AccessDenied exceptions. + """ + + def wrapper(self, *args, **kwargs): + try: + return fun(self, *args, **kwargs) + except EnvironmentError as err: + # support for private module import + if (NoSuchProcess is None or AccessDenied is None or + ZombieProcess is None): + raise + # ENOENT (no such file or directory) gets raised on open(). + # ESRCH (no such process) can get raised on read() if + # process is gone in meantime. + if err.errno in (errno.ENOENT, errno.ESRCH): + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + else: + raise ZombieProcess(self.pid, self._name, self._ppid) + if err.errno in (errno.EPERM, errno.EACCES): + raise AccessDenied(self.pid, self._name) + raise + return wrapper + + +class Process(object): + """Wrapper class around underlying C implementation.""" + + __slots__ = ["pid", "_name", "_ppid", "_procfs_path"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + self._procfs_path = get_procfs_path() + + def oneshot_enter(self): + self._proc_name_and_args.cache_activate() + self._proc_basic_info.cache_activate() + self._proc_cred.cache_activate() + + def oneshot_exit(self): + self._proc_name_and_args.cache_deactivate() + self._proc_basic_info.cache_deactivate() + self._proc_cred.cache_deactivate() + + @memoize_when_activated + def _proc_name_and_args(self): + return cext.proc_name_and_args(self.pid, self._procfs_path) + + @memoize_when_activated + def _proc_basic_info(self): + return cext.proc_basic_info(self.pid, self._procfs_path) + + @memoize_when_activated + def _proc_cred(self): + return cext.proc_cred(self.pid, self._procfs_path) + + @wrap_exceptions + def name(self): + if self.pid == 0: + return "swapper" + # note: this is limited to 15 characters + return self._proc_name_and_args()[0].rstrip("\x00") + + @wrap_exceptions + def exe(self): + # there is no way to get executable path in AIX other than to guess, + # and guessing is more complex than what's in the wrapping class + exe = self.cmdline()[0] + if os.path.sep in exe: + # relative or absolute path + if not os.path.isabs(exe): + # if cwd has changed, we're out of luck - this may be wrong! + exe = os.path.abspath(os.path.join(self.cwd(), exe)) + if (os.path.isabs(exe) and + os.path.isfile(exe) and + os.access(exe, os.X_OK)): + return exe + # not found, move to search in PATH using basename only + exe = os.path.basename(exe) + # search for exe name PATH + for path in os.environ["PATH"].split(":"): + possible_exe = os.path.abspath(os.path.join(path, exe)) + if (os.path.isfile(possible_exe) and + os.access(possible_exe, os.X_OK)): + return possible_exe + return '' + + @wrap_exceptions + def cmdline(self): + return self._proc_name_and_args()[1].split(' ') + + @wrap_exceptions + def create_time(self): + return self._proc_basic_info()[proc_info_map['create_time']] + + @wrap_exceptions + def num_threads(self): + return self._proc_basic_info()[proc_info_map['num_threads']] + + @wrap_exceptions + def threads(self): + rawlist = cext.proc_threads(self.pid) + retlist = [] + for thread_id, utime, stime in rawlist: + ntuple = _common.pthread(thread_id, utime, stime) + retlist.append(ntuple) + # The underlying C implementation retrieves all OS threads + # and filters them by PID. At this point we can't tell whether + # an empty list means there were no connections for process or + # process is no longer active so we force NSP in case the PID + # is no longer there. + if not retlist: + # will raise NSP if process is gone + os.stat('%s/%s' % (self._procfs_path, self.pid)) + return retlist + + @wrap_exceptions + def connections(self, kind='inet'): + ret = net_connections(kind, _pid=self.pid) + # The underlying C implementation retrieves all OS connections + # and filters them by PID. At this point we can't tell whether + # an empty list means there were no connections for process or + # process is no longer active so we force NSP in case the PID + # is no longer there. + if not ret: + # will raise NSP if process is gone + os.stat('%s/%s' % (self._procfs_path, self.pid)) + return ret + + @wrap_exceptions + def nice_get(self): + # For some reason getpriority(3) return ESRCH (no such process) + # for certain low-pid processes, no matter what (even as root). + # The process actually exists though, as it has a name, + # creation time, etc. + # The best thing we can do here appears to be raising AD. + # Note: tested on Solaris 11; on Open Solaris 5 everything is + # fine. + try: + return cext_posix.getpriority(self.pid) + except EnvironmentError as err: + # 48 is 'operation not supported' but errno does not expose + # it. It occurs for low system pids. + if err.errno in (errno.ENOENT, errno.ESRCH, 48): + if pid_exists(self.pid): + raise AccessDenied(self.pid, self._name) + raise + + @wrap_exceptions + def nice_set(self, value): + return cext_posix.setpriority(self.pid, value) + + @wrap_exceptions + def ppid(self): + self._ppid = self._proc_basic_info()[proc_info_map['ppid']] + return self._ppid + + @wrap_exceptions + def uids(self): + real, effective, saved, _, _, _ = self._proc_cred() + return _common.puids(real, effective, saved) + + @wrap_exceptions + def gids(self): + _, _, _, real, effective, saved = self._proc_cred() + return _common.puids(real, effective, saved) + + @wrap_exceptions + def cpu_times(self): + cpu_times = cext.proc_cpu_times(self.pid, self._procfs_path) + return _common.pcputimes(*cpu_times) + + @wrap_exceptions + def terminal(self): + ttydev = self._proc_basic_info()[proc_info_map['ttynr']] + # convert from 64-bit dev_t to 32-bit dev_t and then map the device + ttydev = (((ttydev & 0x0000FFFF00000000) >> 16) | (ttydev & 0xFFFF)) + # try to match rdev of /dev/pts/* files ttydev + for dev in glob.glob("/dev/**/*"): + if os.stat(dev).st_rdev == ttydev: + return dev + return None + + @wrap_exceptions + def cwd(self): + procfs_path = self._procfs_path + try: + result = os.readlink("%s/%s/cwd" % (procfs_path, self.pid)) + return result.rstrip('/') + except OSError as err: + if err.errno == errno.ENOENT: + os.stat("%s/%s" % (procfs_path, self.pid)) # raise NSP or AD + return None + raise + + @wrap_exceptions + def memory_info(self): + ret = self._proc_basic_info() + rss = ret[proc_info_map['rss']] * 1024 + vms = ret[proc_info_map['vms']] * 1024 + return pmem(rss, vms) + + memory_full_info = memory_info + + @wrap_exceptions + def status(self): + code = self._proc_basic_info()[proc_info_map['status']] + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(code, '?') + + def open_files(self): + # TODO rewrite without using procfiles (stat /proc/pid/fd/* and then + # find matching name of the inode) + p = subprocess.Popen(["/usr/bin/procfiles", "-n", str(self.pid)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + if PY3: + stdout, stderr = [x.decode(sys.stdout.encoding) + for x in (stdout, stderr)] + if "no such process" in stderr.lower(): + raise NoSuchProcess(self.pid, self._name) + procfiles = re.findall("(\d+): S_IFREG.*\s*.*name:(.*)\n", stdout) + retlist = [] + for fd, path in procfiles: + path = path.strip() + if path.startswith("//"): + path = path[1:] + if path.lower() == "cannot be retrieved": + continue + retlist.append(_common.popenfile(path, int(fd))) + return retlist + + @wrap_exceptions + def num_fds(self): + if self.pid == 0: # no /proc/0/fd + return 0 + return len(os.listdir("%s/%s/fd" % (self._procfs_path, self.pid))) + + @wrap_exceptions + def wait(self, timeout=None): + try: + return _psposix.wait_pid(self.pid, timeout) + except _psposix.TimeoutExpired: + # support for private module import + if TimeoutExpired is None: + raise + raise TimeoutExpired(timeout, self.pid, self._name) + + @wrap_exceptions + def io_counters(self): + try: + rc, wc, rb, wb = cext.proc_io_counters(self.pid) + except OSError: + # if process is terminated, proc_io_counters returns OSError + # instead of NSP + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) + raise + return _common.pio(rc, wc, rb, wb) diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c new file mode 100644 index 000000000..52a14feb6 --- /dev/null +++ b/psutil/_psutil_aix.c @@ -0,0 +1,930 @@ +/* + * Copyright (c) 2009, Giampaolo Rodola' + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + * + * AIX platform-specific module methods for _psutil_aix + * + */ + +// Useful resources: +// proc filesystem: http://www-01.ibm.com/support/knowledgecenter/ssw_aix_61/com.ibm.aix.files/proc.htm +// libperfstat: http://www-01.ibm.com/support/knowledgecenter/ssw_aix_61/com.ibm.aix.files/libperfstat.h.htm + + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arch/aix/ifaddrs.h" +#include "arch/aix/net_connections.h" +#include "_psutil_common.h" +#include "_psutil_posix.h" + + +#define TV2DOUBLE(t) (((t).tv_nsec * 0.000000001) + (t).tv_sec) + +/* + * Read a file content and fills a C structure with it. + */ +int +psutil_file_to_struct(char *path, void *fstruct, size_t size) { + int fd; + size_t nbytes; + fd = open(path, O_RDONLY); + if (fd == -1) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, path); + return 0; + } + nbytes = read(fd, fstruct, size); + if (nbytes <= 0) { + close(fd); + PyErr_SetFromErrno(PyExc_OSError); + return 0; + } + if (nbytes != size) { + close(fd); + PyErr_SetString(PyExc_RuntimeError, "structure size mismatch"); + return 0; + } + close(fd); + return nbytes; +} + + +/* + * Return process ppid, rss, vms, ctime, nice, nthreads, status and tty + * as a Python tuple. + */ +static PyObject * +psutil_proc_basic_info(PyObject *self, PyObject *args) { + int pid; + char path[100]; + psinfo_t info; + pstatus_t status; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + + sprintf(path, "%s/%i/psinfo", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + + if (info.pr_nlwp == 0 && info.pr_lwp.pr_lwpid == 0) { + // From the /proc docs: "If the process is a zombie, the pr_nlwp + // and pr_lwp.pr_lwpid flags are zero." + status.pr_stat = SZOMB; + } else if (info.pr_flag & SEXIT) { + // "exiting" processes don't have /proc//status + // There are other "exiting" processes that 'ps' shows as "active" + status.pr_stat = SACTIVE; + } else { + sprintf(path, "%s/%i/status", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&status, sizeof(status))) + return NULL; + } + + return Py_BuildValue("KKKdiiiK", + (unsigned long long) info.pr_ppid, // parent pid + (unsigned long long) info.pr_rssize, // rss + (unsigned long long) info.pr_size, // vms + TV2DOUBLE(info.pr_start), // create time + (int) info.pr_lwp.pr_nice, // nice + (int) info.pr_nlwp, // no. of threads + (int) status.pr_stat, // status code + (unsigned long long)info.pr_ttydev // tty nr + ); +} + + +/* + * Return process name and args as a Python tuple. + */ +static PyObject * +psutil_proc_name_and_args(PyObject *self, PyObject *args) { + int pid; + char path[100]; + psinfo_t info; + const char *procfs_path; + PyObject *py_name = NULL; + PyObject *py_args = NULL; + PyObject *py_retlist = NULL; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/psinfo", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + + py_name = PyUnicode_DecodeFSDefault(info.pr_fname); + if (!py_name) + goto error; + py_args = PyUnicode_DecodeFSDefault(info.pr_psargs); + if (!py_args) + goto error; + py_retlist = Py_BuildValue("OO", py_name, py_args); + if (!py_retlist) + goto error; + Py_DECREF(py_name); + Py_DECREF(py_args); + return py_retlist; + +error: + Py_XDECREF(py_name); + Py_XDECREF(py_args); + Py_XDECREF(py_retlist); + return NULL; +} + + +/* + * Retrieves all threads used by process returning a list of tuples + * including thread id, user time and system time. + */ +static PyObject * +psutil_proc_threads(PyObject *self, PyObject *args) { + long pid; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + perfstat_thread_t *threadt = NULL; + perfstat_id_t id; + int i, rc, thread_count; + + if (py_retlist == NULL) + return NULL; + if (! PyArg_ParseTuple(args, "l", &pid)) + goto error; + + /* Get the count of threads */ + thread_count = perfstat_thread(NULL, NULL, sizeof(perfstat_thread_t), 0); + if (thread_count <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + /* Allocate enough memory */ + threadt = (perfstat_thread_t *)calloc(thread_count, + sizeof(perfstat_thread_t)); + if (threadt == NULL) { + PyErr_NoMemory(); + goto error; + } + + strcpy(id.name, ""); + rc = perfstat_thread(&id, threadt, sizeof(perfstat_thread_t), + thread_count); + if (rc <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < thread_count; i++) { + if (threadt[i].pid != pid) + continue; + + py_tuple = Py_BuildValue("Idd", + threadt[i].tid, + threadt[i].ucpu_time, + threadt[i].scpu_time); + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + free(threadt); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (threadt != NULL) + free(threadt); + return NULL; +} + + +static PyObject * +psutil_proc_io_counters(PyObject *self, PyObject *args) { + long pid; + int rc; + perfstat_process_t procinfo; + perfstat_id_t id; + if (! PyArg_ParseTuple(args, "l", &pid)) + return NULL; + + snprintf(id.name, sizeof(id.name), "%ld", pid); + rc = perfstat_process(&id, &procinfo, sizeof(perfstat_process_t), 1); + if (rc <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return Py_BuildValue("(KKKK)", + procinfo.inOps, // XXX always 0 + procinfo.outOps, + procinfo.inBytes, // XXX always 0 + procinfo.outBytes); +} + + +/* + * Return process user and system CPU times as a Python tuple. + */ +static PyObject * +psutil_proc_cpu_times(PyObject *self, PyObject *args) { + int pid; + char path[100]; + pstatus_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/status", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + // results are more precise than os.times() + return Py_BuildValue("dddd", + TV2DOUBLE(info.pr_utime), + TV2DOUBLE(info.pr_stime), + TV2DOUBLE(info.pr_cutime), + TV2DOUBLE(info.pr_cstime)); +} + + +/* + * Return process uids/gids as a Python tuple. + */ +static PyObject * +psutil_proc_cred(PyObject *self, PyObject *args) { + int pid; + char path[100]; + prcred_t info; + const char *procfs_path; + + if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) + return NULL; + sprintf(path, "%s/%i/cred", procfs_path, pid); + if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) + return NULL; + return Py_BuildValue("iiiiii", + info.pr_ruid, info.pr_euid, info.pr_suid, + info.pr_rgid, info.pr_egid, info.pr_sgid); +} + + +/* + * Return users currently connected on the system. + */ +static PyObject * +psutil_users(PyObject *self, PyObject *args) { + struct utmpx *ut; + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; + PyObject *py_user_proc = NULL; + + if (py_retlist == NULL) + return NULL; + + setutxent(); + while (NULL != (ut = getutxent())) { + if (ut->ut_type == USER_PROCESS) + py_user_proc = Py_True; + else + py_user_proc = Py_False; + py_username = PyUnicode_DecodeFSDefault(ut->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(ut->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(ut->ut_host); + if (! py_hostname) + goto error; + py_tuple = Py_BuildValue( + "(OOOfOi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname + (float)ut->ut_tv.tv_sec, // tstamp + py_user_proc, // (bool) user process + ut->ut_pid // process id + ); + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); + Py_DECREF(py_tuple); + } + endutxent(); + + return py_retlist; + +error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (ut != NULL) + endutxent(); + return NULL; +} + + +/* + * Return disk mounted partitions as a list of tuples including device, + * mount point and filesystem type. + */ +static PyObject * +psutil_disk_partitions(PyObject *self, PyObject *args) { + FILE *file = NULL; + struct mntent * mt = NULL; + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; + PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); + + if (py_retlist == NULL) + return NULL; + + file = setmntent(MNTTAB, "rb"); + if (file == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + mt = getmntent(file); + while (mt != NULL) { + py_dev = PyUnicode_DecodeFSDefault(mt->mnt_fsname); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(mt->mnt_dir); + if (! py_mountp) + goto error; + py_tuple = Py_BuildValue( + "(OOss)", + py_dev, // device + py_mountp, // mount point + mt->mnt_type, // fs type + mt->mnt_opts); // options + if (py_tuple == NULL) + goto error; + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_dev); + Py_DECREF(py_mountp); + Py_DECREF(py_tuple); + mt = getmntent(file); + } + endmntent(file); + return py_retlist; + +error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (file != NULL) + endmntent(file); + return NULL; +} + + +/* + * Return a list of tuples for network I/O statistics. + */ +static PyObject * +psutil_net_io_counters(PyObject *self, PyObject *args) { + perfstat_netinterface_t *statp = NULL; + int tot, i; + perfstat_id_t first; + + PyObject *py_retdict = PyDict_New(); + PyObject *py_ifc_info = NULL; + + if (py_retdict == NULL) + return NULL; + + /* check how many perfstat_netinterface_t structures are available */ + tot = perfstat_netinterface( + NULL, NULL, sizeof(perfstat_netinterface_t), 0); + if (tot == 0) { + // no network interfaces - return empty dict + return py_retdict; + } + if (tot < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + statp = (perfstat_netinterface_t *) + malloc(tot * sizeof(perfstat_netinterface_t)); + if (statp == NULL) { + PyErr_NoMemory(); + goto error; + } + strcpy(first.name, FIRST_NETINTERFACE); + tot = perfstat_netinterface(&first, statp, + sizeof(perfstat_netinterface_t), tot); + if (tot < 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < tot; i++) { + py_ifc_info = Py_BuildValue("(KKKKKKKK)", + statp[i].obytes, /* number of bytes sent on interface */ + statp[i].ibytes, /* number of bytes received on interface */ + statp[i].opackets, /* number of packets sent on interface */ + statp[i].ipackets, /* number of packets received on interface */ + statp[i].ierrors, /* number of input errors on interface */ + statp[i].oerrors, /* number of output errors on interface */ + statp[i].if_iqdrops, /* Dropped on input, this interface */ + statp[i].xmitdrops /* number of packets not transmitted */ + ); + if (!py_ifc_info) + goto error; + if (PyDict_SetItemString(py_retdict, statp[i].name, py_ifc_info)) + goto error; + Py_DECREF(py_ifc_info); + } + + free(statp); + return py_retdict; + +error: + if (statp != NULL) + free(statp); + Py_XDECREF(py_ifc_info); + Py_DECREF(py_retdict); + return NULL; +} + + +static PyObject* +psutil_net_if_stats(PyObject* self, PyObject* args) { + char *nic_name; + int sock = 0; + int ret; + int mtu; + struct ifreq ifr; + PyObject *py_is_up = NULL; + PyObject *py_retlist = NULL; + + if (! PyArg_ParseTuple(args, "s", &nic_name)) + return NULL; + + sock = socket(AF_INET, SOCK_DGRAM, 0); + if (sock == -1) + goto error; + + strncpy(ifr.ifr_name, nic_name, sizeof(ifr.ifr_name)); + + // is up? + ret = ioctl(sock, SIOCGIFFLAGS, &ifr); + if (ret == -1) + goto error; + if ((ifr.ifr_flags & IFF_UP) != 0) + py_is_up = Py_True; + else + py_is_up = Py_False; + Py_INCREF(py_is_up); + + // MTU + ret = ioctl(sock, SIOCGIFMTU, &ifr); + if (ret == -1) + goto error; + mtu = ifr.ifr_mtu; + + close(sock); + py_retlist = Py_BuildValue("[Oi]", py_is_up, mtu); + if (!py_retlist) + goto error; + Py_DECREF(py_is_up); + return py_retlist; + +error: + Py_XDECREF(py_is_up); + if (sock != 0) + close(sock); + PyErr_SetFromErrno(PyExc_OSError); + return NULL; +} + + +static PyObject * +psutil_boot_time(PyObject *self, PyObject *args) { + float boot_time = 0.0; + struct utmpx *ut; + + setutxent(); + while (NULL != (ut = getutxent())) { + if (ut->ut_type == BOOT_TIME) { + boot_time = (float)ut->ut_tv.tv_sec; + break; + } + } + endutxent(); + if (boot_time == 0.0) { + /* could not find BOOT_TIME in getutxent loop */ + PyErr_SetString(PyExc_RuntimeError, "can't determine boot time"); + return NULL; + } + return Py_BuildValue("f", boot_time); +} + + +/* + * Return a Python list of tuple representing per-cpu times + */ +static PyObject * +psutil_per_cpu_times(PyObject *self, PyObject *args) { + int ncpu, rc, i; + perfstat_cpu_t *cpu = NULL; + perfstat_id_t id; + PyObject *py_retlist = PyList_New(0); + PyObject *py_cputime = NULL; + + if (py_retlist == NULL) + return NULL; + + /* get the number of cpus in ncpu */ + ncpu = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0); + if (ncpu <= 0){ + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + /* allocate enough memory to hold the ncpu structures */ + cpu = (perfstat_cpu_t *) malloc(ncpu * sizeof(perfstat_cpu_t)); + if (cpu == NULL) { + PyErr_NoMemory(); + goto error; + } + + strcpy(id.name, ""); + rc = perfstat_cpu(&id, cpu, sizeof(perfstat_cpu_t), ncpu); + + if (rc <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < ncpu; i++) { + py_cputime = Py_BuildValue( + "(dddd)", + (double)cpu[i].user, + (double)cpu[i].sys, + (double)cpu[i].idle, + (double)cpu[i].wait); + if (!py_cputime) + goto error; + if (PyList_Append(py_retlist, py_cputime)) + goto error; + Py_DECREF(py_cputime); + } + free(cpu); + return py_retlist; + +error: + Py_XDECREF(py_cputime); + Py_DECREF(py_retlist); + if (cpu != NULL) + free(cpu); + return NULL; +} + + +/* + * Return disk IO statistics. + */ +static PyObject * +psutil_disk_io_counters(PyObject *self, PyObject *args) { + PyObject *py_retdict = PyDict_New(); + PyObject *py_disk_info = NULL; + perfstat_disk_t *diskt = NULL; + perfstat_id_t id; + int i, rc, disk_count; + + if (py_retdict == NULL) + return NULL; + + /* Get the count of disks */ + disk_count = perfstat_disk(NULL, NULL, sizeof(perfstat_disk_t), 0); + if (disk_count <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + /* Allocate enough memory */ + diskt = (perfstat_disk_t *)calloc(disk_count, + sizeof(perfstat_disk_t)); + if (diskt == NULL) { + PyErr_NoMemory(); + goto error; + } + + strcpy(id.name, FIRST_DISK); + rc = perfstat_disk(&id, diskt, sizeof(perfstat_disk_t), + disk_count); + if (rc <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < disk_count; i++) { + py_disk_info = Py_BuildValue( + "KKKKKK", + diskt[i].__rxfers, + diskt[i].xfers - diskt[i].__rxfers, + diskt[i].rblks * diskt[i].bsize, + diskt[i].wblks * diskt[i].bsize, + diskt[i].rserv / 1000 / 1000, // from nano to milli secs + diskt[i].wserv / 1000 / 1000 // from nano to milli secs + ); + if (py_disk_info == NULL) + goto error; + if (PyDict_SetItemString(py_retdict, diskt[i].name, + py_disk_info)) + goto error; + Py_DECREF(py_disk_info); + } + free(diskt); + return py_retdict; + +error: + Py_XDECREF(py_disk_info); + Py_DECREF(py_retdict); + if (diskt != NULL) + free(diskt); + return NULL; +} + + +/* + * Return virtual memory usage statistics. + */ +static PyObject * +psutil_virtual_mem(PyObject *self, PyObject *args) { + int rc; + int pagesize = getpagesize(); + perfstat_memory_total_t memory; + + rc = perfstat_memory_total( + NULL, &memory, sizeof(perfstat_memory_total_t), 1); + if (rc <= 0){ + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return Py_BuildValue("KKKKK", + (unsigned long long) memory.real_total * pagesize, + (unsigned long long) memory.real_avail * pagesize, + (unsigned long long) memory.real_free * pagesize, + (unsigned long long) memory.real_pinned * pagesize, + (unsigned long long) memory.real_inuse * pagesize + ); +} + + +/* + * Return stats about swap memory. + */ +static PyObject * +psutil_swap_mem(PyObject *self, PyObject *args) { + int rc; + int pagesize = getpagesize(); + perfstat_memory_total_t memory; + + rc = perfstat_memory_total( + NULL, &memory, sizeof(perfstat_memory_total_t), 1); + if (rc <= 0){ + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return Py_BuildValue("KKKK", + (unsigned long long) memory.pgsp_total * pagesize, + (unsigned long long) memory.pgsp_free * pagesize, + (unsigned long long) memory.pgins * pagesize, + (unsigned long long) memory.pgouts * pagesize + ); +} + + +/* + * Return CPU statistics. + */ +static PyObject * +psutil_cpu_stats(PyObject *self, PyObject *args) { + int ncpu, rc, i; + // perfstat_cpu_total_t doesn't have invol/vol cswitch, only pswitch + // which is apparently something else. We have to sum over all cpus + perfstat_cpu_t *cpu = NULL; + perfstat_id_t id; + u_longlong_t cswitches = 0; + u_longlong_t devintrs = 0; + u_longlong_t softintrs = 0; + u_longlong_t syscall = 0; + + /* get the number of cpus in ncpu */ + ncpu = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0); + if (ncpu <= 0){ + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + /* allocate enough memory to hold the ncpu structures */ + cpu = (perfstat_cpu_t *) malloc(ncpu * sizeof(perfstat_cpu_t)); + if (cpu == NULL) { + PyErr_NoMemory(); + goto error; + } + + strcpy(id.name, ""); + rc = perfstat_cpu(&id, cpu, sizeof(perfstat_cpu_t), ncpu); + + if (rc <= 0) { + PyErr_SetFromErrno(PyExc_OSError); + goto error; + } + + for (i = 0; i < ncpu; i++) { + cswitches += cpu[i].invol_cswitch + cpu[i].vol_cswitch; + devintrs += cpu[i].devintrs; + softintrs += cpu[i].softintrs; + syscall += cpu[i].syscall; + } + + free(cpu); + + return Py_BuildValue( + "KKKK", + cswitches, + devintrs, + softintrs, + syscall + ); + +error: + if (cpu != NULL) + free(cpu); + return NULL; +} + + +/* + * define the psutil C module methods and initialize the module. + */ +static PyMethodDef +PsutilMethods[] = +{ + // --- process-related functions + {"proc_basic_info", psutil_proc_basic_info, METH_VARARGS, + "Return process ppid, rss, vms, ctime, nice, nthreads, status and tty"}, + {"proc_name_and_args", psutil_proc_name_and_args, METH_VARARGS, + "Return process name and args."}, + {"proc_cpu_times", psutil_proc_cpu_times, METH_VARARGS, + "Return process user and system CPU times."}, + {"proc_cred", psutil_proc_cred, METH_VARARGS, + "Return process uids/gids."}, + {"proc_threads", psutil_proc_threads, METH_VARARGS, + "Return process threads"}, + {"proc_io_counters", psutil_proc_io_counters, METH_VARARGS, + "Get process I/O counters."}, + + // --- system-related functions + {"users", psutil_users, METH_VARARGS, + "Return currently connected users."}, + {"disk_partitions", psutil_disk_partitions, METH_VARARGS, + "Return disk partitions."}, + {"boot_time", psutil_boot_time, METH_VARARGS, + "Return system boot time in seconds since the EPOCH."}, + {"per_cpu_times", psutil_per_cpu_times, METH_VARARGS, + "Return system per-cpu times as a list of tuples"}, + {"disk_io_counters", psutil_disk_io_counters, METH_VARARGS, + "Return a Python dict of tuples for disk I/O statistics."}, + {"virtual_mem", psutil_virtual_mem, METH_VARARGS, + "Return system virtual memory usage statistics"}, + {"swap_mem", psutil_swap_mem, METH_VARARGS, + "Return stats about swap memory, in bytes"}, + {"net_io_counters", psutil_net_io_counters, METH_VARARGS, + "Return a Python dict of tuples for network I/O statistics."}, + {"net_connections", psutil_net_connections, METH_VARARGS, + "Return system-wide connections"}, + {"net_if_stats", psutil_net_if_stats, METH_VARARGS, + "Return NIC stats (isup, mtu)"}, + {"cpu_stats", psutil_cpu_stats, METH_VARARGS, + "Return CPU statistics"}, + + // --- others + {"py_psutil_testing", py_psutil_testing, METH_VARARGS, + "Return True if PSUTIL_TESTING env var is set"}, + + {NULL, NULL, 0, NULL} +}; + + +struct module_state { + PyObject *error; +}; + +#if PY_MAJOR_VERSION >= 3 +#define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) +#else +#define GETSTATE(m) (&_state) +#endif + +#if PY_MAJOR_VERSION >= 3 + +static int +psutil_aix_traverse(PyObject *m, visitproc visit, void *arg) { + Py_VISIT(GETSTATE(m)->error); + return 0; +} + +static int +psutil_aix_clear(PyObject *m) { + Py_CLEAR(GETSTATE(m)->error); + return 0; +} + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "psutil_aix", + NULL, + sizeof(struct module_state), + PsutilMethods, + NULL, + psutil_aix_traverse, + psutil_aix_clear, + NULL +}; + +#define INITERROR return NULL + +PyMODINIT_FUNC PyInit__psutil_aix(void) + +#else +#define INITERROR return + +void init_psutil_aix(void) +#endif +{ +#if PY_MAJOR_VERSION >= 3 + PyObject *module = PyModule_Create(&moduledef); +#else + PyObject *module = Py_InitModule("_psutil_aix", PsutilMethods); +#endif + PyModule_AddIntConstant(module, "version", PSUTIL_VERSION); + + PyModule_AddIntConstant(module, "SIDL", SIDL); + PyModule_AddIntConstant(module, "SZOMB", SZOMB); + PyModule_AddIntConstant(module, "SACTIVE", SACTIVE); + PyModule_AddIntConstant(module, "SSWAP", SSWAP); + PyModule_AddIntConstant(module, "SSTOP", SSTOP); + + PyModule_AddIntConstant(module, "TCPS_CLOSED", TCPS_CLOSED); + PyModule_AddIntConstant(module, "TCPS_CLOSING", TCPS_CLOSING); + PyModule_AddIntConstant(module, "TCPS_CLOSE_WAIT", TCPS_CLOSE_WAIT); + PyModule_AddIntConstant(module, "TCPS_LISTEN", TCPS_LISTEN); + PyModule_AddIntConstant(module, "TCPS_ESTABLISHED", TCPS_ESTABLISHED); + PyModule_AddIntConstant(module, "TCPS_SYN_SENT", TCPS_SYN_SENT); + PyModule_AddIntConstant(module, "TCPS_SYN_RCVD", TCPS_SYN_RECEIVED); + PyModule_AddIntConstant(module, "TCPS_FIN_WAIT_1", TCPS_FIN_WAIT_1); + PyModule_AddIntConstant(module, "TCPS_FIN_WAIT_2", TCPS_FIN_WAIT_2); + PyModule_AddIntConstant(module, "TCPS_LAST_ACK", TCPS_LAST_ACK); + PyModule_AddIntConstant(module, "TCPS_TIME_WAIT", TCPS_TIME_WAIT); + PyModule_AddIntConstant(module, "PSUTIL_CONN_NONE", PSUTIL_CONN_NONE); + + if (module == NULL) + INITERROR; +#if PY_MAJOR_VERSION >= 3 + return module; +#endif +} diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index 80c1b8cba..5268b7215 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -18,6 +18,8 @@ #ifdef PSUTIL_SUNOS10 #include "arch/solaris/v10/ifaddrs.h" +#elif PSUTIL_AIX + #include "arch/aix/ifaddrs.h" #else #include #endif @@ -35,6 +37,8 @@ #elif defined(PSUTIL_SUNOS) #include #include +#elif defined(PSUTIL_AIX) + #include #endif #include "_psutil_common.h" @@ -688,7 +692,7 @@ void init_psutil_posix(void) PyObject *module = Py_InitModule("_psutil_posix", PsutilMethods); #endif -#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) || defined(PSUTIL_SUNOS) +#if defined(PSUTIL_BSD) || defined(PSUTIL_OSX) || defined(PSUTIL_SUNOS) || defined(PSUTIL_AIX) PyModule_AddIntConstant(module, "AF_LINK", AF_LINK); #endif diff --git a/psutil/arch/aix/ifaddrs.c b/psutil/arch/aix/ifaddrs.c new file mode 100644 index 000000000..1a819365a --- /dev/null +++ b/psutil/arch/aix/ifaddrs.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/*! Based on code from + https://lists.samba.org/archive/samba-technical/2009-February/063079.html +!*/ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ifaddrs.h" + +#define MAX(x,y) ((x)>(y)?(x):(y)) +#define SIZE(p) MAX((p).sa_len,sizeof(p)) + + +static struct sockaddr * +sa_dup(struct sockaddr *sa1) +{ + struct sockaddr *sa2; + size_t sz = sa1->sa_len; + sa2 = (struct sockaddr *) calloc(1, sz); + if (sa2 == NULL) + return NULL; + memcpy(sa2, sa1, sz); + return sa2; +} + + +void freeifaddrs(struct ifaddrs *ifp) +{ + if (NULL == ifp) return; + free(ifp->ifa_name); + free(ifp->ifa_addr); + free(ifp->ifa_netmask); + free(ifp->ifa_dstaddr); + freeifaddrs(ifp->ifa_next); + free(ifp); +} + + +int getifaddrs(struct ifaddrs **ifap) +{ + int sd, ifsize; + char *ccp, *ecp; + struct ifconf ifc; + struct ifreq *ifr; + struct ifaddrs *cifa = NULL; /* current */ + struct ifaddrs *pifa = NULL; /* previous */ + const size_t IFREQSZ = sizeof(struct ifreq); + int fam; + + *ifap = NULL; + + sd = socket(AF_INET, SOCK_DGRAM, 0); + if (sd == -1) + goto error; + + /* find how much memory to allocate for the SIOCGIFCONF call */ + if (ioctl(sd, SIOCGSIZIFCONF, (caddr_t)&ifsize) < 0) + goto error; + + ifc.ifc_req = (struct ifreq *) calloc(1, ifsize); + if (ifc.ifc_req == NULL) + goto error; + ifc.ifc_len = ifsize; + + if (ioctl(sd, SIOCGIFCONF, &ifc) < 0) + goto error; + + ccp = (char *)ifc.ifc_req; + ecp = ccp + ifsize; + + while (ccp < ecp) { + + ifr = (struct ifreq *) ccp; + ifsize = sizeof(ifr->ifr_name) + SIZE(ifr->ifr_addr); + fam = ifr->ifr_addr.sa_family; + + if (fam == AF_INET || fam == AF_INET6) { + cifa = (struct ifaddrs *) calloc(1, sizeof(struct ifaddrs)); + if (cifa == NULL) + goto error; + cifa->ifa_next = NULL; + + if (pifa == NULL) *ifap = cifa; /* first one */ + else pifa->ifa_next = cifa; + + cifa->ifa_name = strdup(ifr->ifr_name); + if (cifa->ifa_name == NULL) + goto error; + cifa->ifa_flags = 0; + cifa->ifa_dstaddr = NULL; + + cifa->ifa_addr = sa_dup(&ifr->ifr_addr); + if (cifa->ifa_addr == NULL) + goto error; + + if (fam == AF_INET) { + if (ioctl(sd, SIOCGIFNETMASK, ifr, IFREQSZ) < 0) + goto error; + cifa->ifa_netmask = sa_dup(&ifr->ifr_addr); + if (cifa->ifa_netmask == NULL) + goto error; + } + + if (0 == ioctl(sd, SIOCGIFFLAGS, ifr)) /* optional */ + cifa->ifa_flags = ifr->ifr_flags; + + if (fam == AF_INET) { + if (ioctl(sd, SIOCGIFDSTADDR, ifr, IFREQSZ) < 0) { + if (0 == ioctl(sd, SIOCGIFBRDADDR, ifr, IFREQSZ)) { + cifa->ifa_dstaddr = sa_dup(&ifr->ifr_addr); + if (cifa->ifa_dstaddr == NULL) + goto error; + } + } + else { + cifa->ifa_dstaddr = sa_dup(&ifr->ifr_addr); + if (cifa->ifa_dstaddr == NULL) + goto error; + } + } + pifa = cifa; + } + + ccp += ifsize; + } + free(ifc.ifc_req); + close(sd); + return 0; +error: + if (ifc.ifc_req != NULL) + free(ifc.ifc_req); + if (sd != -1) + close(sd); + freeifaddrs(*ifap); + return (-1); +} \ No newline at end of file diff --git a/psutil/arch/aix/ifaddrs.h b/psutil/arch/aix/ifaddrs.h new file mode 100644 index 000000000..3920c1ccc --- /dev/null +++ b/psutil/arch/aix/ifaddrs.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/*! Based on code from + https://lists.samba.org/archive/samba-technical/2009-February/063079.html +!*/ + + +#ifndef GENERIC_AIX_IFADDRS_H +#define GENERIC_AIX_IFADDRS_H + +#include +#include + +#undef ifa_dstaddr +#undef ifa_broadaddr +#define ifa_broadaddr ifa_dstaddr + +struct ifaddrs { + struct ifaddrs *ifa_next; + char *ifa_name; + unsigned int ifa_flags; + struct sockaddr *ifa_addr; + struct sockaddr *ifa_netmask; + struct sockaddr *ifa_dstaddr; +}; + +extern int getifaddrs(struct ifaddrs **); +extern void freeifaddrs(struct ifaddrs *); + +#endif \ No newline at end of file diff --git a/psutil/arch/aix/net_connections.c b/psutil/arch/aix/net_connections.c new file mode 100644 index 000000000..364cd1b7e --- /dev/null +++ b/psutil/arch/aix/net_connections.c @@ -0,0 +1,346 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/* Baded on code from lsof: + * http://www.ibm.com/developerworks/aix/library/au-lsof.html + * - dialects/aix/dproc.c:gather_proc_info + * - lib/prfp.c:process_file + * - dialects/aix/dsock.c:process_socket + * - dialects/aix/dproc.c:get_kernel_access +*/ + +#include "net_connections.h" +#include +#include +#define _KERNEL 1 +#include +#undef _KERNEL +#include +#include +#include +#include +#include +#include +#include "net_kernel_structs.h" + + + +#define PROCINFO_INCR (256) +#define PROCSIZE (sizeof(struct procentry64)) +#define FDSINFOSIZE (sizeof(struct fdsinfo64)) +#define KMEM "/dev/kmem" +#define NO_SOCKET (PyObject *)(-1) + +typedef u_longlong_t KA_T; +static int PSUTIL_CONN_NONE = 128; + +/* psutil_kread() - read from kernel memory */ +static int +psutil_kread( + int Kd, /* kernel memory file descriptor */ + KA_T addr, /* kernel memory address */ + char *buf, /* buffer to receive data */ + size_t len) { /* length to read */ + int br; + + if (lseek64(Kd, (off64_t)addr, L_SET) == (off64_t)-1) { + PyErr_SetFromErrno(PyExc_OSError); + return 1; + } + br = read(Kd, buf, len); + if (br == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return 1; + } + if (br != len) { + PyErr_SetString(PyExc_RuntimeError, + "size mismatch when reading kernel memory fd"); + return 1; + } + return 0; +} + +static int +read_unp_addr( + int Kd, + KA_T unp_addr, + char *buf, + size_t buflen +) { + struct sockaddr_un *ua = (struct sockaddr_un *)NULL; + struct sockaddr_un un; + struct mbuf64 mb; + int uo; + + if (psutil_kread(Kd, unp_addr, (char *)&mb, sizeof(mb))) { + return 1; + } + + uo = (int)(mb.m_hdr.mh_data - unp_addr); + if ((uo + sizeof(struct sockaddr)) <= sizeof(mb)) + ua = (struct sockaddr_un *)((char *)&mb + uo); + else { + if (psutil_kread(Kd, (KA_T)mb.m_hdr.mh_data, + (char *)&un, sizeof(un))) { + return 1; + } + ua = &un; + } + if (ua && ua->sun_path[0]) { + if (mb.m_len > sizeof(struct sockaddr_un)) + mb.m_len = sizeof(struct sockaddr_un); + *((char *)ua + mb.m_len - 1) = '\0'; + snprintf(buf, buflen, "%s", ua->sun_path); + } + return 0; +} + +static PyObject * +process_file(int Kd, pid32_t pid, int fd, KA_T fp) { + struct file64 f; + struct socket64 s; + struct protosw64 p; + struct domain d; + struct inpcb64 inp; + int fam; + struct tcpcb64 t; + int state = PSUTIL_CONN_NONE; + unsigned char *laddr = (unsigned char *)NULL; + unsigned char *raddr = (unsigned char *)NULL; + int rport, lport; + char laddr_str[INET6_ADDRSTRLEN]; + char raddr_str[INET6_ADDRSTRLEN]; + struct unpcb64 unp; + char unix_laddr_str[PATH_MAX] = { 0 }; + char unix_raddr_str[PATH_MAX] = { 0 }; + + /* Read file structure */ + if (psutil_kread(Kd, fp, (char *)&f, sizeof(f))) { + return NULL; + } + if (!f.f_count || f.f_type != DTYPE_SOCKET) { + return NO_SOCKET; + } + + if (psutil_kread(Kd, (KA_T) f.f_data, (char *) &s, sizeof(s))) { + return NULL; + } + + if (!s.so_type) { + return NO_SOCKET; + } + + if (!s.so_proto) { + PyErr_SetString(PyExc_RuntimeError, "invalid socket protocol handle"); + return NULL; + } + if (psutil_kread(Kd, (KA_T)s.so_proto, (char *)&p, sizeof(p))) { + return NULL; + } + + if (!p.pr_domain) { + PyErr_SetString(PyExc_RuntimeError, "invalid socket protocol domain"); + return NULL; + } + if (psutil_kread(Kd, (KA_T)p.pr_domain, (char *)&d, sizeof(d))) { + return NULL; + } + + fam = d.dom_family; + if (fam == AF_INET || fam == AF_INET6) { + /* Read protocol control block */ + if (!s.so_pcb) { + PyErr_SetString(PyExc_RuntimeError, "invalid socket PCB"); + return NULL; + } + if (psutil_kread(Kd, (KA_T) s.so_pcb, (char *) &inp, sizeof(inp))) { + return NULL; + } + + if (p.pr_protocol == IPPROTO_TCP) { + /* If this is a TCP socket, read its control block */ + if (inp.inp_ppcb + && !psutil_kread(Kd, (KA_T)inp.inp_ppcb, + (char *)&t, sizeof(t))) + state = t.t_state; + } + + if (fam == AF_INET6) { + laddr = (unsigned char *)&inp.inp_laddr6; + if (!IN6_IS_ADDR_UNSPECIFIED(&inp.inp_faddr6)) { + raddr = (unsigned char *)&inp.inp_faddr6; + rport = (int)ntohs(inp.inp_fport); + } + } + if (fam == AF_INET) { + laddr = (unsigned char *)&inp.inp_laddr; + if (inp.inp_faddr.s_addr != INADDR_ANY || inp.inp_fport != 0) { + raddr = (unsigned char *)&inp.inp_faddr; + rport = (int)ntohs(inp.inp_fport); + } + } + lport = (int)ntohs(inp.inp_lport); + + inet_ntop(fam, laddr, laddr_str, sizeof(laddr_str)); + + if (raddr != NULL) { + inet_ntop(fam, raddr, raddr_str, sizeof(raddr_str)); + return Py_BuildValue("(iii(si)(si)ii)", fd, fam, + s.so_type, laddr_str, lport, raddr_str, + rport, state, pid); + } + else { + return Py_BuildValue("(iii(si)()ii)", fd, fam, + s.so_type, laddr_str, lport, state, + pid); + } + } + + + if (fam == AF_UNIX) { + if (psutil_kread(Kd, (KA_T) s.so_pcb, (char *)&unp, sizeof(unp))) { + return NULL; + } + if ((KA_T) f.f_data != (KA_T) unp.unp_socket) { + PyErr_SetString(PyExc_RuntimeError, "unp_socket mismatch"); + return NULL; + } + + if (unp.unp_addr) { + if (read_unp_addr(Kd, unp.unp_addr, unix_laddr_str, + sizeof(unix_laddr_str))) { + return NULL; + } + } + + if (unp.unp_conn) { + if (psutil_kread(Kd, (KA_T) unp.unp_conn, (char *)&unp, + sizeof(unp))) { + return NULL; + } + if (read_unp_addr(Kd, unp.unp_addr, unix_raddr_str, + sizeof(unix_raddr_str))) { + return NULL; + } + } + + return Py_BuildValue("(iiissii)", fd, d.dom_family, + s.so_type, unix_laddr_str, unix_raddr_str, PSUTIL_CONN_NONE, + pid); + } + return NO_SOCKET; +} + +PyObject * +psutil_net_connections(PyObject *self, PyObject *args) { + PyObject *py_retlist = PyList_New(0); + PyObject *py_tuple = NULL; + KA_T fp; + int Kd = -1; + int i, np; + struct procentry64 *p; + struct fdsinfo64 *fds = (struct fdsinfo64 *)NULL; + size_t msz; + pid32_t requested_pid; + pid32_t pid; + int Np = 0; /* number of processes */ + struct procentry64 *processes = (struct procentry64 *)NULL; + /* the process table */ + + if (py_retlist == NULL) + goto error; + if (! PyArg_ParseTuple(args, "i", &requested_pid)) + goto error; + + Kd = open(KMEM, O_RDONLY, 0); + if (Kd < 0) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, KMEM); + goto error; + } + + /* Read the process table */ + msz = (size_t)(PROCSIZE * PROCINFO_INCR); + processes = (struct procentry64 *)malloc(msz); + if (!processes) { + PyErr_NoMemory(); + goto error; + } + Np = PROCINFO_INCR; + np = pid = 0; + p = processes; + while ((i = getprocs64(p, PROCSIZE, (struct fdsinfo64 *)NULL, 0, &pid, + PROCINFO_INCR)) + == PROCINFO_INCR) { + np += PROCINFO_INCR; + if (np >= Np) { + msz = (size_t)(PROCSIZE * (Np + PROCINFO_INCR)); + processes = (struct procentry64 *)realloc((char *)processes, msz); + if (!processes) { + PyErr_NoMemory(); + goto error; + } + Np += PROCINFO_INCR; + } + p = (struct procentry64 *)((char *)processes + (np * PROCSIZE)); + } + + if (i > 0) + np += i; + + /* Loop through processes */ + for (p = processes; np > 0; np--, p++) { + pid = p->pi_pid; + if (requested_pid != -1 && requested_pid != pid) + continue; + if (p->pi_state == 0 || p->pi_state == SZOMB) + continue; + + + if (!fds) { + fds = (struct fdsinfo64 *)malloc((size_t)FDSINFOSIZE); + if (!fds) { + PyErr_NoMemory(); + goto error; + } + } + if (getprocs64((struct procentry64 *)NULL, PROCSIZE, fds, FDSINFOSIZE, + &pid, 1) + != 1) + continue; + + /* loop over file descriptors */ + for (i = 0; i < p->pi_maxofile; i++) { + fp = (KA_T)fds->pi_ufd[i].fp; + if (fp) { + py_tuple = process_file(Kd, p->pi_pid, i, fp); + if (py_tuple == NULL) + goto error; + if (py_tuple != NO_SOCKET) { + if (PyList_Append(py_retlist, py_tuple)) + goto error; + Py_DECREF(py_tuple); + } + } + } + } + close(Kd); + free(processes); + if (fds != NULL) + free(fds); + return py_retlist; + +error: + Py_XDECREF(py_tuple); + Py_DECREF(py_retlist); + if (Kd > 0) + close(Kd); + if (processes != NULL) + free(processes); + if (fds != NULL) + free(fds); + return NULL; +} diff --git a/psutil/arch/aix/net_connections.h b/psutil/arch/aix/net_connections.h new file mode 100644 index 000000000..f6a726cb9 --- /dev/null +++ b/psutil/arch/aix/net_connections.h @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include + +PyObject* psutil_net_connections(PyObject *self, PyObject *args); diff --git a/psutil/arch/aix/net_kernel_structs.h b/psutil/arch/aix/net_kernel_structs.h new file mode 100644 index 000000000..09f320ff5 --- /dev/null +++ b/psutil/arch/aix/net_kernel_structs.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +/* The kernel is always 64 bit but Python is usually compiled as a 32 bit + * process. We're reading the kernel memory to get the network connections, + * so we need the structs we read to be defined with 64 bit "pointers". + * Here are the partial definitions of the structs we use, taken from the + * header files, with data type sizes converted to their 64 bit counterparts, + * and unused data truncated. */ + +#ifdef __64BIT__ +/* In case we're in a 64 bit process after all */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#define file64 file +#define socket64 socket +#define protosw64 protosw +#define inpcb64 inpcb +#define tcpcb64 tcpcb +#define unpcb64 unpcb +#define mbuf64 mbuf +#else + struct file64 { + int f_flag; + int f_count; + int f_options; + int f_type; + u_longlong_t f_data; + }; + + struct socket64 { + short so_type; /* generic type, see socket.h */ + short so_options; /* from socket call, see socket.h */ + ushort so_linger; /* time to linger while closing */ + short so_state; /* internal state flags SS_*, below */ + u_longlong_t so_pcb; /* protocol control block */ + u_longlong_t so_proto; /* protocol handle */ + }; + + struct protosw64 { + short pr_type; /* socket type used for */ + u_longlong_t pr_domain; /* domain protocol a member of */ + short pr_protocol; /* protocol number */ + short pr_flags; /* see below */ + }; + + struct inpcb64 { + u_longlong_t inp_next,inp_prev; + /* pointers to other pcb's */ + u_longlong_t inp_head; /* pointer back to chain of inpcb's + for this protocol */ + u_int32_t inp_iflowinfo; /* input flow label */ + u_short inp_fport; /* foreign port */ + u_int16_t inp_fatype; /* foreign address type */ + union in_addr_6 inp_faddr_6; /* foreign host table entry */ + u_int32_t inp_oflowinfo; /* output flow label */ + u_short inp_lport; /* local port */ + u_int16_t inp_latype; /* local address type */ + union in_addr_6 inp_laddr_6; /* local host table entry */ + u_longlong_t inp_socket; /* back pointer to socket */ + u_longlong_t inp_ppcb; /* pointer to per-protocol pcb */ + u_longlong_t space_rt; + struct sockaddr_in6 spare_dst; + u_longlong_t inp_ifa; /* interface address to use */ + int inp_flags; /* generic IP/datagram flags */ +}; + +struct tcpcb64 { + u_longlong_t seg__next; + u_longlong_t seg__prev; + short t_state; /* state of this connection */ +}; + +struct unpcb64 { + u_longlong_t unp_socket; /* pointer back to socket */ + u_longlong_t unp_vnode; /* if associated with file */ + ino_t unp_vno; /* fake vnode number */ + u_longlong_t unp_conn; /* control block of connected socket */ + u_longlong_t unp_refs; /* referencing socket linked list */ + u_longlong_t unp_nextref; /* link in unp_refs list */ + u_longlong_t unp_addr; /* bound address of socket */ +}; + +struct m_hdr64 +{ + u_longlong_t mh_next; /* next buffer in chain */ + u_longlong_t mh_nextpkt; /* next chain in queue/record */ + long mh_len; /* amount of data in this mbuf */ + u_longlong_t mh_data; /* location of data */ +}; + +struct mbuf64 +{ + struct m_hdr64 m_hdr; +}; + +#define m_len m_hdr.mh_len + +#endif \ No newline at end of file diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index c9cd5006b..033d925e3 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -72,6 +72,7 @@ "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", "HAS_SENSORS_BATTERY", "HAS_BATTERY""HAS_SENSORS_FANS", "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", + "HAS_NUM_CTX_SWITCHES", # subprocesses 'pyrun', 'reap_children', 'get_test_subprocess', 'create_zombie_proc', 'create_proc_children_pair', @@ -156,6 +157,7 @@ HAS_IONICE = hasattr(psutil.Process, "ionice") HAS_MEMORY_FULL_INFO = 'uss' in psutil.Process().memory_full_info()._fields HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps") +HAS_NUM_CTX_SWITCHES = hasattr(psutil.Process, "num_ctx_switches") HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") HAS_RLIMIT = hasattr(psutil.Process, "rlimit") HAS_SENSORS_BATTERY = hasattr(psutil, "sensors_battery") diff --git a/psutil/tests/test_aix.py b/psutil/tests/test_aix.py new file mode 100644 index 000000000..281c9bb0f --- /dev/null +++ b/psutil/tests/test_aix.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python + +# Copyright (c) 2009, Giampaolo Rodola' +# Copyright (c) 2017, Arnon Yaari +# All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""AIX specific tests.""" + +import os +import re + +import psutil +from psutil import AIX +from psutil.tests import retry_before_failing +from psutil.tests import run_test_module_by_name +from psutil.tests import sh +from psutil.tests import unittest + +@unittest.skipIf(not AIX, "AIX only") +class AIXSpecificTestCase(unittest.TestCase): + + def test_virtual_memory(self): + out = sh('/usr/bin/svmon -O unit=KB') + + # example output: + # Unit: KB + # -------------------------------------------------------------------------------------- + # size inuse free pin virtual available mmode + # memory 4194304 1844828 2349476 1250208 1412976 2621596 Ded + # pg space 524288 8304 + re_pattern = "memory\s*" + for field in ("size inuse free pin virtual available mmode").split(): + re_pattern += "(?P<%s>\S+)\s+" % (field,) + matchobj = re.search(re_pattern, out) + + self.assertIsNotNone(matchobj, + "svmon command returned unexpected output") + + KB = 1024 + total = int(matchobj.group("size")) * KB + available = int(matchobj.group("available")) * KB + used = int(matchobj.group("inuse")) * KB + free = int(matchobj.group("free")) * KB + + psutil_result = psutil.virtual_memory() + + # MEMORY_TOLERANCE from psutil.tests is not enough. For some reason + # we're seeing differences of ~1.2 MB. 2 MB is still a good tolerance + # when compared to GBs. + MEMORY_TOLERANCE = 2 * KB * KB # 2 MB + self.assertEqual(psutil_result.total, total) + self.assertAlmostEqual(psutil_result.used, used, + delta=MEMORY_TOLERANCE) + self.assertAlmostEqual(psutil_result.available, available, + delta=MEMORY_TOLERANCE) + self.assertAlmostEqual(psutil_result.free, free, + delta=MEMORY_TOLERANCE) + + def test_swap_memory(self): + out = sh('/usr/sbin/lsps -a') + + # example output: + # Page Space Physical Volume Volume Group Size %Used Active Auto Type Chksum + # hd6 hdisk0 rootvg 512MB 2 yes yes lv 0 + # from the man page, "The size is given in megabytes" so we assume + # we'll always have 'MB' in the result + # TODO maybe try to use "swap -l" to check "used" too, but its units + # are not guaranteed to be "MB" so parsing may not be consistent + matchobj = re.search("(?P\S+)\s+" + "(?P\S+)\s+" + "(?P\S+)\s+" + "(?P\d+)MB", out) + + self.assertIsNotNone(matchobj, + "lsps command returned unexpected output") + + total_mb = int(matchobj.group("size")) + MB = 1024 ** 2 + psutil_result = psutil.swap_memory() + # we divide our result by MB instead of multiplying the lsps value by + # MB because lsps may round down, so we round down too + self.assertEqual(int(psutil_result.total / MB), total_mb) + + def test_cpu_stats(self): + out = sh('/usr/bin/mpstat -a') + + re_pattern = "ALL\s*" + for field in ("min maj mpcs mpcr dev soft dec ph cs ics bound rq " + "push S3pull S3grd S0rd S1rd S2rd S3rd S4rd S5rd " + "sysc").split(): + re_pattern += "(?P<%s>\S+)\s+" % (field,) + matchobj = re.search(re_pattern, out) + + self.assertIsNotNone(matchobj, + "mpstat command returned unexpected output") + + # numbers are usually in the millions so 1000 is ok for tolerance + CPU_STATS_TOLERANCE = 1000 + psutil_result = psutil.cpu_stats() + self.assertAlmostEqual(psutil_result.ctx_switches, + int(matchobj.group("cs")), + delta=CPU_STATS_TOLERANCE) + self.assertAlmostEqual(psutil_result.syscalls, + int(matchobj.group("sysc")), + delta=CPU_STATS_TOLERANCE) + self.assertAlmostEqual(psutil_result.interrupts, + int(matchobj.group("dev")), + delta=CPU_STATS_TOLERANCE) + self.assertAlmostEqual(psutil_result.soft_interrupts, + int(matchobj.group("soft")), + delta=CPU_STATS_TOLERANCE) + + def test_cpu_count_logical(self): + out = sh('/usr/bin/mpstat -a') + mpstat_lcpu = int(re.search("lcpu=(\d+)", out).group(1)) + psutil_lcpu = psutil.cpu_count(logical=True) + self.assertEqual(mpstat_lcpu, psutil_lcpu) + + def test_net_if_addrs_names(self): + out = sh('/etc/ifconfig -l') + ifconfig_names = set(out.split()) + psutil_names = set(psutil.net_if_addrs().keys()) + self.assertSetEqual(ifconfig_names, psutil_names) + + +if __name__ == '__main__': + run_test_module_by_name(__file__) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 65bad757f..13a737e84 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -16,6 +16,7 @@ import traceback from contextlib import closing +from psutil import AIX from psutil import BSD from psutil import FREEBSD from psutil import LINUX @@ -65,7 +66,8 @@ def test_win_service(self): self.assertEqual(hasattr(psutil, "win_service_get"), WINDOWS) def test_PROCFS_PATH(self): - self.assertEqual(hasattr(psutil, "PROCFS_PATH"), LINUX or SUNOS) + self.assertEqual(hasattr(psutil, "PROCFS_PATH"), + LINUX or SUNOS or AIX) def test_win_priority(self): ae = self.assertEqual @@ -159,7 +161,7 @@ def test_proc_cpu_num(self): def test_proc_memory_maps(self): hasit = hasattr(psutil.Process, "memory_maps") - self.assertEqual(hasit, False if OPENBSD or NETBSD else True) + self.assertEqual(hasit, False if OPENBSD or NETBSD or AIX else True) # =================================================================== @@ -372,12 +374,14 @@ def pid(self, ret, proc): self.assertGreaterEqual(ret, 0) def ppid(self, ret, proc): - self.assertIsInstance(ret, int) + self.assertIsInstance(ret, (int, long)) self.assertGreaterEqual(ret, 0) def name(self, ret, proc): self.assertIsInstance(ret, str) - assert ret + # on AIX, "" processes don't have names + if not AIX: + assert ret def create_time(self, ret, proc): self.assertIsInstance(ret, float) @@ -482,7 +486,7 @@ def memory_info(self, ret, proc): for value in ret: self.assertIsInstance(value, (int, long)) self.assertGreaterEqual(value, 0) - if POSIX and ret.vms != 0: + if POSIX and not AIX and ret.vms != 0: # VMS is always supposed to be the highest for name in ret._fields: if name != 'vms': @@ -536,8 +540,8 @@ def connections(self, ret, proc): check_connection_ntuple(conn) def cwd(self, ret, proc): - self.assertIsInstance(ret, str) - if ret is not None: # BSD may return None + if ret: # 'ret' can be None or empty + self.assertIsInstance(ret, str) assert os.path.isabs(ret), ret try: st = os.stat(ret) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 680fe7803..b7638d322 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -24,6 +24,7 @@ import psutil import psutil._common +from psutil import AIX from psutil import LINUX from psutil import OPENBSD from psutil import OSX @@ -38,6 +39,7 @@ from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_IONICE from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_NUM_CTX_SWITCHES from psutil.tests import HAS_PROC_CPU_NUM from psutil.tests import HAS_PROC_IO_COUNTERS from psutil.tests import HAS_RLIMIT @@ -288,6 +290,7 @@ def test_num_fds(self): self.execute(self.proc.num_fds) @skip_if_linux() + @unittest.skipIf(not HAS_NUM_CTX_SWITCHES, "not supported") def test_num_ctx_switches(self): self.execute(self.proc.num_ctx_switches) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 580abdfde..f42a6e637 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -15,6 +15,7 @@ import time import psutil +from psutil import AIX from psutil import BSD from psutil import LINUX from psutil import OPENBSD @@ -48,6 +49,8 @@ def ps(cmd): if SUNOS: cmd = cmd.replace("-o command", "-o comm") cmd = cmd.replace("-o start", "-o stime") + if AIX: + cmd = cmd.replace("-o rss", "-o rssize") output = sh(cmd) if not LINUX: output = output.split('\n')[1].strip() @@ -206,7 +209,9 @@ def test_cmdline(self): # incorrect value (20); the real deal is getpriority(2) which # returns 0; psutil relies on it, see: # https://github.com/giampaolo/psutil/issues/1082 + # AIX has the same issue @unittest.skipIf(SUNOS, "not reliable on SUNOS") + @unittest.skipIf(AIX, "not reliable on AIX") def test_nice(self): ps_nice = ps("ps --no-headers -o nice -p %s" % self.pid) psutil_nice = psutil.Process().nice() @@ -262,7 +267,7 @@ class TestSystemAPIs(unittest.TestCase): def test_pids(self): # Note: this test might fail if the OS is starting/killing # other processes in the meantime - if SUNOS: + if SUNOS or AIX: cmd = ["ps", "-A", "-o", "pid"] else: cmd = ["ps", "ax", "-o", "pid"] @@ -355,6 +360,8 @@ def test_os_waitpid_bad_ret_status(self): psutil._psposix.wait_pid, os.getpid()) assert m.called + # AIX can return '-' in df output instead of numbers, e.g. for /proc + @unittest.skipIf(AIX, "unreliable on AIX") def test_disk_usage(self): def df(device): out = sh("df -k %s" % device).strip() diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 0686ba25b..fab3a4215 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -21,6 +21,7 @@ import psutil +from psutil import AIX from psutil import BSD from psutil import LINUX from psutil import NETBSD @@ -44,6 +45,7 @@ from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_IONICE from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_NUM_CTX_SWITCHES from psutil.tests import HAS_PROC_CPU_NUM from psutil.tests import HAS_PROC_IO_COUNTERS from psutil.tests import HAS_RLIMIT @@ -317,7 +319,7 @@ def test_io_counters(self): with open(PYTHON, 'rb') as f: f.read() io2 = p.io_counters() - if not BSD: + if not BSD and not AIX: self.assertGreater(io2.read_count, io1.read_count) self.assertEqual(io2.write_count, io1.write_count) if LINUX: @@ -994,6 +996,7 @@ def test_num_fds(self): @skip_on_not_implemented(only_if=LINUX) @unittest.skipIf(OPENBSD or NETBSD, "not reliable on OPENBSD & NETBSD") + @unittest.skipIf(not HAS_NUM_CTX_SWITCHES, "not supported") def test_num_ctx_switches(self): p = psutil.Process() before = sum(p.num_ctx_switches()) diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index e93bb6b52..3485fb8f3 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -19,6 +19,7 @@ import time import psutil +from psutil import AIX from psutil import BSD from psutil import FREEBSD from psutil import LINUX @@ -742,7 +743,8 @@ def test_cpu_stats(self): for name in infos._fields: value = getattr(infos, name) self.assertGreaterEqual(value, 0) - if name in ('ctx_switches', 'interrupts'): + # on AIX, ctx_switches is always 0 + if not AIX and name in ('ctx_switches', 'interrupts'): self.assertGreater(value, 0) @unittest.skipIf(not HAS_CPU_FREQ, "not suported") diff --git a/scripts/procinfo.py b/scripts/procinfo.py index d8625560f..54205de36 100755 --- a/scripts/procinfo.py +++ b/scripts/procinfo.py @@ -225,7 +225,8 @@ def run(pid, verbose=False): if 'io_counters' in pinfo: print_('I/O', str_ntuple(pinfo['io_counters'], bytes2human=True)) - print_("ctx-switches", str_ntuple(pinfo['num_ctx_switches'])) + if 'num_ctx_switches' in pinfo: + print_("ctx-switches", str_ntuple(pinfo['num_ctx_switches'])) if pinfo['children']: template = "%-6s %s" print_("children", template % ("PID", "NAME")) diff --git a/setup.py b/setup.py index 5f2683497..1bc940839 100755 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ from _common import POSIX # NOQA from _common import SUNOS # NOQA from _common import WINDOWS # NOQA +from _common import AIX # NOQA macros = [] @@ -239,7 +240,17 @@ def get_ethtool_macro(): ], define_macros=macros, libraries=['kstat', 'nsl', 'socket']) - +# AIX +elif AIX: + macros.append(("PSUTIL_AIX", 1)) + ext = Extension( + 'psutil._psutil_aix', + sources=sources + [ + 'psutil/_psutil_aix.c', + 'psutil/arch/aix/net_connections.c', + 'psutil/arch/aix/ifaddrs.c'], + libraries=['perfstat'], + define_macros=macros) else: sys.exit('platform %s is not supported' % sys.platform) @@ -254,6 +265,8 @@ def get_ethtool_macro(): if platform.release() == '5.10': posix_extension.sources.append('psutil/arch/solaris/v10/ifaddrs.c') posix_extension.define_macros.append(('PSUTIL_SUNOS10', 1)) + elif AIX: + posix_extension.sources.append('psutil/arch/aix/ifaddrs.c') extensions = [ext, posix_extension] else: From 7ab8b570e758f9ef899b1041beafede489847ccb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 27 Sep 2017 14:43:08 +0800 Subject: [PATCH 794/922] update doc --- MANIFEST.in | 8 ++++++++ README.rst | 17 ++++++++++++----- docs/index.rst | 41 +++++++++++++++++++++++------------------ psutil/__init__.py | 5 +++-- 4 files changed, 46 insertions(+), 25 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index e2f8a3192..a2d0a5d02 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -20,14 +20,17 @@ include docs/conf.py include docs/index.rst include docs/make.bat include make.bat +include psutil/TODO.aix include psutil/__init__.py include psutil/_common.py include psutil/_compat.py +include psutil/_psaix.py include psutil/_psbsd.py include psutil/_pslinux.py include psutil/_psosx.py include psutil/_psposix.py include psutil/_pssunos.py +include psutil/_psutil_aix.c include psutil/_psutil_bsd.c include psutil/_psutil_common.c include psutil/_psutil_common.h @@ -38,6 +41,11 @@ include psutil/_psutil_posix.h include psutil/_psutil_sunos.c include psutil/_psutil_windows.c include psutil/_pswindows.py +include psutil/arch/aix/ifaddrs.c +include psutil/arch/aix/ifaddrs.h +include psutil/arch/aix/net_connections.c +include psutil/arch/aix/net_connections.h +include psutil/arch/aix/net_kernel_structs.h include psutil/arch/freebsd/proc_socks.c include psutil/arch/freebsd/proc_socks.h include psutil/arch/freebsd/specific.c diff --git a/README.rst b/README.rst index bd77387b9..1b45c50da 100644 --- a/README.rst +++ b/README.rst @@ -48,13 +48,20 @@ retrieving information on **running processes** and **system utilization** (CPU, memory, disks, network, sensors) in Python. It is useful mainly for **system monitoring**, **profiling and limiting process resources** and **management of running processes**. -It implements many functionalities offered by command line tools such as: +It implements many functionalities offered by UNIX command line tools such as: ps, top, lsof, netstat, ifconfig, who, df, kill, free, nice, ionice, iostat, iotop, uptime, pidof, tty, taskset, pmap. -It currently supports **Linux**, **Windows**, **OSX**, **Sun Solaris**, -**FreeBSD**, **OpenBSD** and **NetBSD**, -both **32-bit** and **64-bit** architectures, with Python versions from **2.6 -to 3.6** (users of Python 2.4 and 2.5 may use +psutil currently supports the following platforms: + +- **Linux** +- **Windows** +- **OSX**, +- **FreeBSD, OpenBSD**, **NetBSD** +- **Sun Solaris** +- **AIX** + +...both **32-bit** and **64-bit** architectures, with Python +versions from **2.6 to 3.6** (users of Python 2.4 and 2.5 may use `2.1.3 `__ version). `PyPy `__ is also known to work. diff --git a/docs/index.rst b/docs/index.rst index 3ab444617..b8becb3a6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,13 +8,13 @@ psutil documentation Quick links ----------- -* `Home page `__ -* `Install `_ -* `Blog `__ -* `Forum `__ -* `Download `__ -* `Development guide `_ -* `What's new `__ +- `Home page `__ +- `Install `_ +- `Blog `__ +- `Forum `__ +- `Download `__ +- `Development guide `_ +- `What's new `__ About ----- @@ -25,11 +25,19 @@ retrieving information on running in **Python**. It is useful mainly for **system monitoring**, **profiling**, **limiting process resources** and the **management of running processes**. -It implements many functionalities offered by command line tools +It implements many functionalities offered by UNIX command line tools such as: *ps, top, lsof, netstat, ifconfig, who, df, kill, free, nice, ionice, iostat, iotop, uptime, pidof, tty, taskset, pmap*. -It currently supports **Linux, Windows, OSX, Sun Solaris, FreeBSD, OpenBSD** -and **NetBSD**, both **32-bit** and **64-bit** architectures, with Python +psutil currently supports the following platforms: + +- **Linux** +- **Windows** +- **OSX**, +- **FreeBSD, OpenBSD**, **NetBSD** +- **Sun Solaris** +- **AIX** + +...both **32-bit** and **64-bit** architectures, with Python versions from **2.6 to 3.6** (users of Python 2.4 and 2.5 may use `2.1.3 `__ version). `PyPy `__ is also known to work. @@ -560,12 +568,8 @@ Network ...] .. note:: - (OSX) :class:`psutil.AccessDenied` is always raised unless running as root. - This is a limitation of the OS and ``lsof`` does the same. - - .. note:: - (AIX) :class:`psutil.AccessDenied` is always raised unless running as root - (lsof does the same). + (OSX and AIX) :class:`psutil.AccessDenied` is always raised unless running + as root. This is a limitation of the OS and ``lsof`` does the same. .. note:: (Solaris) UNIX sockets are not supported. @@ -2135,10 +2139,11 @@ Constants It must be noted that this trick works only for APIs which rely on /proc filesystem (e.g. `memory`_ APIs and most :class:`Process` class methods). - Availability: Linux, Solaris + Availability: Linux, Solaris, AIX .. versionadded:: 3.2.3 .. versionchanged:: 3.4.2 also available on Solaris. + .. versionchanged:: 5.4.0 also available on AIX. .. _const-pstatus: .. data:: STATUS_RUNNING @@ -2551,7 +2556,7 @@ Q&A * Q: What about load average? * A: psutil does not expose any load average function as it's already available in python as - `os.getloadavg `__ + `os.getloadavg `__. Running tests ============= diff --git a/psutil/__init__.py b/psutil/__init__.py index 9c6451e46..006d5f57f 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -11,10 +11,11 @@ - Linux - Windows - OSX - - Sun Solaris - FreeBSD - OpenBSD - NetBSD + - Sun Solaris + - AIX Works with Python versions from 2.6 to 3.X. """ @@ -211,7 +212,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.3.2" +__version__ = "5.4.0" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From a39c3c06d51e17b20088a3b519bcd120250b3219 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 27 Sep 2017 14:47:57 +0800 Subject: [PATCH 795/922] PEP8-ify code --- psutil/_psaix.py | 1 - psutil/tests/test_aix.py | 54 +++++++++++++------------------ psutil/tests/test_memory_leaks.py | 1 - 3 files changed, 23 insertions(+), 33 deletions(-) diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 102e0f5f6..2cbacacb4 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -42,7 +42,6 @@ AF_LINK = cext_posix.AF_LINK PROC_STATUSES = { - cext.SIDL: _common.STATUS_IDLE, cext.SZOMB: _common.STATUS_ZOMBIE, cext.SACTIVE: _common.STATUS_RUNNING, diff --git a/psutil/tests/test_aix.py b/psutil/tests/test_aix.py index 281c9bb0f..7a8a4c334 100644 --- a/psutil/tests/test_aix.py +++ b/psutil/tests/test_aix.py @@ -8,35 +8,27 @@ """AIX specific tests.""" -import os import re -import psutil from psutil import AIX -from psutil.tests import retry_before_failing from psutil.tests import run_test_module_by_name from psutil.tests import sh from psutil.tests import unittest +import psutil + @unittest.skipIf(not AIX, "AIX only") class AIXSpecificTestCase(unittest.TestCase): def test_virtual_memory(self): out = sh('/usr/bin/svmon -O unit=KB') - - # example output: - # Unit: KB - # -------------------------------------------------------------------------------------- - # size inuse free pin virtual available mmode - # memory 4194304 1844828 2349476 1250208 1412976 2621596 Ded - # pg space 524288 8304 re_pattern = "memory\s*" for field in ("size inuse free pin virtual available mmode").split(): re_pattern += "(?P<%s>\S+)\s+" % (field,) matchobj = re.search(re_pattern, out) - self.assertIsNotNone(matchobj, - "svmon command returned unexpected output") + self.assertIsNotNone( + matchobj, "svmon command returned unexpected output") KB = 1024 total = int(matchobj.group("size")) * KB @@ -51,20 +43,16 @@ def test_virtual_memory(self): # when compared to GBs. MEMORY_TOLERANCE = 2 * KB * KB # 2 MB self.assertEqual(psutil_result.total, total) - self.assertAlmostEqual(psutil_result.used, used, - delta=MEMORY_TOLERANCE) - self.assertAlmostEqual(psutil_result.available, available, - delta=MEMORY_TOLERANCE) - self.assertAlmostEqual(psutil_result.free, free, - delta=MEMORY_TOLERANCE) + self.assertAlmostEqual( + psutil_result.used, used, delta=MEMORY_TOLERANCE) + self.assertAlmostEqual( + psutil_result.available, available, delta=MEMORY_TOLERANCE) + self.assertAlmostEqual( + psutil_result.free, free, delta=MEMORY_TOLERANCE) def test_swap_memory(self): out = sh('/usr/sbin/lsps -a') - - # example output: - # Page Space Physical Volume Volume Group Size %Used Active Auto Type Chksum - # hd6 hdisk0 rootvg 512MB 2 yes yes lv 0 - # from the man page, "The size is given in megabytes" so we assume + # From the man page, "The size is given in megabytes" so we assume # we'll always have 'MB' in the result # TODO maybe try to use "swap -l" to check "used" too, but its units # are not guaranteed to be "MB" so parsing may not be consistent @@ -73,8 +61,8 @@ def test_swap_memory(self): "(?P\S+)\s+" "(?P\d+)MB", out) - self.assertIsNotNone(matchobj, - "lsps command returned unexpected output") + self.assertIsNotNone( + matchobj, "lsps command returned unexpected output") total_mb = int(matchobj.group("size")) MB = 1024 ** 2 @@ -93,22 +81,26 @@ def test_cpu_stats(self): re_pattern += "(?P<%s>\S+)\s+" % (field,) matchobj = re.search(re_pattern, out) - self.assertIsNotNone(matchobj, - "mpstat command returned unexpected output") + self.assertIsNotNone( + matchobj, "mpstat command returned unexpected output") # numbers are usually in the millions so 1000 is ok for tolerance CPU_STATS_TOLERANCE = 1000 psutil_result = psutil.cpu_stats() - self.assertAlmostEqual(psutil_result.ctx_switches, + self.assertAlmostEqual( + psutil_result.ctx_switches, int(matchobj.group("cs")), delta=CPU_STATS_TOLERANCE) - self.assertAlmostEqual(psutil_result.syscalls, + self.assertAlmostEqual( + psutil_result.syscalls, int(matchobj.group("sysc")), delta=CPU_STATS_TOLERANCE) - self.assertAlmostEqual(psutil_result.interrupts, + self.assertAlmostEqual( + psutil_result.interrupts, int(matchobj.group("dev")), delta=CPU_STATS_TOLERANCE) - self.assertAlmostEqual(psutil_result.soft_interrupts, + self.assertAlmostEqual( + psutil_result.soft_interrupts, int(matchobj.group("soft")), delta=CPU_STATS_TOLERANCE) diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index b7638d322..76fab357b 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -24,7 +24,6 @@ import psutil import psutil._common -from psutil import AIX from psutil import LINUX from psutil import OPENBSD from psutil import OSX From 6acb8d71e1db4b01ef298408c7423950c867f1c3 Mon Sep 17 00:00:00 2001 From: Prodesire Date: Wed, 27 Sep 2017 09:11:24 -0500 Subject: [PATCH 796/922] =?UTF-8?q?fix=20error=20on=20REHL=205.0:=20expect?= =?UTF-8?q?ed=20specifier-qualifier-list=20before=20=E2=80=98=5F=5Fu3=20(#?= =?UTF-8?q?1139)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- psutil/_psutil_posix.c | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index 5268b7215..ea0fba534 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -26,6 +26,7 @@ #if defined(PSUTIL_LINUX) #include + #include #include #elif defined(PSUTIL_BSD) || defined(PSUTIL_OSX) #include From 12109ee217c247cf68735589fa55e6c721e463fc Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 27 Sep 2017 23:19:14 +0800 Subject: [PATCH 797/922] #1138: update HISTORY --- CREDITS | 4 ++++ HISTORY.rst | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index 3948189d2..e4b4ae630 100644 --- a/CREDITS +++ b/CREDITS @@ -485,3 +485,7 @@ I: 1042, 1079 N: Oleksii Shevchuk W: https://github.com/alxchk I: 1077, 1093, 1091 + +N: Prodesire +W: https://github.com/Prodesire +I: 1138 diff --git a/HISTORY.rst b/HISTORY.rst index 8096e82db..86f72e00e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,12 +2,18 @@ XXXX-XX-XX -5.3.2 +5.4.0 ===== +**Enhancements** + +- 1123_: [AIX] added support for AIX platform. (patch by Arnon Yaari) + *Bug fixes* - 1131_: [SunOS] fix compilation warnings. (patch by Arnon Yaari) +- 1138_: [Linux] can't compile on CentOS 5.0 and RedHat 5.0. + (patch by Prodesire) 2017-09-10 From da68e02d1f84de6821b0468aaaa3e7475c6d0e49 Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Wed, 27 Sep 2017 18:50:46 +0300 Subject: [PATCH 798/922] Fix #1136: check PID in AIX in procfs (#1137) The 'kill' method to check whether processes exist doesn't work on 'wait' processes in AIX. All processes (and only processes) have a "psinfo" file in procfs so we can check whether PIDs exist there. --- psutil/TODO.aix | 1 - psutil/_psaix.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/psutil/TODO.aix b/psutil/TODO.aix index 495f4963d..671372dc0 100644 --- a/psutil/TODO.aix +++ b/psutil/TODO.aix @@ -13,4 +13,3 @@ Known limitations: The following unit tests may fail: test_prog_w_funky_name funky name tests don't work, name is truncated test_cmdline long args are cut from cmdline in /proc/pid/psinfo and getargs - test_pid_exists_2 there are pids in /proc that don't really exist diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 2cbacacb4..663748bb8 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -321,7 +321,7 @@ def pids(): def pid_exists(pid): """Check for the existence of a unix pid.""" - return _psposix.pid_exists(pid) + return os.path.exists(os.path.join(get_procfs_path(), str(pid), "psinfo")) def wrap_exceptions(fun): From 943367f969ac3a49cae2da3186500a68cc3ea6ae Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 28 Sep 2017 01:00:01 +0800 Subject: [PATCH 799/922] 1129: have sensors_temperatures() on Linux skip entry on IOError --- HISTORY.rst | 1 + psutil/_pslinux.py | 21 +++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 86f72e00e..5c301a694 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,7 @@ XXXX-XX-XX *Bug fixes* +- 1129_: [Linux] sensors_temperatures() may crash with IOError. - 1131_: [SunOS] fix compilation warnings. (patch by Arnon Yaari) - 1138_: [Linux] can't compile on CentOS 5.0 and RedHat 5.0. (patch by Prodesire) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index e9ee7df04..c4abacc73 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1142,21 +1142,22 @@ def sensors_temperatures(): basenames = sorted(set([x.split('_')[0] for x in basenames])) for base in basenames: + try: + current = float(cat(base + '_input')) / 1000.0 + except (IOError, OSError) as err: + # A lot of things can go wrong here, so let's just skip the + # whole entry. + # https://github.com/giampaolo/psutil/issues/1009 + # https://github.com/giampaolo/psutil/issues/1101 + # https://github.com/giampaolo/psutil/issues/1129 + warnings.warn("ignoring %r" % err, RuntimeWarning) + continue + unit_name = cat(os.path.join(os.path.dirname(base), 'name'), binary=False) high = cat(base + '_max', fallback=None) critical = cat(base + '_crit', fallback=None) label = cat(base + '_label', fallback='', binary=False) - try: - current = float(cat(base + '_input')) / 1000.0 - except OSError as err: - # https://github.com/giampaolo/psutil/issues/1009 - # https://github.com/giampaolo/psutil/issues/1101 - if err.errno in (errno.EIO, errno.ENODEV): - warnings.warn("ignoring %r" % err, RuntimeWarning) - continue - else: - raise if high is not None: high = float(high) / 1000.0 From d9247ed08bab0c6795492fb2cb847f6e9e6572f7 Mon Sep 17 00:00:00 2001 From: Sebastian Saip Date: Thu, 28 Sep 2017 16:29:58 +0200 Subject: [PATCH 800/922] 1129: have sensors_fans() on Linux skip entry on IOError (#1141) --- psutil/_pslinux.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index c4abacc73..154a8d636 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1191,7 +1191,11 @@ def sensors_fans(): unit_name = cat(os.path.join(os.path.dirname(base), 'name'), binary=False) label = cat(base + '_label', fallback='', binary=False) - current = int(cat(base + '_input')) + try: + current = int(cat(base + '_input')) + except (IOError, OSError) as err: + warnings.warn("ignoring %r" % err, RuntimeWarning) + continue ret[unit_name].append(_common.sfan(label, current)) From 9632fb1969c526129eec63a267a6bcf91010228d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 28 Sep 2017 22:40:19 +0800 Subject: [PATCH 801/922] #1141: sensors_fans / linux: skip unreadable _input label for first --- CREDITS | 4 ++++ HISTORY.rst | 3 ++- psutil/_pslinux.py | 11 +++++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CREDITS b/CREDITS index e4b4ae630..c9b3434dd 100644 --- a/CREDITS +++ b/CREDITS @@ -489,3 +489,7 @@ I: 1077, 1093, 1091 N: Prodesire W: https://github.com/Prodesire I: 1138 + +N: Sebastian Saip +W: https://github.com/ssaip +I: 1141 diff --git a/HISTORY.rst b/HISTORY.rst index 5c301a694..f2878b08f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,7 +11,8 @@ XXXX-XX-XX *Bug fixes* -- 1129_: [Linux] sensors_temperatures() may crash with IOError. +- 1009_: [Linux] sensors_temperatures() may crash with IOError. +- 1129_: [Linux] sensors_fans() may crash with IOError. - 1131_: [SunOS] fix compilation warnings. (patch by Arnon Yaari) - 1138_: [Linux] can't compile on CentOS 5.0 and RedHat 5.0. (patch by Prodesire) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 154a8d636..2fec8ea78 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1170,8 +1170,8 @@ def sensors_temperatures(): def sensors_fans(): - """Return hardware (CPU and others) fans as a dict - including hardware label, current speed. + """Return hardware fans info (for CPU and other peripherals) as a + dict including hardware label and current speed. Implementation notes: - /sys/class/hwmon looks like the most recent interface to @@ -1188,15 +1188,14 @@ def sensors_fans(): basenames = sorted(set([x.split('_')[0] for x in basenames])) for base in basenames: - unit_name = cat(os.path.join(os.path.dirname(base), 'name'), - binary=False) - label = cat(base + '_label', fallback='', binary=False) try: current = int(cat(base + '_input')) except (IOError, OSError) as err: warnings.warn("ignoring %r" % err, RuntimeWarning) continue - + unit_name = cat(os.path.join(os.path.dirname(base), 'name'), + binary=False) + label = cat(base + '_label', fallback='', binary=False) ret[unit_name].append(_common.sfan(label, current)) return dict(ret) From cbbf1bdfd163af486fb4975445edd777c75cb758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Max=20B=C3=A9langer?= Date: Thu, 28 Sep 2017 07:45:31 -0700 Subject: [PATCH 802/922] first pass (#1133) --- psutil/arch/windows/ntextapi.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/psutil/arch/windows/ntextapi.h b/psutil/arch/windows/ntextapi.h index 1bbbf2ac0..ea23ddb72 100644 --- a/psutil/arch/windows/ntextapi.h +++ b/psutil/arch/windows/ntextapi.h @@ -163,13 +163,15 @@ typedef enum _KWAIT_REASON { } KWAIT_REASON, *PKWAIT_REASON; -typedef struct _CLIENT_ID { +typedef struct _CLIENT_ID2 { HANDLE UniqueProcess; HANDLE UniqueThread; -} CLIENT_ID, *PCLIENT_ID; +} CLIENT_ID2, *PCLIENT_ID2; +#define CLIENT_ID CLIENT_ID2 +#define PCLIENT_ID PCLIENT_ID2 -typedef struct _SYSTEM_THREAD_INFORMATION { +typedef struct _SYSTEM_THREAD_INFORMATION2 { LARGE_INTEGER KernelTime; LARGE_INTEGER UserTime; LARGE_INTEGER CreateTime; @@ -181,8 +183,10 @@ typedef struct _SYSTEM_THREAD_INFORMATION { ULONG ContextSwitches; ULONG ThreadState; KWAIT_REASON WaitReason; -} SYSTEM_THREAD_INFORMATION, *PSYSTEM_THREAD_INFORMATION; +} SYSTEM_THREAD_INFORMATION2, *PSYSTEM_THREAD_INFORMATION2; +#define SYSTEM_THREAD_INFORMATION SYSTEM_THREAD_INFORMATION2 +#define PSYSTEM_THREAD_INFORMATION PSYSTEM_THREAD_INFORMATION2 typedef struct _TEB *PTEB; From 9a7af565ebd0df2bab15e18b2f70f61025377fd9 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 28 Sep 2017 22:51:33 +0800 Subject: [PATCH 803/922] #1129, #1133: give CREDITS --- CREDITS | 2 +- HISTORY.rst | 34 ++++++++++++++++++---------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/CREDITS b/CREDITS index c9b3434dd..b9759b9a7 100644 --- a/CREDITS +++ b/CREDITS @@ -443,7 +443,7 @@ I: 919 N: Max Bélanger W: https://github.com/maxbelanger -I: 936 +I: 936, 1133 N: Pierre Fersing C: France diff --git a/HISTORY.rst b/HISTORY.rst index f2878b08f..bcc1a7ab4 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,33 +1,35 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* -XXXX-XX-XX - 5.4.0 ===== +*XXXX-XX-XX* + **Enhancements** - 1123_: [AIX] added support for AIX platform. (patch by Arnon Yaari) -*Bug fixes* +**Bug fixes** - 1009_: [Linux] sensors_temperatures() may crash with IOError. -- 1129_: [Linux] sensors_fans() may crash with IOError. +- 1129_: [Linux] sensors_fans() may crash with IOError. (patch by Sebastian + Saip) - 1131_: [SunOS] fix compilation warnings. (patch by Arnon Yaari) +- 1133_: [Windows] can't compile on newer versions of Visual Studio 2017 15.4. + (patch by Max Bélanger) - 1138_: [Linux] can't compile on CentOS 5.0 and RedHat 5.0. (patch by Prodesire) - -2017-09-10 - 5.3.1 ===== +*2017-09-10* + **Enhancements** - 1124_: documentation moved to http://psutil.readthedocs.io -**Big fixes** +**Bug fixes** - 1105_: [FreeBSD] psutil does not compile on FreeBSD 12. - 1125_: [BSD] net_connections() raises TypeError. @@ -37,11 +39,11 @@ XXXX-XX-XX - 1120_: .exe files for Windows are no longer uploaded on PYPI as per PEP-527; only wheels are provided. -*2017-09-01* - 5.3.0 ===== +*2017-09-01* + **Enhancements** - 802_: disk_io_counters() and net_io_counters() numbers no longer wrap @@ -143,11 +145,11 @@ XXXX-XX-XX - WindowsService.display_name() - WindowsService.username() -*2017-04-10* - 5.2.2 ===== +*2017-04-10* + **Bug fixes** - 1000_: fixed some setup.py warnings. @@ -159,11 +161,11 @@ XXXX-XX-XX - 1009_: [Linux] sensors_temperatures() may raise OSError. - 1010_: [Linux] virtual_memory() may raise ValueError on Ubuntu 14.04. -*2017-03-24* - 5.2.1 ===== +*2017-03-24* + **Bug fixes** - 981_: [Linux] cpu_freq() may return an empty list. @@ -173,11 +175,11 @@ XXXX-XX-XX - 997_: [FreeBSD] virtual_memory() may fail due to missing sysctl parameter on FreeBSD 12. -*2017-03-05* - 5.2.0 ===== +*2017-03-05* + **Enhancements** - 971_: [Linux] Add psutil.sensors_fans() function. (patch by Nicolas Hennion) From beda43bea5e64c28e3e6e1367761b28a112253be Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 6 Oct 2017 08:15:39 +0200 Subject: [PATCH 804/922] fix #1012: disk_io_counters()'s read_time and write_time were expressed in tens of micro seconds instead of milliseconds --- HISTORY.rst | 2 ++ psutil/_psutil_windows.c | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bcc1a7ab4..5888e6003 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,6 +12,8 @@ **Bug fixes** - 1009_: [Linux] sensors_temperatures() may crash with IOError. +- 1012_: [Windows] disk_io_counters()'s read_time and write_time were expressed + in tens of micro seconds instead of milliseconds. - 1129_: [Linux] sensors_fans() may crash with IOError. (patch by Sebastian Saip) - 1131_: [SunOS] fix compilation warnings. (patch by Arnon Yaari) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 74e7cfef0..ac6fa9cb7 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2393,15 +2393,14 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { diskPerformance.WriteCount, diskPerformance.BytesRead, diskPerformance.BytesWritten, - (unsigned long long)(diskPerformance.ReadTime.QuadPart * 10) / 1000, - (unsigned long long)(diskPerformance.WriteTime.QuadPart * 10) / 1000); + (unsigned long long) + (diskPerformance.ReadTime.QuadPart) / 1000000, + (unsigned long long) + (diskPerformance.WriteTime.QuadPart) / 1000000); if (!py_tuple) goto error; - if (PyDict_SetItemString(py_retdict, szDeviceDisplay, - py_tuple)) - { + if (PyDict_SetItemString(py_retdict, szDeviceDisplay, py_tuple)) goto error; - } Py_XDECREF(py_tuple); } else { From f6b064e930ad771187f2fefb1fdfefe892d76521 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 6 Oct 2017 09:55:59 +0200 Subject: [PATCH 805/922] #1012: properly convert to ms --- psutil/_psutil_windows.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index ac6fa9cb7..bca6cac0e 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2393,10 +2393,12 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { diskPerformance.WriteCount, diskPerformance.BytesRead, diskPerformance.BytesWritten, + // convert to ms: + // https://github.com/giampaolo/psutil/issues/1012 (unsigned long long) - (diskPerformance.ReadTime.QuadPart) / 1000000, + (diskPerformance.ReadTime.QuadPart) / 10000000, (unsigned long long) - (diskPerformance.WriteTime.QuadPart) / 1000000); + (diskPerformance.WriteTime.QuadPart) / 10000000); if (!py_tuple) goto error; if (PyDict_SetItemString(py_retdict, szDeviceDisplay, py_tuple)) From 241109c51a02709612ba275e410532ac06ea3b65 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 6 Oct 2017 14:02:17 +0200 Subject: [PATCH 806/922] investigate travis failure --- psutil/tests/test_process.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index fab3a4215..bf0ca6ec6 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -870,6 +870,9 @@ def test_cpu_affinity(self): # CPUs on get): # AssertionError: Lists differ: [0, 1, 2, 3, 4, 5, 6, ... != [0] for n in all_cpus: + # XXX + if hasattr(os, "sched_setaffinity"): + os.sched_setaffinity(os.getpid(), [n]) try: p.cpu_affinity([n]) except ValueError as err: From 5e33068e2a515efe5a434a7771b316b61519cc2d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 6 Oct 2017 16:15:08 +0200 Subject: [PATCH 807/922] investigat travis failure --- psutil/tests/test_process.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index bf0ca6ec6..86c7760a3 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -869,8 +869,11 @@ def test_cpu_affinity(self): # setting on travis doesn't seem to work (always return all # CPUs on get): # AssertionError: Lists differ: [0, 1, 2, 3, 4, 5, 6, ... != [0] + # XXX + print("cpu_count = %s" % psutil.cpu_count()) + print("all_cpus = %s" % all_cpus) for n in all_cpus: - # XXX + print(n) if hasattr(os, "sched_setaffinity"): os.sched_setaffinity(os.getpid(), [n]) try: From ad4b22f51b064c4dcc07f3f3c019a46c63e35119 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 6 Oct 2017 16:31:00 +0200 Subject: [PATCH 808/922] investigat travis failure --- psutil/tests/test_process.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 86c7760a3..9f9ff17b7 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -872,7 +872,8 @@ def test_cpu_affinity(self): # XXX print("cpu_count = %s" % psutil.cpu_count()) print("all_cpus = %s" % all_cpus) - for n in all_cpus: + print("initial affinity = %s" % initial) + for n in all_cpus if not TRAVIS else initial: print(n) if hasattr(os, "sched_setaffinity"): os.sched_setaffinity(os.getpid(), [n]) From 632dac16e9a3d8cb79270dabe7033143b0b112ba Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 6 Oct 2017 16:43:33 +0200 Subject: [PATCH 809/922] work around travis failure --- psutil/tests/test_process.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 9f9ff17b7..45e79bbba 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -866,25 +866,10 @@ def test_cpu_affinity(self): self.assertEqual(len(initial), len(set(initial))) all_cpus = list(range(len(psutil.cpu_percent(percpu=True)))) - # setting on travis doesn't seem to work (always return all - # CPUs on get): - # AssertionError: Lists differ: [0, 1, 2, 3, 4, 5, 6, ... != [0] - # XXX - print("cpu_count = %s" % psutil.cpu_count()) - print("all_cpus = %s" % all_cpus) - print("initial affinity = %s" % initial) + # Work around travis failure: + # https://travis-ci.org/giampaolo/psutil/builds/284173194 for n in all_cpus if not TRAVIS else initial: - print(n) - if hasattr(os, "sched_setaffinity"): - os.sched_setaffinity(os.getpid(), [n]) - try: - p.cpu_affinity([n]) - except ValueError as err: - if TRAVIS and LINUX and "not eligible" in str(err): - # https://travis-ci.org/giampaolo/psutil/jobs/279890461 - continue - else: - raise + p.cpu_affinity([n]) self.assertEqual(p.cpu_affinity(), [n]) if hasattr(os, "sched_getaffinity"): self.assertEqual(p.cpu_affinity(), From 9c75262c01266a3e2e31cd09d5d9c710d0270e18 Mon Sep 17 00:00:00 2001 From: Jakub Bacic Date: Wed, 11 Oct 2017 16:23:46 +0200 Subject: [PATCH 810/922] #1127 Fix reference counting bug in open_files impl on OSX (#1144) --- psutil/_psutil_osx.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 450f1db08..2924aa399 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1129,7 +1129,6 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { iterations = (pidinfo_result / PROC_PIDLISTFD_SIZE); for (i = 0; i < iterations; i++) { - py_tuple = NULL; fdp_pointer = &fds_pointer[i]; if (fdp_pointer->proc_fdtype == PROX_FDTYPE_VNODE) { @@ -1167,7 +1166,9 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { if (PyList_Append(py_retlist, py_tuple)) goto error; Py_DECREF(py_tuple); + py_tuple = NULL; Py_DECREF(py_path); + py_path = NULL; // --- /construct python list } } From 1d8399a9a0822e0f2216409f316aa1f2e966efc6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 11 Oct 2017 16:25:40 +0200 Subject: [PATCH 811/922] #1127: give credits to @jakub-bacic --- CREDITS | 4 ++++ HISTORY.rst | 2 ++ 2 files changed, 6 insertions(+) diff --git a/CREDITS b/CREDITS index b9759b9a7..17d5e66d7 100644 --- a/CREDITS +++ b/CREDITS @@ -493,3 +493,7 @@ I: 1138 N: Sebastian Saip W: https://github.com/ssaip I: 1141 + +N: Jakub Bacic +W: https://github.com/jakub-bacic +I: 1127 diff --git a/HISTORY.rst b/HISTORY.rst index 5888e6003..da9894604 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -14,6 +14,8 @@ - 1009_: [Linux] sensors_temperatures() may crash with IOError. - 1012_: [Windows] disk_io_counters()'s read_time and write_time were expressed in tens of micro seconds instead of milliseconds. +- 1127_: [OSX] invalid reference counting in Process.open_files() may lead to + segfault. (patch by Jakub Bacic) - 1129_: [Linux] sensors_fans() may crash with IOError. (patch by Sebastian Saip) - 1131_: [SunOS] fix compilation warnings. (patch by Arnon Yaari) From 38f43df717386e767de0b1e044bb17510b0b1a6c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 12 Oct 2017 09:20:50 +0200 Subject: [PATCH 812/922] pre release --- HISTORY.rst | 2 +- Makefile | 6 +++--- docs/index.rst | 4 ++++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index da9894604..5a682a4c8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.4.0 ===== -*XXXX-XX-XX* +*2017-10-12* **Enhancements** diff --git a/Makefile b/Makefile index 1ed62c393..8c9e8e015 100644 --- a/Makefile +++ b/Makefile @@ -237,9 +237,6 @@ win-upload-exes: # All the necessary steps before making a release. pre-release: - ${MAKE} generate-manifest - git diff MANIFEST.in > /dev/null # ...otherwise 'git diff-index HEAD' will complain - @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else sys.exit(0);" ${MAKE} install @PYTHONWARNINGS=all $(PYTHON) -c \ "from psutil import __version__ as ver; \ @@ -248,6 +245,9 @@ pre-release: assert ver in doc, '%r not in docs/index.rst' % ver; \ assert ver in history, '%r not in HISTORY.rst' % ver; \ assert 'XXXX' not in history, 'XXXX in HISTORY.rst';" + ${MAKE} generate-manifest + git diff MANIFEST.in > /dev/null # ...otherwise 'git diff-index HEAD' will complain + @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else sys.exit(0);" ${MAKE} win-download-exes ${MAKE} sdist diff --git a/docs/index.rst b/docs/index.rst index b8becb3a6..e630d0b0a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2580,6 +2580,10 @@ take a look at the Timeline ======== +- 2017-10-12: + `5.4.0 `__ - + `what's new `__ - + `diff `__ - 2017-09-10: `5.3.1 `__ - `what's new `__ - From 2fd5679f6b61cb8cc5b358a2d298f6338c1c915b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 12 Oct 2017 09:25:52 +0200 Subject: [PATCH 813/922] make pre-release is deleting wheels --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8c9e8e015..5a8603656 100644 --- a/Makefile +++ b/Makefile @@ -217,7 +217,6 @@ install-git-hooks: # Generate tar.gz source distribution. sdist: - ${MAKE} clean ${MAKE} generate-manifest PYTHONWARNINGS=all $(PYTHON) setup.py sdist @@ -255,7 +254,7 @@ pre-release: # upload doc, git tag release. release: ${MAKE} pre-release - PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/* # upload tar.gz and Windows wheels on PYPI + $(PYTHON) -m twine upload dist/* # upload tar.gz and Windows wheels on PYPI ${MAKE} git-tag-release # Print announce of new release. From 3e5e9b780c9e2809f7552f6218e5ae86cbd35e28 Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Thu, 19 Oct 2017 11:42:57 +0300 Subject: [PATCH 814/922] fix or skip remaining AIX unit tests (#1145) * create_exe should use default code if c_code is True * fix or skip remaining AIX unit tests --- IDEAS | 1 - psutil/TODO.aix | 4 ---- psutil/tests/__init__.py | 3 ++- psutil/tests/test_posix.py | 35 +++++++++++++++++++++++++---------- psutil/tests/test_process.py | 4 +++- 5 files changed, 30 insertions(+), 17 deletions(-) diff --git a/IDEAS b/IDEAS index d122be871..4932ad728 100644 --- a/IDEAS +++ b/IDEAS @@ -9,7 +9,6 @@ PLATFORMS ========= - #355: Android (with patch) -- #605: AIX (with branch) - #82: Cygwin (PR at #998) - #276: GNU/Hurd - #693: Windows Nano diff --git a/psutil/TODO.aix b/psutil/TODO.aix index 671372dc0..e1d428bd9 100644 --- a/psutil/TODO.aix +++ b/psutil/TODO.aix @@ -9,7 +9,3 @@ Known limitations: reading basic process info may fail or return incorrect values when process is starting (see IBM APAR IV58499 - fixed in newer AIX versions) sockets and pipes may not be counted in num_fds (fixed in newer AIX versions) - -The following unit tests may fail: - test_prog_w_funky_name funky name tests don't work, name is truncated - test_cmdline long args are cut from cmdline in /proc/pid/psinfo and getargs diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 033d925e3..5bb540174 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -690,7 +690,7 @@ def create_exe(outpath, c_code=None): if c_code: if not which("gcc"): raise ValueError("gcc is not installed") - if c_code is None: + if isinstance(c_code, bool): # c_code is True c_code = textwrap.dedent( """ #include @@ -699,6 +699,7 @@ def create_exe(outpath, c_code=None): return 1; } """) + assert isinstance(c_code, str), c_code with tempfile.NamedTemporaryFile( suffix='.c', delete=False, mode='wt') as f: f.write(c_code) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index f42a6e637..d1f9e54a3 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -47,7 +47,6 @@ def ps(cmd): if not LINUX: cmd = cmd.replace(" --no-headers ", " ") if SUNOS: - cmd = cmd.replace("-o command", "-o comm") cmd = cmd.replace("-o start", "-o stime") if AIX: cmd = cmd.replace("-o rss", "-o rssize") @@ -59,6 +58,28 @@ def ps(cmd): except ValueError: return output +# ps "-o" field names differ wildly between platforms. +# "comm" means "only executable name" but is not available on BSD platforms. +# "args" means "command with all its arguments", and is also not available +# on BSD platforms. +# "command" is like "args" on most platforms, but like "comm" on AIX, +# and not available on SUNOS. +# so for the executable name we can use "comm" on Solaris and split "command" +# on other platforms. +# to get the cmdline (with args) we have to use "args" on AIX and +# Solaris, and can use "command" on all others. + +def ps_name(pid): + field = "command" + if SUNOS: + field = "comm" + return ps("ps --no-headers -o %s -p %s" % (field, pid)).split(' ')[0] + +def ps_args(pid): + field = "command" + if AIX or SUNOS: + field = "args" + return ps("ps --no-headers -o %s -p %s" % (field, pid)) @unittest.skipIf(not POSIX, "POSIX only") class TestProcess(unittest.TestCase): @@ -124,9 +145,7 @@ def test_vsz_memory(self): self.assertEqual(vsz_ps, vsz_psutil) def test_name(self): - # use command + arg since "comm" keyword not supported on all platforms - name_ps = ps("ps --no-headers -o command -p %s" % ( - self.pid)).split(' ')[0] + name_ps = ps_name(self.pid) # remove path if there is any, from the command name_ps = os.path.basename(name_ps).lower() name_psutil = psutil.Process(self.pid).name().lower() @@ -182,8 +201,7 @@ def test_create_time(self): self.assertIn(time_ps, [time_psutil_tstamp, round_time_psutil_tstamp]) def test_exe(self): - ps_pathname = ps("ps --no-headers -o command -p %s" % - self.pid).split(' ')[0] + ps_pathname = ps_name(self.pid) psutil_pathname = psutil.Process(self.pid).exe() try: self.assertEqual(ps_pathname, psutil_pathname) @@ -198,11 +216,8 @@ def test_exe(self): self.assertEqual(ps_pathname, adjusted_ps_pathname) def test_cmdline(self): - ps_cmdline = ps("ps --no-headers -o command -p %s" % self.pid) + ps_cmdline = ps_args(self.pid) psutil_cmdline = " ".join(psutil.Process(self.pid).cmdline()) - if SUNOS: - # ps on Solaris only shows the first part of the cmdline - psutil_cmdline = psutil_cmdline.split(" ")[0] self.assertEqual(ps_cmdline, psutil_cmdline) # On SUNOS "ps" reads niceness /proc/pid/psinfo which returns an diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 45e79bbba..bd13c2d27 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -724,7 +724,8 @@ def test_cmdline(self): # and Open BSD returns a truncated string. # Also /proc/pid/cmdline behaves the same so it looks # like this is a kernel bug. - if NETBSD or OPENBSD: + # XXX - AIX truncates long arguments in /proc/pid/cmdline + if NETBSD or OPENBSD or AIX: self.assertEqual( psutil.Process(sproc.pid).cmdline()[0], PYTHON) else: @@ -738,6 +739,7 @@ def test_name(self): # XXX @unittest.skipIf(SUNOS, "broken on SUNOS") + @unittest.skipIf(AIX, "broken on AIX") def test_prog_w_funky_name(self): # Test that name(), exe() and cmdline() correctly handle programs # with funky chars such as spaces and ")", see: From 548656a636da14636c44fd1ce64c677dbcb6e15d Mon Sep 17 00:00:00 2001 From: Akos Kiss Date: Fri, 20 Oct 2017 09:52:52 +0200 Subject: [PATCH 815/922] Terminate Windows processes with SIGTERM code rather than 0 (#1150) If the TerminateProcess WinAPI function is called with 0, then the exit code of the terminated process (e.g., observed by its parent) will be 0. However, this is usually associated with successful execution. Any other exit code could be used to signal forced termination, but perhaps the value of SIGTERM is the most meaningful. --- psutil/_psutil_windows.c | 3 ++- psutil/tests/test_process.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index bca6cac0e..d908a1c7d 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -25,6 +25,7 @@ #include #include #include +#include // Link with Iphlpapi.lib #pragma comment(lib, "IPHLPAPI.lib") @@ -349,7 +350,7 @@ psutil_proc_kill(PyObject *self, PyObject *args) { } // kill the process - if (! TerminateProcess(hProcess, 0)) { + if (! TerminateProcess(hProcess, SIGTERM)) { err = GetLastError(); // See: https://github.com/giampaolo/psutil/issues/1099 if (err != ERROR_ACCESS_DENIED) { diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index bd13c2d27..fa07f5a92 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -151,7 +151,7 @@ def test_wait(self): if POSIX: self.assertEqual(code, -signal.SIGKILL) else: - self.assertEqual(code, 0) + self.assertEqual(code, signal.SIGTERM) self.assertFalse(p.is_running()) sproc = get_test_subprocess() @@ -161,7 +161,7 @@ def test_wait(self): if POSIX: self.assertEqual(code, -signal.SIGTERM) else: - self.assertEqual(code, 0) + self.assertEqual(code, signal.SIGTERM) self.assertFalse(p.is_running()) # check sys.exit() code @@ -207,8 +207,8 @@ def test_wait_non_children(self): # to get None. self.assertEqual(ret2, None) else: - self.assertEqual(ret1, 0) - self.assertEqual(ret1, 0) + self.assertEqual(ret1, signal.SIGTERM) + self.assertEqual(ret1, signal.SIGTERM) def test_wait_timeout_0(self): sproc = get_test_subprocess() @@ -227,7 +227,7 @@ def test_wait_timeout_0(self): if POSIX: self.assertEqual(code, -signal.SIGKILL) else: - self.assertEqual(code, 0) + self.assertEqual(code, signal.SIGTERM) self.assertFalse(p.is_running()) def test_cpu_percent(self): From d286ac5eef2a41fc1e34bb122d583892aaf432ce Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 20 Oct 2017 09:56:52 +0200 Subject: [PATCH 816/922] #1150 give credits to @akosthekiss --- CREDITS | 4 ++++ HISTORY.rst | 10 ++++++++++ docs/index.rst | 2 +- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index 17d5e66d7..114bd0bc1 100644 --- a/CREDITS +++ b/CREDITS @@ -497,3 +497,7 @@ I: 1141 N: Jakub Bacic W: https://github.com/jakub-bacic I: 1127 + +N: Akos Kiss +W: https://github.com/akosthekiss +I: 1150 diff --git a/HISTORY.rst b/HISTORY.rst index 5a682a4c8..1740e0da3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,15 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.4.1 +===== + +*XXXX-XX-XX* + +**Bug fixes** + +- 1150_: [Windows] when a process is terminate()d now the exit code is set to + SIGTERM instead of 0. (patch by Akos Kiss) + 5.4.0 ===== diff --git a/docs/index.rst b/docs/index.rst index e630d0b0a..8dcef1f99 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1153,7 +1153,7 @@ Process class Availability: Linux, OSX, Windows, SunOS .. versionadded:: 4.0.0 - .. versionchanged:: 5.3.0: added SunOS support + .. versionchanged:: 5.3.0 added SunOS support .. method:: create_time() From 5444c5b548650d6c3550fca2cfb324109b8fa4aa Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 20 Oct 2017 10:03:53 +0200 Subject: [PATCH 817/922] automatically set PSUTIL_TEST env var during tests --- appveyor.yml | 2 +- psutil/tests/__init__.py | 2 +- scripts/internal/winmake.py | 12 ------------ 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b18677242..3396cb906 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -91,7 +91,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "set PSUTIL_TESTING=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" + - "%WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 5bb540174..86930100e 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -744,7 +744,7 @@ def __str__(self): def _setup_tests(): - assert 'PSUTIL_TESTING' in os.environ + os.environ['PSUTIL_TESTING'] = '1' assert psutil._psplatform.cext.py_psutil_testing() diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index c2ee2ab04..185db7ee0 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -328,7 +328,6 @@ def flake8(): def test(): """Run tests""" install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa %s" % (PYTHON, TSCRIPT)) @@ -337,7 +336,6 @@ def coverage(): """Run coverage tests.""" # Note: coverage options are controlled by .coveragerc file install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m coverage run %s" % (PYTHON, TSCRIPT)) sh("%s -m coverage report" % PYTHON) sh("%s -m coverage html" % PYTHON) @@ -348,7 +346,6 @@ def coverage(): def test_process(): """Run process tests""" install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_process" % PYTHON) @@ -356,7 +353,6 @@ def test_process(): def test_system(): """Run system tests""" install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_system" % PYTHON) @@ -364,7 +360,6 @@ def test_system(): def test_platform(): """Run windows only tests""" install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_windows" % PYTHON) @@ -372,7 +367,6 @@ def test_platform(): def test_misc(): """Run misc tests""" install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_misc" % PYTHON) @@ -380,7 +374,6 @@ def test_misc(): def test_unicode(): """Run unicode tests""" install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_unicode" % PYTHON) @@ -388,7 +381,6 @@ def test_unicode(): def test_connections(): """Run connections tests""" install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_connections" % PYTHON) @@ -396,7 +388,6 @@ def test_connections(): def test_contracts(): """Run contracts tests""" install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v psutil.tests.test_contracts" % PYTHON) @@ -409,7 +400,6 @@ def test_by_name(): except IndexError: sys.exit('second arg missing') install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa -m unittest -v %s" % (PYTHON, name)) @@ -422,7 +412,6 @@ def test_script(): except IndexError: sys.exit('second arg missing') install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa %s" % (PYTHON, name)) @@ -430,7 +419,6 @@ def test_script(): def test_memleaks(): """Run memory leaks tests""" install() - os.environ['PSUTIL_TESTING'] = '1' sh("%s -Wa psutil\\tests\\test_memory_leaks.py" % PYTHON) From 88740189e0d846df8163fd172f4b4e34f7981e22 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 20 Oct 2017 12:41:10 +0200 Subject: [PATCH 818/922] pep8 --- .ci/travis/run.sh | 2 +- psutil/tests/test_posix.py | 3 +++ scripts/internal/winmake.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.ci/travis/run.sh b/.ci/travis/run.sh index b27e6695c..1501387a5 100755 --- a/.ci/travis/run.sh +++ b/.ci/travis/run.sh @@ -30,6 +30,6 @@ if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then PSUTIL_TESTING=1 python -Wa psutil/tests/test_memory_leaks.py # run linter (on Linux only) if [[ "$(uname -s)" != 'Darwin' ]]; then - python -Wa -m flake8 + python -m flake8 fi fi diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index d1f9e54a3..aaeef7472 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -69,18 +69,21 @@ def ps(cmd): # to get the cmdline (with args) we have to use "args" on AIX and # Solaris, and can use "command" on all others. + def ps_name(pid): field = "command" if SUNOS: field = "comm" return ps("ps --no-headers -o %s -p %s" % (field, pid)).split(' ')[0] + def ps_args(pid): field = "command" if AIX or SUNOS: field = "args" return ps("ps --no-headers -o %s -p %s" % (field, pid)) + @unittest.skipIf(not POSIX, "POSIX only") class TestProcess(unittest.TestCase): """Compare psutil results against 'ps' command line utility (mainly).""" diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 185db7ee0..5b2bea989 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -321,7 +321,7 @@ def flake8(): py_files = py_files.decode() py_files = [x for x in py_files.split() if x.endswith('.py')] py_files = ' '.join(py_files) - sh("%s -Wa -m flake8 %s" % (PYTHON, py_files), nolog=True) + sh("%s -m flake8 %s" % (PYTHON, py_files), nolog=True) @cmd From d24d73840d2ff0dccbdb1c7b55e3477c1466f508 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 20 Oct 2017 14:24:56 +0200 Subject: [PATCH 819/922] update history for #1151 --- HISTORY.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY.rst b/HISTORY.rst index 1740e0da3..0309bbfd2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ - 1150_: [Windows] when a process is terminate()d now the exit code is set to SIGTERM instead of 0. (patch by Akos Kiss) +- 1151_: python -m psutil.tests fail 5.4.0 ===== From 5ed29588e0f8d95ffac21cf3128489383691961b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 20 Oct 2017 14:48:03 +0200 Subject: [PATCH 820/922] try to fix appveyor --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 3396cb906..b18677242 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -91,7 +91,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "%WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" + - "set PSUTIL_TESTING=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" From f645d8f8b74b69933aa25fe673c7b2c8179937e6 Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Thu, 26 Oct 2017 17:55:03 +0300 Subject: [PATCH 821/922] Small changes (#1158) * fix valid_types in memory_percent pfullmem._fields is always added twice to valid_types so the message about invalid memtype lists the types twice too. pfullmem is available on all platforms and is always the same as or a superset of pmem. We can look at its fields only to get all valid_types. Also we can check whether to use memory_full_info or not by checking the fields of pfullmem vs. pmem instead of using hard coded mem types. * remove workaround made for Solaris on AIX The problem described in the comment doesn't apply for AIX * update "oneshot" table in documentation * Removed "nice" and "ionice" which are not boosted * Removed "~Process." prefix which was only on a few methods and not others * Added AIX Fixes #1157 * small AIX additions to docs --- CREDITS | 2 ++ DEVGUIDE.rst | 2 +- docs/index.rst | 78 +++++++++++++++++++++++----------------------- psutil/__init__.py | 6 ++-- psutil/_psaix.py | 17 +--------- 5 files changed, 45 insertions(+), 60 deletions(-) diff --git a/CREDITS b/CREDITS index 114bd0bc1..b54fac21b 100644 --- a/CREDITS +++ b/CREDITS @@ -43,6 +43,8 @@ Github usernames of people to CC on github when in need of help. - SunOS: - wiggin15, Arnon Yaari - alxchk, Oleksii Shevchuk +- AIX: + - wiggin15, Arnon Yaari Contributors ============ diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index da5c9d799..904f4b8e5 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -142,7 +142,7 @@ Two icons in the home page (README) always show the build status: :target: https://ci.appveyor.com/project/giampaolo/psutil :alt: Windows tests (Appveyor) -OSX, BSD and Solaris are currently tested manually (sigh!). +OSX, BSD, AIX and Solaris are currently tested manually (sigh!). Test coverage ------------- diff --git a/docs/index.rst b/docs/index.rst index 8dcef1f99..feb9fd78f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1061,45 +1061,45 @@ Process class The last column (speedup) shows an approximation of the speedup you can get if you call all the methods together (best case scenario). - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | Linux | Windows | OSX | BSD | SunOS | - +==============================+===============================+==============================+==============================+==========================+ - | :meth:`cpu_num` | :meth:`~Process.cpu_percent` | :meth:`~Process.cpu_percent` | :meth:`cpu_num` | :meth:`name` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`~Process.cpu_percent` | :meth:`~Process.cpu_times` | :meth:`~Process.cpu_times` | :meth:`~Process.cpu_percent` | :meth:`cmdline` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`~Process.cpu_times` | :meth:`io_counters()` | :meth:`memory_info` | :meth:`~Process.cpu_times` | :meth:`create_time` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`create_time` | :meth:`ionice` | :meth:`memory_percent` | :meth:`create_time` | | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`name` | :meth:`memory_info` | :meth:`num_ctx_switches` | :meth:`gids` | :meth:`memory_info` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`ppid` | :meth:`nice` | :meth:`num_threads` | :meth:`io_counters` | :meth:`memory_percent` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`status` | :meth:`memory_maps` | | :meth:`name` | :meth:`nice` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`terminal` | :meth:`num_ctx_switches` | :meth:`create_time` | :meth:`memory_info` | :meth:`num_threads` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | | :meth:`num_handles` | :meth:`gids` | :meth:`memory_percent` | :meth:`ppid` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`gids` | :meth:`num_threads` | :meth:`name` | :meth:`num_ctx_switches` | :meth:`status` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`num_ctx_switches` | :meth:`username` | :meth:`ppid` | :meth:`ppid` | :meth:`terminal` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`num_threads` | | :meth:`status` | :meth:`status` | | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`uids` | | :meth:`terminal` | :meth:`terminal` | :meth:`gids` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`username` | | :meth:`uids` | :meth:`uids` | :meth:`uids` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | | | :meth:`username` | :meth:`username` | :meth:`username` | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`memory_full_info` | | | | | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | :meth:`memory_maps` | | | | | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ - | *speedup: +2.6x* | *speedup: +1.8x / +6.5x* | *speedup: +1.9x* | *speedup: +2.0x* | *speedup: +1.3x* | - +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+ + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | Linux | Windows | OSX | BSD | SunOS | AIX | + +==============================+===============================+==============================+==============================+==========================+==========================+ + | :meth:`cpu_num` | :meth:`cpu_percent` | :meth:`cpu_percent` | :meth:`cpu_num` | :meth:`name` | :meth:`name` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`cpu_percent` | :meth:`cpu_times` | :meth:`cpu_times` | :meth:`cpu_percent` | :meth:`cmdline` | :meth:`cmdline` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`cpu_times` | :meth:`io_counters()` | :meth:`memory_info` | :meth:`cpu_times` | :meth:`create_time` | :meth:`create_time` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`create_time` | :meth:`memory_info` | :meth:`memory_percent` | :meth:`create_time` | | | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`name` | :meth:`memory_maps` | :meth:`num_ctx_switches` | :meth:`gids` | :meth:`memory_info` | :meth:`memory_info` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`ppid` | :meth:`num_ctx_switches` | :meth:`num_threads` | :meth:`io_counters` | :meth:`memory_percent` | :meth:`memory_percent` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`status` | :meth:`num_handles` | | :meth:`name` | :meth:`num_threads` | :meth:`num_threads` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`terminal` | :meth:`num_threads` | :meth:`create_time` | :meth:`memory_info` | :meth:`ppid` | :meth:`ppid` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | | :meth:`username` | :meth:`gids` | :meth:`memory_percent` | :meth:`status` | :meth:`status` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`gids` | | :meth:`name` | :meth:`num_ctx_switches` | :meth:`terminal` | :meth:`terminal` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`num_ctx_switches` | | :meth:`ppid` | :meth:`ppid` | | | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`num_threads` | | :meth:`status` | :meth:`status` | :meth:`gids` | :meth:`gids` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`uids` | | :meth:`terminal` | :meth:`terminal` | :meth:`uids` | :meth:`uids` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`username` | | :meth:`uids` | :meth:`uids` | :meth:`username` | :meth:`username` | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | | | :meth:`username` | :meth:`username` | | | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`memory_full_info` | | | | | | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | :meth:`memory_maps` | | | | | | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ + | *speedup: +2.6x* | *speedup: +1.8x / +6.5x* | *speedup: +1.9x* | *speedup: +2.0x* | *speedup: +1.3x* | *speedup: +1.3x* | + +------------------------------+-------------------------------+------------------------------+------------------------------+--------------------------+--------------------------+ .. versionadded:: 5.0.0 diff --git a/psutil/__init__.py b/psutil/__init__.py index 006d5f57f..bbc1df6de 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1160,13 +1160,11 @@ def memory_percent(self, memtype="rss"): ('rss', 'vms', 'shared', 'text', 'lib', 'data', 'dirty', 'uss', 'pss') """ valid_types = list(_psplatform.pfullmem._fields) - if hasattr(_psplatform, "pfullmem"): - valid_types.extend(list(_psplatform.pfullmem._fields)) if memtype not in valid_types: raise ValueError("invalid memtype %r; valid types are %r" % ( memtype, tuple(valid_types))) - fun = self.memory_full_info if memtype in ('uss', 'pss', 'swap') else \ - self.memory_info + fun = self.memory_info if memtype in _psplatform.pmem._fields else \ + self.memory_full_info metrics = fun() value = getattr(metrics, memtype) diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 663748bb8..a4703c032 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -459,22 +459,7 @@ def connections(self, kind='inet'): @wrap_exceptions def nice_get(self): - # For some reason getpriority(3) return ESRCH (no such process) - # for certain low-pid processes, no matter what (even as root). - # The process actually exists though, as it has a name, - # creation time, etc. - # The best thing we can do here appears to be raising AD. - # Note: tested on Solaris 11; on Open Solaris 5 everything is - # fine. - try: - return cext_posix.getpriority(self.pid) - except EnvironmentError as err: - # 48 is 'operation not supported' but errno does not expose - # it. It occurs for low system pids. - if err.errno in (errno.ENOENT, errno.ESRCH, 48): - if pid_exists(self.pid): - raise AccessDenied(self.pid, self._name) - raise + return cext_posix.getpriority(self.pid) @wrap_exceptions def nice_set(self, value): From 8d72368934542e78e7dcf1e31ecab3821677bbb2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Oct 2017 23:12:42 +0200 Subject: [PATCH 822/922] print full mod representation when running test --- psutil/tests/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 86930100e..50063bc92 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -729,9 +729,11 @@ class TestCase(unittest.TestCase): # Print a full path representation of the single unit tests # being run. def __str__(self): + mod = self.__class__.__module__ + if mod == '__main__': + mod = __file__.split('.')[0] return "%s.%s.%s" % ( - self.__class__.__module__, self.__class__.__name__, - self._testMethodName) + mod, self.__class__.__name__, self._testMethodName) # assertRaisesRegexp renamed to assertRaisesRegex in 3.3; # add support for the new name. From 516269a43646693a3ef4423ba0c2890a816e89bc Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 26 Oct 2017 23:15:29 +0200 Subject: [PATCH 823/922] revert last commit --- psutil/tests/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 50063bc92..86930100e 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -729,11 +729,9 @@ class TestCase(unittest.TestCase): # Print a full path representation of the single unit tests # being run. def __str__(self): - mod = self.__class__.__module__ - if mod == '__main__': - mod = __file__.split('.')[0] return "%s.%s.%s" % ( - mod, self.__class__.__name__, self._testMethodName) + self.__class__.__module__, self.__class__.__name__, + self._testMethodName) # assertRaisesRegexp renamed to assertRaisesRegex in 3.3; # add support for the new name. From be8abbdf9cbab6ea82a1b16ad144a55783c5f45b Mon Sep 17 00:00:00 2001 From: Adrian Page Date: Sat, 28 Oct 2017 18:34:35 +0100 Subject: [PATCH 824/922] Fix pre-commit hook for python 3.x. (#1159) sh() return value is a string due to Popen(universal_newlines=True). Traceback (most recent call last): File ".git/hooks/pre-commit", line 118, in main() File ".git/hooks/pre-commit", line 78, in main py_files = [x for x in out.split(b'\n') if x.endswith(b'.py') and TypeError: must be str or None, not bytes --- .git-pre-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.git-pre-commit b/.git-pre-commit index a2f2d18e4..c3c605e05 100755 --- a/.git-pre-commit +++ b/.git-pre-commit @@ -75,7 +75,7 @@ def sh(cmd): def main(): out = sh("git diff --cached --name-only") - py_files = [x for x in out.split(b'\n') if x.endswith(b'.py') and + py_files = [x for x in out.split('\n') if x.endswith('.py') and os.path.exists(x)] lineno = 0 From 3cec63006e0f3f3498862310268d2ca6e21af39d Mon Sep 17 00:00:00 2001 From: Adrian Page Date: Sat, 28 Oct 2017 18:37:07 +0100 Subject: [PATCH 825/922] Fix network tests for newer ifconfig versions. (#1160) Arch Linux and Ubuntu 17.10 use a newer ifconfig version than other distributions and that changes the statistics output text formatting, causing the following tests to fail: psutil.tests.test_linux.TestSystemNetwork.test_net_if_stats psutil.tests.test_linux.TestSystemNetwork.test_net_io_counters MTU becomes lower case, colons are replaced with spaces, and packets and bytes are on the same line. Example ifconfig output: enp2s0f0: flags=4163 mtu 1500 ether a8:20:66:04:4c:45 txqueuelen 1000 (Ethernet) RX packets 1396351 bytes 1928499072 (1.7 GiB) RX errors 0 dropped 0 overruns 0 frame 0 TX packets 750492 bytes 185338978 (176.7 MiB) TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0 device interrupt 16 --- psutil/tests/test_linux.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 468b3c663..7bb37a8df 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -762,21 +762,25 @@ def test_net_if_stats(self): # Not always reliable. # self.assertEqual(stats.isup, 'RUNNING' in out, msg=out) self.assertEqual(stats.mtu, - int(re.findall(r'MTU:(\d+)', out)[0])) + int(re.findall(r'(?i)MTU[: ](\d+)', out)[0])) @retry_before_failing() def test_net_io_counters(self): def ifconfig(nic): ret = {} out = sh("ifconfig %s" % name) - ret['packets_recv'] = int(re.findall(r'RX packets:(\d+)', out)[0]) - ret['packets_sent'] = int(re.findall(r'TX packets:(\d+)', out)[0]) - ret['errin'] = int(re.findall(r'errors:(\d+)', out)[0]) - ret['errout'] = int(re.findall(r'errors:(\d+)', out)[1]) - ret['dropin'] = int(re.findall(r'dropped:(\d+)', out)[0]) - ret['dropout'] = int(re.findall(r'dropped:(\d+)', out)[1]) - ret['bytes_recv'] = int(re.findall(r'RX bytes:(\d+)', out)[0]) - ret['bytes_sent'] = int(re.findall(r'TX bytes:(\d+)', out)[0]) + ret['packets_recv'] = int( + re.findall(r'RX packets[: ](\d+)', out)[0]) + ret['packets_sent'] = int( + re.findall(r'TX packets[: ](\d+)', out)[0]) + ret['errin'] = int(re.findall(r'errors[: ](\d+)', out)[0]) + ret['errout'] = int(re.findall(r'errors[: ](\d+)', out)[1]) + ret['dropin'] = int(re.findall(r'dropped[: ](\d+)', out)[0]) + ret['dropout'] = int(re.findall(r'dropped[: ](\d+)', out)[1]) + ret['bytes_recv'] = int( + re.findall(r'RX (?:packets \d+ +)?bytes[: ](\d+)', out)[0]) + ret['bytes_sent'] = int( + re.findall(r'TX (?:packets \d+ +)?bytes[: ](\d+)', out)[0]) return ret nio = psutil.net_io_counters(pernic=True, nowrap=False) From 5c51581f1ea80b3d3ab5652ac0d904f8cb65fd77 Mon Sep 17 00:00:00 2001 From: Adrian Page Date: Sat, 28 Oct 2017 18:40:29 +0100 Subject: [PATCH 826/922] Fix test asserts due to leftover test subprocesses (#1161) * Reap test subprocess. The leftover child process was triggering an assert in a later test: ====================================================================== FAIL: psutil.tests.test_connections.TestConnectedSocketPairs.test_combos ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/ade/projects/psutil/psutil/tests/__init__.py", line 792, in wrapper return fun(*args, **kwargs) File "/home/ade/projects/psutil/psutil/tests/test_connections.py", line 330, in test_combos self.assertEqual(len(cons), 1) AssertionError: 0 != 1 * Inherit from Base so that reap_children() cleans up the test processes. Fixes memory leak test asserts. psutil/tests/test_memory_leaks.py:334: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ psutil/tests/test_memory_leaks.py:125: in execute self.assertEqual(thisproc.children(), []) E AssertionError: Lists differ: [ E E + [] E - [, E - , E - , E - , E - , E - , E - , E - , E - , E - ] --- psutil/tests/test_connections.py | 2 +- psutil/tests/test_unicode.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index 203ddebba..d248af577 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -418,7 +418,7 @@ def test_multi_sockets_filtering(self): # ===================================================================== -class TestSystemWideConnections(unittest.TestCase): +class TestSystemWideConnections(Base, unittest.TestCase): """Tests for net_connections().""" @skip_on_access_denied() diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 2aa752216..4e2289bf0 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -352,6 +352,7 @@ def test_proc_environ(self): self.assertIsInstance(k, str) self.assertIsInstance(v, str) self.assertEqual(env['FUNNY_ARG'], funky_str) + reap_children() if __name__ == '__main__': From 20bb2e7ac54ecd0fd22d13aac3a3890da19d2d63 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 28 Oct 2017 19:44:04 +0200 Subject: [PATCH 827/922] give CREDITS to @adpag for #1159, #1160 and #1161 --- CREDITS | 4 ++++ psutil/tests/test_unicode.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index b54fac21b..f6e2b1991 100644 --- a/CREDITS +++ b/CREDITS @@ -503,3 +503,7 @@ I: 1127 N: Akos Kiss W: https://github.com/akosthekiss I: 1150 + +N: Adrian Page +W: https://github.com/adpag +I: 1159, 1160, 1161 diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 4e2289bf0..e491d3dc7 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -335,6 +335,9 @@ def test_name_type(self): class TestNonFSAPIS(unittest.TestCase): """Unicode tests for non fs-related APIs.""" + def tearDown(self): + reap_children() + @unittest.skipIf(not HAS_ENVIRON, "not supported") def test_proc_environ(self): # Note: differently from others, this test does not deal @@ -352,7 +355,6 @@ def test_proc_environ(self): self.assertIsInstance(k, str) self.assertIsInstance(v, str) self.assertEqual(env['FUNNY_ARG'], funky_str) - reap_children() if __name__ == '__main__': From 21e323f51d05cc95eb129704f8eb1c27a47a697d Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Sat, 28 Oct 2017 20:45:09 +0300 Subject: [PATCH 828/922] Fix #1154: remove 'threads' method on older AIX (#1156) --- psutil/__init__.py | 16 +++++++++------- psutil/_psaix.py | 35 +++++++++++++++++++---------------- psutil/_psutil_aix.c | 4 ++++ psutil/tests/__init__.py | 1 + psutil/tests/test_process.py | 3 +++ 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index bbc1df6de..b8b94e7ed 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -911,13 +911,15 @@ def num_threads(self): """Return the number of threads used by this process.""" return self._proc.num_threads() - def threads(self): - """Return threads opened by process as a list of - (id, user_time, system_time) namedtuples representing - thread id and thread CPU times (user/system). - On OpenBSD this method requires root access. - """ - return self._proc.threads() + if hasattr(_psplatform.Process, "threads"): + + def threads(self): + """Return threads opened by process as a list of + (id, user_time, system_time) namedtuples representing + thread id and thread CPU times (user/system). + On OpenBSD this method requires root access. + """ + return self._proc.threads() @_assert_pid_not_reused def children(self, recursive=False): diff --git a/psutil/_psaix.py b/psutil/_psaix.py index a4703c032..5cd088e55 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -38,6 +38,8 @@ # ===================================================================== +HAS_THREADS = hasattr(cext, "proc_threads") + PAGE_SIZE = os.sysconf('SC_PAGE_SIZE') AF_LINK = cext_posix.AF_LINK @@ -427,22 +429,23 @@ def create_time(self): def num_threads(self): return self._proc_basic_info()[proc_info_map['num_threads']] - @wrap_exceptions - def threads(self): - rawlist = cext.proc_threads(self.pid) - retlist = [] - for thread_id, utime, stime in rawlist: - ntuple = _common.pthread(thread_id, utime, stime) - retlist.append(ntuple) - # The underlying C implementation retrieves all OS threads - # and filters them by PID. At this point we can't tell whether - # an empty list means there were no connections for process or - # process is no longer active so we force NSP in case the PID - # is no longer there. - if not retlist: - # will raise NSP if process is gone - os.stat('%s/%s' % (self._procfs_path, self.pid)) - return retlist + if HAS_THREADS: + @wrap_exceptions + def threads(self): + rawlist = cext.proc_threads(self.pid) + retlist = [] + for thread_id, utime, stime in rawlist: + ntuple = _common.pthread(thread_id, utime, stime) + retlist.append(ntuple) + # The underlying C implementation retrieves all OS threads + # and filters them by PID. At this point we can't tell whether + # an empty list means there were no connections for process or + # process is no longer active so we force NSP in case the PID + # is no longer there. + if not retlist: + # will raise NSP if process is gone + os.stat('%s/%s' % (self._procfs_path, self.pid)) + return retlist @wrap_exceptions def connections(self, kind='inet'): diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c index 52a14feb6..76ed736b2 100644 --- a/psutil/_psutil_aix.c +++ b/psutil/_psutil_aix.c @@ -157,6 +157,7 @@ psutil_proc_name_and_args(PyObject *self, PyObject *args) { } +#ifdef CURR_VERSION_THREAD /* * Retrieves all threads used by process returning a list of tuples * including thread id, user time and system time. @@ -222,6 +223,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { free(threadt); return NULL; } +#endif static PyObject * @@ -813,8 +815,10 @@ PsutilMethods[] = "Return process user and system CPU times."}, {"proc_cred", psutil_proc_cred, METH_VARARGS, "Return process uids/gids."}, +#ifdef CURR_VERSION_THREAD {"proc_threads", psutil_proc_threads, METH_VARARGS, "Return process threads"}, +#endif {"proc_io_counters", psutil_proc_io_counters, METH_VARARGS, "Get process I/O counters."}, diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 86930100e..1d109af0e 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -160,6 +160,7 @@ HAS_NUM_CTX_SWITCHES = hasattr(psutil.Process, "num_ctx_switches") HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") HAS_RLIMIT = hasattr(psutil.Process, "rlimit") +HAS_THREADS = hasattr(psutil.Process, "threads") HAS_SENSORS_BATTERY = hasattr(psutil, "sensors_battery") HAS_BATTERY = HAS_SENSORS_BATTERY and psutil.sensors_battery() HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans") diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index fa07f5a92..3eeaa7c3e 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -49,6 +49,7 @@ from psutil.tests import HAS_PROC_CPU_NUM from psutil.tests import HAS_PROC_IO_COUNTERS from psutil.tests import HAS_RLIMIT +from psutil.tests import HAS_THREADS from psutil.tests import mock from psutil.tests import PYPY from psutil.tests import PYTHON @@ -528,6 +529,7 @@ def test_num_handles(self): p = psutil.Process() self.assertGreater(p.num_handles(), 0) + @unittest.skipIf(not HAS_THREADS, 'not supported') def test_threads(self): p = psutil.Process() if OPENBSD: @@ -552,6 +554,7 @@ def test_threads(self): @retry_before_failing() @skip_on_access_denied(only_if=OSX) + @unittest.skipIf(not HAS_THREADS, 'not supported') def test_threads_2(self): sproc = get_test_subprocess() p = psutil.Process(sproc.pid) From 4789fb72315d6ed102762673232189b4e42af5fb Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 29 Oct 2017 11:51:11 -0700 Subject: [PATCH 829/922] Remove trove classifiers for untested and unsupported platforms (#1162) Helps library users know, at a glance, what platforms are tested and supported. This helps users know if the library is suitable for integration in an existing project. --- setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.py b/setup.py index 1bc940839..d11dd782d 100755 --- a/setup.py +++ b/setup.py @@ -317,9 +317,6 @@ def main(): 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.0', - 'Programming Language :: Python :: 3.1', - 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', From 35bd6fdfd752e61da41ca4325bc30bc85de1cbce Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 29 Oct 2017 21:14:57 +0100 Subject: [PATCH 830/922] try to limit occasional appveyor failure --- CREDITS | 10 +++++----- HISTORY.rst | 1 + psutil/tests/test_process.py | 12 +++++++++++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CREDITS b/CREDITS index f6e2b1991..e02ba7eeb 100644 --- a/CREDITS +++ b/CREDITS @@ -44,7 +44,7 @@ Github usernames of people to CC on github when in need of help. - wiggin15, Arnon Yaari - alxchk, Oleksii Shevchuk - AIX: - - wiggin15, Arnon Yaari + - wiggin15, Arnon Yaari (maintainer) Contributors ============ @@ -55,6 +55,10 @@ E: jloden@gmail.com D: original co-author, initial design/bootstrap and occasional bug fixes W: http://www.jayloden.com +N: Arnon Yaari (wiggin15) +W: https://github.com/wiggin15 +I: 517, 607, 610, 1131, 1123, 1130, 1154 + N: Jeff Tang W: https://github.com/mrjefftang I: 340, 529, 616, 653, 654, 648, 641 @@ -365,10 +369,6 @@ N: maozguttman W: https://github.com/maozguttman I: 659 -N: Arnon Yaari (wiggin15) -W: https://github.com/wiggin15 -I: 517, 607, 610, 1131, 1123, 1130 - N: dasumin W: https://github.com/dasumin I: 541 diff --git a/HISTORY.rst b/HISTORY.rst index 0309bbfd2..12d6d184c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ - 1150_: [Windows] when a process is terminate()d now the exit code is set to SIGTERM instead of 0. (patch by Akos Kiss) - 1151_: python -m psutil.tests fail +- 1154_: [AIX] psutil won't compile on AIX 6.1.0. (patch by Arnon Yaari) 5.4.0 ===== diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 3eeaa7c3e..b14981924 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -747,9 +747,19 @@ def test_prog_w_funky_name(self): # Test that name(), exe() and cmdline() correctly handle programs # with funky chars such as spaces and ")", see: # https://github.com/giampaolo/psutil/issues/628 + + def rm(): + # Try to limit occasional failures on Appveyor: + # https://ci.appveyor.com/project/giampaolo/psutil/build/1350/ + # job/lbo3bkju55le850n + try: + safe_rmpath(funky_path) + except OSError: + pass + funky_path = TESTFN + 'foo bar )' create_exe(funky_path) - self.addCleanup(safe_rmpath, funky_path) + self.addCleanup(rm) cmdline = [funky_path, "-c", "import time; [time.sleep(0.01) for x in range(3000)];" "arg1", "arg2", "", "arg3", ""] From d6c54a02431d7ad6408231bcc7c8b1a3534c788b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 29 Oct 2017 22:44:39 +0100 Subject: [PATCH 831/922] update README --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1b45c50da..4559e55c4 100644 --- a/README.rst +++ b/README.rst @@ -84,7 +84,9 @@ and `doc recipes `__. Projects using psutil ===================== -At the time of writing there are over +At the time of writing psutil has roughly +`2.9 milion downloads `__ +per month and there are over `6000 open source projects `__ on github which depend from psutil. Here's some I find particularly interesting: From fe53d1a582a8174204ae08b5e301a3dcfdf09901 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sun, 29 Oct 2017 15:51:44 -0700 Subject: [PATCH 832/922] Fix test_emulate_energy_full_not_avail (#1163) The value may come from two different files, must mock both. --- psutil/tests/test_linux.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 7bb37a8df..4658dd21e 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1326,7 +1326,9 @@ def test_emulate_energy_full_not_avail(self): # Emulate a case where energy_full file does not exist. # Expected fallback on /capacity. def open_mock(name, *args, **kwargs): - if name.startswith("/sys/class/power_supply/BAT0/energy_full"): + energy_full = "/sys/class/power_supply/BAT0/energy_full" + charge_full = "/sys/class/power_supply/BAT0/charge_full" + if name.startswith(energy_full) or name.startswith(charge_full): raise IOError(errno.ENOENT, "") elif name.startswith("/sys/class/power_supply/BAT0/capacity"): return io.BytesIO(b"88") From b30bef830aaa0adb0f3904ed805aa3174aaa2d49 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Oct 2017 00:05:58 +0100 Subject: [PATCH 833/922] add DEVNOTES file; move TODO.aix into _psutil_aix.c --- MANIFEST.in | 2 +- psutil/DEVNOTES | 5 +++++ psutil/TODO.aix | 11 ----------- psutil/_psutil_aix.c | 24 ++++++++++++++++++------ 4 files changed, 24 insertions(+), 18 deletions(-) create mode 100644 psutil/DEVNOTES delete mode 100644 psutil/TODO.aix diff --git a/MANIFEST.in b/MANIFEST.in index a2d0a5d02..a07f16da4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -20,7 +20,7 @@ include docs/conf.py include docs/index.rst include docs/make.bat include make.bat -include psutil/TODO.aix +include psutil/DEVNOTES include psutil/__init__.py include psutil/_common.py include psutil/_compat.py diff --git a/psutil/DEVNOTES b/psutil/DEVNOTES new file mode 100644 index 000000000..4fd15ea31 --- /dev/null +++ b/psutil/DEVNOTES @@ -0,0 +1,5 @@ +API REFERENCES +============== + +- psutil.sensors_battery: + https://github.com/Kentzo/Power/ diff --git a/psutil/TODO.aix b/psutil/TODO.aix deleted file mode 100644 index e1d428bd9..000000000 --- a/psutil/TODO.aix +++ /dev/null @@ -1,11 +0,0 @@ -AIX support is experimental at this time. -The following functions and methods are unsupported on the AIX platform: - - psutil.Process.memory_maps - psutil.Process.num_ctx_switches - -Known limitations: - psutil.Process.io_counters read count is always 0 - reading basic process info may fail or return incorrect values when process is starting - (see IBM APAR IV58499 - fixed in newer AIX versions) - sockets and pipes may not be counted in num_fds (fixed in newer AIX versions) diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c index 76ed736b2..8ffc8f47a 100644 --- a/psutil/_psutil_aix.c +++ b/psutil/_psutil_aix.c @@ -4,16 +4,28 @@ * All rights reserved. * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. + /* + +/* + * AIX support is experimental at this time. + * The following functions and methods are unsupported on the AIX platform: + * - psutil.Process.memory_maps + * - psutil.Process.num_ctx_switches * - * AIX platform-specific module methods for _psutil_aix + * Known limitations: + * - psutil.Process.io_counters read count is always 0 + * - reading basic process info may fail or return incorrect values when + * process is starting (see IBM APAR IV58499 - fixed in newer AIX versions) + * - sockets and pipes may not be counted in num_fds (fixed in newer AIX + * versions) * + * Useful resources: + * - proc filesystem: http://www-01.ibm.com/support/knowledgecenter/ + * ssw_aix_61/com.ibm.aix.files/proc.htm + * - libperfstat: http://www-01.ibm.com/support/knowledgecenter/ + * ssw_aix_61/com.ibm.aix.files/libperfstat.h.htm */ -// Useful resources: -// proc filesystem: http://www-01.ibm.com/support/knowledgecenter/ssw_aix_61/com.ibm.aix.files/proc.htm -// libperfstat: http://www-01.ibm.com/support/knowledgecenter/ssw_aix_61/com.ibm.aix.files/libperfstat.h.htm - - #include #include From da32baf6ce6f648b709bc5cbd537e904022fc790 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Oct 2017 01:04:15 +0100 Subject: [PATCH 834/922] improve logic to determine python exe location --- psutil/tests/__init__.py | 23 +++++++++++++++++------ psutil/tests/test_posix.py | 4 ++-- psutil/tests/test_process.py | 33 +++++++++++++++++---------------- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 1d109af0e..72f3076ea 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -65,7 +65,7 @@ __all__ = [ # constants 'APPVEYOR', 'DEVNULL', 'GLOBAL_TIMEOUT', 'MEMORY_TOLERANCE', 'NO_RETRIES', - 'PYPY', 'PYTHON', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFILE_PREFIX', + 'PYPY', 'PYTHON_EXE', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFILE_PREFIX', 'TESTFN', 'TESTFN_UNICODE', 'TOX', 'TRAVIS', 'VALID_PROC_STATUSES', 'VERBOSITY', "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", @@ -168,7 +168,18 @@ # --- misc -PYTHON = os.path.realpath(sys.executable) + +def _get_py_exe(): + exe = os.path.realpath(sys.executable) + if not os.path.exists(exe): + # It seems this only occurs on OSX. + exe = which("python%s.%s" % sys.version_info[:2]) + if not exe or not os.path.exists(exe): + ValueError("can't find python exe real abspath") + return exe + + +PYTHON_EXE = _get_py_exe() DEVNULL = open(os.devnull, 'r+') VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil) if x.startswith('STATUS_')] @@ -288,7 +299,7 @@ def get_test_subprocess(cmd=None, **kwds): pyline = "from time import sleep;" \ "open(r'%s', 'w').close();" \ "sleep(60);" % _TESTFN - cmd = [PYTHON, "-c", pyline] + cmd = [PYTHON_EXE, "-c", pyline] sproc = subprocess.Popen(cmd, **kwds) _subprocesses_started.add(sproc) wait_for_file(_TESTFN, delete=True, empty=True) @@ -310,13 +321,13 @@ def create_proc_children_pair(): _TESTFN2 = os.path.basename(_TESTFN) + '2' # need to be relative s = textwrap.dedent("""\ import subprocess, os, sys, time - PYTHON = os.path.realpath(sys.executable) + PYTHON_EXE = os.path.realpath(sys.executable) s = "import os, time;" s += "f = open('%s', 'w');" s += "f.write(str(os.getpid()));" s += "f.close();" s += "time.sleep(60);" - subprocess.Popen([PYTHON, '-c', s]) + subprocess.Popen([PYTHON_EXE, '-c', s]) time.sleep(60) """ % _TESTFN2) # On Windows if we create a subprocess with CREATE_NO_WINDOW flag @@ -384,7 +395,7 @@ def pyrun(src, **kwds): _testfiles_created.add(f.name) f.write(src) f.flush() - subp = get_test_subprocess([PYTHON, f.name], **kwds) + subp = get_test_subprocess([PYTHON_EXE, f.name], **kwds) wait_for_pid(subp.pid) return subp diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index aaeef7472..ac68b8d3d 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -28,7 +28,7 @@ from psutil.tests import get_kernel_version from psutil.tests import get_test_subprocess from psutil.tests import mock -from psutil.tests import PYTHON +from psutil.tests import PYTHON_EXE from psutil.tests import reap_children from psutil.tests import retry_before_failing from psutil.tests import run_test_module_by_name @@ -90,7 +90,7 @@ class TestProcess(unittest.TestCase): @classmethod def setUpClass(cls): - cls.pid = get_test_subprocess([PYTHON, "-E", "-O"], + cls.pid = get_test_subprocess([PYTHON_EXE, "-E", "-O"], stdin=subprocess.PIPE).pid wait_for_pid(cls.pid) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index b14981924..4d0a783c3 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -52,7 +52,7 @@ from psutil.tests import HAS_THREADS from psutil.tests import mock from psutil.tests import PYPY -from psutil.tests import PYTHON +from psutil.tests import PYTHON_EXE from psutil.tests import reap_children from psutil.tests import retry_before_failing from psutil.tests import run_test_module_by_name @@ -167,7 +167,7 @@ def test_wait(self): # check sys.exit() code code = "import time, sys; time.sleep(0.01); sys.exit(5);" - sproc = get_test_subprocess([PYTHON, "-c", code]) + sproc = get_test_subprocess([PYTHON_EXE, "-c", code]) p = psutil.Process(sproc.pid) self.assertEqual(p.wait(), 5) self.assertFalse(p.is_running()) @@ -176,7 +176,7 @@ def test_wait(self): # It is not supposed to raise NSP when the process is gone. # On UNIX this should return None, on Windows it should keep # returning the exit code. - sproc = get_test_subprocess([PYTHON, "-c", code]) + sproc = get_test_subprocess([PYTHON_EXE, "-c", code]) p = psutil.Process(sproc.pid) self.assertEqual(p.wait(), 5) self.assertIn(p.wait(), (5, None)) @@ -317,7 +317,7 @@ def test_io_counters(self): # test reads io1 = p.io_counters() - with open(PYTHON, 'rb') as f: + with open(PYTHON_EXE, 'rb') as f: f.read() io2 = p.io_counters() if not BSD and not AIX: @@ -692,12 +692,12 @@ def test_exe(self): sproc = get_test_subprocess() exe = psutil.Process(sproc.pid).exe() try: - self.assertEqual(exe, PYTHON) + self.assertEqual(exe, PYTHON_EXE) except AssertionError: - if WINDOWS and len(exe) == len(PYTHON): + if WINDOWS and len(exe) == len(PYTHON_EXE): # on Windows we don't care about case sensitivity normcase = os.path.normcase - self.assertEqual(normcase(exe), normcase(PYTHON)) + self.assertEqual(normcase(exe), normcase(PYTHON_EXE)) else: # certain platforms such as BSD are more accurate returning: # "/usr/local/bin/python2.7" @@ -708,7 +708,7 @@ def test_exe(self): ver = "%s.%s" % (sys.version_info[0], sys.version_info[1]) try: self.assertEqual(exe.replace(ver, ''), - PYTHON.replace(ver, '')) + PYTHON_EXE.replace(ver, '')) except AssertionError: # Tipically OSX. Really not sure what to do here. pass @@ -717,7 +717,7 @@ def test_exe(self): self.assertEqual(out, 'hey') def test_cmdline(self): - cmdline = [PYTHON, "-c", "import time; time.sleep(60)"] + cmdline = [PYTHON_EXE, "-c", "import time; time.sleep(60)"] sproc = get_test_subprocess(cmdline) try: self.assertEqual(' '.join(psutil.Process(sproc.pid).cmdline()), @@ -730,12 +730,12 @@ def test_cmdline(self): # XXX - AIX truncates long arguments in /proc/pid/cmdline if NETBSD or OPENBSD or AIX: self.assertEqual( - psutil.Process(sproc.pid).cmdline()[0], PYTHON) + psutil.Process(sproc.pid).cmdline()[0], PYTHON_EXE) else: raise def test_name(self): - sproc = get_test_subprocess(PYTHON) + sproc = get_test_subprocess(PYTHON_EXE) name = psutil.Process(sproc.pid).name().lower() pyexe = os.path.basename(os.path.realpath(sys.executable)).lower() assert pyexe.startswith(name), (pyexe, name) @@ -864,7 +864,8 @@ def test_cwd(self): self.assertEqual(p.cwd(), os.getcwd()) def test_cwd_2(self): - cmd = [PYTHON, "-c", "import os, time; os.chdir('..'); time.sleep(60)"] + cmd = [PYTHON_EXE, "-c", + "import os, time; os.chdir('..'); time.sleep(60)"] sproc = get_test_subprocess(cmd) p = psutil.Process(sproc.pid) call_until(p.cwd, "ret == os.path.dirname(os.getcwd())") @@ -949,7 +950,7 @@ def test_open_files(self): # another process cmdline = "import time; f = open(r'%s', 'r'); time.sleep(60);" % TESTFN - sproc = get_test_subprocess([PYTHON, "-c", cmdline]) + sproc = get_test_subprocess([PYTHON_EXE, "-c", cmdline]) p = psutil.Process(sproc.pid) for x in range(100): @@ -1506,7 +1507,7 @@ def test_misc(self): # XXX this test causes a ResourceWarning on Python 3 because # psutil.__subproc instance doesn't get propertly freed. # Not sure what to do though. - cmd = [PYTHON, "-c", "import time; time.sleep(60);"] + cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] with psutil.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: proc.name() @@ -1517,7 +1518,7 @@ def test_misc(self): proc.terminate() def test_ctx_manager(self): - with psutil.Popen([PYTHON, "-V"], + with psutil.Popen([PYTHON_EXE, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) as proc: @@ -1531,7 +1532,7 @@ def test_kill_terminate(self): # subprocess.Popen()'s terminate(), kill() and send_signal() do # not raise exception after the process is gone. psutil.Popen # diverges from that. - cmd = [PYTHON, "-c", "import time; time.sleep(60);"] + cmd = [PYTHON_EXE, "-c", "import time; time.sleep(60);"] with psutil.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: proc.terminate() From 1781c243e590d4dfdb3abff9a658fdc8bbec80b4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 30 Oct 2017 01:13:22 +0100 Subject: [PATCH 835/922] use new PYTHON_EXE --- psutil/tests/__init__.py | 7 +++---- psutil/tests/__main__.py | 12 ++++++------ psutil/tests/test_misc.py | 4 ++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 72f3076ea..cb46a463e 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -321,15 +321,14 @@ def create_proc_children_pair(): _TESTFN2 = os.path.basename(_TESTFN) + '2' # need to be relative s = textwrap.dedent("""\ import subprocess, os, sys, time - PYTHON_EXE = os.path.realpath(sys.executable) s = "import os, time;" s += "f = open('%s', 'w');" s += "f.write(str(os.getpid()));" s += "f.close();" s += "time.sleep(60);" - subprocess.Popen([PYTHON_EXE, '-c', s]) + subprocess.Popen(['%s', '-c', s]) time.sleep(60) - """ % _TESTFN2) + """ % (_TESTFN2, PYTHON_EXE)) # On Windows if we create a subprocess with CREATE_NO_WINDOW flag # set (which is the default) a "conhost.exe" extra process will be # spawned as a child. We don't want that. @@ -721,7 +720,7 @@ def create_exe(outpath, c_code=None): safe_rmpath(f.name) else: # copy python executable - shutil.copyfile(sys.executable, outpath) + shutil.copyfile(PYTHON_EXE, outpath) if POSIX: st = os.stat(outpath) os.chmod(outpath, st.st_mode | stat.S_IEXEC) diff --git a/psutil/tests/__main__.py b/psutil/tests/__main__.py index 475e6b81c..2cdf5c425 100755 --- a/psutil/tests/__main__.py +++ b/psutil/tests/__main__.py @@ -21,11 +21,11 @@ except ImportError: from urllib2 import urlopen +from psutil.tests import PYTHON_EXE from psutil.tests import run_suite HERE = os.path.abspath(os.path.dirname(__file__)) -PYTHON = os.path.basename(sys.executable) GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" TEST_DEPS = [] if sys.version_info[:2] == (2, 6): @@ -54,7 +54,7 @@ def install_pip(): f.flush() print("installing pip") - code = os.system('%s %s --user' % (sys.executable, f.name)) + code = os.system('%s %s --user' % (PYTHON_EXE, f.name)) return code @@ -68,12 +68,12 @@ def install_test_deps(deps=None): opts = "--user" if not is_venv else "" install_pip() code = os.system('%s -m pip install %s --upgrade %s' % ( - sys.executable, opts, " ".join(deps))) + PYTHON_EXE, opts, " ".join(deps))) return code def main(): - usage = "%s -m psutil.tests [opts]" % PYTHON + usage = "%s -m psutil.tests [opts]" % PYTHON_EXE parser = optparse.OptionParser(usage=usage, description="run unit tests") parser.add_option("-i", "--install-deps", action="store_true", default=False, @@ -88,8 +88,8 @@ def main(): try: __import__(dep.split("==")[0]) except ImportError: - sys.exit("%r lib is not installed; run:\n" - "%s -m psutil.tests --install-deps" % (dep, PYTHON)) + sys.exit("%r lib is not installed; run %s -m psutil.tests " + "--install-deps" % (dep, PYTHON_EXE)) run_suite() diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 85bab84c7..43b589eb3 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -18,7 +18,6 @@ import pickle import socket import stat -import sys from psutil import LINUX from psutil import POSIX @@ -49,6 +48,7 @@ from psutil.tests import import_module_by_path from psutil.tests import is_namedtuple from psutil.tests import mock +from psutil.tests import PYTHON_EXE from psutil.tests import reap_children from psutil.tests import reload_module from psutil.tests import retry @@ -652,7 +652,7 @@ def assert_stdout(exe, args=None, **kwds): if args: exe = exe + ' ' + args try: - out = sh(sys.executable + ' ' + exe, **kwds).strip() + out = sh(PYTHON_EXE + ' ' + exe, **kwds).strip() except RuntimeError as err: if 'AccessDenied' in str(err): return str(err) From 65a52341b55faaab41f68ebc4ed31f18f0929754 Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Tue, 31 Oct 2017 11:46:08 +0200 Subject: [PATCH 836/922] AIX: implement num_ctx_switches (#1164) * small changes * AIX: implement num_ctx_switches --- MANIFEST.in | 2 + docs/index.rst | 2 +- psutil/__init__.py | 12 ++--- psutil/_psaix.py | 5 ++ psutil/_psutil_aix.c | 44 ++++++++++++++++- psutil/_psutil_sunos.c | 2 +- psutil/arch/aix/common.c | 79 +++++++++++++++++++++++++++++++ psutil/arch/aix/common.h | 31 ++++++++++++ psutil/arch/aix/net_connections.c | 79 ++++--------------------------- psutil/arch/aix/net_connections.h | 5 ++ psutil/arch/solaris/v10/ifaddrs.h | 4 +- psutil/tests/__init__.py | 2 - psutil/tests/test_contracts.py | 2 +- psutil/tests/test_memory_leaks.py | 2 - psutil/tests/test_process.py | 2 - setup.py | 1 + 16 files changed, 185 insertions(+), 89 deletions(-) create mode 100644 psutil/arch/aix/common.c create mode 100644 psutil/arch/aix/common.h diff --git a/MANIFEST.in b/MANIFEST.in index a07f16da4..59a102a55 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -45,6 +45,8 @@ include psutil/arch/aix/ifaddrs.c include psutil/arch/aix/ifaddrs.h include psutil/arch/aix/net_connections.c include psutil/arch/aix/net_connections.h +include psutil/arch/aix/common.c +include psutil/arch/aix/common.h include psutil/arch/aix/net_kernel_structs.h include psutil/arch/freebsd/proc_socks.c include psutil/arch/freebsd/proc_socks.h diff --git a/docs/index.rst b/docs/index.rst index feb9fd78f..ac591d303 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1376,7 +1376,7 @@ Process class The number voluntary and involuntary context switches performed by this process (cumulative). - Availability: all platforms except AIX + .. versionchanged:: 5.4.1 added AIX support .. method:: num_fds() diff --git a/psutil/__init__.py b/psutil/__init__.py index b8b94e7ed..97d80940a 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -899,13 +899,11 @@ def num_handles(self): """ return self._proc.num_handles() - if hasattr(_psplatform.Process, "num_ctx_switches"): - - def num_ctx_switches(self): - """Return the number of voluntary and involuntary context - switches performed by this process. - """ - return self._proc.num_ctx_switches() + def num_ctx_switches(self): + """Return the number of voluntary and involuntary context + switches performed by this process. + """ + return self._proc.num_ctx_switches() def num_threads(self): """Return the number of threads used by this process.""" diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 5cd088e55..c78922b06 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -554,6 +554,11 @@ def num_fds(self): return 0 return len(os.listdir("%s/%s/fd" % (self._procfs_path, self.pid))) + @wrap_exceptions + def num_ctx_switches(self): + return _common.pctxsw( + *cext.proc_num_ctx_switches(self.pid)) + @wrap_exceptions def wait(self, timeout=None): try: diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c index 8ffc8f47a..0834726dd 100644 --- a/psutil/_psutil_aix.c +++ b/psutil/_psutil_aix.c @@ -4,16 +4,16 @@ * All rights reserved. * Use of this source code is governed by a BSD-style license that can be * found in the LICENSE file. - /* + */ /* * AIX support is experimental at this time. * The following functions and methods are unsupported on the AIX platform: * - psutil.Process.memory_maps - * - psutil.Process.num_ctx_switches * * Known limitations: * - psutil.Process.io_counters read count is always 0 + * - psutil.Process.threads may not be available on older AIX versions * - reading basic process info may fail or return incorrect values when * process is starting (see IBM APAR IV58499 - fixed in newer AIX versions) * - sockets and pipes may not be counted in num_fds (fixed in newer AIX @@ -49,6 +49,7 @@ #include "arch/aix/ifaddrs.h" #include "arch/aix/net_connections.h" +#include "arch/aix/common.h" #include "_psutil_common.h" #include "_psutil_posix.h" @@ -307,6 +308,43 @@ psutil_proc_cred(PyObject *self, PyObject *args) { } +/* + * Return process voluntary and involuntary context switches as a Python tuple. + */ +static PyObject * +psutil_proc_num_ctx_switches(PyObject *self, PyObject *args) { + PyObject *py_tuple = NULL; + pid32_t requested_pid; + pid32_t pid = 0; + int np = 0; + struct procentry64 *processes = (struct procentry64 *)NULL; + struct procentry64 *p; + + if (! PyArg_ParseTuple(args, "i", &requested_pid)) + return NULL; + + processes = psutil_read_process_table(&np); + if (!processes) + return NULL; + + /* Loop through processes */ + for (p = processes; np > 0; np--, p++) { + pid = p->pi_pid; + if (requested_pid != pid) + continue; + py_tuple = Py_BuildValue("LL", + (long long) p->pi_ru.ru_nvcsw, /* voluntary context switches */ + (long long) p->pi_ru.ru_nivcsw); /* involuntary */ + free(processes); + return py_tuple; + } + + /* finished iteration without finding requested pid */ + free(processes); + return NoSuchProcess(); +} + + /* * Return users currently connected on the system. */ @@ -833,6 +871,8 @@ PsutilMethods[] = #endif {"proc_io_counters", psutil_proc_io_counters, METH_VARARGS, "Get process I/O counters."}, + {"proc_num_ctx_switches", psutil_proc_num_ctx_switches, METH_VARARGS, + "Get process I/O counters."}, // --- system-related functions {"users", psutil_users, METH_VARARGS, diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 8f9773424..083b78cb1 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -360,7 +360,7 @@ psutil_proc_cred(PyObject *self, PyObject *args) { /* - * Return process uids/gids as a Python tuple. + * Return process voluntary and involuntary context switches as a Python tuple. */ static PyObject * psutil_proc_num_ctx_switches(PyObject *self, PyObject *args) { diff --git a/psutil/arch/aix/common.c b/psutil/arch/aix/common.c new file mode 100644 index 000000000..6115a15db --- /dev/null +++ b/psutil/arch/aix/common.c @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include +#include +#include +#include "common.h" + +/* psutil_kread() - read from kernel memory */ +int +psutil_kread( + int Kd, /* kernel memory file descriptor */ + KA_T addr, /* kernel memory address */ + char *buf, /* buffer to receive data */ + size_t len) { /* length to read */ + int br; + + if (lseek64(Kd, (off64_t)addr, L_SET) == (off64_t)-1) { + PyErr_SetFromErrno(PyExc_OSError); + return 1; + } + br = read(Kd, buf, len); + if (br == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return 1; + } + if (br != len) { + PyErr_SetString(PyExc_RuntimeError, + "size mismatch when reading kernel memory fd"); + return 1; + } + return 0; +} + +struct procentry64 * +psutil_read_process_table(int * num) { + size_t msz; + pid32_t pid = 0; + struct procentry64 *processes = (struct procentry64 *)NULL; + struct procentry64 *p; + int Np = 0; /* number of processes allocated in 'processes' */ + int np = 0; /* number of processes read into 'processes' */ + int i; /* number of processes read in current iteration */ + + msz = (size_t)(PROCSIZE * PROCINFO_INCR); + processes = (struct procentry64 *)malloc(msz); + if (!processes) { + PyErr_NoMemory(); + return NULL; + } + Np = PROCINFO_INCR; + p = processes; + while ((i = getprocs64(p, PROCSIZE, (struct fdsinfo64 *)NULL, 0, &pid, + PROCINFO_INCR)) + == PROCINFO_INCR) { + np += PROCINFO_INCR; + if (np >= Np) { + msz = (size_t)(PROCSIZE * (Np + PROCINFO_INCR)); + processes = (struct procentry64 *)realloc((char *)processes, msz); + if (!processes) { + PyErr_NoMemory(); + return NULL; + } + Np += PROCINFO_INCR; + } + p = (struct procentry64 *)((char *)processes + (np * PROCSIZE)); + } + + /* add the number of processes read in the last iteration */ + if (i > 0) + np += i; + + *num = np; + return processes; +} \ No newline at end of file diff --git a/psutil/arch/aix/common.h b/psutil/arch/aix/common.h new file mode 100644 index 000000000..b677d8c29 --- /dev/null +++ b/psutil/arch/aix/common.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2017, Arnon Yaari + * All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#ifndef __PSUTIL_AIX_COMMON_H__ +#define __PSUTIL_AIX_COMMON_H__ + +#include + +#define PROCINFO_INCR (256) +#define PROCSIZE (sizeof(struct procentry64)) +#define FDSINFOSIZE (sizeof(struct fdsinfo64)) +#define KMEM "/dev/kmem" + +typedef u_longlong_t KA_T; + +/* psutil_kread() - read from kernel memory */ +int psutil_kread(int Kd, /* kernel memory file descriptor */ + KA_T addr, /* kernel memory address */ + char *buf, /* buffer to receive data */ + size_t len); /* length to read */ + +struct procentry64 * +psutil_read_process_table( + int * num /* out - number of processes read */ +); + +#endif /* __PSUTIL_AIX_COMMON_H__ */ diff --git a/psutil/arch/aix/net_connections.c b/psutil/arch/aix/net_connections.c index 364cd1b7e..69b438920 100644 --- a/psutil/arch/aix/net_connections.c +++ b/psutil/arch/aix/net_connections.c @@ -13,57 +13,26 @@ * - dialects/aix/dproc.c:get_kernel_access */ -#include "net_connections.h" +#include +#include #include -#include -#define _KERNEL 1 +#define _KERNEL #include #undef _KERNEL -#include +#include #include #include #include #include #include -#include "net_kernel_structs.h" - +#include "../../_psutil_common.h" +#include "net_kernel_structs.h" +#include "net_connections.h" +#include "common.h" -#define PROCINFO_INCR (256) -#define PROCSIZE (sizeof(struct procentry64)) -#define FDSINFOSIZE (sizeof(struct fdsinfo64)) -#define KMEM "/dev/kmem" #define NO_SOCKET (PyObject *)(-1) -typedef u_longlong_t KA_T; -static int PSUTIL_CONN_NONE = 128; - -/* psutil_kread() - read from kernel memory */ -static int -psutil_kread( - int Kd, /* kernel memory file descriptor */ - KA_T addr, /* kernel memory address */ - char *buf, /* buffer to receive data */ - size_t len) { /* length to read */ - int br; - - if (lseek64(Kd, (off64_t)addr, L_SET) == (off64_t)-1) { - PyErr_SetFromErrno(PyExc_OSError); - return 1; - } - br = read(Kd, buf, len); - if (br == -1) { - PyErr_SetFromErrno(PyExc_OSError); - return 1; - } - if (br != len) { - PyErr_SetString(PyExc_RuntimeError, - "size mismatch when reading kernel memory fd"); - return 1; - } - return 0; -} - static int read_unp_addr( int Kd, @@ -244,10 +213,8 @@ psutil_net_connections(PyObject *self, PyObject *args) { int i, np; struct procentry64 *p; struct fdsinfo64 *fds = (struct fdsinfo64 *)NULL; - size_t msz; pid32_t requested_pid; pid32_t pid; - int Np = 0; /* number of processes */ struct procentry64 *processes = (struct procentry64 *)NULL; /* the process table */ @@ -262,34 +229,9 @@ psutil_net_connections(PyObject *self, PyObject *args) { goto error; } - /* Read the process table */ - msz = (size_t)(PROCSIZE * PROCINFO_INCR); - processes = (struct procentry64 *)malloc(msz); - if (!processes) { - PyErr_NoMemory(); + processes = psutil_read_process_table(&np); + if (!processes) goto error; - } - Np = PROCINFO_INCR; - np = pid = 0; - p = processes; - while ((i = getprocs64(p, PROCSIZE, (struct fdsinfo64 *)NULL, 0, &pid, - PROCINFO_INCR)) - == PROCINFO_INCR) { - np += PROCINFO_INCR; - if (np >= Np) { - msz = (size_t)(PROCSIZE * (Np + PROCINFO_INCR)); - processes = (struct procentry64 *)realloc((char *)processes, msz); - if (!processes) { - PyErr_NoMemory(); - goto error; - } - Np += PROCINFO_INCR; - } - p = (struct procentry64 *)((char *)processes + (np * PROCSIZE)); - } - - if (i > 0) - np += i; /* Loop through processes */ for (p = processes; np > 0; np--, p++) { @@ -299,7 +241,6 @@ psutil_net_connections(PyObject *self, PyObject *args) { if (p->pi_state == 0 || p->pi_state == SZOMB) continue; - if (!fds) { fds = (struct fdsinfo64 *)malloc((size_t)FDSINFOSIZE); if (!fds) { diff --git a/psutil/arch/aix/net_connections.h b/psutil/arch/aix/net_connections.h index f6a726cb9..222bcaf35 100644 --- a/psutil/arch/aix/net_connections.h +++ b/psutil/arch/aix/net_connections.h @@ -5,6 +5,11 @@ * found in the LICENSE file. */ +#ifndef __NET_CONNECTIONS_H__ +#define __NET_CONNECTIONS_H__ + #include PyObject* psutil_net_connections(PyObject *self, PyObject *args); + +#endif /* __NET_CONNECTIONS_H__ */ \ No newline at end of file diff --git a/psutil/arch/solaris/v10/ifaddrs.h b/psutil/arch/solaris/v10/ifaddrs.h index d27711935..0953a9b99 100644 --- a/psutil/arch/solaris/v10/ifaddrs.h +++ b/psutil/arch/solaris/v10/ifaddrs.h @@ -1,8 +1,8 @@ /* Reference: https://lists.samba.org/archive/samba-technical/2009-February/063079.html */ -#ifndef __IFADDRS_H___ -#define __IFADDRS_H___ +#ifndef __IFADDRS_H__ +#define __IFADDRS_H__ #include #include diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index cb46a463e..14f1b53f0 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -72,7 +72,6 @@ "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", "HAS_SENSORS_BATTERY", "HAS_BATTERY""HAS_SENSORS_FANS", "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", - "HAS_NUM_CTX_SWITCHES", # subprocesses 'pyrun', 'reap_children', 'get_test_subprocess', 'create_zombie_proc', 'create_proc_children_pair', @@ -157,7 +156,6 @@ HAS_IONICE = hasattr(psutil.Process, "ionice") HAS_MEMORY_FULL_INFO = 'uss' in psutil.Process().memory_full_info()._fields HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps") -HAS_NUM_CTX_SWITCHES = hasattr(psutil.Process, "num_ctx_switches") HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") HAS_RLIMIT = hasattr(psutil.Process, "rlimit") HAS_THREADS = hasattr(psutil.Process, "threads") diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 13a737e84..d9633339b 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -611,7 +611,7 @@ def nice(self, ret, proc): def num_ctx_switches(self, ret, proc): assert is_namedtuple(ret) for value in ret: - self.assertIsInstance(value, int) + self.assertIsInstance(value, (int, long)) self.assertGreaterEqual(value, 0) def rlimit(self, ret, proc): diff --git a/psutil/tests/test_memory_leaks.py b/psutil/tests/test_memory_leaks.py index 76fab357b..680fe7803 100755 --- a/psutil/tests/test_memory_leaks.py +++ b/psutil/tests/test_memory_leaks.py @@ -38,7 +38,6 @@ from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_IONICE from psutil.tests import HAS_MEMORY_MAPS -from psutil.tests import HAS_NUM_CTX_SWITCHES from psutil.tests import HAS_PROC_CPU_NUM from psutil.tests import HAS_PROC_IO_COUNTERS from psutil.tests import HAS_RLIMIT @@ -289,7 +288,6 @@ def test_num_fds(self): self.execute(self.proc.num_fds) @skip_if_linux() - @unittest.skipIf(not HAS_NUM_CTX_SWITCHES, "not supported") def test_num_ctx_switches(self): self.execute(self.proc.num_ctx_switches) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 4d0a783c3..1e01aea55 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -45,7 +45,6 @@ from psutil.tests import HAS_ENVIRON from psutil.tests import HAS_IONICE from psutil.tests import HAS_MEMORY_MAPS -from psutil.tests import HAS_NUM_CTX_SWITCHES from psutil.tests import HAS_PROC_CPU_NUM from psutil.tests import HAS_PROC_IO_COUNTERS from psutil.tests import HAS_RLIMIT @@ -1004,7 +1003,6 @@ def test_num_fds(self): @skip_on_not_implemented(only_if=LINUX) @unittest.skipIf(OPENBSD or NETBSD, "not reliable on OPENBSD & NETBSD") - @unittest.skipIf(not HAS_NUM_CTX_SWITCHES, "not supported") def test_num_ctx_switches(self): p = psutil.Process() before = sum(p.num_ctx_switches()) diff --git a/setup.py b/setup.py index d11dd782d..a170d2def 100755 --- a/setup.py +++ b/setup.py @@ -248,6 +248,7 @@ def get_ethtool_macro(): sources=sources + [ 'psutil/_psutil_aix.c', 'psutil/arch/aix/net_connections.c', + 'psutil/arch/aix/common.c', 'psutil/arch/aix/ifaddrs.c'], libraries=['perfstat'], define_macros=macros) From 90847b3e9d3f9a0ae39accea4597e408c4e80bbf Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 31 Oct 2017 10:47:48 +0100 Subject: [PATCH 837/922] #1164 give CREDITS to @wiggin15 --- CREDITS | 2 +- HISTORY.rst | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index e02ba7eeb..95e4bd345 100644 --- a/CREDITS +++ b/CREDITS @@ -57,7 +57,7 @@ W: http://www.jayloden.com N: Arnon Yaari (wiggin15) W: https://github.com/wiggin15 -I: 517, 607, 610, 1131, 1123, 1130, 1154 +I: 517, 607, 610, 1131, 1123, 1130, 1154, 1164 N: Jeff Tang W: https://github.com/mrjefftang diff --git a/HISTORY.rst b/HISTORY.rst index 12d6d184c..0fba703e1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,11 @@ *XXXX-XX-XX* +**Enhancements** + +- 1164_: [AIX] add support for Process.num_ctx_switches(). (patch by Arnon + Yaari) + **Bug fixes** - 1150_: [Windows] when a process is terminate()d now the exit code is set to From a4f00398f923bf1db2cd575af0074870138d328a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 31 Oct 2017 11:03:30 +0100 Subject: [PATCH 838/922] fix test --- psutil/tests/test_misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 43b589eb3..85bab84c7 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -18,6 +18,7 @@ import pickle import socket import stat +import sys from psutil import LINUX from psutil import POSIX @@ -48,7 +49,6 @@ from psutil.tests import import_module_by_path from psutil.tests import is_namedtuple from psutil.tests import mock -from psutil.tests import PYTHON_EXE from psutil.tests import reap_children from psutil.tests import reload_module from psutil.tests import retry @@ -652,7 +652,7 @@ def assert_stdout(exe, args=None, **kwds): if args: exe = exe + ' ' + args try: - out = sh(PYTHON_EXE + ' ' + exe, **kwds).strip() + out = sh(sys.executable + ' ' + exe, **kwds).strip() except RuntimeError as err: if 'AccessDenied' in str(err): return str(err) From 3f74e32f0c97e67e4647903df6f400c0e1491445 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 3 Nov 2017 10:27:19 +0100 Subject: [PATCH 839/922] update Makefile --- MANIFEST.in | 4 ++-- Makefile | 28 ++++++++++------------------ psutil/__init__.py | 2 +- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 59a102a55..11945017e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -41,12 +41,12 @@ include psutil/_psutil_posix.h include psutil/_psutil_sunos.c include psutil/_psutil_windows.c include psutil/_pswindows.py +include psutil/arch/aix/common.c +include psutil/arch/aix/common.h include psutil/arch/aix/ifaddrs.c include psutil/arch/aix/ifaddrs.h include psutil/arch/aix/net_connections.c include psutil/arch/aix/net_connections.h -include psutil/arch/aix/common.c -include psutil/arch/aix/common.h include psutil/arch/aix/net_kernel_structs.h include psutil/arch/freebsd/proc_socks.c include psutil/arch/freebsd/proc_socks.h diff --git a/Makefile b/Makefile index 5a8603656..f3691048b 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,6 @@ PYTHON = python TSCRIPT = psutil/tests/__main__.py ARGS = - # List of nice-to-have dev libs. DEPS = \ argparse \ @@ -60,7 +59,6 @@ clean: _: - # Compile without installing. build: _ # make sure setuptools is installed (needed for 'develop' / edit mode) @@ -87,7 +85,7 @@ uninstall: # Install PIP (only if necessary). install-pip: - PYTHONWARNINGS=all $(PYTHON) -c \ + $(PYTHON) -c \ "import sys, ssl, os, pkgutil, tempfile, atexit; \ sys.exit(0) if pkgutil.find_loader('pip') else None; \ pyexc = 'from urllib.request import urlopen' if sys.version_info[0] == 3 else 'from urllib2 import urlopen'; \ @@ -159,7 +157,7 @@ test-posix: # Run specific platform tests only. test-platform: ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS") if getattr(psutil, x)][0])'`.py + PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py # Memory leak tests. test-memleaks: @@ -218,12 +216,12 @@ install-git-hooks: # Generate tar.gz source distribution. sdist: ${MAKE} generate-manifest - PYTHONWARNINGS=all $(PYTHON) setup.py sdist + $(PYTHON) setup.py sdist # Upload source tarball on https://pypi.python.org/pypi/psutil. upload-src: ${MAKE} sdist - PYTHONWARNINGS=all $(PYTHON) setup.py sdist upload + $(PYTHON) setup.py sdist upload # Download exes/wheels hosted on appveyor. win-download-exes: @@ -231,13 +229,13 @@ win-download-exes: # Upload exes/wheels in dist/* directory to PYPI. win-upload-exes: - PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/*.exe - PYTHONWARNINGS=all $(PYTHON) -m twine upload dist/*.whl + $(PYTHON) -m twine upload dist/*.exe + $(PYTHON) -m twine upload dist/*.whl # All the necessary steps before making a release. pre-release: ${MAKE} install - @PYTHONWARNINGS=all $(PYTHON) -c \ + $(PYTHON) -c \ "from psutil import __version__ as ver; \ doc = open('docs/index.rst').read(); \ history = open('HISTORY.rst').read(); \ @@ -246,7 +244,7 @@ pre-release: assert 'XXXX' not in history, 'XXXX in HISTORY.rst';" ${MAKE} generate-manifest git diff MANIFEST.in > /dev/null # ...otherwise 'git diff-index HEAD' will complain - @PYTHONWARNINGS=all $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else sys.exit(0);" + $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else sys.exit(0);" ${MAKE} win-download-exes ${MAKE} sdist @@ -267,11 +265,11 @@ print-timeline: # Inspect MANIFEST.in file. check-manifest: - PYTHONWARNINGS=all $(PYTHON) -m check_manifest -v $(ARGS) + $(PYTHON) -m check_manifest -v $(ARGS) # Generates MANIFEST.in file. generate-manifest: - @PYTHONWARNINGS=all $(PYTHON) scripts/internal/generate_manifest.py > MANIFEST.in + $(PYTHON) scripts/internal/generate_manifest.py > MANIFEST.in # =================================================================== # Misc @@ -290,12 +288,6 @@ bench-oneshot-2: ${MAKE} install PYTHONWARNINGS=all $(PYTHON) scripts/internal/bench_oneshot_2.py -# generate a doc.zip file and manually upload it to PYPI. -doc: - cd docs && make html && cd _build/html/ && zip doc.zip -r . - mv docs/_build/html/doc.zip . - @echo "done; now manually upload doc.zip from here: https://pypi.python.org/pypi?:action=pkg_edit&name=psutil" - # check whether the links mentioned in some files are valid. check-broken-links: git ls-files | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py diff --git a/psutil/__init__.py b/psutil/__init__.py index 97d80940a..01934ac3b 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -212,7 +212,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.4.0" +__version__ = "5.4.1" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From a2957e6a4a95ffee461c5b4d37a4b16c3d13c7d2 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 4 Nov 2017 10:23:59 +0100 Subject: [PATCH 840/922] fix failure on osx/travis --- psutil/tests/test_misc.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 85bab84c7..f7305a0db 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -643,6 +643,10 @@ def test_cache_clear_public_apis(self): @unittest.skipIf(TOX, "can't test on TOX") +# See: https://travis-ci.org/giampaolo/psutil/jobs/295224806 +@unittest.skipIf(TRAVIS and not + os.path.exists(os.path.join(SCRIPTS_DIR, 'free.py')), + "can't locate scripts directory") class TestScripts(unittest.TestCase): """Tests for scripts in the "scripts" directory.""" From 657d028799ee071be2c2ad50e2526f53e0dfdc9f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 4 Nov 2017 10:28:24 +0100 Subject: [PATCH 841/922] unicode tests: use different name for test dir --- psutil/tests/test_unicode.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index e491d3dc7..94ebd1632 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -192,13 +192,15 @@ def test_proc_cmdline(self): self.assertEqual(cmdline, [self.funky_name]) def test_proc_cwd(self): - safe_mkdir(self.funky_name) - with chdir(self.funky_name): + dname = self.funky_name + "2" + self.addCleanup(safe_rmpath, dname) + safe_mkdir(dname) + with chdir(dname): p = psutil.Process() cwd = p.cwd() self.assertIsInstance(p.cwd(), str) if self.expect_exact_path_match(): - self.assertEqual(cwd, self.funky_name) + self.assertEqual(cwd, dname) def test_proc_open_files(self): p = psutil.Process() @@ -260,8 +262,10 @@ def find_sock(cons): self.assertEqual(conn.laddr, name) def test_disk_usage(self): - safe_mkdir(self.funky_name) - psutil.disk_usage(self.funky_name) + dname = self.funky_name + "2" + self.addCleanup(safe_rmpath, dname) + safe_mkdir(dname) + psutil.disk_usage(dname) @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") @unittest.skipIf(not PY3, "ctypes does not support unicode on PY2") From 02d1d199e6a439d423165320b88a5955561ddc76 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 4 Nov 2017 10:32:46 +0100 Subject: [PATCH 842/922] reap_children() in a finally block in order to limit false positives --- psutil/tests/test_unicode.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 94ebd1632..3aaca4365 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -125,8 +125,9 @@ def subprocess_supports_unicode(name): except UnicodeEncodeError: return False else: - reap_children() return True + finally: + reap_children() # An invalid unicode string. From e73ddf4057ed9ea07747b633bf539f6553366049 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 4 Nov 2017 10:42:40 +0100 Subject: [PATCH 843/922] try to limit false positives on appveyor/windows --- psutil/tests/test_unicode.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index 3aaca4365..bbb763f31 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -91,8 +91,6 @@ def safe_rmpath(path): - # XXX - return _safe_rmpath(path) if APPVEYOR: # TODO - this is quite random and I'm not sure why it happens, # nor I can reproduce it locally: @@ -146,18 +144,23 @@ def subprocess_supports_unicode(name): class _BaseFSAPIsTests(object): funky_name = None - def setUp(self): - safe_rmpath(self.funky_name) + @classmethod + def setUpClass(cls): + safe_rmpath(cls.funky_name) + create_exe(cls.funky_name) + + @classmethod + def tearDownClass(cls): + reap_children() + safe_rmpath(cls.funky_name) def tearDown(self): reap_children() - safe_rmpath(self.funky_name) def expect_exact_path_match(self): raise NotImplementedError("must be implemented in subclass") def test_proc_exe(self): - create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) exe = p.exe() @@ -166,7 +169,6 @@ def test_proc_exe(self): self.assertEqual(exe, self.funky_name) def test_proc_name(self): - create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) if WINDOWS: # On Windows name() is determined from exe() first, because @@ -183,7 +185,6 @@ def test_proc_name(self): self.assertEqual(name, os.path.basename(self.funky_name)) def test_proc_cmdline(self): - create_exe(self.funky_name) subp = get_test_subprocess(cmd=[self.funky_name]) p = psutil.Process(subp.pid) cmdline = p.cmdline() From a0a196e8e08f426fc83a3ffea20e1d5158a60b31 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 6 Nov 2017 13:52:53 +0100 Subject: [PATCH 844/922] ifconfig.py humanize bytes --- scripts/ifconfig.py | 56 ++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 18 deletions(-) diff --git a/scripts/ifconfig.py b/scripts/ifconfig.py index b823b3740..e2a9ce536 100755 --- a/scripts/ifconfig.py +++ b/scripts/ifconfig.py @@ -10,34 +10,34 @@ $ python scripts/ifconfig.py lo: stats : speed=0MB, duplex=?, mtu=65536, up=yes - incoming : bytes=6889336, pkts=84032, errs=0, drops=0 - outgoing : bytes=6889336, pkts=84032, errs=0, drops=0 + incoming : bytes=1.95M, pkts=22158, errs=0, drops=0 + outgoing : bytes=1.95M, pkts=22158, errs=0, drops=0 IPv4 address : 127.0.0.1 netmask : 255.0.0.0 IPv6 address : ::1 netmask : ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff MAC address : 00:00:00:00:00:00 -vboxnet0: - stats : speed=10MB, duplex=full, mtu=1500, up=yes - incoming : bytes=0, pkts=0, errs=0, drops=0 - outgoing : bytes=1622766, pkts=9102, errs=0, drops=0 - IPv4 address : 192.168.33.1 - broadcast : 192.168.33.255 - netmask : 255.255.255.0 - IPv6 address : fe80::800:27ff:fe00:0%vboxnet0 +docker0: + stats : speed=0MB, duplex=?, mtu=1500, up=yes + incoming : bytes=3.48M, pkts=65470, errs=0, drops=0 + outgoing : bytes=164.06M, pkts=112993, errs=0, drops=0 + IPv4 address : 172.17.0.1 + broadcast : 172.17.0.1 + netmask : 255.255.0.0 + IPv6 address : fe80::42:27ff:fe5e:799e%docker0 netmask : ffff:ffff:ffff:ffff:: - MAC address : 0a:00:27:00:00:00 + MAC address : 02:42:27:5e:79:9e broadcast : ff:ff:ff:ff:ff:ff -eth0: +wlp3s0: stats : speed=0MB, duplex=?, mtu=1500, up=yes - incoming : bytes=18905596301, pkts=15178374, errs=0, drops=21 - outgoing : bytes=1913720087, pkts=9543981, errs=0, drops=0 - IPv4 address : 10.0.0.3 + incoming : bytes=7.04G, pkts=5637208, errs=0, drops=0 + outgoing : bytes=372.01M, pkts=3200026, errs=0, drops=0 + IPv4 address : 10.0.0.2 broadcast : 10.255.255.255 netmask : 255.0.0.0 - IPv6 address : fe80::7592:1dcf:bcb7:98d6%wlp3s0 + IPv6 address : fe80::ecb3:1584:5d17:937%wlp3s0 netmask : ffff:ffff:ffff:ffff:: MAC address : 48:45:20:59:a4:0c broadcast : ff:ff:ff:ff:ff:ff @@ -62,6 +62,24 @@ } +def bytes2human(n): + """ + >>> bytes2human(10000) + '9.8 K' + >>> bytes2human(100001221) + '95.4 M' + """ + symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') + prefix = {} + for i, s in enumerate(symbols): + prefix[s] = 1 << (i + 1) * 10 + for s in reversed(symbols): + if n >= prefix[s]: + value = float(n) / prefix[s] + return '%.2f%s' % (value, s) + return '%.2fB' % (n) + + def main(): stats = psutil.net_if_stats() io_counters = psutil.net_io_counters(pernic=True) @@ -77,10 +95,12 @@ def main(): io = io_counters[nic] print(" incoming : ", end='') print("bytes=%s, pkts=%s, errs=%s, drops=%s" % ( - io.bytes_recv, io.packets_recv, io.errin, io.dropin)) + bytes2human(io.bytes_recv), io.packets_recv, io.errin, + io.dropin)) print(" outgoing : ", end='') print("bytes=%s, pkts=%s, errs=%s, drops=%s" % ( - io.bytes_sent, io.packets_sent, io.errout, io.dropout)) + bytes2human(io.bytes_sent), io.packets_sent, io.errout, + io.dropout)) for addr in addrs: print(" %-4s" % af_map.get(addr.family, addr.family), end="") print(" address : %s" % addr.address) From bbab187d20908cca7a51ee3ca0267ab69f7c8c9d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 6 Nov 2017 19:16:09 +0100 Subject: [PATCH 845/922] provide a 'make help' command --- DEVGUIDE.rst | 1 + Makefile | 116 ++++++++++++++++++--------------------------------- 2 files changed, 42 insertions(+), 75 deletions(-) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index 904f4b8e5..9b26fba9d 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -23,6 +23,7 @@ If you plan on hacking on psutil this is what you're supposed to do first: (see `make.bat `_). - do not use ``sudo``; ``make install`` installs psutil as a limited user in "edit" mode; also ``make setup-dev-env`` installs deps as a limited user. +- use `make help` to see the list of available commands. ============ Coding style diff --git a/Makefile b/Makefile index f3691048b..1d6b15be0 100644 --- a/Makefile +++ b/Makefile @@ -33,8 +33,7 @@ all: test # Install # =================================================================== -# Remove all build files. -clean: +clean: ## Remove all build files. rm -rf `find . -type d -name __pycache__ \ -o -type f -name \*.bak \ -o -type f -name \*.orig \ @@ -59,8 +58,7 @@ clean: _: -# Compile without installing. -build: _ +build: _ ## Compile without installing. # make sure setuptools is installed (needed for 'develop' / edit mode) $(PYTHON) -c "import setuptools" PYTHONWARNINGS=all $(PYTHON) setup.py build @@ -71,20 +69,15 @@ build: _ rm -rf tmp $(PYTHON) -c "import psutil" # make sure it actually worked -# Install this package + GIT hooks. Install is done: -# - as the current user, in order to avoid permission issues -# - in development / edit mode, so that source can be modified on the fly -install: +install: ## Install this package as current user in "edit" mode. ${MAKE} build PYTHONWARNINGS=all $(PYTHON) setup.py develop $(INSTALL_OPTS) rm -rf tmp -# Uninstall this package via pip. -uninstall: +uninstall: ## Uninstall this package via pip. cd ..; $(PYTHON) -m pip uninstall -y -v psutil -# Install PIP (only if necessary). -install-pip: +install-pip: ## Install pip (no-op if already installed). $(PYTHON) -c \ "import sys, ssl, os, pkgutil, tempfile, atexit; \ sys.exit(0) if pkgutil.find_loader('pip') else None; \ @@ -103,8 +96,7 @@ install-pip: f.close(); \ sys.exit(code);" -# Install GIT hooks, pip, test deps (also upgrades them). -setup-dev-env: +setup-dev-env: ## Install GIT hooks, pip, test deps (also upgrades them). ${MAKE} install-git-hooks ${MAKE} install-pip $(PYTHON) -m pip install $(INSTALL_OPTS) --upgrade pip @@ -114,64 +106,51 @@ setup-dev-env: # Tests # =================================================================== -# Run all tests. -test: +test: ## Run all tests. ${MAKE} install PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) $(TSCRIPT) -# Run process-related API tests. -test-process: +test-process: ## Run process-related API tests. ${MAKE} install PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_process -# Run system-related API tests. -test-system: +test-system: ## Run system-related API tests. ${MAKE} install PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_system -# Run miscellaneous tests. -test-misc: +test-misc: ## Run miscellaneous tests. ${MAKE} install PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_misc.py -# Test APIs dealing with strings. -test-unicode: +test-unicode: ## Test APIs dealing with strings. ${MAKE} install PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_unicode.py -# APIs sanity tests. -test-contracts: +test-contracts: ## APIs sanity tests. ${MAKE} install PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_contracts.py -# Test net_connections() and Process.connections(). -test-connections: +test-connections: ## Test net_connections() and Process.connections(). ${MAKE} install PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_connections.py -# POSIX specific tests. -test-posix: +test-posix: ## POSIX specific tests. ${MAKE} install PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_posix.py -# Run specific platform tests only. -test-platform: +test-platform: ## Run specific platform tests only. ${MAKE} install PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py -# Memory leak tests. -test-memleaks: +test-memleaks: ## Memory leak tests. ${MAKE} install PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_memory_leaks.py -# Run a specific test by name, e.g. -# make test-by-name psutil.tests.test_system.TestSystemAPIs.test_cpu_times -test-by-name: +test-by-name: ## e.g. make test-by-name ARGS=psutil.tests.test_system.TestSystemAPIs ${MAKE} install @PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v $(ARGS) -# Run test coverage. -coverage: +test-coverage: ## Run test coverage. ${MAKE} install # Note: coverage options are controlled by .coveragerc file rm -rf .coverage htmlcov @@ -185,27 +164,25 @@ coverage: # Linters # =================================================================== -pep8: +pep8: ## PEP8 linter. @git ls-files | grep \\.py$ | xargs $(PYTHON) -m pep8 -pyflakes: +pyflakes: ## Pyflakes linter. @export PYFLAKES_NODOCTEST=1 && \ git ls-files | grep \\.py$ | xargs $(PYTHON) -m pyflakes -flake8: +flake8: ## flake8 linter. @git ls-files | grep \\.py$ | xargs $(PYTHON) -m flake8 # =================================================================== # GIT # =================================================================== -# git-tag a new release -git-tag-release: +git-tag-release: ## Git-tag a new release. git tag -a release-`python -c "import setup; print(setup.get_version())"` -m `git rev-list HEAD --count`:`git rev-parse --short HEAD` git push --follow-tags -# Install GIT pre-commit hook. -install-git-hooks: +install-git-hooks: ## Install GIT pre-commit hook. ln -sf ../../.git-pre-commit .git/hooks/pre-commit chmod +x .git/hooks/pre-commit @@ -213,27 +190,21 @@ install-git-hooks: # Distribution # =================================================================== -# Generate tar.gz source distribution. -sdist: +sdist: ## Generate tar.gz source distribution. ${MAKE} generate-manifest $(PYTHON) setup.py sdist -# Upload source tarball on https://pypi.python.org/pypi/psutil. -upload-src: +upload-src: ## Upload source tarball on https://pypi.python.org/pypi/psutil. ${MAKE} sdist $(PYTHON) setup.py sdist upload -# Download exes/wheels hosted on appveyor. -win-download-exes: +win-download-exes: ## Download exes/wheels hosted on appveyor. PYTHONWARNINGS=all $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil -# Upload exes/wheels in dist/* directory to PYPI. -win-upload-exes: - $(PYTHON) -m twine upload dist/*.exe +win-upload-exes: ## Upload wheels in dist/* directory on PYPI. $(PYTHON) -m twine upload dist/*.whl -# All the necessary steps before making a release. -pre-release: +pre-release: ## Check if we're ready to produce a new release. ${MAKE} install $(PYTHON) -c \ "from psutil import __version__ as ver; \ @@ -248,46 +219,41 @@ pre-release: ${MAKE} win-download-exes ${MAKE} sdist -# Create a release: creates tar.gz and exes/wheels, uploads them, -# upload doc, git tag release. -release: +release: ## Create a release (down/uploads tar.gz, wheels, git tag release). + ${MAKE} pre-release $(PYTHON) -m twine upload dist/* # upload tar.gz and Windows wheels on PYPI ${MAKE} git-tag-release -# Print announce of new release. -print-announce: +print-announce: ## Print announce of new release. @PYTHONWARNINGS=all $(PYTHON) scripts/internal/print_announce.py -# Print releases' timeline. -print-timeline: +print-timeline: ## Print releases' timeline. @PYTHONWARNINGS=all $(PYTHON) scripts/internal/print_timeline.py -# Inspect MANIFEST.in file. -check-manifest: +check-manifest: ## Inspect MANIFEST.in file. $(PYTHON) -m check_manifest -v $(ARGS) -# Generates MANIFEST.in file. -generate-manifest: +generate-manifest: ## Generates MANIFEST.in file. $(PYTHON) scripts/internal/generate_manifest.py > MANIFEST.in # =================================================================== # Misc # =================================================================== -grep-todos: +grep-todos: ## Look for TODOs in the source files. git grep -EIn "TODO|FIXME|XXX" -# run script which benchmarks oneshot() ctx manager (see #799) -bench-oneshot: +bench-oneshot: ## Benchmarks for oneshot() ctx manager (see #799). ${MAKE} install PYTHONWARNINGS=all $(PYTHON) scripts/internal/bench_oneshot.py -# same as above but using perf module (supposed to be more precise) -bench-oneshot-2: +bench-oneshot-2: ## Same as above but using perf module (supposed to be more precise) ${MAKE} install PYTHONWARNINGS=all $(PYTHON) scripts/internal/bench_oneshot_2.py -# check whether the links mentioned in some files are valid. -check-broken-links: +check-broken-links: ## Look for broken links in source files. git ls-files | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py + +help: ## Display callable targets. + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' From 62c845a7d97eea0c2740b8f488a62b6c08eb4331 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 7 Nov 2017 11:23:04 +0100 Subject: [PATCH 846/922] fix #1166 (doc mistake) --- DEVGUIDE.rst | 2 +- docs/index.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index 9b26fba9d..2d0af7fc2 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -44,7 +44,7 @@ Some useful make commands:: $ make setup-dev-env # install useful dev libs (pyflakes, unittest2, etc.) $ make test # run unit tests $ make test-memleaks # run memory leak tests - $ make coverage # run test coverage + $ make test-coverage # run test coverage $ make flake8 # run PEP8 linter There are some differences between ``make`` on UNIX and Windows. diff --git a/docs/index.rst b/docs/index.rst index ac591d303..f9f4f3988 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2429,13 +2429,13 @@ resources. for p in procs: p.terminate() gone, alive = psutil.wait_procs(procs, timeout=timeout, callback=on_terminate) - if not alive: + if alive: # send SIGKILL for p in alive: print("process {} survived SIGTERM; trying SIGKILL" % p) p.kill() gone, alive = psutil.wait_procs(alive, timeout=timeout, callback=on_terminate) - if not alive: + if alive: # give up for p in alive: print("process {} survived SIGKILL; giving up" % p) From e3f911e6ab90be10a37edf00c5da2c204513c242 Mon Sep 17 00:00:00 2001 From: Matthew Long Date: Wed, 8 Nov 2017 06:05:12 -0500 Subject: [PATCH 847/922] Including non-unicast packets in packet count calculation (#1167) --- psutil/_psutil_windows.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index d908a1c7d..1d1fd939a 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2304,8 +2304,8 @@ psutil_net_io_counters(PyObject *self, PyObject *args) { py_nic_info = Py_BuildValue("(KKKKKKKK)", pIfRow->OutOctets, pIfRow->InOctets, - pIfRow->OutUcastPkts, - pIfRow->InUcastPkts, + (pIfRow->OutUcastPkts + pIfRow->OutNUcastPkts), + (pIfRow->InUcastPkts + pIfRow->InNUcastPkts), pIfRow->InErrors, pIfRow->OutErrors, pIfRow->InDiscards, @@ -2314,8 +2314,8 @@ psutil_net_io_counters(PyObject *self, PyObject *args) { py_nic_info = Py_BuildValue("(kkkkkkkk)", pIfRow->dwOutOctets, pIfRow->dwInOctets, - pIfRow->dwOutUcastPkts, - pIfRow->dwInUcastPkts, + (pIfRow->dwOutUcastPkts + pIfRow->dwOutNUcastPkts), + (pIfRow->dwInUcastPkts + pIfRow->dwInNUcastPkts), pIfRow->dwInErrors, pIfRow->dwOutErrors, pIfRow->dwInDiscards, From 0c25f9a6b20e11059805f55928cec5bcd18fbf6b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 8 Nov 2017 12:07:56 +0100 Subject: [PATCH 848/922] #1167 give CREDITS to @matray --- CREDITS | 4 ++++ HISTORY.rst | 2 ++ Makefile | 3 +-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CREDITS b/CREDITS index 95e4bd345..890b82230 100644 --- a/CREDITS +++ b/CREDITS @@ -507,3 +507,7 @@ I: 1150 N: Adrian Page W: https://github.com/adpag I: 1159, 1160, 1161 + +N: Matthew Long +W: https://github.com/matray +I: 1167 diff --git a/HISTORY.rst b/HISTORY.rst index 0fba703e1..db11d4ab1 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -16,6 +16,8 @@ SIGTERM instead of 0. (patch by Akos Kiss) - 1151_: python -m psutil.tests fail - 1154_: [AIX] psutil won't compile on AIX 6.1.0. (patch by Arnon Yaari) +- 1167_: [Windows] net_io_counter() packets count now include also non-unicast + packets. (patch by Matthew Long) 5.4.0 ===== diff --git a/Makefile b/Makefile index 1d6b15be0..035c72f30 100644 --- a/Makefile +++ b/Makefile @@ -220,7 +220,6 @@ pre-release: ## Check if we're ready to produce a new release. ${MAKE} sdist release: ## Create a release (down/uploads tar.gz, wheels, git tag release). - ${MAKE} pre-release $(PYTHON) -m twine upload dist/* # upload tar.gz and Windows wheels on PYPI ${MAKE} git-tag-release @@ -256,4 +255,4 @@ check-broken-links: ## Look for broken links in source files. git ls-files | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py help: ## Display callable targets. - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' From c95f317d586bbd4313f705c5618f0fd5477cb210 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 8 Nov 2017 12:26:54 +0100 Subject: [PATCH 849/922] try to fix appveyor failure; also refactor generate_manifest.py --- psutil/tests/test_unicode.py | 2 +- scripts/internal/generate_manifest.py | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index bbb763f31..c2a2f8479 100755 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -207,7 +207,7 @@ def test_proc_cwd(self): def test_proc_open_files(self): p = psutil.Process() start = set(p.open_files()) - with open(self.funky_name, 'wb'): + with open(self.funky_name, 'rb'): new = set(p.open_files()) path = (new - start).pop().path self.assertIsInstance(path, str) diff --git a/scripts/internal/generate_manifest.py b/scripts/internal/generate_manifest.py index 8b6b4f5fa..3511b7492 100755 --- a/scripts/internal/generate_manifest.py +++ b/scripts/internal/generate_manifest.py @@ -12,6 +12,10 @@ import subprocess +IGNORED_EXTS = ('.png', '.jpg', '.jpeg') +IGNORED_FILES = ('.travis.yml', 'appveyor.yml') + + def sh(cmd): return subprocess.check_output( cmd, shell=True, universal_newlines=True).strip() @@ -21,8 +25,8 @@ def main(): files = sh("git ls-files").split('\n') for file in files: if file.startswith('.ci/') or \ - os.path.splitext(file)[1] in ('.png', '.jpg') or \ - file in ('.travis.yml', 'appveyor.yml'): + os.path.splitext(file)[1].lower() in IGNORED_EXTS or \ + file in IGNORED_FILES: continue print("include " + file) From 810498c36895e03a1b4e947659a2b30d8188557b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 8 Nov 2017 14:15:15 +0100 Subject: [PATCH 850/922] #1053: drop python 3.3 support --- .travis.yml | 1 - HISTORY.rst | 2 ++ appveyor.yml | 8 -------- scripts/internal/download_exes.py | 2 +- scripts/internal/winmake.py | 4 ++-- setup.py | 1 - 6 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0f1919385..e08b3a39c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ matrix: include: - python: 2.6 - python: 2.7 - - python: 3.3 - python: 3.4 - python: 3.5 - python: 3.6 diff --git a/HISTORY.rst b/HISTORY.rst index db11d4ab1..980be8633 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ - 1164_: [AIX] add support for Process.num_ctx_switches(). (patch by Arnon Yaari) +- 1053_: abandon Python 3.3 support (psutil still works but it's no longer + tested). **Bug fixes** diff --git a/appveyor.yml b/appveyor.yml index b18677242..5fb1d7a3c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,10 +20,6 @@ environment: PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python33" - PYTHON_VERSION: "3.3.x" - PYTHON_ARCH: "32" - - PYTHON: "C:\\Python34" PYTHON_VERSION: "3.4.x" PYTHON_ARCH: "32" @@ -42,10 +38,6 @@ environment: PYTHON_VERSION: "2.7.x" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python33-x64" - PYTHON_VERSION: "3.3.x" - PYTHON_ARCH: "64" - - PYTHON: "C:\\Python34-x64" PYTHON_VERSION: "3.4.x" PYTHON_ARCH: "64" diff --git a/scripts/internal/download_exes.py b/scripts/internal/download_exes.py index 5c2d70acd..1b0044288 100755 --- a/scripts/internal/download_exes.py +++ b/scripts/internal/download_exes.py @@ -25,7 +25,7 @@ BASE_URL = 'https://ci.appveyor.com/api' -PY_VERSIONS = ['2.7', '3.3', '3.4', '3.5', '3.6'] +PY_VERSIONS = ['2.7', '3.4', '3.5', '3.6'] TIMEOUT = 30 COLORS = True diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 5b2bea989..a09e28960 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -448,8 +448,8 @@ def set_python(s): # try to look for a python installation orig = s s = s.replace('.', '') - vers = ('26', '27', '33', '34', '35', '36', '37', - '26-64', '27-64', '33-64', '34-64', '35-64', '36-64', '37-64') + vers = ('26', '27', '34', '35', '36', '37', + '26-64', '27-64', '34-64', '35-64', '36-64', '37-64') for v in vers: if s == v: path = 'C:\\python%s\python.exe' % s diff --git a/setup.py b/setup.py index a170d2def..1625a3eb4 100755 --- a/setup.py +++ b/setup.py @@ -318,7 +318,6 @@ def main(): 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', From 147de27ff4a1f9f078b7e3783dae48a8c9eb8da6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 8 Nov 2017 14:49:51 +0100 Subject: [PATCH 851/922] pre-release --- HISTORY.rst | 2 +- docs/index.rst | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 980be8633..8693bcf5c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.4.1 ===== -*XXXX-XX-XX* +*2017-11-08* **Enhancements** diff --git a/docs/index.rst b/docs/index.rst index f9f4f3988..7690dfea9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2580,6 +2580,10 @@ take a look at the Timeline ======== +- 2017-11-08: + `5.4.1 `__ - + `what's new `__ - + `diff `__ - 2017-10-12: `5.4.0 `__ - `what's new `__ - From b0209d0acd50fd95723a30bf24e7d06b9378835e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 8 Nov 2017 15:13:39 +0100 Subject: [PATCH 852/922] update doc --- README.rst | 3 +-- docs/index.rst | 11 +++++++++-- scripts/internal/print_announce.py | 7 +++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 4559e55c4..4ac707aba 100644 --- a/README.rst +++ b/README.rst @@ -61,8 +61,7 @@ psutil currently supports the following platforms: - **AIX** ...both **32-bit** and **64-bit** architectures, with Python -versions from **2.6 to 3.6** (users of Python 2.4 and 2.5 may use -`2.1.3 `__ version). +versions from **2.6 to 3.6**. `PyPy `__ is also known to work. ==================== diff --git a/docs/index.rst b/docs/index.rst index 7690dfea9..ce798e923 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2520,8 +2520,8 @@ Top 3 processes opening more file descriptors:: (2721, {'name': 'chrome', 'num_fds': 185}), (2650, {'name': 'chrome', 'num_fds': 354})] -Q&A -=== +FAQs +==== * Q: What Windows versions are supported? * A: From Windows **Vista** onwards, both 32 and 64 bit versions. @@ -2534,6 +2534,13 @@ Q&A ---- +* Q: What Python versions are supported? +* A: From 2.6 to 3.6, both 32 and 64 bit versions. Last version supporting + Python 2.4 and 2.5 is `psutil 2.1.3 `__. + PyPy is also known to work. + +---- + * Q: What SunOS versions are supported? * A: From Solaris 10 onwards. diff --git a/scripts/internal/print_announce.py b/scripts/internal/print_announce.py index 9d2cbb62c..1c2b9e113 100755 --- a/scripts/internal/print_announce.py +++ b/scripts/internal/print_announce.py @@ -39,10 +39,9 @@ running processes. It implements many functionalities offered by command \ line tools such as: ps, top, lsof, netstat, ifconfig, who, df, kill, free, \ nice, ionice, iostat, iotop, uptime, pidof, tty, taskset, pmap. It \ -currently supports Linux, Windows, OSX, Sun Solaris, FreeBSD, OpenBSD and \ -NetBSD, both 32-bit and 64-bit architectures, with Python versions from 2.6 \ -to 3.5 (users of Python 2.4 and 2.5 may use 2.1.3 version). PyPy is also \ -known to work. +currently supports Linux, Windows, OSX, Sun Solaris, FreeBSD, OpenBSD, NetBSD \ +and AIX, both 32-bit and 64-bit architectures, with Python versions from 2.6 \ +to 3.6. PyPy is also known to work. What's new ========== From 02e25d765040d3f36ecc66249700a8777b175c7a Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Sat, 11 Nov 2017 03:34:22 +0200 Subject: [PATCH 853/922] skip cpu_freq tests if not available (#1170) cpu_freq is not always available on Linux --- psutil/tests/test_contracts.py | 5 ++++- psutil/tests/test_linux.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index d9633339b..5e5c2e9a0 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -110,7 +110,10 @@ def test_linux_rlimit(self): ae(hasattr(psutil, "RLIMIT_SIGPENDING"), hasit) def test_cpu_freq(self): - self.assertEqual(hasattr(psutil, "cpu_freq"), LINUX or OSX or WINDOWS) + linux = (LINUX and + (os.path.exists("/sys/devices/system/cpu/cpufreq") or + os.path.exists("/sys/devices/system/cpu/cpu0/cpufreq"))) + self.assertEqual(hasattr(psutil, "cpu_freq"), linux or OSX or WINDOWS) def test_sensors_temperatures(self): self.assertEqual(hasattr(psutil, "sensors_temperatures"), LINUX) diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 4658dd21e..71d428c31 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -29,6 +29,7 @@ from psutil._compat import u from psutil.tests import call_until from psutil.tests import HAS_BATTERY +from psutil.tests import HAS_CPU_FREQ from psutil.tests import HAS_RLIMIT from psutil.tests import MEMORY_TOLERANCE from psutil.tests import mock @@ -607,11 +608,13 @@ def test_cpu_count_physical_mocked(self): self.assertIsNone(psutil._pslinux.cpu_count_physical()) assert m.called + @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq_no_result(self): with mock.patch("psutil._pslinux.glob.glob", return_value=[]): self.assertIsNone(psutil.cpu_freq()) @unittest.skipIf(TRAVIS, "fails on Travis") + @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq_use_second_file(self): # https://github.com/giampaolo/psutil/issues/981 def glob_mock(pattern): @@ -629,6 +632,7 @@ def glob_mock(pattern): assert psutil.cpu_freq() self.assertEqual(len(flags), 2) + @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq_emulate_data(self): def open_mock(name, *args, **kwargs): if name.endswith('/scaling_cur_freq'): @@ -651,6 +655,7 @@ def open_mock(name, *args, **kwargs): self.assertEqual(freq.min, 600.0) self.assertEqual(freq.max, 700.0) + @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq_emulate_multi_cpu(self): def open_mock(name, *args, **kwargs): if name.endswith('/scaling_cur_freq'): @@ -675,6 +680,7 @@ def open_mock(name, *args, **kwargs): self.assertEqual(freq.max, 300.0) @unittest.skipIf(TRAVIS, "fails on Travis") + @unittest.skipIf(not HAS_CPU_FREQ, "not supported") def test_cpu_freq_no_scaling_cur_freq_file(self): # See: https://github.com/giampaolo/psutil/issues/1071 def open_mock(name, *args, **kwargs): From ffde3264f09c5412ce3987617130f2c9d5d7a22b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 11 Nov 2017 13:22:09 +0100 Subject: [PATCH 854/922] try to set PSUTIL_TESTING env var from python before failing --- Makefile | 35 ++++++++++++++++++----------------- psutil/tests/__init__.py | 6 ++++-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 035c72f30..a3fef692a 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ DEPS = \ # In not in a virtualenv, add --user options for install commands. INSTALL_OPTS = `$(PYTHON) -c "import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"` +TEST_PREFIX = PYTHONWARNINGS=all PSUTIL_TESTING=1 all: test @@ -108,53 +109,53 @@ setup-dev-env: ## Install GIT hooks, pip, test deps (also upgrades them). test: ## Run all tests. ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) $(TSCRIPT) + $(TEST_PREFIX) $(PYTHON) $(TSCRIPT) test-process: ## Run process-related API tests. ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_process + $(TEST_PREFIX) $(PYTHON) -m unittest -v psutil.tests.test_process test-system: ## Run system-related API tests. ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v psutil.tests.test_system + $(TEST_PREFIX) $(PYTHON) -m unittest -v psutil.tests.test_system test-misc: ## Run miscellaneous tests. ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_misc.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_misc.py test-unicode: ## Test APIs dealing with strings. ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_unicode.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_unicode.py test-contracts: ## APIs sanity tests. ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_contracts.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_contracts.py test-connections: ## Test net_connections() and Process.connections(). ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_connections.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_connections.py test-posix: ## POSIX specific tests. ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_posix.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_posix.py test-platform: ## Run specific platform tests only. ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_`$(PYTHON) -c 'import psutil; print([x.lower() for x in ("LINUX", "BSD", "OSX", "SUNOS", "WINDOWS", "AIX") if getattr(psutil, x)][0])'`.py test-memleaks: ## Memory leak tests. ${MAKE} install - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) psutil/tests/test_memory_leaks.py + $(TEST_PREFIX) $(PYTHON) psutil/tests/test_memory_leaks.py test-by-name: ## e.g. make test-by-name ARGS=psutil.tests.test_system.TestSystemAPIs ${MAKE} install - @PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m unittest -v $(ARGS) + @$(TEST_PREFIX) $(PYTHON) -m unittest -v $(ARGS) test-coverage: ## Run test coverage. ${MAKE} install # Note: coverage options are controlled by .coveragerc file rm -rf .coverage htmlcov - PSUTIL_TESTING=1 PYTHONWARNINGS=all $(PYTHON) -m coverage run $(TSCRIPT) + $(TEST_PREFIX) $(PYTHON) -m coverage run $(TSCRIPT) $(PYTHON) -m coverage report @echo "writing results to htmlcov/index.html" $(PYTHON) -m coverage html @@ -199,7 +200,7 @@ upload-src: ## Upload source tarball on https://pypi.python.org/pypi/psutil. $(PYTHON) setup.py sdist upload win-download-exes: ## Download exes/wheels hosted on appveyor. - PYTHONWARNINGS=all $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil + $(TEST_PREFIX) $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil win-upload-exes: ## Upload wheels in dist/* directory on PYPI. $(PYTHON) -m twine upload dist/*.whl @@ -225,10 +226,10 @@ release: ## Create a release (down/uploads tar.gz, wheels, git tag release). ${MAKE} git-tag-release print-announce: ## Print announce of new release. - @PYTHONWARNINGS=all $(PYTHON) scripts/internal/print_announce.py + @$(TEST_PREFIX) $(PYTHON) scripts/internal/print_announce.py print-timeline: ## Print releases' timeline. - @PYTHONWARNINGS=all $(PYTHON) scripts/internal/print_timeline.py + @$(TEST_PREFIX) $(PYTHON) scripts/internal/print_timeline.py check-manifest: ## Inspect MANIFEST.in file. $(PYTHON) -m check_manifest -v $(ARGS) @@ -245,11 +246,11 @@ grep-todos: ## Look for TODOs in the source files. bench-oneshot: ## Benchmarks for oneshot() ctx manager (see #799). ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) scripts/internal/bench_oneshot.py + $(TEST_PREFIX) $(PYTHON) scripts/internal/bench_oneshot.py bench-oneshot-2: ## Same as above but using perf module (supposed to be more precise) ${MAKE} install - PYTHONWARNINGS=all $(PYTHON) scripts/internal/bench_oneshot_2.py + $(TEST_PREFIX) $(PYTHON) scripts/internal/bench_oneshot_2.py check-broken-links: ## Look for broken links in source files. git ls-files | xargs $(PYTHON) -Wa scripts/internal/check_broken_links.py diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 14f1b53f0..b94cfe6c6 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -753,8 +753,10 @@ def __str__(self): def _setup_tests(): - os.environ['PSUTIL_TESTING'] = '1' - assert psutil._psplatform.cext.py_psutil_testing() + if 'PSUTIL_TESTING' not in os.environ: + os.environ['PSUTIL_TESTING'] = '1' # not guaranteed to work + if not psutil._psplatform.cext.py_psutil_testing(): + raise AssertionError('PSUTIL_TESTING env var is not set') def get_suite(): From 0662915f58949a4084f9fb0c278d143708cf3ede Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 11 Nov 2017 21:35:41 +0100 Subject: [PATCH 855/922] get rid of PSUTIL_TESTING env var: it must be necessarily set from cmdline, hence 'python -m psutil.tests' won't work out of the box --- psutil/_psutil_aix.c | 6 ++++-- psutil/_psutil_bsd.c | 6 ++++-- psutil/_psutil_common.c | 18 ++++++++++-------- psutil/_psutil_common.h | 3 ++- psutil/_psutil_linux.c | 6 ++++-- psutil/_psutil_osx.c | 6 ++++-- psutil/_psutil_sunos.c | 6 ++++-- psutil/_psutil_windows.c | 6 ++++-- psutil/tests/__init__.py | 4 ++-- 9 files changed, 38 insertions(+), 23 deletions(-) diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c index 0834726dd..3b188b85e 100644 --- a/psutil/_psutil_aix.c +++ b/psutil/_psutil_aix.c @@ -899,8 +899,10 @@ PsutilMethods[] = "Return CPU statistics"}, // --- others - {"py_psutil_testing", py_psutil_testing, METH_VARARGS, - "Return True if PSUTIL_TESTING env var is set"}, + {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, + "Return True if psutil is in testing mode"}, + {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 3527b6667..7b0f140e9 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -984,8 +984,10 @@ PsutilMethods[] = { #endif // --- others - {"py_psutil_testing", py_psutil_testing, METH_VARARGS, - "Return True if PSUTIL_TESTING env var is set"}, + {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, + "Return True if psutil is in testing mode"}, + {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index dace4724b..1fd2344ec 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -37,7 +37,7 @@ AccessDenied(void) { } -static int _psutil_testing = -1; +static int _psutil_testing = 0; /* @@ -45,12 +45,6 @@ static int _psutil_testing = -1; */ int psutil_testing(void) { - if (_psutil_testing == -1) { - if (getenv("PSUTIL_TESTING") != NULL) - _psutil_testing = 1; - else - _psutil_testing = 0; - } return _psutil_testing; } @@ -59,7 +53,7 @@ psutil_testing(void) { * Return True if PSUTIL_TESTING env var is set else False. */ PyObject * -py_psutil_testing(PyObject *self, PyObject *args) { +py_psutil_is_testing(PyObject *self, PyObject *args) { PyObject *res; res = psutil_testing() ? Py_True : Py_False; Py_INCREF(res); @@ -67,6 +61,14 @@ py_psutil_testing(PyObject *self, PyObject *args) { } +PyObject * +py_psutil_set_testing(PyObject *self, PyObject *args) { + _psutil_testing = 1; + Py_INCREF(Py_None); + return Py_None; +} + + /* * Backport of unicode FS APIs from Python 3. * On Python 2 we just return a plain byte string diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 134045327..09999bbaf 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -12,7 +12,8 @@ static const int PSUTIL_CONN_NONE = 128; PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); int psutil_testing(void); -PyObject* py_psutil_testing(PyObject *self, PyObject *args); +PyObject* py_psutil_is_testing(PyObject *self, PyObject *args); +PyObject* py_psutil_set_testing(PyObject *self, PyObject *args); #if PY_MAJOR_VERSION < 3 PyObject* PyUnicode_DecodeFSDefault(char *s); PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index a15ebe5cf..2391a67a4 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -607,8 +607,10 @@ PsutilMethods[] = { #endif // --- others - {"py_psutil_testing", py_psutil_testing, METH_VARARGS, - "Return True if PSUTIL_TESTING env var is set"}, + {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, + "Return True if psutil is in testing mode"}, + {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 2924aa399..9908d0332 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1845,8 +1845,10 @@ PsutilMethods[] = { "Return CPU statistics"}, // --- others - {"py_psutil_testing", py_psutil_testing, METH_VARARGS, - "Return True if PSUTIL_TESTING env var is set"}, + {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, + "Return True if psutil is in testing mode"}, + {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 083b78cb1..2abcd8295 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -1592,8 +1592,10 @@ PsutilMethods[] = { "Return CPU statistics"}, // --- others - {"py_psutil_testing", py_psutil_testing, METH_VARARGS, - "Return True if PSUTIL_TESTING env var is set"}, + {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, + "Return True if psutil is in testing mode"}, + {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 1d1fd939a..e1110b538 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -3624,8 +3624,10 @@ PsutilMethods[] = { "QueryDosDevice binding"}, // --- others - {"py_psutil_testing", py_psutil_testing, METH_VARARGS, - "Return True if PSUTIL_TESTING env var is set"}, + {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, + "Return True if psutil is in testing mode"}, + {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} }; diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index b94cfe6c6..9f943d7ad 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -755,8 +755,8 @@ def __str__(self): def _setup_tests(): if 'PSUTIL_TESTING' not in os.environ: os.environ['PSUTIL_TESTING'] = '1' # not guaranteed to work - if not psutil._psplatform.cext.py_psutil_testing(): - raise AssertionError('PSUTIL_TESTING env var is not set') + psutil._psplatform.cext.py_psutil_set_testing() + assert psutil._psplatform.cext.py_psutil_is_testing() def get_suite(): From 159cb8c55254048c797816b0bc6a7f26b84146ac Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 11 Nov 2017 21:52:04 +0100 Subject: [PATCH 856/922] update README, bump up version --- HISTORY.rst | 9 +++++++++ psutil/__init__.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8693bcf5c..e0f790803 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,14 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.4.2 +===== + +*XXXX-XX-XX* + +**Bug fixes** + +- 1172_: [Windows] `make test` does not work. + 5.4.1 ===== diff --git a/psutil/__init__.py b/psutil/__init__.py index 01934ac3b..f80079d01 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -212,7 +212,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.4.1" +__version__ = "5.4.2" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED From 666cf81bd916aa14a9c363ab4933472b1622ad20 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 11 Nov 2017 21:57:40 +0100 Subject: [PATCH 857/922] fix #1169: (Linux) users() hostname returns username instead --- CREDITS | 4 ++++ HISTORY.rst | 2 ++ psutil/_psutil_linux.c | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index 890b82230..ff242998b 100644 --- a/CREDITS +++ b/CREDITS @@ -511,3 +511,7 @@ I: 1159, 1160, 1161 N: Matthew Long W: https://github.com/matray I: 1167 + +N: janderbrain +W: https://github.com/janderbrain +I: 1169 diff --git a/HISTORY.rst b/HISTORY.rst index e0f790803..d14253e2a 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,8 @@ **Bug fixes** +- 1169_: [Linux] users() "hostname" returns username instead. (patch by + janderbrain) - 1172_: [Windows] `make test` does not work. 5.4.1 diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 2391a67a4..6232fe508 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -479,7 +479,7 @@ psutil_users(PyObject *self, PyObject *args) { "(OOOfOi)", py_username, // username py_tty, // tty - py_username, // hostname + py_hostname, // hostname (float)ut->ut_tv.tv_sec, // tstamp py_user_proc, // (bool) user process ut->ut_pid // process id From b77021bc220b046de6d4d449ab8101885de1b6a0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 11 Nov 2017 22:02:27 +0100 Subject: [PATCH 858/922] travis / OSX: run py 3.6 instead of 3.4 --- .travis.yml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index e08b3a39c..10ddd73d0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,30 +3,19 @@ language: python cache: pip matrix: include: + # Linux - python: 2.6 - python: 2.7 - python: 3.4 - python: 3.5 - python: 3.6 - - "pypy" - # XXX - commented because OSX builds are deadly slow - # - language: generic - # os: osx - # env: PYVER=py26 + # OSX - language: generic os: osx env: PYVER=py27 - # XXX - commented because OSX builds are deadly slow - # - language: generic - # os: osx - # env: PYVER=py33 - language: generic os: osx - env: PYVER=py34 - # XXX - not supported yet - # - language: generic - # os: osx - # env: PYVER=py35 + env: PYVER=py36 install: - ./.ci/travis/install.sh script: From 500b80bcf7deb77ceeaa9165fa538e0f7ad134c8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 11 Nov 2017 22:04:40 +0100 Subject: [PATCH 859/922] disable IPv6 tests if IPv6 is not supported --- psutil/tests/test_connections.py | 2 ++ psutil/tests/test_misc.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/psutil/tests/test_connections.py b/psutil/tests/test_connections.py index d248af577..176e26648 100755 --- a/psutil/tests/test_connections.py +++ b/psutil/tests/test_connections.py @@ -152,6 +152,7 @@ def test_tcp_v4(self): assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_LISTEN) + @unittest.skipIf(not supports_ipv6(), "IPv6 not supported") def test_tcp_v6(self): addr = ("::1", get_free_port()) with closing(bind_socket(AF_INET6, SOCK_STREAM, addr=addr)) as sock: @@ -166,6 +167,7 @@ def test_udp_v4(self): assert not conn.raddr self.assertEqual(conn.status, psutil.CONN_NONE) + @unittest.skipIf(not supports_ipv6(), "IPv6 not supported") def test_udp_v6(self): addr = ("::1", get_free_port()) with closing(bind_socket(AF_INET6, SOCK_DGRAM, addr=addr)) as sock: diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index f7305a0db..7dc887353 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -1018,7 +1018,8 @@ def test_create_sockets(self): # work around http://bugs.python.org/issue30204 types[s.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE)] += 1 self.assertGreaterEqual(fams[socket.AF_INET], 2) - self.assertGreaterEqual(fams[socket.AF_INET6], 2) + if supports_ipv6(): + self.assertGreaterEqual(fams[socket.AF_INET6], 2) if POSIX and HAS_CONNECTIONS_UNIX: self.assertGreaterEqual(fams[socket.AF_UNIX], 2) self.assertGreaterEqual(types[socket.SOCK_STREAM], 2) From 53fb242fbb7025ac892abf615b3bdf5acc5a6156 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 11 Nov 2017 22:41:37 +0100 Subject: [PATCH 860/922] change make cmds --- .travis.yml | 2 +- Makefile | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 10ddd73d0..9289eb6b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ matrix: env: PYVER=py27 - language: generic os: osx - env: PYVER=py36 + env: PYVER=py34 install: - ./.ci/travis/install.sh script: diff --git a/Makefile b/Makefile index a3fef692a..f3d0275c7 100644 --- a/Makefile +++ b/Makefile @@ -191,21 +191,21 @@ install-git-hooks: ## Install GIT pre-commit hook. # Distribution # =================================================================== -sdist: ## Generate tar.gz source distribution. +dist-source: ## Generate tar.gz source distribution. ${MAKE} generate-manifest $(PYTHON) setup.py sdist -upload-src: ## Upload source tarball on https://pypi.python.org/pypi/psutil. +dist-upload-src: ## Upload source tarball on https://pypi.python.org/pypi/psutil. ${MAKE} sdist $(PYTHON) setup.py sdist upload -win-download-exes: ## Download exes/wheels hosted on appveyor. +dist-download-win-wheels: ## Download wheels hosted on appveyor. $(TEST_PREFIX) $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil -win-upload-exes: ## Upload wheels in dist/* directory on PYPI. +dist-upload-win-wheels: ## Upload wheels in dist/* directory on PYPI. $(PYTHON) -m twine upload dist/*.whl -pre-release: ## Check if we're ready to produce a new release. +dist-pre-release: ## Check if we're ready to produce a new release. ${MAKE} install $(PYTHON) -c \ "from psutil import __version__ as ver; \ @@ -220,7 +220,7 @@ pre-release: ## Check if we're ready to produce a new release. ${MAKE} win-download-exes ${MAKE} sdist -release: ## Create a release (down/uploads tar.gz, wheels, git tag release). +dist-release: ## Create a release (down/uploads tar.gz, wheels, git tag release). ${MAKE} pre-release $(PYTHON) -m twine upload dist/* # upload tar.gz and Windows wheels on PYPI ${MAKE} git-tag-release From 672253e844933e8cd88535d1da8e0b9402ce0be1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 11 Nov 2017 23:33:57 +0100 Subject: [PATCH 861/922] upgrade dist cmds --- Makefile | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index f3d0275c7..e8ff51ca3 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ DEPS = \ sphinx \ twine \ unittest2 \ - requests + wheel # In not in a virtualenv, add --user options for install commands. INSTALL_OPTS = `$(PYTHON) -c "import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"` @@ -191,22 +191,32 @@ install-git-hooks: ## Install GIT pre-commit hook. # Distribution # =================================================================== -dist-source: ## Generate tar.gz source distribution. +# --- create + +dist-source: ## Create tar.gz source distribution. ${MAKE} generate-manifest $(PYTHON) setup.py sdist -dist-upload-src: ## Upload source tarball on https://pypi.python.org/pypi/psutil. - ${MAKE} sdist - $(PYTHON) setup.py sdist upload +dist-wheel: ## Generate wheel. + $(PYTHON) setup.py bdist_wheel -dist-download-win-wheels: ## Download wheels hosted on appveyor. +dist-win-download-wheels: ## Download wheels hosted on appveyor. $(TEST_PREFIX) $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil +# --- upload + +dist-upload-src: ## Upload source tarball on https://pypi.python.org/pypi/psutil. + ${MAKE} dist-source + $(PYTHON) setup.py sdist upload + dist-upload-win-wheels: ## Upload wheels in dist/* directory on PYPI. $(PYTHON) -m twine upload dist/*.whl -dist-pre-release: ## Check if we're ready to produce a new release. +# --- others + +pre-release: ## Check if we're ready to produce a new release. ${MAKE} install + ${MAKE} dist-source $(PYTHON) -c \ "from psutil import __version__ as ver; \ doc = open('docs/index.rst').read(); \ @@ -217,10 +227,10 @@ dist-pre-release: ## Check if we're ready to produce a new release. ${MAKE} generate-manifest git diff MANIFEST.in > /dev/null # ...otherwise 'git diff-index HEAD' will complain $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else sys.exit(0);" - ${MAKE} win-download-exes + ${MAKE} dist-win-download-wheels ${MAKE} sdist -dist-release: ## Create a release (down/uploads tar.gz, wheels, git tag release). +release: ## Create a release (down/uploads tar.gz, wheels, git tag release). ${MAKE} pre-release $(PYTHON) -m twine upload dist/* # upload tar.gz and Windows wheels on PYPI ${MAKE} git-tag-release From ab90e4e6ac73c249cf7aea7e92aec2b6a07ef041 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 00:51:40 +0100 Subject: [PATCH 862/922] #1152 / win / disk_io_counters(): DeviceIOControl errors were ignored; che return value and retry call on ERROR_INSUFFICIENT_BUFFER --- HISTORY.rst | 1 + psutil/_psutil_windows.c | 69 +++++++++++++++++++++++----------------- 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index d14253e2a..48adbe523 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -7,6 +7,7 @@ **Bug fixes** +- 1152_: [Windows] disk_io_counters() may return an empty dict. - 1169_: [Linux] users() "hostname" returns username instead. (patch by janderbrain) - 1172_: [Windows] `make test` does not work. diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index e1110b538..7cf8f2c93 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2366,6 +2366,9 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { char szDevice[MAX_PATH]; char szDeviceDisplay[MAX_PATH]; int devNum; + int i; + size_t ioctrlSize; + BOOL WINAPI ret; PyObject *py_retdict = PyDict_New(); PyObject *py_tuple = NULL; @@ -2380,39 +2383,47 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { sprintf_s(szDevice, MAX_PATH, "\\\\.\\PhysicalDrive%d", devNum); hDevice = CreateFile(szDevice, 0, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_EXISTING, 0, NULL); - if (hDevice == INVALID_HANDLE_VALUE) continue; - if (DeviceIoControl(hDevice, IOCTL_DISK_PERFORMANCE, NULL, 0, - &diskPerformance, sizeof(diskPerformance), - &dwSize, NULL)) - { - sprintf_s(szDeviceDisplay, MAX_PATH, "PhysicalDrive%d", devNum); - py_tuple = Py_BuildValue( - "(IILLKK)", - diskPerformance.ReadCount, - diskPerformance.WriteCount, - diskPerformance.BytesRead, - diskPerformance.BytesWritten, - // convert to ms: - // https://github.com/giampaolo/psutil/issues/1012 - (unsigned long long) - (diskPerformance.ReadTime.QuadPart) / 10000000, - (unsigned long long) - (diskPerformance.WriteTime.QuadPart) / 10000000); - if (!py_tuple) - goto error; - if (PyDict_SetItemString(py_retdict, szDeviceDisplay, py_tuple)) - goto error; - Py_XDECREF(py_tuple); - } - else { - // XXX we might get here with ERROR_INSUFFICIENT_BUFFER when - // compiling with mingw32; not sure what to do. - // return PyErr_SetFromWindowsErr(0); - ;; + + i = 0; + ioctrlSize = sizeof(diskPerformance); + while (1) { + i += 1; + ret = DeviceIoControl( + hDevice, IOCTL_DISK_PERFORMANCE, NULL, 0, &diskPerformance, + ioctrlSize, &dwSize, NULL); + if (ret != 0) + break; // OK! + if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + if (i <= 1024) { // prevent looping forever + ioctrlSize *= 2; + continue; + } + } + PyErr_SetFromWindowsErr(0); + goto error; } + sprintf_s(szDeviceDisplay, MAX_PATH, "PhysicalDrive%d", devNum); + py_tuple = Py_BuildValue( + "(IILLKK)", + diskPerformance.ReadCount, + diskPerformance.WriteCount, + diskPerformance.BytesRead, + diskPerformance.BytesWritten, + // convert to ms: + // https://github.com/giampaolo/psutil/issues/1012 + (unsigned long long) + (diskPerformance.ReadTime.QuadPart) / 10000000, + (unsigned long long) + (diskPerformance.WriteTime.QuadPart) / 10000000); + if (!py_tuple) + goto error; + if (PyDict_SetItemString(py_retdict, szDeviceDisplay, py_tuple)) + goto error; + Py_XDECREF(py_tuple); + CloseHandle(hDevice); } From 988dcd737da2855bc3eb10904f46e6456a664f83 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 01:47:20 +0100 Subject: [PATCH 863/922] #1152: (DeviceIOControl), skip disk on ERROR_INVALID_FUNCTION and ERROR_NOT_SUPPORTED --- psutil/_psutil_windows.c | 25 ++++++++++++++++++++++++- psutil/tests/test_system.py | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 7cf8f2c93..e28c12e38 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2386,6 +2386,7 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { if (hDevice == INVALID_HANDLE_VALUE) continue; + // DeviceIoControl() sucks! i = 0; ioctrlSize = sizeof(diskPerformance); while (1) { @@ -2396,11 +2397,32 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { if (ret != 0) break; // OK! if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - if (i <= 1024) { // prevent looping forever + // Retry with a bigger buffer (+ limit for retries). + if (i <= 1024) { ioctrlSize *= 2; continue; } } + else if (GetLastError() == ERROR_INVALID_FUNCTION) { + // This happens on AppVeyor: + // https://ci.appveyor.com/project/giampaolo/psutil/build/ + // 1364/job/ascpdi271b06jle3 + // Assume it means we're dealing with some exotic disk + // and go on. + goto next; + } + else if (GetLastError() == ERROR_NOT_SUPPORTED) { + // Again, let's assume we're dealing with some exotic disk. + goto next; + } + // XXX: it seems we should also catch ERROR_INVALID_PARAMETER: + // https://sites.ualberta.ca/dept/aict/uts/software/openbsd/ + // ports/4.1/i386/openafs/w-openafs-1.4.14-transarc/ + // openafs-1.4.14/src/usd/usd_nt.c + + // XXX: we can also bump into ERROR_MORE_DATA in which case + // (quoting doc) we're supposed to retry with a bigger buffer + // and specify a new "starting point", whatever it means. PyErr_SetFromWindowsErr(0); goto error; } @@ -2424,6 +2446,7 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { goto error; Py_XDECREF(py_tuple); +next: CloseHandle(hDevice); } diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 3485fb8f3..2da4d60df 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -670,7 +670,7 @@ def test_net_if_stats(self): @unittest.skipIf(LINUX and not os.path.exists('/proc/diskstats'), '/proc/diskstats not available on this linux version') - @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") # no visible disks + # @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") # no visible disks def test_disk_io_counters(self): def check_ntuple(nt): self.assertEqual(nt[0], nt.read_count) From 9383be7bf0bde4d4f4a7425ee3db8aaab0337a30 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 01:55:16 +0100 Subject: [PATCH 864/922] inspect PSUTIL_TESTING env var from C again --- psutil/_psutil_common.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 1fd2344ec..52aee48a5 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -37,20 +37,27 @@ AccessDenied(void) { } -static int _psutil_testing = 0; +static int _psutil_testing = -1; /* - * Return 1 if PSUTIL_TESTING env var is set else 0. + * Return 1 if PSUTIL_TESTING env var is set or if testing mode was + * enabled with py_psutil_set_testing. */ int psutil_testing(void) { + if (_psutil_testing == -1) { + if (getenv("PSUTIL_TESTING") != NULL) + _psutil_testing = 1; + else + _psutil_testing = 0; + } return _psutil_testing; } /* - * Return True if PSUTIL_TESTING env var is set else False. + * Same as above but return a Python bool. */ PyObject * py_psutil_is_testing(PyObject *self, PyObject *args) { @@ -61,6 +68,11 @@ py_psutil_is_testing(PyObject *self, PyObject *args) { } +/* + * Enable testing mode. This has the same effect as setting PSUTIL_TESTING + * env var. The dual method exists because updating os.environ on + * Windows has no effect. + */ PyObject * py_psutil_set_testing(PyObject *self, PyObject *args) { _psutil_testing = 1; From 0797f32c20b2b722cfb93752359fc4e67d5bdd88 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 02:04:41 +0100 Subject: [PATCH 865/922] refactor PSUTIL_TESTING C APIs --- psutil/_psutil_aix.c | 4 +--- psutil/_psutil_bsd.c | 4 +--- psutil/_psutil_common.c | 4 ++-- psutil/_psutil_common.h | 2 +- psutil/_psutil_linux.c | 4 +--- psutil/_psutil_osx.c | 4 +--- psutil/_psutil_sunos.c | 4 +--- psutil/_psutil_windows.c | 4 +--- psutil/tests/__init__.py | 6 +++--- 9 files changed, 12 insertions(+), 24 deletions(-) diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c index 3b188b85e..a4ea584e7 100644 --- a/psutil/_psutil_aix.c +++ b/psutil/_psutil_aix.c @@ -899,9 +899,7 @@ PsutilMethods[] = "Return CPU statistics"}, // --- others - {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, - "Return True if psutil is in testing mode"}, - {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + {"set_testing", psutil_set_testing, METH_NOARGS, "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 7b0f140e9..edb790a9b 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -984,9 +984,7 @@ PsutilMethods[] = { #endif // --- others - {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, - "Return True if psutil is in testing mode"}, - {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + {"set_testing", psutil_set_testing, METH_NOARGS, "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 52aee48a5..970acb321 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -42,7 +42,7 @@ static int _psutil_testing = -1; /* * Return 1 if PSUTIL_TESTING env var is set or if testing mode was - * enabled with py_psutil_set_testing. + * enabled with psutil_set_testing. */ int psutil_testing(void) { @@ -74,7 +74,7 @@ py_psutil_is_testing(PyObject *self, PyObject *args) { * Windows has no effect. */ PyObject * -py_psutil_set_testing(PyObject *self, PyObject *args) { +psutil_set_testing(PyObject *self, PyObject *args) { _psutil_testing = 1; Py_INCREF(Py_None); return Py_None; diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 09999bbaf..0b6066982 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -13,7 +13,7 @@ PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); int psutil_testing(void); PyObject* py_psutil_is_testing(PyObject *self, PyObject *args); -PyObject* py_psutil_set_testing(PyObject *self, PyObject *args); +PyObject* psutil_set_testing(PyObject *self, PyObject *args); #if PY_MAJOR_VERSION < 3 PyObject* PyUnicode_DecodeFSDefault(char *s); PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 6232fe508..254208b4a 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -607,9 +607,7 @@ PsutilMethods[] = { #endif // --- others - {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, - "Return True if psutil is in testing mode"}, - {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + {"set_testing", psutil_set_testing, METH_NOARGS, "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 9908d0332..93ebd849c 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1845,9 +1845,7 @@ PsutilMethods[] = { "Return CPU statistics"}, // --- others - {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, - "Return True if psutil is in testing mode"}, - {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + {"set_testing", psutil_set_testing, METH_NOARGS, "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 2abcd8295..7b47ef1b8 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -1592,9 +1592,7 @@ PsutilMethods[] = { "Return CPU statistics"}, // --- others - {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, - "Return True if psutil is in testing mode"}, - {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + {"set_testing", psutil_set_testing, METH_NOARGS, "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index e28c12e38..1e7b3ad42 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -3658,9 +3658,7 @@ PsutilMethods[] = { "QueryDosDevice binding"}, // --- others - {"py_psutil_is_testing", py_psutil_is_testing, METH_VARARGS, - "Return True if psutil is in testing mode"}, - {"py_psutil_set_testing", py_psutil_set_testing, METH_VARARGS, + {"set_testing", psutil_set_testing, METH_NOARGS, "Set psutil in testing mode"}, {NULL, NULL, 0, NULL} diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 9f943d7ad..d2368ae53 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -754,9 +754,9 @@ def __str__(self): def _setup_tests(): if 'PSUTIL_TESTING' not in os.environ: - os.environ['PSUTIL_TESTING'] = '1' # not guaranteed to work - psutil._psplatform.cext.py_psutil_set_testing() - assert psutil._psplatform.cext.py_psutil_is_testing() + # This won't work on Windows but set_testing() below will do it. + os.environ['PSUTIL_TESTING'] = '1' + psutil._psplatform.cext.set_testing() def get_suite(): From d32973d145bd50f22407d4b8d936f1da6be7313c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 02:23:32 +0100 Subject: [PATCH 866/922] re-enable test on appveyor; remove unused C code --- psutil/_psutil_common.c | 12 ------------ psutil/tests/test_system.py | 4 +++- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 970acb321..694aa147f 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -56,18 +56,6 @@ psutil_testing(void) { } -/* - * Same as above but return a Python bool. - */ -PyObject * -py_psutil_is_testing(PyObject *self, PyObject *args) { - PyObject *res; - res = psutil_testing() ? Py_True : Py_False; - Py_INCREF(res); - return res; -} - - /* * Enable testing mode. This has the same effect as setting PSUTIL_TESTING * env var. The dual method exists because updating os.environ on diff --git a/psutil/tests/test_system.py b/psutil/tests/test_system.py index 2da4d60df..20b132a91 100755 --- a/psutil/tests/test_system.py +++ b/psutil/tests/test_system.py @@ -670,7 +670,8 @@ def test_net_if_stats(self): @unittest.skipIf(LINUX and not os.path.exists('/proc/diskstats'), '/proc/diskstats not available on this linux version') - # @unittest.skipIf(APPVEYOR, "unreliable on APPVEYOR") # no visible disks + @unittest.skipIf(APPVEYOR and psutil.disk_io_counters() is None, + "unreliable on APPVEYOR") # no visible disks def test_disk_io_counters(self): def check_ntuple(nt): self.assertEqual(nt[0], nt.read_count) @@ -690,6 +691,7 @@ def check_ntuple(nt): assert getattr(nt, name) >= 0, nt ret = psutil.disk_io_counters(perdisk=False) + assert ret is not None, "no disks on this system?" check_ntuple(ret) ret = psutil.disk_io_counters(perdisk=True) # make sure there are no duplicates From c11c6ab12886283200b0ae5897d12fe64255a824 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 02:27:16 +0100 Subject: [PATCH 867/922] rename C func --- psutil/_psutil_common.c | 2 +- psutil/_psutil_common.h | 3 +-- psutil/arch/windows/process_info.c | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 694aa147f..499925cdf 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -45,7 +45,7 @@ static int _psutil_testing = -1; * enabled with psutil_set_testing. */ int -psutil_testing(void) { +psutil_is_testing(void) { if (_psutil_testing == -1) { if (getenv("PSUTIL_TESTING") != NULL) _psutil_testing = 1; diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 0b6066982..c76b1ecd4 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -11,8 +11,7 @@ static const int PSUTIL_CONN_NONE = 128; PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); -int psutil_testing(void); -PyObject* py_psutil_is_testing(PyObject *self, PyObject *args); +int psutil_is_testing(void); PyObject* psutil_set_testing(PyObject *self, PyObject *args); #if PY_MAJOR_VERSION < 3 PyObject* PyUnicode_DecodeFSDefault(char *s); diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index a9687f9cd..c7410a183 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -337,7 +337,7 @@ psutil_get_pids(DWORD *numberOfReturnedPIDs) { int psutil_assert_pid_exists(DWORD pid, char *err) { - if (psutil_testing()) { + if (psutil_is_testing()) { if (psutil_pid_in_pids(pid) == 0) { PyErr_SetString(PyExc_AssertionError, err); return 0; @@ -349,7 +349,7 @@ psutil_assert_pid_exists(DWORD pid, char *err) { int psutil_assert_pid_not_exists(DWORD pid, char *err) { - if (psutil_testing()) { + if (psutil_is_testing()) { if (psutil_pid_in_pids(pid) == 1) { PyErr_SetString(PyExc_AssertionError, err); return 0; From ca73eebc911ebfb3db0f1bcc9305d3270524519d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 02:37:58 +0100 Subject: [PATCH 868/922] move PyUnicode compt fun definition up in the file --- psutil/_psutil_common.c | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 499925cdf..7a2665d7a 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -9,6 +9,27 @@ #include #include + +/* + * Backport of unicode FS APIs from Python 3. + * On Python 2 we just return a plain byte string + * which is never supposed to raise decoding errors. + * See: https://github.com/giampaolo/psutil/issues/1040 + */ +#if PY_MAJOR_VERSION < 3 +PyObject * +PyUnicode_DecodeFSDefault(char *s) { + return PyString_FromString(s); +} + + +PyObject * +PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size) { + return PyString_FromStringAndSize(s, size); +} +#endif + + /* * Set OSError(errno=ESRCH, strerror="No such process") Python exception. */ @@ -67,23 +88,3 @@ psutil_set_testing(PyObject *self, PyObject *args) { Py_INCREF(Py_None); return Py_None; } - - -/* - * Backport of unicode FS APIs from Python 3. - * On Python 2 we just return a plain byte string - * which is never supposed to raise decoding errors. - * See: https://github.com/giampaolo/psutil/issues/1040 - */ -#if PY_MAJOR_VERSION < 3 -PyObject * -PyUnicode_DecodeFSDefault(char *s) { - return PyString_FromString(s); -} - - -PyObject * -PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size) { - return PyString_FromStringAndSize(s, size); -} -#endif From 128f38dc84bc0fc2f2f50c255e8a431edf3930cc Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 02:51:03 +0100 Subject: [PATCH 869/922] define a setup() function which is called on import by all C modules --- psutil/_psutil_aix.c | 2 ++ psutil/_psutil_bsd.c | 2 ++ psutil/_psutil_common.c | 18 +++++++++++------- psutil/_psutil_common.h | 11 +++++++---- psutil/_psutil_linux.c | 2 ++ psutil/_psutil_osx.c | 2 ++ psutil/_psutil_sunos.c | 2 ++ psutil/_psutil_windows.c | 1 + 8 files changed, 29 insertions(+), 11 deletions(-) diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c index a4ea584e7..4d522ba29 100644 --- a/psutil/_psutil_aix.c +++ b/psutil/_psutil_aix.c @@ -978,6 +978,8 @@ void init_psutil_aix(void) PyModule_AddIntConstant(module, "TCPS_TIME_WAIT", TCPS_TIME_WAIT); PyModule_AddIntConstant(module, "PSUTIL_CONN_NONE", PSUTIL_CONN_NONE); + psutil_setup(); + if (module == NULL) INITERROR; #if PY_MAJOR_VERSION >= 3 diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index edb790a9b..f1adb1d37 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -1088,6 +1088,8 @@ void init_psutil_bsd(void) // PSUTIL_CONN_NONE PyModule_AddIntConstant(module, "PSUTIL_CONN_NONE", 128); + psutil_setup(); + if (module == NULL) INITERROR; #if PY_MAJOR_VERSION >= 3 diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 7a2665d7a..97b1494eb 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -58,7 +58,7 @@ AccessDenied(void) { } -static int _psutil_testing = -1; +static int _psutil_testing = 0; /* @@ -67,12 +67,6 @@ static int _psutil_testing = -1; */ int psutil_is_testing(void) { - if (_psutil_testing == -1) { - if (getenv("PSUTIL_TESTING") != NULL) - _psutil_testing = 1; - else - _psutil_testing = 0; - } return _psutil_testing; } @@ -88,3 +82,13 @@ psutil_set_testing(PyObject *self, PyObject *args) { Py_INCREF(Py_None); return Py_None; } + + +/* + * Called on module import on all platforms. + */ +void +psutil_setup(void) { + if (getenv("PSUTIL_TESTING") != NULL) + _psutil_testing = 1; +} diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index c76b1ecd4..2b71caa40 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -9,11 +9,14 @@ // a signaler for connections without an actual status static const int PSUTIL_CONN_NONE = 128; -PyObject* AccessDenied(void); -PyObject* NoSuchProcess(void); -int psutil_is_testing(void); -PyObject* psutil_set_testing(PyObject *self, PyObject *args); #if PY_MAJOR_VERSION < 3 PyObject* PyUnicode_DecodeFSDefault(char *s); PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); #endif + +PyObject* AccessDenied(void); +PyObject* NoSuchProcess(void); + +int psutil_is_testing(void); +PyObject* psutil_set_testing(PyObject *self, PyObject *args); +void psutil_setup(void); diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 254208b4a..d1f0d1455 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -713,6 +713,8 @@ void init_psutil_linux(void) PyModule_AddIntConstant(module, "DUPLEX_FULL", DUPLEX_FULL); PyModule_AddIntConstant(module, "DUPLEX_UNKNOWN", DUPLEX_UNKNOWN); + psutil_setup(); + if (module == NULL) INITERROR; #if PY_MAJOR_VERSION >= 3 diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 93ebd849c..4ff301d4f 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -1927,6 +1927,8 @@ init_psutil_osx(void) PyModule_AddIntConstant(module, "TCPS_TIME_WAIT", TCPS_TIME_WAIT); PyModule_AddIntConstant(module, "PSUTIL_CONN_NONE", PSUTIL_CONN_NONE); + psutil_setup(); + if (module == NULL) INITERROR; #if PY_MAJOR_VERSION >= 3 diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 7b47ef1b8..15508461c 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -1679,6 +1679,8 @@ void init_psutil_sunos(void) PyModule_AddIntConstant(module, "TCPS_BOUND", TCPS_BOUND); PyModule_AddIntConstant(module, "PSUTIL_CONN_NONE", PSUTIL_CONN_NONE); + psutil_setup(); + if (module == NULL) INITERROR; #if PY_MAJOR_VERSION >= 3 diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 1e7b3ad42..574885b64 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -3805,6 +3805,7 @@ void init_psutil_windows(void) // set SeDebug for the current process psutil_set_se_debug(); + psutil_setup(); #if PY_MAJOR_VERSION >= 3 return module; From 39c40cb155c0dfd463178dbeb26778f4c446c650 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 02:57:18 +0100 Subject: [PATCH 870/922] fix unicode err --- psutil/tests/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index d2368ae53..4f7995dcd 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -193,7 +193,11 @@ def _get_py_exe(): def _cleanup_files(): DEVNULL.close() for name in os.listdir(u('.')): - if name.startswith(u(TESTFILE_PREFIX)): + if isinstance(name, unicode): + prefix = u(TESTFILE_PREFIX) + else: + prefix = TESTFILE_PREFIX + if name.startswith(prefix): try: safe_rmpath(name) except Exception: From a8dd6ac992152dc84e0fdf87655355085f465c6f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 03:14:12 +0100 Subject: [PATCH 871/922] use a C global variable to figure out whether we're in testing mode --- psutil/_psutil_common.c | 25 ++++++++----------------- psutil/_psutil_common.h | 3 ++- psutil/arch/windows/process_info.c | 5 +++-- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index 97b1494eb..cf9899f11 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -10,6 +10,10 @@ #include +// Global vars. +int PSUTIL_TESTING = 0; + + /* * Backport of unicode FS APIs from Python 3. * On Python 2 we just return a plain byte string @@ -58,27 +62,14 @@ AccessDenied(void) { } -static int _psutil_testing = 0; - - -/* - * Return 1 if PSUTIL_TESTING env var is set or if testing mode was - * enabled with psutil_set_testing. - */ -int -psutil_is_testing(void) { - return _psutil_testing; -} - - /* * Enable testing mode. This has the same effect as setting PSUTIL_TESTING - * env var. The dual method exists because updating os.environ on - * Windows has no effect. + * env var. This dual method exists because updating os.environ on + * Windows has no effect. Called on unit tests setup. */ PyObject * psutil_set_testing(PyObject *self, PyObject *args) { - _psutil_testing = 1; + PSUTIL_TESTING = 1; Py_INCREF(Py_None); return Py_None; } @@ -90,5 +81,5 @@ psutil_set_testing(PyObject *self, PyObject *args) { void psutil_setup(void) { if (getenv("PSUTIL_TESTING") != NULL) - _psutil_testing = 1; + PSUTIL_TESTING = 1; } diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 2b71caa40..a609b886a 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -6,6 +6,8 @@ #include +extern int PSUTIL_TESTING; + // a signaler for connections without an actual status static const int PSUTIL_CONN_NONE = 128; @@ -17,6 +19,5 @@ PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); -int psutil_is_testing(void); PyObject* psutil_set_testing(PyObject *self, PyObject *args); void psutil_setup(void); diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index c7410a183..92095fe52 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -337,7 +337,8 @@ psutil_get_pids(DWORD *numberOfReturnedPIDs) { int psutil_assert_pid_exists(DWORD pid, char *err) { - if (psutil_is_testing()) { + if (PSUTIL_TESTING) { + printf("testing\n"); if (psutil_pid_in_pids(pid) == 0) { PyErr_SetString(PyExc_AssertionError, err); return 0; @@ -349,7 +350,7 @@ psutil_assert_pid_exists(DWORD pid, char *err) { int psutil_assert_pid_not_exists(DWORD pid, char *err) { - if (psutil_is_testing()) { + if (PSUTIL_TESTING) { if (psutil_pid_in_pids(pid) == 1) { PyErr_SetString(PyExc_AssertionError, err); return 0; From 083f711e577764a7de82394469aa790a20ca058a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 04:27:21 +0100 Subject: [PATCH 872/922] refactor winmake.py --- psutil/arch/windows/process_info.c | 1 - scripts/internal/winmake.py | 41 +++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 92095fe52..9a54d8544 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -338,7 +338,6 @@ psutil_get_pids(DWORD *numberOfReturnedPIDs) { int psutil_assert_pid_exists(DWORD pid, char *err) { if (PSUTIL_TESTING) { - printf("testing\n"); if (psutil_pid_in_pids(pid) == 0) { PyErr_SetString(PyExc_AssertionError, err); return 0; diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index a09e28960..35ce059cf 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -174,6 +174,11 @@ def recursive_rm(*patterns): safe_rmtree(os.path.join(root, dir)) +def test_setup(): + os.environ['PYTHONWARNINGS'] = 'all' + os.environ['PSUTIL_TESTING'] = '1' + + # =================================================================== # commands # =================================================================== @@ -328,7 +333,8 @@ def flake8(): def test(): """Run tests""" install() - sh("%s -Wa %s" % (PYTHON, TSCRIPT)) + test_setup() + sh("%s %s" % (PYTHON, TSCRIPT)) @cmd @@ -336,7 +342,8 @@ def coverage(): """Run coverage tests.""" # Note: coverage options are controlled by .coveragerc file install() - sh("%s -Wa -m coverage run %s" % (PYTHON, TSCRIPT)) + test_setup() + sh("%s -m coverage run %s" % (PYTHON, TSCRIPT)) sh("%s -m coverage report" % PYTHON) sh("%s -m coverage html" % PYTHON) sh("%s -m webbrowser -t htmlcov/index.html" % PYTHON) @@ -346,49 +353,56 @@ def coverage(): def test_process(): """Run process tests""" install() - sh("%s -Wa -m unittest -v psutil.tests.test_process" % PYTHON) + test_setup() + sh("%s -m unittest -v psutil.tests.test_process" % PYTHON) @cmd def test_system(): """Run system tests""" install() - sh("%s -Wa -m unittest -v psutil.tests.test_system" % PYTHON) + test_setup() + sh("%s -m unittest -v psutil.tests.test_system" % PYTHON) @cmd def test_platform(): """Run windows only tests""" install() - sh("%s -Wa -m unittest -v psutil.tests.test_windows" % PYTHON) + test_setup() + sh("%s -m unittest -v psutil.tests.test_windows" % PYTHON) @cmd def test_misc(): """Run misc tests""" install() - sh("%s -Wa -m unittest -v psutil.tests.test_misc" % PYTHON) + test_setup() + sh("%s -m unittest -v psutil.tests.test_misc" % PYTHON) @cmd def test_unicode(): """Run unicode tests""" install() - sh("%s -Wa -m unittest -v psutil.tests.test_unicode" % PYTHON) + test_setup() + sh("%s -m unittest -v psutil.tests.test_unicode" % PYTHON) @cmd def test_connections(): """Run connections tests""" install() - sh("%s -Wa -m unittest -v psutil.tests.test_connections" % PYTHON) + test_setup() + sh("%s -m unittest -v psutil.tests.test_connections" % PYTHON) @cmd def test_contracts(): """Run contracts tests""" install() - sh("%s -Wa -m unittest -v psutil.tests.test_contracts" % PYTHON) + test_setup() + sh("%s -m unittest -v psutil.tests.test_contracts" % PYTHON) @cmd @@ -400,7 +414,8 @@ def test_by_name(): except IndexError: sys.exit('second arg missing') install() - sh("%s -Wa -m unittest -v %s" % (PYTHON, name)) + test_setup() + sh("%s -m unittest -v %s" % (PYTHON, name)) @cmd @@ -412,14 +427,16 @@ def test_script(): except IndexError: sys.exit('second arg missing') install() - sh("%s -Wa %s" % (PYTHON, name)) + test_setup() + sh("%s %s" % (PYTHON, name)) @cmd def test_memleaks(): """Run memory leaks tests""" install() - sh("%s -Wa psutil\\tests\\test_memory_leaks.py" % PYTHON) + test_setup() + sh("%s psutil\\tests\\test_memory_leaks.py" % PYTHON) @cmd From 1c3a15f637521ba5c0031283da39c733fda53e4c Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 04:34:06 +0100 Subject: [PATCH 873/922] appveyor: enable python warnings when running tests --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 5fb1d7a3c..f17b5b878 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -83,7 +83,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "set PSUTIL_TESTING=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" + - "set PYTHONWARNINGS=all && set PSUTIL_TESTING=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" From fe68b30dacec3255b023fcefac5c9095d96a692f Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Mon, 13 Nov 2017 00:38:12 +0200 Subject: [PATCH 874/922] Move exceptions to separate file (#1174) --- psutil/__init__.py | 110 +++--------------------------------------- psutil/_exceptions.py | 94 ++++++++++++++++++++++++++++++++++++ psutil/_psaix.py | 10 ++-- psutil/_psbsd.py | 10 ++-- psutil/_pslinux.py | 10 ++-- psutil/_psosx.py | 10 ++-- psutil/_pssunos.py | 10 ++-- psutil/_pswindows.py | 8 ++- 8 files changed, 123 insertions(+), 139 deletions(-) create mode 100644 psutil/_exceptions.py diff --git a/psutil/__init__.py b/psutil/__init__.py index f80079d01..d14c5e908 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -85,6 +85,12 @@ from ._common import SUNOS from ._common import WINDOWS +from ._exceptions import Error +from ._exceptions import NoSuchProcess +from ._exceptions import ZombieProcess +from ._exceptions import AccessDenied +from ._exceptions import TimeoutExpired + if LINUX: # This is public API and it will be retrieved from _pslinux.py # via sys.modules. @@ -243,110 +249,6 @@ raise ImportError(msg) -# ===================================================================== -# --- exceptions -# ===================================================================== - - -class Error(Exception): - """Base exception class. All other psutil exceptions inherit - from this one. - """ - - def __init__(self, msg=""): - Exception.__init__(self, msg) - self.msg = msg - - def __repr__(self): - ret = "%s.%s %s" % (self.__class__.__module__, - self.__class__.__name__, self.msg) - return ret.strip() - - __str__ = __repr__ - - -class NoSuchProcess(Error): - """Exception raised when a process with a certain PID doesn't - or no longer exists. - """ - - def __init__(self, pid, name=None, msg=None): - Error.__init__(self, msg) - self.pid = pid - self.name = name - self.msg = msg - if msg is None: - if name: - details = "(pid=%s, name=%s)" % (self.pid, repr(self.name)) - else: - details = "(pid=%s)" % self.pid - self.msg = "process no longer exists " + details - - -class ZombieProcess(NoSuchProcess): - """Exception raised when querying a zombie process. This is - raised on OSX, BSD and Solaris only, and not always: depending - on the query the OS may be able to succeed anyway. - On Linux all zombie processes are querable (hence this is never - raised). Windows doesn't have zombie processes. - """ - - def __init__(self, pid, name=None, ppid=None, msg=None): - NoSuchProcess.__init__(self, msg) - self.pid = pid - self.ppid = ppid - self.name = name - self.msg = msg - if msg is None: - args = ["pid=%s" % pid] - if name: - args.append("name=%s" % repr(self.name)) - if ppid: - args.append("ppid=%s" % self.ppid) - details = "(%s)" % ", ".join(args) - self.msg = "process still exists but it's a zombie " + details - - -class AccessDenied(Error): - """Exception raised when permission to perform an action is denied.""" - - def __init__(self, pid=None, name=None, msg=None): - Error.__init__(self, msg) - self.pid = pid - self.name = name - self.msg = msg - if msg is None: - if (pid is not None) and (name is not None): - self.msg = "(pid=%s, name=%s)" % (pid, repr(name)) - elif (pid is not None): - self.msg = "(pid=%s)" % self.pid - else: - self.msg = "" - - -class TimeoutExpired(Error): - """Raised on Process.wait(timeout) if timeout expires and process - is still alive. - """ - - def __init__(self, seconds, pid=None, name=None): - Error.__init__(self, "timeout after %s seconds" % seconds) - self.seconds = seconds - self.pid = pid - self.name = name - if (pid is not None) and (name is not None): - self.msg += " (pid=%s, name=%s)" % (pid, repr(name)) - elif (pid is not None): - self.msg += " (pid=%s)" % self.pid - - -# push exception classes into platform specific module namespace -_psplatform.NoSuchProcess = NoSuchProcess -_psplatform.ZombieProcess = ZombieProcess -_psplatform.AccessDenied = AccessDenied -_psplatform.TimeoutExpired = TimeoutExpired - - # ===================================================================== # --- Process class # ===================================================================== diff --git a/psutil/_exceptions.py b/psutil/_exceptions.py new file mode 100644 index 000000000..c08e6d83c --- /dev/null +++ b/psutil/_exceptions.py @@ -0,0 +1,94 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + + +class Error(Exception): + """Base exception class. All other psutil exceptions inherit + from this one. + """ + + def __init__(self, msg=""): + Exception.__init__(self, msg) + self.msg = msg + + def __repr__(self): + ret = "psutil.%s %s" % (self.__class__.__name__, self.msg) + return ret.strip() + + __str__ = __repr__ + + +class NoSuchProcess(Error): + """Exception raised when a process with a certain PID doesn't + or no longer exists. + """ + + def __init__(self, pid, name=None, msg=None): + Error.__init__(self, msg) + self.pid = pid + self.name = name + self.msg = msg + if msg is None: + if name: + details = "(pid=%s, name=%s)" % (self.pid, repr(self.name)) + else: + details = "(pid=%s)" % self.pid + self.msg = "process no longer exists " + details + + +class ZombieProcess(NoSuchProcess): + """Exception raised when querying a zombie process. This is + raised on OSX, BSD and Solaris only, and not always: depending + on the query the OS may be able to succeed anyway. + On Linux all zombie processes are querable (hence this is never + raised). Windows doesn't have zombie processes. + """ + + def __init__(self, pid, name=None, ppid=None, msg=None): + NoSuchProcess.__init__(self, msg) + self.pid = pid + self.ppid = ppid + self.name = name + self.msg = msg + if msg is None: + args = ["pid=%s" % pid] + if name: + args.append("name=%s" % repr(self.name)) + if ppid: + args.append("ppid=%s" % self.ppid) + details = "(%s)" % ", ".join(args) + self.msg = "process still exists but it's a zombie " + details + + +class AccessDenied(Error): + """Exception raised when permission to perform an action is denied.""" + + def __init__(self, pid=None, name=None, msg=None): + Error.__init__(self, msg) + self.pid = pid + self.name = name + self.msg = msg + if msg is None: + if (pid is not None) and (name is not None): + self.msg = "(pid=%s, name=%s)" % (pid, repr(name)) + elif (pid is not None): + self.msg = "(pid=%s)" % self.pid + else: + self.msg = "" + + +class TimeoutExpired(Error): + """Raised on Process.wait(timeout) if timeout expires and process + is still alive. + """ + + def __init__(self, seconds, pid=None, name=None): + Error.__init__(self, "timeout after %s seconds" % seconds) + self.seconds = seconds + self.pid = pid + self.name = name + if (pid is not None) and (name is not None): + self.msg += " (pid=%s, name=%s)" % (pid, repr(name)) + elif (pid is not None): + self.msg += " (pid=%s)" % self.pid diff --git a/psutil/_psaix.py b/psutil/_psaix.py index c78922b06..f63e66ed6 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -28,6 +28,10 @@ from ._common import socktype_to_enum from ._common import usage_percent from ._compat import PY3 +from ._exceptions import NoSuchProcess +from ._exceptions import ZombieProcess +from ._exceptions import AccessDenied +from ._exceptions import TimeoutExpired __extra__all__ = ["PROCFS_PATH"] @@ -76,12 +80,6 @@ status=6, ttynr=7) -# these get overwritten on "import psutil" from the __init__.py file -NoSuchProcess = None -ZombieProcess = None -AccessDenied = None -TimeoutExpired = None - # ===================================================================== # --- named tuples diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index 6517f2446..a9d164504 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -27,6 +27,10 @@ from ._common import socktype_to_enum from ._common import usage_percent from ._compat import which +from ._exceptions import NoSuchProcess +from ._exceptions import ZombieProcess +from ._exceptions import AccessDenied +from ._exceptions import TimeoutExpired __extra__all__ = [] @@ -128,12 +132,6 @@ name=24, ) -# these get overwritten on "import psutil" from the __init__.py file -NoSuchProcess = None -ZombieProcess = None -AccessDenied = None -TimeoutExpired = None - # ===================================================================== # --- named tuples diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 2fec8ea78..228dbd987 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -41,6 +41,10 @@ from ._compat import basestring from ._compat import long from ._compat import PY3 +from ._exceptions import NoSuchProcess +from ._exceptions import ZombieProcess +from ._exceptions import AccessDenied +from ._exceptions import TimeoutExpired if sys.version_info >= (3, 4): import enum @@ -137,12 +141,6 @@ class IOPriority(enum.IntEnum): "0B": _common.CONN_CLOSING } -# these get overwritten on "import psutil" from the __init__.py file -NoSuchProcess = None -ZombieProcess = None -AccessDenied = None -TimeoutExpired = None - # ===================================================================== # --- named tuples diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 8093e8149..c2a8b3afe 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -23,6 +23,10 @@ from ._common import sockfam_to_enum from ._common import socktype_to_enum from ._common import usage_percent +from ._exceptions import NoSuchProcess +from ._exceptions import ZombieProcess +from ._exceptions import AccessDenied +from ._exceptions import TimeoutExpired __extra__all__ = [] @@ -84,12 +88,6 @@ volctxsw=7, ) -# these get overwritten on "import psutil" from the __init__.py file -NoSuchProcess = None -ZombieProcess = None -AccessDenied = None -TimeoutExpired = None - # ===================================================================== # --- named tuples diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index 06e8bbbaa..aafb3c683 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -24,6 +24,10 @@ from ._common import usage_percent from ._compat import b from ._compat import PY3 +from ._exceptions import NoSuchProcess +from ._exceptions import ZombieProcess +from ._exceptions import AccessDenied +from ._exceptions import TimeoutExpired __extra__all__ = ["CONN_IDLE", "CONN_BOUND", "PROCFS_PATH"] @@ -78,12 +82,6 @@ status=6, ttynr=7) -# these get overwritten on "import psutil" from the __init__.py file -NoSuchProcess = None -ZombieProcess = None -AccessDenied = None -TimeoutExpired = None - # ===================================================================== # --- named tuples diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 8903a3072..ee30308b4 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -45,6 +45,9 @@ from ._compat import PY3 from ._compat import unicode from ._compat import xrange +from ._exceptions import NoSuchProcess +from ._exceptions import AccessDenied +from ._exceptions import TimeoutExpired from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS from ._psutil_windows import HIGH_PRIORITY_CLASS @@ -139,11 +142,6 @@ class Priority(enum.IntEnum): mem_private=21, ) -# these get overwritten on "import psutil" from the __init__.py file -NoSuchProcess = None -AccessDenied = None -TimeoutExpired = None - # ===================================================================== # --- named tuples From 100391f880ef2a2c5b124bba4b0722623f3edb3e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 12 Nov 2017 23:41:30 +0100 Subject: [PATCH 875/922] sort imports by name --- psutil/__init__.py | 4 ++-- psutil/_psaix.py | 4 ++-- psutil/_psbsd.py | 4 ++-- psutil/_pslinux.py | 4 ++-- psutil/_psosx.py | 4 ++-- psutil/_pssunos.py | 4 ++-- psutil/_pswindows.py | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index d14c5e908..12cd2c4cc 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -85,11 +85,11 @@ from ._common import SUNOS from ._common import WINDOWS +from ._exceptions import AccessDenied from ._exceptions import Error from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess -from ._exceptions import AccessDenied from ._exceptions import TimeoutExpired +from ._exceptions import ZombieProcess if LINUX: # This is public API and it will be retrieved from _pslinux.py diff --git a/psutil/_psaix.py b/psutil/_psaix.py index f63e66ed6..369896236 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -28,10 +28,10 @@ from ._common import socktype_to_enum from ._common import usage_percent from ._compat import PY3 -from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess from ._exceptions import AccessDenied +from ._exceptions import NoSuchProcess from ._exceptions import TimeoutExpired +from ._exceptions import ZombieProcess __extra__all__ = ["PROCFS_PATH"] diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index a9d164504..c26300a31 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -27,10 +27,10 @@ from ._common import socktype_to_enum from ._common import usage_percent from ._compat import which -from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess from ._exceptions import AccessDenied +from ._exceptions import NoSuchProcess from ._exceptions import TimeoutExpired +from ._exceptions import ZombieProcess __extra__all__ = [] diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 228dbd987..fc38fb141 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -41,10 +41,10 @@ from ._compat import basestring from ._compat import long from ._compat import PY3 -from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess from ._exceptions import AccessDenied +from ._exceptions import NoSuchProcess from ._exceptions import TimeoutExpired +from ._exceptions import ZombieProcess if sys.version_info >= (3, 4): import enum diff --git a/psutil/_psosx.py b/psutil/_psosx.py index c2a8b3afe..583ed3556 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -23,10 +23,10 @@ from ._common import sockfam_to_enum from ._common import socktype_to_enum from ._common import usage_percent -from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess from ._exceptions import AccessDenied +from ._exceptions import NoSuchProcess from ._exceptions import TimeoutExpired +from ._exceptions import ZombieProcess __extra__all__ = [] diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index aafb3c683..d1f5df791 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -24,10 +24,10 @@ from ._common import usage_percent from ._compat import b from ._compat import PY3 -from ._exceptions import NoSuchProcess -from ._exceptions import ZombieProcess from ._exceptions import AccessDenied +from ._exceptions import NoSuchProcess from ._exceptions import TimeoutExpired +from ._exceptions import ZombieProcess __extra__all__ = ["CONN_IDLE", "CONN_BOUND", "PROCFS_PATH"] diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index ee30308b4..83936d08b 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -45,8 +45,8 @@ from ._compat import PY3 from ._compat import unicode from ._compat import xrange -from ._exceptions import NoSuchProcess from ._exceptions import AccessDenied +from ._exceptions import NoSuchProcess from ._exceptions import TimeoutExpired from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS From 40573cbe58407a3f8dfcb0c3b71237444b10fc0a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Nov 2017 00:00:30 +0100 Subject: [PATCH 876/922] #1174: use TimeoutExpired in wait_pid() --- CREDITS | 2 +- psutil/_psaix.py | 9 +-------- psutil/_psbsd.py | 6 +----- psutil/_pslinux.py | 6 +----- psutil/_psosx.py | 6 +----- psutil/_psposix.py | 12 ++++-------- psutil/_pssunos.py | 6 +----- 7 files changed, 10 insertions(+), 37 deletions(-) diff --git a/CREDITS b/CREDITS index ff242998b..811f48fe9 100644 --- a/CREDITS +++ b/CREDITS @@ -57,7 +57,7 @@ W: http://www.jayloden.com N: Arnon Yaari (wiggin15) W: https://github.com/wiggin15 -I: 517, 607, 610, 1131, 1123, 1130, 1154, 1164 +I: 517, 607, 610, 1131, 1123, 1130, 1154, 1164, 1174 N: Jeff Tang W: https://github.com/mrjefftang diff --git a/psutil/_psaix.py b/psutil/_psaix.py index 369896236..9abc8d17e 100644 --- a/psutil/_psaix.py +++ b/psutil/_psaix.py @@ -30,7 +30,6 @@ from ._compat import PY3 from ._exceptions import AccessDenied from ._exceptions import NoSuchProcess -from ._exceptions import TimeoutExpired from ._exceptions import ZombieProcess @@ -559,13 +558,7 @@ def num_ctx_switches(self): @wrap_exceptions def wait(self, timeout=None): - try: - return _psposix.wait_pid(self.pid, timeout) - except _psposix.TimeoutExpired: - # support for private module import - if TimeoutExpired is None: - raise - raise TimeoutExpired(timeout, self.pid, self._name) + return _psposix.wait_pid(self.pid, timeout, self._name) @wrap_exceptions def io_counters(self): diff --git a/psutil/_psbsd.py b/psutil/_psbsd.py index c26300a31..0553401a5 100644 --- a/psutil/_psbsd.py +++ b/psutil/_psbsd.py @@ -29,7 +29,6 @@ from ._compat import which from ._exceptions import AccessDenied from ._exceptions import NoSuchProcess -from ._exceptions import TimeoutExpired from ._exceptions import ZombieProcess __extra__all__ = [] @@ -758,10 +757,7 @@ def connections(self, kind='inet'): @wrap_exceptions def wait(self, timeout=None): - try: - return _psposix.wait_pid(self.pid, timeout) - except _psposix.TimeoutExpired: - raise TimeoutExpired(timeout, self.pid, self._name) + return _psposix.wait_pid(self.pid, timeout, self._name) @wrap_exceptions def nice_get(self): diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index fc38fb141..3fe62c5c4 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -43,7 +43,6 @@ from ._compat import PY3 from ._exceptions import AccessDenied from ._exceptions import NoSuchProcess -from ._exceptions import TimeoutExpired from ._exceptions import ZombieProcess if sys.version_info >= (3, 4): @@ -1534,10 +1533,7 @@ def cpu_num(self): @wrap_exceptions def wait(self, timeout=None): - try: - return _psposix.wait_pid(self.pid, timeout) - except _psposix.TimeoutExpired: - raise TimeoutExpired(timeout, self.pid, self._name) + return _psposix.wait_pid(self.pid, timeout, self._name) @wrap_exceptions def create_time(self): diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 583ed3556..e9b6ed824 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -25,7 +25,6 @@ from ._common import usage_percent from ._exceptions import AccessDenied from ._exceptions import NoSuchProcess -from ._exceptions import TimeoutExpired from ._exceptions import ZombieProcess @@ -500,10 +499,7 @@ def num_fds(self): @wrap_exceptions def wait(self, timeout=None): - try: - return _psposix.wait_pid(self.pid, timeout) - except _psposix.TimeoutExpired: - raise TimeoutExpired(timeout, self.pid, self._name) + return _psposix.wait_pid(self.pid, timeout, self._name) @wrap_exceptions def nice_get(self): diff --git a/psutil/_psposix.py b/psutil/_psposix.py index 66d81a3d1..6bb8444d8 100644 --- a/psutil/_psposix.py +++ b/psutil/_psposix.py @@ -15,14 +15,10 @@ from ._common import usage_percent from ._compat import PY3 from ._compat import unicode +from ._exceptions import TimeoutExpired -__all__ = ['TimeoutExpired', 'pid_exists', 'wait_pid', 'disk_usage', - 'get_terminal_map'] - - -class TimeoutExpired(Exception): - pass +__all__ = ['pid_exists', 'wait_pid', 'disk_usage', 'get_terminal_map'] def pid_exists(pid): @@ -53,7 +49,7 @@ def pid_exists(pid): return True -def wait_pid(pid, timeout=None): +def wait_pid(pid, timeout=None, proc_name=None): """Wait for process with pid 'pid' to terminate and return its exit status code as an integer. @@ -67,7 +63,7 @@ def wait_pid(pid, timeout=None): def check_timeout(delay): if timeout is not None: if timer() >= stop_at: - raise TimeoutExpired() + raise TimeoutExpired(timeout, pid=pid, name=proc_name) time.sleep(delay) return min(delay * 2, 0.04) diff --git a/psutil/_pssunos.py b/psutil/_pssunos.py index d1f5df791..5471d5aa4 100644 --- a/psutil/_pssunos.py +++ b/psutil/_pssunos.py @@ -26,7 +26,6 @@ from ._compat import PY3 from ._exceptions import AccessDenied from ._exceptions import NoSuchProcess -from ._exceptions import TimeoutExpired from ._exceptions import ZombieProcess @@ -723,7 +722,4 @@ def num_ctx_switches(self): @wrap_exceptions def wait(self, timeout=None): - try: - return _psposix.wait_pid(self.pid, timeout) - except _psposix.TimeoutExpired: - raise TimeoutExpired(timeout, self.pid, self._name) + return _psposix.wait_pid(self.pid, timeout, self._name) From 4d3f5ba337c5a91efa642699cc96331718889b4f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Nov 2017 00:26:35 +0100 Subject: [PATCH 877/922] update MANIFEST --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 11945017e..7a92a4e5a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,6 +24,7 @@ include psutil/DEVNOTES include psutil/__init__.py include psutil/_common.py include psutil/_compat.py +include psutil/_exceptions.py include psutil/_psaix.py include psutil/_psbsd.py include psutil/_pslinux.py From c907d406d96f4c79a28e1a999e3d9ddc4888653e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 13 Nov 2017 01:08:15 +0100 Subject: [PATCH 878/922] code style --- psutil/_psutil_windows.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 574885b64..82ad26b7c 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -1038,7 +1038,6 @@ psutil_per_cpu_times(PyObject *self, PyObject *args) { /* * Return process current working directory as a Python string. */ - static PyObject * psutil_proc_cwd(PyObject *self, PyObject *args) { long pid; @@ -1064,6 +1063,7 @@ int psutil_proc_suspend_or_resume(DWORD pid, int suspend) { // a huge thanks to http://www.codeproject.com/KB/threads/pausep.aspx HANDLE hThreadSnap = NULL; + HANDLE hThread; THREADENTRY32 te32 = {0}; if (pid == 0) { @@ -1089,20 +1089,17 @@ psutil_proc_suspend_or_resume(DWORD pid, int suspend) { // Walk the thread snapshot to find all threads of the process. // If the thread belongs to the process, add its information // to the display list. - do - { - if (te32.th32OwnerProcessID == pid) - { - HANDLE hThread = OpenThread(THREAD_SUSPEND_RESUME, FALSE, - te32.th32ThreadID); + do { + if (te32.th32OwnerProcessID == pid) { + hThread = OpenThread(THREAD_SUSPEND_RESUME, FALSE, + te32.th32ThreadID); if (hThread == NULL) { PyErr_SetFromWindowsErr(0); CloseHandle(hThread); CloseHandle(hThreadSnap); return FALSE; } - if (suspend == 1) - { + if (suspend == 1) { if (SuspendThread(hThread) == (DWORD) - 1) { PyErr_SetFromWindowsErr(0); CloseHandle(hThread); @@ -1110,8 +1107,7 @@ psutil_proc_suspend_or_resume(DWORD pid, int suspend) { return FALSE; } } - else - { + else { if (ResumeThread(hThread) == (DWORD) - 1) { PyErr_SetFromWindowsErr(0); CloseHandle(hThread); From 26c77800591a09718064d3bcea4b04c9a8544dd1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 14 Nov 2017 23:35:19 +0100 Subject: [PATCH 879/922] 1173 debug mode (#1176) * implement PSUTIL_DEBUG from C module * update doc * add psutil_debug() utility function * update doc * enable PSUTIL_DEBUG for tests * update appveyor.yml * change psutil_debug() signature so that it can accept variable num of args * provide DEBUG info in psutil_raise_for_pid() * properly print debug message * do not print too much --- HISTORY.rst | 5 +++++ Makefile | 2 +- appveyor.yml | 2 +- docs/index.rst | 23 +++++++++++++++++++++++ psutil/_psutil_bsd.c | 2 +- psutil/_psutil_common.c | 17 +++++++++++++++++ psutil/_psutil_common.h | 2 ++ psutil/_psutil_osx.c | 13 +++++++------ psutil/_psutil_posix.c | 13 +++++++++---- psutil/_psutil_windows.c | 8 +++++++- psutil/arch/freebsd/proc_socks.c | 2 +- psutil/arch/freebsd/specific.c | 6 +++--- psutil/arch/netbsd/specific.c | 2 +- psutil/arch/openbsd/specific.c | 4 ++-- psutil/arch/osx/process_info.c | 2 +- scripts/internal/winmake.py | 1 + 16 files changed, 82 insertions(+), 22 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 48adbe523..6dadb0312 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,11 @@ *XXXX-XX-XX* +**Enhancements** + +- 1173_: introduced PSUTIL_DEBUG environment variable which can be set in order + to print useful debug messages on stderr (useful in case of nasty errors). + **Bug fixes** - 1152_: [Windows] disk_io_counters() may return an empty dict. diff --git a/Makefile b/Makefile index e8ff51ca3..7b2eca867 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ DEPS = \ # In not in a virtualenv, add --user options for install commands. INSTALL_OPTS = `$(PYTHON) -c "import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"` -TEST_PREFIX = PYTHONWARNINGS=all PSUTIL_TESTING=1 +TEST_PREFIX = PYTHONWARNINGS=all PSUTIL_TESTING=1 PSUTIL_DEBUG=1 all: test diff --git a/appveyor.yml b/appveyor.yml index f17b5b878..092dc23a8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -83,7 +83,7 @@ build: off test_script: - "%WITH_COMPILER% %PYTHON%/python -V" - - "set PYTHONWARNINGS=all && set PSUTIL_TESTING=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" + - "set PYTHONWARNINGS=all && set PSUTIL_TESTING=1 && set PSUTIL_DEBUG=1 && %WITH_COMPILER% %PYTHON%/python psutil/tests/__main__.py" after_test: - "%WITH_COMPILER% %PYTHON%/python setup.py bdist_wheel" diff --git a/docs/index.rst b/docs/index.rst index ce798e923..620fea464 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2300,6 +2300,29 @@ Constants ---- +Debug mode +========== + +In case you bump into nasty errors which look like being psutil's fault you may +want to run psutil in debug mode. psutil may (or may not) print some useful +message on stderr before crashing with an exception +(see `original motivation `__). +To enable debug mode on UNIX: + +.. code-block:: bash + + PSUTIL_DEBUG=1 python script.py + +On Windows: + +.. code-block:: bat + + set PSUTIL_DEBUG=1 && C:\python36\python.exe script.py + +.. versionadded:: 5.4.2 + +---- + Unicode ======= diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index f1adb1d37..9a2ed04bc 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -475,7 +475,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); goto error; } diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index cf9899f11..e9fce85e6 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -11,6 +11,7 @@ // Global vars. +int PSUTIL_DEBUG = 0; int PSUTIL_TESTING = 0; @@ -75,11 +76,27 @@ psutil_set_testing(PyObject *self, PyObject *args) { } +/* + * Print a debug message on stderr. No-op if PSUTIL_DEBUG env var is not set. + */ +void +psutil_debug(const char* format, ...) { + va_list argptr; + va_start(argptr, format); + fprintf(stderr, "psutil-dubug> "); + vfprintf(stderr, format, argptr); + fprintf(stderr, "\n"); + va_end(argptr); +} + + /* * Called on module import on all platforms. */ void psutil_setup(void) { + if (getenv("PSUTIL_DEBUG") != NULL) + PSUTIL_DEBUG = 1; if (getenv("PSUTIL_TESTING") != NULL) PSUTIL_TESTING = 1; } diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index a609b886a..965966af7 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -7,6 +7,7 @@ #include extern int PSUTIL_TESTING; +extern int PSUTIL_DEBUG; // a signaler for connections without an actual status static const int PSUTIL_CONN_NONE = 128; @@ -20,4 +21,5 @@ PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); PyObject* psutil_set_testing(PyObject *self, PyObject *args); +void psutil_debug(const char* format, ...); void psutil_setup(void); diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 4ff301d4f..bb98c1cc1 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -266,7 +266,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (pid == 0) AccessDenied(); else - psutil_raise_for_pid(pid, "proc_pidpath() syscall failed"); + psutil_raise_for_pid(pid, "proc_pidpath()"); return NULL; } return PyUnicode_DecodeFSDefault(buf); @@ -336,7 +336,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { err = task_for_pid(mach_task_self(), (pid_t)pid, &task); if (err != KERN_SUCCESS) { - psutil_raise_for_pid(pid, "task_for_pid() failed"); + psutil_raise_for_pid(pid, "task_for_pid()"); goto error; } @@ -376,8 +376,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { errno = 0; proc_regionfilename((pid_t)pid, address, buf, sizeof(buf)); if ((errno != 0) || ((sizeof(buf)) <= 0)) { - psutil_raise_for_pid( - pid, "proc_regionfilename() syscall failed"); + psutil_raise_for_pid(pid, "proc_regionfilename()"); goto error; } @@ -1147,7 +1146,8 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { continue; } else { - psutil_raise_for_pid(pid, "proc_pidinfo() syscall failed"); + psutil_raise_for_pid( + pid, "proc_pidinfo(PROC_PIDFDVNODEPATHINFO)"); goto error; } } @@ -1259,7 +1259,8 @@ psutil_proc_connections(PyObject *self, PyObject *args) { continue; } else { - psutil_raise_for_pid(pid, "proc_pidinfo() syscall failed"); + psutil_raise_for_pid( + pid, "proc_pidinfo(PROC_PIDFDSOCKETINFO)"); goto error; } } diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index ea0fba534..76cf2db03 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -112,16 +112,21 @@ psutil_pid_exists(long pid) { * This will always set a Python exception and return NULL. */ int -psutil_raise_for_pid(long pid, char *msg) { +psutil_raise_for_pid(long pid, char *syscall_name) { // Set exception to AccessDenied if pid exists else NoSuchProcess. if (errno != 0) { + // Unlikely we get here. PyErr_SetFromErrno(PyExc_OSError); return 0; } - if (psutil_pid_exists(pid) == 0) + else if (psutil_pid_exists(pid) == 0) { + psutil_debug("%s syscall failed and PID %i no longer exists; " + "assume NoSuchProcess", syscall_name, pid); NoSuchProcess(); - else - PyErr_SetString(PyExc_RuntimeError, msg); + } + else { + PyErr_Format(PyExc_RuntimeError, "%s syscall failed", syscall_name); + } return 0; } diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 574885b64..e95ab45c2 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -341,6 +341,8 @@ psutil_proc_kill(PyObject *self, PyObject *args) { if (hProcess == NULL) { if (GetLastError() == ERROR_INVALID_PARAMETER) { // see https://github.com/giampaolo/psutil/issues/24 + psutil_debug("OpenProcess -> ERROR_INVALID_PARAMETER turned " + "into NoSuchProcess"); NoSuchProcess(); } else { @@ -2409,10 +2411,14 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { // 1364/job/ascpdi271b06jle3 // Assume it means we're dealing with some exotic disk // and go on. + psutil_debug("DeviceIoControl -> ERROR_INVALID_FUNCTION; " + "ignore PhysicalDrive%i", devNum); goto next; } else if (GetLastError() == ERROR_NOT_SUPPORTED) { // Again, let's assume we're dealing with some exotic disk. + psutil_debug("DeviceIoControl -> ERROR_NOT_SUPPORTED; " + "ignore PhysicalDrive%i", devNum); goto next; } // XXX: it seems we should also catch ERROR_INVALID_PARAMETER: @@ -2427,7 +2433,7 @@ psutil_disk_io_counters(PyObject *self, PyObject *args) { goto error; } - sprintf_s(szDeviceDisplay, MAX_PATH, "PhysicalDrive%d", devNum); + sprintf_s(szDeviceDisplay, MAX_PATH, "PhysicalDrive%i", devNum); py_tuple = Py_BuildValue( "(IILLKK)", diskPerformance.ReadCount, diff --git a/psutil/arch/freebsd/proc_socks.c b/psutil/arch/freebsd/proc_socks.c index c5b19a0de..a458a01e5 100644 --- a/psutil/arch/freebsd/proc_socks.c +++ b/psutil/arch/freebsd/proc_socks.c @@ -212,7 +212,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); goto error; } diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index 8d09ad89a..ff128e65f 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -548,7 +548,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); goto error; } @@ -597,7 +597,7 @@ psutil_proc_num_fds(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); return NULL; } free(freep); @@ -765,7 +765,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getvmmap(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getvmmap() failed"); + psutil_raise_for_pid(pid, "kinfo_getvmmap()"); goto error; } for (i = 0; i < cnt; i++) { diff --git a/psutil/arch/netbsd/specific.c b/psutil/arch/netbsd/specific.c index 1dc2080ee..0a32139d3 100644 --- a/psutil/arch/netbsd/specific.c +++ b/psutil/arch/netbsd/specific.c @@ -502,7 +502,7 @@ psutil_proc_num_fds(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); return NULL; } free(freep); diff --git a/psutil/arch/openbsd/specific.c b/psutil/arch/openbsd/specific.c index de30c4d7d..2a0d30cea 100644 --- a/psutil/arch/openbsd/specific.c +++ b/psutil/arch/openbsd/specific.c @@ -404,7 +404,7 @@ psutil_proc_num_fds(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); return NULL; } free(freep); @@ -509,7 +509,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { errno = 0; freep = kinfo_getfile(pid, &cnt); if (freep == NULL) { - psutil_raise_for_pid(pid, "kinfo_getfile() failed"); + psutil_raise_for_pid(pid, "kinfo_getfile()"); goto error; } diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/osx/process_info.c index 7c715be81..f0a011320 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -354,7 +354,7 @@ psutil_proc_pidinfo(long pid, int flavor, uint64_t arg, void *pti, int size) { errno = 0; int ret = proc_pidinfo((int)pid, flavor, arg, pti, size); if ((ret <= 0) || ((unsigned long)ret < sizeof(pti))) { - psutil_raise_for_pid(pid, "proc_pidinfo() syscall failed"); + psutil_raise_for_pid(pid, "proc_pidinfo()"); return 0; } return ret; diff --git a/scripts/internal/winmake.py b/scripts/internal/winmake.py index 35ce059cf..548f7a8ed 100755 --- a/scripts/internal/winmake.py +++ b/scripts/internal/winmake.py @@ -177,6 +177,7 @@ def recursive_rm(*patterns): def test_setup(): os.environ['PYTHONWARNINGS'] = 'all' os.environ['PSUTIL_TESTING'] = '1' + os.environ['PSUTIL_DEBUG'] = '1' # =================================================================== From ca3eb35e8b5b76f8c8f0230071e08ecf86d83486 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 18 Nov 2017 03:00:50 +0100 Subject: [PATCH 880/922] fix doc indentation --- docs/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 620fea464..f352d4061 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2559,8 +2559,8 @@ FAQs * Q: What Python versions are supported? * A: From 2.6 to 3.6, both 32 and 64 bit versions. Last version supporting - Python 2.4 and 2.5 is `psutil 2.1.3 `__. - PyPy is also known to work. + Python 2.4 and 2.5 is `psutil 2.1.3 `__. + PyPy is also known to work. ---- From a7e7b4e83f66576f507ca5fa2759f092cffc18b7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 18 Nov 2017 03:09:58 +0100 Subject: [PATCH 881/922] syntax highlight in doc files --- DEVGUIDE.rst | 2 +- INSTALL.rst | 55 +++++++++++++++++++++++++++++++++++----------------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index 2d0af7fc2..3892e75ee 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -4,7 +4,7 @@ Setup and running tests If you plan on hacking on psutil this is what you're supposed to do first: -- clone the GIT repository:: +- clone the GIT repository: $ git clone git@github.com:giampaolo/psutil.git diff --git a/INSTALL.rst b/INSTALL.rst index 8737c94a1..4cd9bc9e7 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -4,16 +4,22 @@ Install pip pip is the easiest way to install psutil. It is shipped by default with Python 2.7.9+ and 3.4+. For other Python versions you can install it manually. -On Linux or via wget:: +On Linux or via wget: + +.. code-block:: bash wget https://bootstrap.pypa.io/get-pip.py -O - | python -On OSX or via curl:: +On OSX or via curl: + +.. code-block:: bash python < <(curl -s https://bootstrap.pypa.io/get-pip.py) On Windows, `download pip `__, open -cmd.exe and install it:: +cmd.exe and install it: + +.. code-block:: bat C:\Python27\python.exe get-pip.py @@ -25,25 +31,30 @@ If you're not or you bump into permission errors you can either: * prepend ``sudo``, e.g.: -:: +.. code-block:: bash sudo pip install psutil * install psutil for your user only (not at system level): -:: +.. code-block:: bash pip install --user psutil Linux ===== -Ubuntu / Debian:: +Ubuntu / Debian: + +.. code-block:: bash sudo apt-get install gcc python-dev python-pip pip install psutil -RedHat / CentOS:: +RedHat / CentOS: + + +.. code-block:: bash sudo yum install gcc python-devel python-pip pip install psutil @@ -51,11 +62,15 @@ RedHat / CentOS:: If you're on Python 3 use ``python3-dev`` and ``python3-pip`` instead. Major Linux distros also provide binary distributions of psutil so, for -instance, on Ubuntu and Debian you can also do:: +instance, on Ubuntu and Debian you can also do: + +.. code-block:: bash sudo apt-get install python-psutil -On RedHat and CentOS:: +On RedHat and CentOS: + +.. code-block:: bash sudo yum install python-psutil @@ -68,7 +83,7 @@ OSX Install `Xcode `__ first, then: -:: +.. code-block:: bash pip install psutil @@ -77,7 +92,9 @@ Windows The easiest way to install psutil on Windows is to just use the pre-compiled exe/wheel installers hosted on -`PYPI `__ via pip:: +`PYPI `__ via pip: + +.. code-block:: bat C:\Python27\python.exe -m pip install psutil @@ -92,7 +109,9 @@ Compiling 64 bit versions of Python 2.6 and 2.7 with VS 2008 requires `Windows SDK and .NET Framework 3.5 SP1 `__. Once installed run vcvars64.bat, then you can finally compile (see `here `__). -To compile / install psutil from sources on Windows run:: +To compile / install psutil from sources on Windows run: + +.. code-block:: bat make.bat build make.bat install @@ -100,7 +119,7 @@ To compile / install psutil from sources on Windows run:: FreeBSD ======= -:: +.. code-block:: bash pkg install python gcc python -m pip install psutil @@ -108,7 +127,7 @@ FreeBSD OpenBSD ======= -:: +.. code-block:: bash export PKG_PATH="http://ftp.openbsd.org/pub/OpenBSD/`uname -r`/packages/`arch -s`/" pkg_add -v python gcc @@ -117,7 +136,7 @@ OpenBSD NetBSD ====== -:: +.. code-block:: bash export PKG_PATH="ftp.netbsd.org/pub/pkgsrc/packages/NetBSD/`uname -m`/`uname -r`/All" pkg_add -v pkgin @@ -129,13 +148,13 @@ Solaris If ``cc`` compiler is not installed create a symlink to ``gcc``: -:: +.. code-block:: bash sudo ln -s /usr/bin/gcc /usr/local/bin/cc Install: -:: +.. code-block:: bash pkg install gcc python -m pip install psutil @@ -143,7 +162,7 @@ Install: Install from sources ==================== -:: +.. code-block:: bash git clone https://github.com/giampaolo/psutil.git cd psutil From d56f2a9dcc8b532b0f0d4bda78aa2f227d38a2d1 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 18 Nov 2017 03:13:21 +0100 Subject: [PATCH 882/922] syntax highlight in doc files --- DEVGUIDE.rst | 56 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/DEVGUIDE.rst b/DEVGUIDE.rst index 3892e75ee..2d48ced27 100644 --- a/DEVGUIDE.rst +++ b/DEVGUIDE.rst @@ -6,15 +6,21 @@ If you plan on hacking on psutil this is what you're supposed to do first: - clone the GIT repository: +.. code-block:: bash + $ git clone git@github.com:giampaolo/psutil.git -- install test deps and GIT hooks:: +- install test deps and GIT hooks: + +.. code-block:: bash + + make setup-dev-env - $ make setup-dev-env +- run tests: -- run tests:: +.. code-block:: bash - $ make test + make test - bear in mind that ``make`` (see `Makefile `_) @@ -38,34 +44,46 @@ Coding style Makefile ======== -Some useful make commands:: +Some useful make commands: + +.. code-block:: bash - $ make install # install - $ make setup-dev-env # install useful dev libs (pyflakes, unittest2, etc.) - $ make test # run unit tests - $ make test-memleaks # run memory leak tests - $ make test-coverage # run test coverage - $ make flake8 # run PEP8 linter + make install # install + make setup-dev-env # install useful dev libs (pyflakes, unittest2, etc.) + make test # run unit tests + make test-memleaks # run memory leak tests + make test-coverage # run test coverage + make flake8 # run PEP8 linter There are some differences between ``make`` on UNIX and Windows. -For instance, to run a specific Python version. On UNIX:: +For instance, to run a specific Python version. On UNIX: + +.. code-block:: bash make test PYTHON=python3.5 -On Windows:: +On Windows: + +.. code-block:: bat set PYTHON=C:\python35\python.exe && make test - # ...or: +...or: + +.. code-block:: bat make -p 35 test If you want to modify psutil and run a script on the fly which uses it do -(on UNIX):: +(on UNIX): + +.. code-block:: bash make test TSCRIPT=foo.py -On Windows:: +On Windows: + +.. code-block:: bat set TSCRIPT=foo.py && make test @@ -75,7 +93,7 @@ Adding a new feature Usually the files involved when adding a new functionality are: -.. code-block:: plain +.. code-block:: bash psutil/__init__.py # main psutil namespace psutil/_ps{platform}.py # python platform wrapper @@ -185,7 +203,7 @@ FreeBSD notes .. code-block:: bash - $ pkg install python python3 gcc git vim screen bash - $ chsh -s /usr/local/bin/bash user # set bash as default shell + pkg install python python3 gcc git vim screen bash + chsh -s /usr/local/bin/bash user # set bash as default shell - ``/usr/src`` contains the source codes for all installed CLI tools (grep in it). From 36b3a2b00aafa9a2e3c31533a7897aa83e8c4d01 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 18 Nov 2017 03:23:21 +0100 Subject: [PATCH 883/922] do not mention apt-get as method of installation as it's not recommended --- INSTALL.rst | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/INSTALL.rst b/INSTALL.rst index 4cd9bc9e7..f2a80eede 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -61,22 +61,6 @@ RedHat / CentOS: If you're on Python 3 use ``python3-dev`` and ``python3-pip`` instead. -Major Linux distros also provide binary distributions of psutil so, for -instance, on Ubuntu and Debian you can also do: - -.. code-block:: bash - - sudo apt-get install python-psutil - -On RedHat and CentOS: - -.. code-block:: bash - - sudo yum install python-psutil - -This is not recommended though as Linux distros usually ship older psutil -versions. - OSX === From 67c73ad93a748972bfe528e66117c766f05a4859 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 18 Nov 2017 11:08:55 +0100 Subject: [PATCH 884/922] add debug messages --- psutil/_psutil_osx.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index bb98c1cc1..467516263 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -336,6 +336,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { err = task_for_pid(mach_task_self(), (pid_t)pid, &task); if (err != KERN_SUCCESS) { + psutil_debug("task_for_pid() failed"); // TODO temporary psutil_raise_for_pid(pid, "task_for_pid()"); goto error; } @@ -347,8 +348,15 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { err = vm_region_recurse_64(task, &address, &size, &depth, (vm_region_info_64_t)&info, &count); - if (err == KERN_INVALID_ADDRESS) + if (err == KERN_INVALID_ADDRESS) { + // TODO temporary + psutil_debug("vm_region_recurse_64 returned KERN_INVALID_ADDRESS"); break; + } + if (err != KERN_SUCCESS) { + psutil_debug("vm_region_recurse_64 returned != KERN_SUCCESS"); + } + if (info.is_submap) { depth++; } @@ -376,6 +384,8 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { errno = 0; proc_regionfilename((pid_t)pid, address, buf, sizeof(buf)); if ((errno != 0) || ((sizeof(buf)) <= 0)) { + // TODO temporary + psutil_debug("proc_regionfilename() failed"); psutil_raise_for_pid(pid, "proc_regionfilename()"); goto error; } From e0df5da398abfc0fc773736a3dfcc553d2eaa40d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 18 Nov 2017 11:09:53 +0100 Subject: [PATCH 885/922] improve error msg for old windows systems #811 --- psutil/_pswindows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 83936d08b..b6c58c936 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -26,7 +26,8 @@ # but if we get here it means this this was a wheel (or exe). msg = "this Windows version is too old (< Windows Vista); " msg += "psutil 3.4.2 is the latest version which supports Windows " - msg += "2000, XP and 2003 server" + msg += "2000, XP and 2003 server; it may be possible that psutil " + msg += "will work if compiled from sources though" raise RuntimeError(msg) else: raise From 3a3598e433adec73e2a9c4d5f08e658516fb1d32 Mon Sep 17 00:00:00 2001 From: wiggin15 Date: Sun, 19 Nov 2017 20:06:18 +0200 Subject: [PATCH 886/922] OSX: implement sensors_battery (#1177) --- psutil/__init__.py | 2 +- psutil/_psosx.py | 23 +++++++++ psutil/_psutil_osx.c | 90 ++++++++++++++++++++++++++++++++++ psutil/tests/__init__.py | 2 +- psutil/tests/test_contracts.py | 2 +- psutil/tests/test_osx.py | 13 +++++ 6 files changed, 129 insertions(+), 3 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 12cd2c4cc..c8da0a3dc 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -2210,7 +2210,7 @@ def sensors_fans(): __all__.append("sensors_fans") -# Linux, Windows, FreeBSD +# Linux, Windows, FreeBSD, OSX if hasattr(_psplatform, "sensors_battery"): def sensors_battery(): diff --git a/psutil/_psosx.py b/psutil/_psosx.py index e9b6ed824..9fa7716df 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -207,6 +207,29 @@ def disk_partitions(all=False): return retlist +# ===================================================================== +# --- sensors +# ===================================================================== + + +def sensors_battery(): + """Return battery information. + """ + try: + percent, minsleft, power_plugged = cext.sensors_battery() + except NotImplementedError: + # no power source - return None according to interface + return None + power_plugged = power_plugged == 1 + if power_plugged: + secsleft = _common.POWER_TIME_UNLIMITED + elif minsleft == -1: + secsleft = _common.POWER_TIME_UNKNOWN + else: + secsleft = minsleft * 60 + return _common.sbattery(percent, secsleft, power_plugged) + + # ===================================================================== # --- network # ===================================================================== diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 467516263..6c520e5d8 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -38,6 +38,8 @@ #include #include #include +#include +#include #include "_psutil_common.h" #include "_psutil_posix.h" @@ -1788,6 +1790,92 @@ psutil_cpu_stats(PyObject *self, PyObject *args) { } +/* + * Return battery information. + */ +static PyObject * +psutil_sensors_battery(PyObject *self, PyObject *args) { + PyObject *py_tuple = NULL; + CFTypeRef power_info = NULL; + CFArrayRef power_sources_list = NULL; + CFDictionaryRef power_sources_information = NULL; + CFNumberRef capacity_ref = NULL; + CFNumberRef time_to_empty_ref = NULL; + CFStringRef ps_state_ref = NULL; + uint32_t capacity; /* units are percent */ + int time_to_empty; /* units are minutes */ + int is_power_plugged; + + power_info = IOPSCopyPowerSourcesInfo(); + + if (!power_info) { + PyErr_SetString(PyExc_RuntimeError, + "IOPSCopyPowerSourcesInfo() syscall failed"); + goto error; + } + + power_sources_list = IOPSCopyPowerSourcesList(power_info); + if (!power_sources_list) { + PyErr_SetString(PyExc_RuntimeError, + "IOPSCopyPowerSourcesList() syscall failed"); + goto error; + } + + /* Should only get one source. But in practice, check for > 0 sources */ + if (!CFArrayGetCount(power_sources_list)) { + PyErr_SetString(PyExc_NotImplementedError, "no battery"); + goto error; + } + + power_sources_information = IOPSGetPowerSourceDescription( + power_info, CFArrayGetValueAtIndex(power_sources_list, 0)); + + capacity_ref = (CFNumberRef) CFDictionaryGetValue( + power_sources_information, CFSTR(kIOPSCurrentCapacityKey)); + if (!CFNumberGetValue(capacity_ref, kCFNumberSInt32Type, &capacity)) { + PyErr_SetString(PyExc_RuntimeError, + "No battery capacity infomration in power sources info"); + goto error; + } + + ps_state_ref = (CFStringRef) CFDictionaryGetValue( + power_sources_information, CFSTR(kIOPSPowerSourceStateKey)); + is_power_plugged = CFStringCompare( + ps_state_ref, CFSTR(kIOPSACPowerValue), 0) + == kCFCompareEqualTo; + + time_to_empty_ref = (CFNumberRef) CFDictionaryGetValue( + power_sources_information, CFSTR(kIOPSTimeToEmptyKey)); + if (!CFNumberGetValue(time_to_empty_ref, + kCFNumberIntType, &time_to_empty)) { + /* This value is recommended for non-Apple power sources, so it's not + * an error if it doesn't exist. We'll return -1 for "unknown" */ + /* A value of -1 indicates "Still Calculating the Time" also for + * apple power source */ + time_to_empty = -1; + } + + py_tuple = Py_BuildValue("Iii", + capacity, time_to_empty, is_power_plugged); + if (!py_tuple) { + goto error; + } + + CFRelease(power_info); + CFRelease(power_sources_list); + /* Caller should NOT release power_sources_information */ + + return py_tuple; + +error: + if (power_info) + CFRelease(power_info); + if (power_sources_list) + CFRelease(power_sources_list); + Py_XDECREF(py_tuple); + return NULL; +} + /* * define the psutil C module methods and initialize the module. @@ -1854,6 +1942,8 @@ PsutilMethods[] = { "Return currently connected users as a list of tuples"}, {"cpu_stats", psutil_cpu_stats, METH_VARARGS, "Return CPU statistics"}, + {"sensors_battery", psutil_sensors_battery, METH_VARARGS, + "Return battery information."}, // --- others {"set_testing", psutil_set_testing, METH_NOARGS, diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 4f7995dcd..7c58ab3c4 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -70,7 +70,7 @@ 'VERBOSITY', "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", - "HAS_SENSORS_BATTERY", "HAS_BATTERY""HAS_SENSORS_FANS", + "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS", "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO", # subprocesses 'pyrun', 'reap_children', 'get_test_subprocess', 'create_zombie_proc', diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 5e5c2e9a0..4c57b25a8 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -123,7 +123,7 @@ def test_sensors_fans(self): def test_battery(self): self.assertEqual(hasattr(psutil, "sensors_battery"), - LINUX or WINDOWS or FREEBSD) + LINUX or WINDOWS or FREEBSD or OSX) def test_proc_environ(self): self.assertEqual(hasattr(psutil.Process, "environ"), diff --git a/psutil/tests/test_osx.py b/psutil/tests/test_osx.py index c8214f14c..bcb2ba4e1 100755 --- a/psutil/tests/test_osx.py +++ b/psutil/tests/test_osx.py @@ -14,6 +14,7 @@ from psutil import OSX from psutil.tests import create_zombie_proc from psutil.tests import get_test_subprocess +from psutil.tests import HAS_BATTERY from psutil.tests import MEMORY_TOLERANCE from psutil.tests import reap_children from psutil.tests import retry_before_failing @@ -285,6 +286,18 @@ def test_net_if_stats(self): self.assertEqual(stats.mtu, int(re.findall(r'mtu (\d+)', out)[0])) + # --- sensors_battery + + @unittest.skipIf(not HAS_BATTERY, "no battery") + def test_sensors_battery(self): + out = sh("pmset -g batt") + percent = re.search("(\d+)%", out).group(1) + drawing_from = re.search("Now drawing from '([^']+)'", out).group(1) + power_plugged = drawing_from == "AC Power" + psutil_result = psutil.sensors_battery() + self.assertEqual(psutil_result.power_plugged, power_plugged) + self.assertEqual(psutil_result.percent, int(percent)) + if __name__ == '__main__': run_test_module_by_name(__file__) From b0c1b9f1d7e23ff32df5e500e22cc400387f9a8d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 21 Nov 2017 22:08:05 +0100 Subject: [PATCH 887/922] #1177: give credits to @wiggin15 --- CREDITS | 2 +- HISTORY.rst | 1 + docs/index.rst | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CREDITS b/CREDITS index 811f48fe9..d215c60fe 100644 --- a/CREDITS +++ b/CREDITS @@ -57,7 +57,7 @@ W: http://www.jayloden.com N: Arnon Yaari (wiggin15) W: https://github.com/wiggin15 -I: 517, 607, 610, 1131, 1123, 1130, 1154, 1164, 1174 +I: 517, 607, 610, 1131, 1123, 1130, 1154, 1164, 1174, 1177 N: Jeff Tang W: https://github.com/mrjefftang diff --git a/HISTORY.rst b/HISTORY.rst index 6dadb0312..fb3311bb3 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,7 @@ - 1173_: introduced PSUTIL_DEBUG environment variable which can be set in order to print useful debug messages on stderr (useful in case of nasty errors). +- 1177_: added support for sensors_battery() on OSX. (patch by Arnon Yaari) **Bug fixes** diff --git a/docs/index.rst b/docs/index.rst index f352d4061..d34a1c457 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -766,6 +766,8 @@ Sensors .. versionadded:: 5.1.0 + .. versionchanged:: 5.4.2 added OSX support + .. warning:: this API is experimental. Backward incompatible changes may occur if From 4d785835929d5e95089b42fd31c8534ff21e71a3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Wed, 22 Nov 2017 19:24:16 +0100 Subject: [PATCH 888/922] try to use PYTHON_EXE instead of sys.executable --- psutil/tests/test_misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 7dc887353..1e23ab6b2 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -18,7 +18,6 @@ import pickle import socket import stat -import sys from psutil import LINUX from psutil import POSIX @@ -49,6 +48,7 @@ from psutil.tests import import_module_by_path from psutil.tests import is_namedtuple from psutil.tests import mock +from psutil.tests import PYTHON_EXE from psutil.tests import reap_children from psutil.tests import reload_module from psutil.tests import retry @@ -656,7 +656,7 @@ def assert_stdout(exe, args=None, **kwds): if args: exe = exe + ' ' + args try: - out = sh(sys.executable + ' ' + exe, **kwds).strip() + out = sh(PYTHON_EXE + ' ' + exe, **kwds).strip() except RuntimeError as err: if 'AccessDenied' in str(err): return str(err) From 546d6df9ddc43e7cf16dfd63be3ad780e901ae50 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 23 Nov 2017 16:15:37 +0100 Subject: [PATCH 889/922] fix travis failures --- psutil/tests/test_misc.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 1e23ab6b2..6acc3828b 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -20,6 +20,7 @@ import stat from psutil import LINUX +from psutil import OSX from psutil import POSIX from psutil import WINDOWS from psutil._common import memoize @@ -365,6 +366,8 @@ def check(ret): def test_setup_script(self): setup_py = os.path.join(ROOT_DIR, 'setup.py') + if TRAVIS and not os.path.exists(setup_py): + return self.skipTest("can't find setup.py") module = import_module_by_path(setup_py) self.assertRaises(SystemExit, module.setup) self.assertEqual(module.get_version(), psutil.__version__) @@ -662,6 +665,11 @@ def assert_stdout(exe, args=None, **kwds): return str(err) else: raise + except ImportError: + if OSX and TRAVIS: + pass + else: + raise assert out, out return out From c0a35fd6b5f6b2cad4b370d31d2d175db316faf8 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 23 Nov 2017 16:23:31 +0100 Subject: [PATCH 890/922] try to fix travis failure --- psutil/tests/test_misc.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index 6acc3828b..d0ec8a7d7 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -20,7 +20,6 @@ import stat from psutil import LINUX -from psutil import OSX from psutil import POSIX from psutil import WINDOWS from psutil._common import memoize @@ -654,22 +653,18 @@ class TestScripts(unittest.TestCase): """Tests for scripts in the "scripts" directory.""" @staticmethod - def assert_stdout(exe, args=None, **kwds): - exe = '"%s"' % os.path.join(SCRIPTS_DIR, exe) - if args: - exe = exe + ' ' + args + def assert_stdout(exe, *args, **kwargs): + exe = '%s' % os.path.join(SCRIPTS_DIR, exe) + cmd = [PYTHON_EXE, exe] + for arg in args: + cmd.append(arg) try: - out = sh(PYTHON_EXE + ' ' + exe, **kwds).strip() + out = sh(cmd, **kwargs).strip() except RuntimeError as err: if 'AccessDenied' in str(err): return str(err) else: raise - except ImportError: - if OSX and TRAVIS: - pass - else: - raise assert out, out return out @@ -712,7 +707,7 @@ def test_meminfo(self): self.assert_stdout('meminfo.py') def test_procinfo(self): - self.assert_stdout('procinfo.py', args=str(os.getpid())) + self.assert_stdout('procinfo.py', str(os.getpid())) # can't find users on APPVEYOR or TRAVIS @unittest.skipIf(APPVEYOR or TRAVIS and not psutil.users(), @@ -736,7 +731,7 @@ def test_ifconfig(self): @unittest.skipIf(not HAS_MEMORY_MAPS, "not supported") def test_pmap(self): - self.assert_stdout('pmap.py', args=str(os.getpid())) + self.assert_stdout('pmap.py', str(os.getpid())) @unittest.skipIf(not HAS_MEMORY_FULL_INFO, "not supported") def test_procsmem(self): @@ -755,7 +750,7 @@ def test_iotop(self): self.assert_syntax('iotop.py') def test_pidof(self): - output = self.assert_stdout('pidof.py', args=psutil.Process().name()) + output = self.assert_stdout('pidof.py', psutil.Process().name()) self.assertIn(str(os.getpid()), output) @unittest.skipIf(not WINDOWS, "WINDOWS only") From ecab2b3aa54ec5dbee9d7f7664ad4ed359048cd5 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 23 Nov 2017 16:51:55 +0100 Subject: [PATCH 891/922] do not test platf specific modules on wheelhouse --- psutil/tests/__init__.py | 11 +++++++---- psutil/tests/test_misc.py | 3 +-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 7c58ab3c4..5b4f6f37d 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -764,11 +764,14 @@ def _setup_tests(): def get_suite(): - testmodules = [os.path.splitext(x)[0] for x in os.listdir(HERE) - if x.endswith('.py') and x.startswith('test_') and not - x.startswith('test_memory_leaks')] + testmods = [os.path.splitext(x)[0] for x in os.listdir(HERE) + if x.endswith('.py') and x.startswith('test_') and not + x.startswith('test_memory_leaks')] + if "WHEELHOUSE_UPLOADER_USERNAME" in os.environ: + testmods = [x for x in testmods if not x.endswith(( + "osx", "posix", "linux"))] suite = unittest.TestSuite() - for tm in testmodules: + for tm in testmods: # ...so that the full test paths are printed on screen tm = "psutil.tests.%s" % tm suite.addTest(unittest.defaultTestLoader.loadTestsFromName(tm)) diff --git a/psutil/tests/test_misc.py b/psutil/tests/test_misc.py index d0ec8a7d7..f67c0e4cd 100755 --- a/psutil/tests/test_misc.py +++ b/psutil/tests/test_misc.py @@ -646,8 +646,7 @@ def test_cache_clear_public_apis(self): @unittest.skipIf(TOX, "can't test on TOX") # See: https://travis-ci.org/giampaolo/psutil/jobs/295224806 -@unittest.skipIf(TRAVIS and not - os.path.exists(os.path.join(SCRIPTS_DIR, 'free.py')), +@unittest.skipIf(TRAVIS and not os.path.exists(SCRIPTS_DIR), "can't locate scripts directory") class TestScripts(unittest.TestCase): """Tests for scripts in the "scripts" directory.""" From d108baf4a27f0f3b115aae5e6109eaf109302bc3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 23 Nov 2017 18:24:53 +0100 Subject: [PATCH 892/922] be smarter in searching python exe --- psutil/tests/__init__.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/psutil/tests/__init__.py b/psutil/tests/__init__.py index 5b4f6f37d..9e8d8596b 100644 --- a/psutil/tests/__init__.py +++ b/psutil/tests/__init__.py @@ -168,13 +168,28 @@ def _get_py_exe(): - exe = os.path.realpath(sys.executable) - if not os.path.exists(exe): - # It seems this only occurs on OSX. - exe = which("python%s.%s" % sys.version_info[:2]) - if not exe or not os.path.exists(exe): - ValueError("can't find python exe real abspath") - return exe + def attempt(exe): + try: + subprocess.check_call( + [exe, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except Exception: + return None + else: + return exe + + if OSX: + exe = \ + attempt(sys.executable) or \ + attempt(os.path.realpath(sys.executable)) or \ + attempt(which("python%s.%s" % sys.version_info[:2])) or \ + attempt(psutil.Process().exe()) + if not exe: + raise ValueError("can't find python exe real abspath") + return exe + else: + exe = os.path.realpath(sys.executable) + assert os.path.exists(exe), exe + return exe PYTHON_EXE = _get_py_exe() From 71c4f5683460e110693c976b21b06ab4034ae8a3 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Nov 2017 08:26:41 +0100 Subject: [PATCH 893/922] fix travis failure https://travis-ci.org/giampaolo/psutil/jobs/306424509 --- psutil/tests/test_posix.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index ac68b8d3d..0c3f64348 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -10,6 +10,7 @@ import datetime import errno import os +import re import subprocess import sys import time @@ -152,6 +153,9 @@ def test_name(self): # remove path if there is any, from the command name_ps = os.path.basename(name_ps).lower() name_psutil = psutil.Process(self.pid).name().lower() + # ...because of how we calculate PYTHON_EXE; on OSX this may + # be "pythonX.Y". + name_psutil = re.sub(r"\d.\d", "", name_psutil) self.assertEqual(name_ps, name_psutil) def test_name_long(self): From 34fb3666859bda349afdc9b4397b0b7a715f08de Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Nov 2017 14:07:06 +0100 Subject: [PATCH 894/922] Arguments for NoSuchProcess and AccessDenied for the C ext (#1180) * change NoSuchProcess and AccessDenied C exceptions signatures * fix arg call on win --- psutil/_psutil_aix.c | 2 +- psutil/_psutil_common.c | 14 ++++++++------ psutil/_psutil_common.h | 4 ++-- psutil/_psutil_osx.c | 12 ++++++------ psutil/_psutil_posix.c | 2 +- psutil/_psutil_sunos.c | 2 +- psutil/_psutil_windows.c | 28 ++++++++++++++-------------- psutil/arch/freebsd/specific.c | 8 ++++---- psutil/arch/netbsd/specific.c | 8 ++++---- psutil/arch/openbsd/specific.c | 6 +++--- psutil/arch/osx/process_info.c | 6 +++--- psutil/arch/windows/process_info.c | 6 +++--- 12 files changed, 50 insertions(+), 48 deletions(-) diff --git a/psutil/_psutil_aix.c b/psutil/_psutil_aix.c index 4d522ba29..916254d5a 100644 --- a/psutil/_psutil_aix.c +++ b/psutil/_psutil_aix.c @@ -341,7 +341,7 @@ psutil_proc_num_ctx_switches(PyObject *self, PyObject *args) { /* finished iteration without finding requested pid */ free(processes); - return NoSuchProcess(); + return NoSuchProcess(""); } diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index e9fce85e6..908dbf14a 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -37,12 +37,13 @@ PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size) { /* * Set OSError(errno=ESRCH, strerror="No such process") Python exception. + * If msg != "" the exception message will change in accordance. */ PyObject * -NoSuchProcess(void) { +NoSuchProcess(char *msg) { PyObject *exc; - char *msg = strerror(ESRCH); - exc = PyObject_CallFunction(PyExc_OSError, "(is)", ESRCH, msg); + exc = PyObject_CallFunction( + PyExc_OSError, "(is)", ESRCH, strlen(msg) ? msg : strerror(ESRCH)); PyErr_SetObject(PyExc_OSError, exc); Py_XDECREF(exc); return NULL; @@ -51,12 +52,13 @@ NoSuchProcess(void) { /* * Set OSError(errno=EACCES, strerror="Permission denied") Python exception. + * If msg != "" the exception message will change in accordance. */ PyObject * -AccessDenied(void) { +AccessDenied(char *msg) { PyObject *exc; - char *msg = strerror(EACCES); - exc = PyObject_CallFunction(PyExc_OSError, "(is)", EACCES, msg); + exc = PyObject_CallFunction( + PyExc_OSError, "(is)", EACCES, strlen(msg) ? msg : strerror(EACCES)); PyErr_SetObject(PyExc_OSError, exc); Py_XDECREF(exc); return NULL; diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 965966af7..3db3f5ede 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -17,8 +17,8 @@ PyObject* PyUnicode_DecodeFSDefault(char *s); PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); #endif -PyObject* AccessDenied(void); -PyObject* NoSuchProcess(void); +PyObject* AccessDenied(char *msg); +PyObject* NoSuchProcess(char *msg); PyObject* psutil_set_testing(PyObject *self, PyObject *args); void psutil_debug(const char* format, ...); diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index 6c520e5d8..fef61ca85 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -266,7 +266,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { ret = proc_pidpath((pid_t)pid, &buf, sizeof(buf)); if (ret == 0) { if (pid == 0) - AccessDenied(); + AccessDenied(""); else psutil_raise_for_pid(pid, "proc_pidpath()"); return NULL; @@ -571,9 +571,9 @@ psutil_proc_memory_uss(PyObject *self, PyObject *args) { err = task_for_pid(mach_task_self(), (pid_t)pid, &task); if (err != KERN_SUCCESS) { if (psutil_pid_exists(pid) == 0) - NoSuchProcess(); + NoSuchProcess(""); else - AccessDenied(); + AccessDenied(""); return NULL; } @@ -1018,9 +1018,9 @@ psutil_proc_threads(PyObject *self, PyObject *args) { err = task_for_pid(mach_task_self(), (pid_t)pid, &task); if (err != KERN_SUCCESS) { if (psutil_pid_exists(pid) == 0) - NoSuchProcess(); + NoSuchProcess(""); else - AccessDenied(); + AccessDenied(""); goto error; } @@ -1030,7 +1030,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { if (err != KERN_SUCCESS) { // errcode 4 is "invalid argument" (access denied) if (err == 4) { - AccessDenied(); + AccessDenied(""); } else { // otherwise throw a runtime error with appropriate error code diff --git a/psutil/_psutil_posix.c b/psutil/_psutil_posix.c index 76cf2db03..cc827273c 100644 --- a/psutil/_psutil_posix.c +++ b/psutil/_psutil_posix.c @@ -122,7 +122,7 @@ psutil_raise_for_pid(long pid, char *syscall_name) { else if (psutil_pid_exists(pid) == 0) { psutil_debug("%s syscall failed and PID %i no longer exists; " "assume NoSuchProcess", syscall_name, pid); - NoSuchProcess(); + NoSuchProcess(""); } else { PyErr_Format(PyExc_RuntimeError, "%s syscall failed", syscall_name); diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 15508461c..c6673642c 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -185,7 +185,7 @@ psutil_proc_environ(PyObject *self, PyObject *args) { goto error; if (! info.pr_envp) { - AccessDenied(); + AccessDenied(""); goto error; } diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 1f7573854..c840a2b7c 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -335,7 +335,7 @@ psutil_proc_kill(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; if (pid == 0) - return AccessDenied(); + return AccessDenied(""); hProcess = OpenProcess(PROCESS_TERMINATE, FALSE, pid); if (hProcess == NULL) { @@ -343,7 +343,7 @@ psutil_proc_kill(PyObject *self, PyObject *args) { // see https://github.com/giampaolo/psutil/issues/24 psutil_debug("OpenProcess -> ERROR_INVALID_PARAMETER turned " "into NoSuchProcess"); - NoSuchProcess(); + NoSuchProcess(""); } else { PyErr_SetFromWindowsErr(0); @@ -381,7 +381,7 @@ psutil_proc_wait(PyObject *self, PyObject *args) { if (! PyArg_ParseTuple(args, "ll", &pid, &timeout)) return NULL; if (pid == 0) - return AccessDenied(); + return AccessDenied(""); hProcess = OpenProcess(SYNCHRONIZE | PROCESS_QUERY_INFORMATION, FALSE, pid); @@ -444,7 +444,7 @@ psutil_proc_cpu_times(PyObject *self, PyObject *args) { if (GetLastError() == ERROR_ACCESS_DENIED) { // usually means the process has died so we throw a NoSuchProcess // here - return NoSuchProcess(); + return NoSuchProcess(""); } else { return PyErr_SetFromWindowsErr(0); @@ -498,7 +498,7 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { if (GetLastError() == ERROR_ACCESS_DENIED) { // usually means the process has died so we throw a // NoSuchProcess here - return NoSuchProcess(); + return NoSuchProcess(""); } else { return PyErr_SetFromWindowsErr(0); @@ -517,7 +517,7 @@ psutil_proc_create_time(PyObject *self, PyObject *args) { CloseHandle(hProcess); if (ret != 0) { if (exitCode != STILL_ACTIVE) - return NoSuchProcess(); + return NoSuchProcess(""); } else { // Ignore access denied as it means the process is still alive. @@ -631,7 +631,7 @@ psutil_proc_cmdline(PyObject *self, PyObject *args) { pid_return = psutil_pid_is_running(pid); if (pid_return == 0) - return NoSuchProcess(); + return NoSuchProcess(""); if (pid_return == -1) return NULL; @@ -654,7 +654,7 @@ psutil_proc_environ(PyObject *self, PyObject *args) { pid_return = psutil_pid_is_running(pid); if (pid_return == 0) - return NoSuchProcess(); + return NoSuchProcess(""); if (pid_return == -1) return NULL; @@ -719,7 +719,7 @@ psutil_proc_name(PyObject *self, PyObject *args) { } CloseHandle(hSnapShot); - NoSuchProcess(); + NoSuchProcess(""); return NULL; } @@ -1050,7 +1050,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { pid_return = psutil_pid_is_running(pid); if (pid_return == 0) - return NoSuchProcess(); + return NoSuchProcess(""); if (pid_return == -1) return NULL; @@ -1069,7 +1069,7 @@ psutil_proc_suspend_or_resume(DWORD pid, int suspend) { THREADENTRY32 te32 = {0}; if (pid == 0) { - AccessDenied(); + AccessDenied(""); return FALSE; } @@ -1171,13 +1171,13 @@ psutil_proc_threads(PyObject *self, PyObject *args) { if (pid == 0) { // raise AD instead of returning 0 as procexp is able to // retrieve useful information somehow - AccessDenied(); + AccessDenied(""); goto error; } pid_return = psutil_pid_is_running(pid); if (pid_return == 0) { - NoSuchProcess(); + NoSuchProcess(""); goto error; } if (pid_return == -1) @@ -1568,7 +1568,7 @@ psutil_net_connections(PyObject *self, PyObject *args) { pid_return = psutil_pid_is_running(pid); if (pid_return == 0) { _psutil_conn_decref_objs(); - return NoSuchProcess(); + return NoSuchProcess(""); } else if (pid_return == -1) { _psutil_conn_decref_objs(); diff --git a/psutil/arch/freebsd/specific.c b/psutil/arch/freebsd/specific.c index ff128e65f..2c8944ddd 100644 --- a/psutil/arch/freebsd/specific.c +++ b/psutil/arch/freebsd/specific.c @@ -59,7 +59,7 @@ psutil_kinfo_proc(const pid_t pid, struct kinfo_proc *proc) { // sysctl stores 0 in the size if we can't find the process information. if (size == 0) { - NoSuchProcess(); + NoSuchProcess(""); return -1; } return 0; @@ -297,7 +297,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (ret == -1) return NULL; else if (ret == 0) - return NoSuchProcess(); + return NoSuchProcess(""); else strcpy(pathname, ""); } @@ -354,7 +354,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { goto error; } if (size == 0) { - NoSuchProcess(); + NoSuchProcess(""); goto error; } @@ -370,7 +370,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { goto error; } if (size == 0) { - NoSuchProcess(); + NoSuchProcess(""); goto error; } diff --git a/psutil/arch/netbsd/specific.c b/psutil/arch/netbsd/specific.c index 0a32139d3..cab60d608 100644 --- a/psutil/arch/netbsd/specific.c +++ b/psutil/arch/netbsd/specific.c @@ -72,7 +72,7 @@ psutil_kinfo_proc(pid_t pid, kinfo_proc *proc) { } // sysctl stores 0 in the size if we can't find the process information. if (size == 0) { - NoSuchProcess(); + NoSuchProcess(""); return -1; } return 0; @@ -157,7 +157,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (ret == -1) return NULL; else if (ret == 0) - return NoSuchProcess(); + return NoSuchProcess(""); else strcpy(pathname, ""); } @@ -209,7 +209,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { goto error; } if (size == 0) { - NoSuchProcess(); + NoSuchProcess(""); goto error; } @@ -226,7 +226,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { goto error; } if (size == 0) { - NoSuchProcess(); + NoSuchProcess(""); goto error; } diff --git a/psutil/arch/openbsd/specific.c b/psutil/arch/openbsd/specific.c index 2a0d30cea..33ebdeecb 100644 --- a/psutil/arch/openbsd/specific.c +++ b/psutil/arch/openbsd/specific.c @@ -67,7 +67,7 @@ psutil_kinfo_proc(pid_t pid, struct kinfo_proc *proc) { } // sysctl stores 0 in the size if we can't find the process information. if (size == 0) { - NoSuchProcess(); + NoSuchProcess(""); return -1; } return 0; @@ -242,7 +242,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { kd = kvm_openfiles(0, 0, 0, O_RDONLY, errbuf); if (! kd) { if (strstr(errbuf, "Permission denied") != NULL) - AccessDenied(); + AccessDenied(""); else PyErr_Format(PyExc_RuntimeError, "kvm_openfiles() syscall failed"); goto error; @@ -253,7 +253,7 @@ psutil_proc_threads(PyObject *self, PyObject *args) { sizeof(*kp), &nentries); if (! kp) { if (strstr(errbuf, "Permission denied") != NULL) - AccessDenied(); + AccessDenied(""); else PyErr_Format(PyExc_RuntimeError, "kvm_getprocs() syscall failed"); goto error; diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/osx/process_info.c index f0a011320..40c79a2cd 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -144,7 +144,7 @@ psutil_get_cmdline(long pid) { // In case of zombie process we'll get EINVAL. We translate it // to NSP and _psosx.py will translate it to ZP. if ((errno == EINVAL) && (psutil_pid_exists(pid))) - NoSuchProcess(); + NoSuchProcess(""); else PyErr_SetFromErrno(PyExc_OSError); goto error; @@ -238,7 +238,7 @@ psutil_get_environ(long pid) { // In case of zombie process we'll get EINVAL. We translate it // to NSP and _psosx.py will translate it to ZP. if ((errno == EINVAL) && (psutil_pid_exists(pid))) - NoSuchProcess(); + NoSuchProcess(""); else PyErr_SetFromErrno(PyExc_OSError); goto error; @@ -338,7 +338,7 @@ psutil_get_kinfo_proc(long pid, struct kinfo_proc *kp) { // sysctl succeeds but len is zero, happens when process has gone away if (len == 0) { - NoSuchProcess(); + NoSuchProcess(""); return -1; } return 0; diff --git a/psutil/arch/windows/process_info.c b/psutil/arch/windows/process_info.c index 9a54d8544..ffd3c80ef 100644 --- a/psutil/arch/windows/process_info.c +++ b/psutil/arch/windows/process_info.c @@ -256,7 +256,7 @@ psutil_check_phandle(HANDLE hProcess, DWORD pid) { if (ret == 1) return hProcess; else if (ret == 0) - return NoSuchProcess(); + return NoSuchProcess(""); else if (ret == -1) return PyErr_SetFromWindowsErr(0); else // -2 @@ -277,7 +277,7 @@ psutil_handle_from_pid_waccess(DWORD pid, DWORD dwDesiredAccess) { if (pid == 0) { // otherwise we'd get NoSuchProcess - return AccessDenied(); + return AccessDenied(""); } hProcess = OpenProcess(dwDesiredAccess, FALSE, pid); @@ -959,7 +959,7 @@ psutil_get_proc_info(DWORD pid, PSYSTEM_PROCESS_INFORMATION *retProcess, } } while ( (process = PSUTIL_NEXT_PROCESS(process)) ); - NoSuchProcess(); + NoSuchProcess(""); goto error; error: From 04e867218f565bbf3fd37eb14dd68699092e99b0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Nov 2017 14:08:20 +0100 Subject: [PATCH 895/922] fix posix failure --- psutil/tests/test_posix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index 0c3f64348..b4b23084f 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -155,6 +155,7 @@ def test_name(self): name_psutil = psutil.Process(self.pid).name().lower() # ...because of how we calculate PYTHON_EXE; on OSX this may # be "pythonX.Y". + name_ps = re.sub(r"\d.\d", "", name_ps) name_psutil = re.sub(r"\d.\d", "", name_psutil) self.assertEqual(name_ps, name_psutil) From d43ee3003350df56c6d1a96dac9784f064daad46 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 24 Nov 2017 15:05:33 +0100 Subject: [PATCH 896/922] fix #1181: raise AD if task_for_pid() fails with 5 and errno == ENOENT --- HISTORY.rst | 1 + psutil/_psutil_osx.c | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index fb3311bb3..430fb5e4f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,7 @@ - 1169_: [Linux] users() "hostname" returns username instead. (patch by janderbrain) - 1172_: [Windows] `make test` does not work. +- 1181_: [OSX] Process.memory_maps() may raise ENOENT. 5.4.1 ===== diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index fef61ca85..55dd64ca5 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -338,8 +338,16 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { err = task_for_pid(mach_task_self(), (pid_t)pid, &task); if (err != KERN_SUCCESS) { - psutil_debug("task_for_pid() failed"); // TODO temporary - psutil_raise_for_pid(pid, "task_for_pid()"); + if ((err == 5) && (errno == ENOENT)) { + // See: https://github.com/giampaolo/psutil/issues/1181 + psutil_debug("task_for_pid(MACH_PORT_NULL) failed; err=%i, " + "errno=%i, msg='%s'\n", err, errno, + mach_error_string(err)); + AccessDenied(""); + } + else { + psutil_raise_for_pid(pid, "task_for_pid(MACH_PORT_NULL)"); + } goto error; } From bb27cbf65c5abf91ad0b6c2725c0527b8132f75d Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sun, 26 Nov 2017 09:06:14 +0100 Subject: [PATCH 897/922] set x bit to test_aix.py --- psutil/tests/test_aix.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 psutil/tests/test_aix.py diff --git a/psutil/tests/test_aix.py b/psutil/tests/test_aix.py old mode 100644 new mode 100755 From 7c6b6c2a6d89a1a270d6357be3b77fd8e0e2cbe7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 28 Nov 2017 11:10:42 +0100 Subject: [PATCH 898/922] fix #1179 / linux / cmdline: handle processes erroneously overwriting /proc/pid/cmdline by using spaces instead of null bytes as args separator --- HISTORY.rst | 3 +++ psutil/_pslinux.py | 12 ++++++++++-- psutil/tests/test_linux.py | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 430fb5e4f..bfaa2568c 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,6 +17,9 @@ - 1169_: [Linux] users() "hostname" returns username instead. (patch by janderbrain) - 1172_: [Windows] `make test` does not work. +- 1179_: [Linux] Process.cmdline() correctly splits cmdline args for + misbehaving processes who overwrite /proc/pid/cmdline by using spaces + instead of null bytes as args separator. - 1181_: [OSX] Process.memory_maps() may raise ENOENT. 5.4.1 diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 3fe62c5c4..a5a3fd891 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1471,9 +1471,17 @@ def cmdline(self): if not data: # may happen in case of zombie process return [] - if data.endswith('\x00'): + # 'man proc' states that args are separated by null bytes '\0' + # and last char is supposed to be a null byte. Nevertheless + # some processes may change their cmdline after being started + # (via setproctitle() or similar), they are usually not + # compliant with this rule and use spaces instead. Google + # Chrome process is an example. See: + # https://github.com/giampaolo/psutil/issues/1179 + sep = '\x00' if data.endswith('\x00') else ' ' + if data.endswith(sep): data = data[:-1] - return [x for x in data.split('\x00')] + return [x for x in data.split(sep)] @wrap_exceptions def environ(self): diff --git a/psutil/tests/test_linux.py b/psutil/tests/test_linux.py index 71d428c31..6ba17b254 100755 --- a/psutil/tests/test_linux.py +++ b/psutil/tests/test_linux.py @@ -1585,6 +1585,20 @@ def test_cmdline_mocked(self): self.assertEqual(p.cmdline(), ['foo', 'bar', '']) assert m.called + def test_cmdline_spaces_mocked(self): + # see: https://github.com/giampaolo/psutil/issues/1179 + p = psutil.Process() + fake_file = io.StringIO(u('foo bar ')) + with mock.patch('psutil._pslinux.open', + return_value=fake_file, create=True) as m: + self.assertEqual(p.cmdline(), ['foo', 'bar']) + assert m.called + fake_file = io.StringIO(u('foo bar ')) + with mock.patch('psutil._pslinux.open', + return_value=fake_file, create=True) as m: + self.assertEqual(p.cmdline(), ['foo', 'bar', '']) + assert m.called + def test_readlink_path_deleted_mocked(self): with mock.patch('psutil._pslinux.os.readlink', return_value='/home/foo (deleted)'): From a840dba69cbb68d814c413664a7327e82b2bedfb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 30 Nov 2017 15:09:30 +0100 Subject: [PATCH 899/922] update HISTORY --- HISTORY.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bfaa2568c..f8a7d3015 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,8 +17,8 @@ - 1169_: [Linux] users() "hostname" returns username instead. (patch by janderbrain) - 1172_: [Windows] `make test` does not work. -- 1179_: [Linux] Process.cmdline() correctly splits cmdline args for - misbehaving processes who overwrite /proc/pid/cmdline by using spaces +- 1179_: [Linux] Process.cmdline() is now able to splits cmdline args for + misbehaving processes which overwrite /proc/pid/cmdline and use spaces instead of null bytes as args separator. - 1181_: [OSX] Process.memory_maps() may raise ENOENT. From 8f8b4d79b1b2f0d6a94107f0038796efcc098278 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 30 Nov 2017 15:17:35 +0100 Subject: [PATCH 900/922] update doc --- docs/index.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index d34a1c457..0119b4233 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1716,9 +1716,8 @@ Process class .. method:: children(recursive=False) - Return the children of this process as a list of :Class:`Process` objects, - preemptively checking whether PID has been reused. If recursive is `True` - return all the parent descendants. + Return the children of this process as a list of :Class:`Process` objects. + If recursive is `True` return all the parent descendants. Pseudo code example assuming *A == this process*: :: @@ -1738,7 +1737,7 @@ Process class Note that in the example above if process X disappears process Y won't be returned either as the reference to process A is lost. This concept is well summaried by this - `unit test `__. + `unit test `__. See also how to `kill a process tree <#kill-process-tree>`__ and `terminate my children <#terminate-my-children>`__. From f0094db79ad9e4f2246997cb8c2046b71c465a29 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 1 Dec 2017 14:49:53 +0100 Subject: [PATCH 901/922] Speedup Process.children() (#1185) * update HISTORY * update doc * #1183: speedup Process.children() by 2.2x * fix windows err * #1083 / #1084: implement linux-specific ppid_map() function speending things up from 2x to 2.4x * add ESRCH to err handling * update doc --- HISTORY.rst | 5 +-- docs/index.rst | 8 ++--- psutil/__init__.py | 82 +++++++++++++++++++++++----------------------- psutil/_pslinux.py | 24 ++++++++++++++ 4 files changed, 72 insertions(+), 47 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bfaa2568c..8bc3f7847 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ - 1173_: introduced PSUTIL_DEBUG environment variable which can be set in order to print useful debug messages on stderr (useful in case of nasty errors). - 1177_: added support for sensors_battery() on OSX. (patch by Arnon Yaari) +- 1183_: Process.children() is 2x faster on UNIX and 2.4x faster on Linux. **Bug fixes** @@ -17,8 +18,8 @@ - 1169_: [Linux] users() "hostname" returns username instead. (patch by janderbrain) - 1172_: [Windows] `make test` does not work. -- 1179_: [Linux] Process.cmdline() correctly splits cmdline args for - misbehaving processes who overwrite /proc/pid/cmdline by using spaces +- 1179_: [Linux] Process.cmdline() is now able to splits cmdline args for + misbehaving processes which overwrite /proc/pid/cmdline and use spaces instead of null bytes as args separator. - 1181_: [OSX] Process.memory_maps() may raise ENOENT. diff --git a/docs/index.rst b/docs/index.rst index d34a1c457..ea2384bb3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1716,9 +1716,9 @@ Process class .. method:: children(recursive=False) - Return the children of this process as a list of :Class:`Process` objects, - preemptively checking whether PID has been reused. If recursive is `True` - return all the parent descendants. + Return the children of this process as a list of :class:`Process` + instances. + If recursive is `True` return all the parent descendants. Pseudo code example assuming *A == this process*: :: @@ -1738,7 +1738,7 @@ Process class Note that in the example above if process X disappears process Y won't be returned either as the reference to process A is lost. This concept is well summaried by this - `unit test `__. + `unit test `__. See also how to `kill a process tree <#kill-process-tree>`__ and `terminate my children <#terminate-my-children>`__. diff --git a/psutil/__init__.py b/psutil/__init__.py index c8da0a3dc..a84479738 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -250,10 +250,31 @@ # ===================================================================== -# --- Process class +# --- Utils # ===================================================================== +if hasattr(_psplatform, 'ppid_map'): + # Faster version (Windows and Linux). + _ppid_map = _psplatform.ppid_map +else: + def _ppid_map(): + """Return a {pid: ppid, ...} dict for all running processes in + one shot. Used to speed up Process.children(). + """ + ret = {} + for pid in pids(): + try: + proc = _psplatform.Process(pid) + ppid = proc.ppid() + except (NoSuchProcess, AccessDenied): + # Note: AccessDenied is unlikely to happen. + pass + else: + ret[pid] = ppid + return ret + + def _assert_pid_not_reused(fun): """Decorator which raises NoSuchProcess in case a process is no longer running or its PID has been reused. @@ -266,6 +287,11 @@ def wrapper(self, *args, **kwargs): return wrapper +# ===================================================================== +# --- Process class +# ===================================================================== + + class Process(object): """Represents an OS process with the given PID. If PID is omitted current process PID (os.getpid()) is used. @@ -848,55 +874,29 @@ def children(self, recursive=False): process Y won't be listed as the reference to process A is lost. """ - if hasattr(_psplatform, 'ppid_map'): - # Windows only: obtain a {pid:ppid, ...} dict for all running - # processes in one shot (faster). - ppid_map = _psplatform.ppid_map() - else: - ppid_map = None - + ppid_map = _ppid_map() ret = [] if not recursive: - if ppid_map is None: - # 'slow' version, common to all platforms except Windows - for p in process_iter(): + for pid, ppid in ppid_map.items(): + if ppid == self.pid: try: - if p.ppid() == self.pid: - # if child happens to be older than its parent - # (self) it means child's PID has been reused - if self.create_time() <= p.create_time(): - ret.append(p) + child = Process(pid) + # if child happens to be older than its parent + # (self) it means child's PID has been reused + if self.create_time() <= child.create_time(): + ret.append(child) except (NoSuchProcess, ZombieProcess): pass - else: # pragma: no cover - # Windows only (faster) - for pid, ppid in ppid_map.items(): - if ppid == self.pid: - try: - child = Process(pid) - # if child happens to be older than its parent - # (self) it means child's PID has been reused - if self.create_time() <= child.create_time(): - ret.append(child) - except (NoSuchProcess, ZombieProcess): - pass else: # construct a dict where 'values' are all the processes # having 'key' as their parent table = collections.defaultdict(list) - if ppid_map is None: - for p in process_iter(): - try: - table[p.ppid()].append(p) - except (NoSuchProcess, ZombieProcess): - pass - else: # pragma: no cover - for pid, ppid in ppid_map.items(): - try: - p = Process(pid) - table[ppid].append(p) - except (NoSuchProcess, ZombieProcess): - pass + for pid, ppid in ppid_map.items(): + try: + p = Process(pid) + table[ppid].append(p) + except (NoSuchProcess, ZombieProcess): + pass # At this point we have a mapping table where table[self.pid] # are the current process' children. # Below, we look for all descendants recursively, similarly diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index a5a3fd891..b57adb34e 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1356,6 +1356,30 @@ def pid_exists(pid): return pid in pids() +def ppid_map(): + """Obtain a {pid: ppid, ...} dict for all running processes in + one shot. Used to speed up Process.children(). + """ + ret = {} + procfs_path = get_procfs_path() + for pid in pids(): + try: + with open_binary("%s/%s/stat" % (procfs_path, pid)) as f: + data = f.read() + except EnvironmentError as err: + # Note: we should be able to access /stat for all processes + # so we won't bump into EPERM, which is good. + if err.errno not in (errno.ENOENT, errno.ESRCH, + errno.EPERM, errno.EACCES): + raise + else: + rpar = data.rfind(b')') + dset = data[rpar + 2:].split() + ppid = int(dset[1]) + ret[pid] = ppid + return ret + + def wrap_exceptions(fun): """Decorator which translates bare OSError and IOError exceptions into NoSuchProcess and AccessDenied. From f1d94cc22afdba6cb61c5a0ab246e877a83080ab Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 2 Dec 2017 10:07:06 +0100 Subject: [PATCH 902/922] Faster Process.children(recursive=True) (#1186) Before: ``` $ python -m timeit -s "import psutil; p = psutil.Process()" "list(p.children(recursive=True))" 10 loops, best of 3: 29.6 msec per loop ``` After: ``` $ python -m timeit -s "import psutil; p = psutil.Process()" "list(p.children(recursive=True))" 100 loops, best of 3: 12.4 msec per loop ``` For reference, non-recursive: ``` $ python -m timeit -s "import psutil; p = psutil.Process()" "list(p.children())" 100 loops, best of 3: 12.2 msec per loop ``` --- psutil/__init__.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index a84479738..9a422cda6 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -888,33 +888,33 @@ def children(self, recursive=False): except (NoSuchProcess, ZombieProcess): pass else: - # construct a dict where 'values' are all the processes - # having 'key' as their parent - table = collections.defaultdict(list) + # Construct a {pid: [child pids]} dict + reverse_ppid_map = collections.defaultdict(list) for pid, ppid in ppid_map.items(): - try: - p = Process(pid) - table[ppid].append(p) - except (NoSuchProcess, ZombieProcess): - pass - # At this point we have a mapping table where table[self.pid] - # are the current process' children. - # Below, we look for all descendants recursively, similarly - # to a recursive function call. - checkpids = [self.pid] - for pid in checkpids: - for child in table[pid]: + reverse_ppid_map[ppid].append(pid) + # Recursively traverse that dict, starting from self.pid, + # such that we only call Process() on actual children + seen = set() + stack = [self.pid] + while stack: + pid = stack.pop() + if pid in seen: + # Since pids can be reused while the ppid_map is + # constructed, there may be rare instances where + # there's a cycle in the recorded process "tree". + continue + seen.add(pid) + for child_pid in reverse_ppid_map[pid]: try: + child = Process(child_pid) # if child happens to be older than its parent # (self) it means child's PID has been reused intime = self.create_time() <= child.create_time() - except (NoSuchProcess, ZombieProcess): - pass - else: if intime: ret.append(child) - if child.pid not in checkpids: - checkpids.append(child.pid) + stack.append(child_pid) + except (NoSuchProcess, ZombieProcess): + pass return ret def cpu_percent(self, interval=None): From fce2564643ded6d837a98d00ffaf285e4e484806 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 2 Dec 2017 11:22:52 +0100 Subject: [PATCH 903/922] refactor Process.__repr__ --- psutil/__init__.py | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 9a422cda6..7a170937c 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -24,6 +24,7 @@ import collections import contextlib +import datetime import errno import functools import os @@ -287,6 +288,17 @@ def wrapper(self, *args, **kwargs): return wrapper +def _pprint_secs(secs): + """Format seconds in a human readable form.""" + now = time.time() + secs_ago = int(now - secs) + if secs_ago < 60 * 60 * 24: + fmt = "%H:%M:%S" + else: + fmt = "%Y-%m-%d %H:%M:%S" + return datetime.datetime.fromtimestamp(secs).strftime(fmt) + + # ===================================================================== # --- Process class # ===================================================================== @@ -377,21 +389,26 @@ def _init(self, pid, _ignore_nsp=False): def __str__(self): try: - pid = self.pid - name = repr(self.name()) + info = collections.OrderedDict() + except AttributeError: + info = {} # Python 2.6 + info["pid"] = self.pid + try: + info["name"] = self.name() + if self._create_time: + info['started'] = _pprint_secs(self._create_time) except ZombieProcess: - details = "(pid=%s (zombie))" % self.pid + info["status"] = "zombie" except NoSuchProcess: - details = "(pid=%s (terminated))" % self.pid + info["status"] = "terminated" except AccessDenied: - details = "(pid=%s)" % (self.pid) - else: - details = "(pid=%s, name=%s)" % (pid, name) - return "%s.%s%s" % (self.__class__.__module__, - self.__class__.__name__, details) + pass + return "%s.%s(%s)" % ( + self.__class__.__module__, + self.__class__.__name__, + ", ".join(["%s=%r" % (k, v) for k, v in info.items()])) - def __repr__(self): - return "<%s at %s>" % (self.__str__(), id(self)) + __repr__ = __str__ def __eq__(self, other): # Test for equality with another Process object based @@ -2280,8 +2297,6 @@ def test(): # pragma: no cover """List info of all currently running processes emulating ps aux output. """ - import datetime - today_day = datetime.date.today() templ = "%-10s %5s %4s %7s %7s %-13s %5s %7s %s" attrs = ['pid', 'memory_percent', 'name', 'cpu_times', 'create_time', From cdb2e2be8fa0cfce867800419f1746ea87dd5cac Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 2 Dec 2017 22:08:00 +0100 Subject: [PATCH 904/922] change assert in test --- psutil/tests/test_posix.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/psutil/tests/test_posix.py b/psutil/tests/test_posix.py index b4b23084f..c59f9a1c7 100755 --- a/psutil/tests/test_posix.py +++ b/psutil/tests/test_posix.py @@ -313,11 +313,7 @@ def test_pids(self): # on OSX and OPENBSD ps doesn't show pid 0 if OSX or OPENBSD and 0 not in pids_ps: pids_ps.insert(0, 0) - - if pids_ps != pids_psutil: - difference = [x for x in pids_psutil if x not in pids_ps] + \ - [x for x in pids_ps if x not in pids_psutil] - self.fail("difference: " + str(difference)) + self.assertEqual(pids_ps, pids_psutil) # for some reason ifconfig -a does not report all interfaces # returned by psutil From 98f0e70fdc7f44cc982f7496557dcdc31a9f99e6 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 4 Dec 2017 13:50:38 +0100 Subject: [PATCH 905/922] Fix OSX pid 0 bug (#1187) * debug * change travis conf * more debug info * work around pid 0 on osx * fix pid 0 bug * skip failing test on OSX + TRAVIS which started failing all of the sudden * skip failing test on osx + travis --- HISTORY.rst | 1 + psutil/_psosx.py | 18 +++++++++++++++++- psutil/tests/test_process.py | 12 ++++++++---- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 8bc3f7847..7b8f7f41f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -22,6 +22,7 @@ misbehaving processes which overwrite /proc/pid/cmdline and use spaces instead of null bytes as args separator. - 1181_: [OSX] Process.memory_maps() may raise ENOENT. +- 1187_: [OSX] pids() does not return PID 0 on recent OSX versions. 5.4.1 ===== diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 9fa7716df..4c97af71e 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -301,7 +301,23 @@ def users(): # ===================================================================== -pids = cext.pids +def pids(): + ls = cext.pids() + if 0 not in ls: + # On certain OSX versions pids() C doesn't return PID 0 but + # "ps" does and the process is querable via sysctl(): + # https://travis-ci.org/giampaolo/psutil/jobs/309619941 + try: + Process(0).create_time() + except NoSuchProcess: + return False + except AccessDenied: + ls.append(0) + else: + ls.append(0) + return ls + + pid_exists = _psposix.pid_exists diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 1e01aea55..3987943a5 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1314,10 +1314,14 @@ def succeed_or_zombie_p_exc(fun, *args, **kwargs): # self.assertEqual(zpid.ppid(), os.getpid()) # ...and all other APIs should be able to deal with it self.assertTrue(psutil.pid_exists(zpid)) - self.assertIn(zpid, psutil.pids()) - self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) - psutil._pmap = {} - self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) + if not TRAVIS and OSX: + # For some reason this started failing all of the sudden. + # Maybe they upgraded OSX version? + # https://travis-ci.org/giampaolo/psutil/jobs/310896404 + self.assertIn(zpid, psutil.pids()) + self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) + psutil._pmap = {} + self.assertIn(zpid, [x.pid for x in psutil.process_iter()]) @unittest.skipIf(not POSIX, 'POSIX only') def test_zombie_process_is_running_w_exc(self): From c0b1ee00b770bff34c49b1ad9242f86e24cc5a34 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 4 Dec 2017 23:14:58 +0100 Subject: [PATCH 906/922] refactor environ() test --- psutil/tests/test_process.py | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 3987943a5..3a4e8b8d1 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -62,7 +62,6 @@ from psutil.tests import TESTFILE_PREFIX from psutil.tests import TESTFN from psutil.tests import ThreadTask -from psutil.tests import TOX from psutil.tests import TRAVIS from psutil.tests import unittest from psutil.tests import wait_for_pid @@ -1384,28 +1383,23 @@ def test_pid_0(self): @unittest.skipIf(not HAS_ENVIRON, "not supported") def test_environ(self): + def clean_dict(d): + # Most of these are problematic on Travis. + d.pop("PSUTIL_TESTING", None) + d.pop("PLAT", None) + d.pop("HOME", None) + if OSX: + d.pop("__CF_USER_TEXT_ENCODING") + d.pop("VERSIONER_PYTHON_PREFER_32_BIT") + d.pop("VERSIONER_PYTHON_VERSION") + return dict( + [(k.rstrip("\r\n"), v.rstrip("\r\n")) for k, v in d.items()]) + self.maxDiff = None p = psutil.Process() - d = p.environ() - d2 = os.environ.copy() - - removes = [] - if 'PSUTIL_TESTING' in os.environ: - removes.append('PSUTIL_TESTING') - if OSX: - removes.extend([ - "__CF_USER_TEXT_ENCODING", - "VERSIONER_PYTHON_PREFER_32_BIT", - "VERSIONER_PYTHON_VERSION"]) - if LINUX or OSX: - removes.extend(['PLAT']) - if TOX: - removes.extend(['HOME']) - for key in removes: - d.pop(key, None) - d2.pop(key, None) - - self.assertEqual(d, d2) + d1 = clean_dict(p.environ()) + d2 = clean_dict(os.environ.copy()) + self.assertEqual(d1, d2) @unittest.skipIf(not HAS_ENVIRON, "not supported") @unittest.skipIf(not POSIX, "POSIX only") From 82c77dc888e8a44a4f86aecc387c30b42150eb7a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 5 Dec 2017 09:58:07 +0100 Subject: [PATCH 907/922] fix test --- psutil/tests/test_process.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index 3a4e8b8d1..d5c2cd85c 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -1389,9 +1389,9 @@ def clean_dict(d): d.pop("PLAT", None) d.pop("HOME", None) if OSX: - d.pop("__CF_USER_TEXT_ENCODING") - d.pop("VERSIONER_PYTHON_PREFER_32_BIT") - d.pop("VERSIONER_PYTHON_VERSION") + d.pop("__CF_USER_TEXT_ENCODING", None) + d.pop("VERSIONER_PYTHON_PREFER_32_BIT", None) + d.pop("VERSIONER_PYTHON_VERSION", None) return dict( [(k.rstrip("\r\n"), v.rstrip("\r\n")) for k, v in d.items()]) From c3767da76a366cabbe1c84ad9cef007ae51c400e Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 7 Dec 2017 11:02:35 +0100 Subject: [PATCH 908/922] Use FutureWarning instead of DeprecationWarning (#1188) * make Process.memory_info_ex() raise FutureWarning instead of DeprecationWarning because the latter is suppressed by default * update HISTORY * add test case --- HISTORY.rst | 2 ++ psutil/_common.py | 4 ++-- psutil/tests/test_contracts.py | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 7b8f7f41f..a8bc31025 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -11,6 +11,8 @@ to print useful debug messages on stderr (useful in case of nasty errors). - 1177_: added support for sensors_battery() on OSX. (patch by Arnon Yaari) - 1183_: Process.children() is 2x faster on UNIX and 2.4x faster on Linux. +- 1188_: deprecated method Process.memory_info_ex() now warns by using + FutureWarning instead of DeprecationWarning. **Bug fixes** diff --git a/psutil/_common.py b/psutil/_common.py index 2d562f93e..870971e41 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -461,14 +461,14 @@ def deprecated_method(replacement): 'replcement' is the method name which will be called instead. """ def outer(fun): - msg = "%s() is deprecated; use %s() instead" % ( + msg = "%s() is deprecated and will be removed; use %s() instead" % ( fun.__name__, replacement) if fun.__doc__ is None: fun.__doc__ = msg @functools.wraps(fun) def inner(self, *args, **kwargs): - warnings.warn(msg, category=DeprecationWarning, stacklevel=2) + warnings.warn(msg, category=FutureWarning, stacklevel=2) return getattr(self, replacement)(*args, **kwargs) return inner return outer diff --git a/psutil/tests/test_contracts.py b/psutil/tests/test_contracts.py index 4c57b25a8..855b53bf9 100755 --- a/psutil/tests/test_contracts.py +++ b/psutil/tests/test_contracts.py @@ -14,6 +14,7 @@ import stat import time import traceback +import warnings from contextlib import closing from psutil import AIX @@ -167,6 +168,22 @@ def test_proc_memory_maps(self): self.assertEqual(hasit, False if OPENBSD or NETBSD or AIX else True) +# =================================================================== +# --- Test deprecations +# =================================================================== + + +class TestDeprecations(unittest.TestCase): + + def test_memory_info_ex(self): + with warnings.catch_warnings(record=True) as ws: + psutil.Process().memory_info_ex() + w = ws[0] + self.assertIsInstance(w.category(), FutureWarning) + self.assertIn("memory_info_ex() is deprecated", str(w.message)) + self.assertIn("use memory_info() instead", str(w.message)) + + # =================================================================== # --- System API types # =================================================================== From 8ed14183cc72791b42249de4bd0d72da90cff41f Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 7 Dec 2017 12:59:45 +0100 Subject: [PATCH 909/922] pre-release; also get rid of PSUTIL_DEBUG doc instructions (it's kinda useless for the user after all) --- HISTORY.rst | 2 +- Makefile | 16 ++++++++-------- docs/index.rst | 27 ++++----------------------- 3 files changed, 13 insertions(+), 32 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index a8bc31025..b28e52bb6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.4.2 ===== -*XXXX-XX-XX* +*2017-12-07* **Enhancements** diff --git a/Makefile b/Makefile index 7b2eca867..5e784beef 100644 --- a/Makefile +++ b/Makefile @@ -193,30 +193,30 @@ install-git-hooks: ## Install GIT pre-commit hook. # --- create -dist-source: ## Create tar.gz source distribution. +source: ## Create tar.gz source distribution. ${MAKE} generate-manifest $(PYTHON) setup.py sdist -dist-wheel: ## Generate wheel. +wheel: ## Generate wheel. $(PYTHON) setup.py bdist_wheel -dist-win-download-wheels: ## Download wheels hosted on appveyor. +win-download-wheels: ## Download wheels hosted on appveyor. $(TEST_PREFIX) $(PYTHON) scripts/internal/download_exes.py --user giampaolo --project psutil # --- upload -dist-upload-src: ## Upload source tarball on https://pypi.python.org/pypi/psutil. - ${MAKE} dist-source +upload-src: ## Upload source tarball on https://pypi.python.org/pypi/psutil. + ${MAKE} source $(PYTHON) setup.py sdist upload -dist-upload-win-wheels: ## Upload wheels in dist/* directory on PYPI. +upload-win-wheels: ## Upload wheels in dist/* directory on PYPI. $(PYTHON) -m twine upload dist/*.whl # --- others pre-release: ## Check if we're ready to produce a new release. ${MAKE} install - ${MAKE} dist-source + ${MAKE} source $(PYTHON) -c \ "from psutil import __version__ as ver; \ doc = open('docs/index.rst').read(); \ @@ -227,7 +227,7 @@ pre-release: ## Check if we're ready to produce a new release. ${MAKE} generate-manifest git diff MANIFEST.in > /dev/null # ...otherwise 'git diff-index HEAD' will complain $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else sys.exit(0);" - ${MAKE} dist-win-download-wheels + ${MAKE} win-download-wheels ${MAKE} sdist release: ## Create a release (down/uploads tar.gz, wheels, git tag release). diff --git a/docs/index.rst b/docs/index.rst index ea2384bb3..c09045f77 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2302,29 +2302,6 @@ Constants ---- -Debug mode -========== - -In case you bump into nasty errors which look like being psutil's fault you may -want to run psutil in debug mode. psutil may (or may not) print some useful -message on stderr before crashing with an exception -(see `original motivation `__). -To enable debug mode on UNIX: - -.. code-block:: bash - - PSUTIL_DEBUG=1 python script.py - -On Windows: - -.. code-block:: bat - - set PSUTIL_DEBUG=1 && C:\python36\python.exe script.py - -.. versionadded:: 5.4.2 - ----- - Unicode ======= @@ -2612,6 +2589,10 @@ take a look at the Timeline ======== +- 2017-12-07: + `5.4.1 `__ - + `what's new `__ - + `diff `__ - 2017-11-08: `5.4.1 `__ - `what's new `__ - From 9db307472b1fce4847b01ecf64e3978c85cb3a86 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 7 Dec 2017 13:02:30 +0100 Subject: [PATCH 910/922] pre release --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5e784beef..080e334cf 100644 --- a/Makefile +++ b/Makefile @@ -193,7 +193,7 @@ install-git-hooks: ## Install GIT pre-commit hook. # --- create -source: ## Create tar.gz source distribution. +sdist: ## Create tar.gz source distribution. ${MAKE} generate-manifest $(PYTHON) setup.py sdist @@ -206,7 +206,7 @@ win-download-wheels: ## Download wheels hosted on appveyor. # --- upload upload-src: ## Upload source tarball on https://pypi.python.org/pypi/psutil. - ${MAKE} source + ${MAKE} sdist $(PYTHON) setup.py sdist upload upload-win-wheels: ## Upload wheels in dist/* directory on PYPI. @@ -215,8 +215,8 @@ upload-win-wheels: ## Upload wheels in dist/* directory on PYPI. # --- others pre-release: ## Check if we're ready to produce a new release. + rm -rf dist ${MAKE} install - ${MAKE} source $(PYTHON) -c \ "from psutil import __version__ as ver; \ doc = open('docs/index.rst').read(); \ From a406eb0616fab64c1abd0582b800f4d4da6ddc08 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 7 Dec 2017 13:06:48 +0100 Subject: [PATCH 911/922] update doc --- docs/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index c09045f77..74e7d2323 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2590,7 +2590,7 @@ Timeline ======== - 2017-12-07: - `5.4.1 `__ - + `5.4.2 `__ - `what's new `__ - `diff `__ - 2017-11-08: From 090ae20078e7135fa0fe95db2b83366079da9802 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 7 Dec 2017 13:13:36 +0100 Subject: [PATCH 912/922] what a stupid bug! (#1190) --- HISTORY.rst | 9 +++++++++ psutil/__init__.py | 2 +- psutil/_psosx.py | 5 ++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b28e52bb6..eabe7cc6f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -1,5 +1,14 @@ *Bug tracker at https://github.com/giampaolo/psutil/issues* +5.4.3 +===== + +*XXXX-XX-XX* + +**Bug fixes** + +- 1193_: pids() may return False on OSX. + 5.4.2 ===== diff --git a/psutil/__init__.py b/psutil/__init__.py index 7a170937c..5e29a7fc5 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -219,7 +219,7 @@ ] __all__.extend(_psplatform.__extra__all__) __author__ = "Giampaolo Rodola'" -__version__ = "5.4.2" +__version__ = "5.4.3" version_info = tuple([int(num) for num in __version__.split('.')]) AF_LINK = _psplatform.AF_LINK POWER_TIME_UNLIMITED = _common.POWER_TIME_UNLIMITED diff --git a/psutil/_psosx.py b/psutil/_psosx.py index 4c97af71e..308756a81 100644 --- a/psutil/_psosx.py +++ b/psutil/_psosx.py @@ -309,12 +309,11 @@ def pids(): # https://travis-ci.org/giampaolo/psutil/jobs/309619941 try: Process(0).create_time() + ls.append(0) except NoSuchProcess: - return False + pass except AccessDenied: ls.append(0) - else: - ls.append(0) return ls From e46803bbd1c4a0bcaf099a0b66c1fcc28b6ec031 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Sat, 9 Dec 2017 13:13:56 +0100 Subject: [PATCH 913/922] add test for cpu_affinity --- psutil/tests/test_process.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/psutil/tests/test_process.py b/psutil/tests/test_process.py index d5c2cd85c..a629cae52 100755 --- a/psutil/tests/test_process.py +++ b/psutil/tests/test_process.py @@ -9,6 +9,7 @@ import collections import errno import getpass +import itertools import os import signal import socket @@ -910,8 +911,6 @@ def test_cpu_affinity(self): p.cpu_affinity(set(all_cpus)) p.cpu_affinity(tuple(all_cpus)) - # TODO: temporary, see: https://github.com/MacPython/psutil/issues/1 - @unittest.skipIf(LINUX, "temporary") @unittest.skipIf(not HAS_CPU_AFFINITY, 'not supported') def test_cpu_affinity_errs(self): sproc = get_test_subprocess() @@ -922,6 +921,24 @@ def test_cpu_affinity_errs(self): self.assertRaises(TypeError, p.cpu_affinity, [0, "1"]) self.assertRaises(ValueError, p.cpu_affinity, [0, -1]) + @unittest.skipIf(not HAS_CPU_AFFINITY, 'not supported') + def test_cpu_affinity_all_combinations(self): + p = psutil.Process() + initial = p.cpu_affinity() + assert initial, initial + self.addCleanup(p.cpu_affinity, initial) + + # All possible CPU set combinations. + combos = [] + for l in range(0, len(initial) + 1): + for subset in itertools.combinations(initial, l): + if subset: + combos.append(list(subset)) + + for combo in combos: + p.cpu_affinity(combo) + self.assertEqual(p.cpu_affinity(), combo) + # TODO: #595 @unittest.skipIf(BSD, "broken on BSD") # can't find any process file on Appveyor From 87f88101cb3d31f71939d70b15ef8402acf77062 Mon Sep 17 00:00:00 2001 From: Jake Omann Date: Tue, 12 Dec 2017 07:18:44 -0600 Subject: [PATCH 914/922] Add mount points to disk_partitions() in Windows (#775) (#1192) * POC for adding mountpoints to disk_partitions() with all=True * Refactor check for mount points - Less code duplication - Show even with all=False like in Linux * Goto error and close mp handler --- CREDITS | 4 ++-- HISTORY.rst | 4 ++++ psutil/_psutil_windows.c | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/CREDITS b/CREDITS index d215c60fe..cf9ce493c 100644 --- a/CREDITS +++ b/CREDITS @@ -414,8 +414,8 @@ E: khanzf@gmail.com I: 823 N: Jake Omann -E: https://github.com/jhomann -I: 816 +E: https://github.com/jomann09 +I: 816, 775 N: Jeremy Humble W: https://github.com/jhumble diff --git a/HISTORY.rst b/HISTORY.rst index eabe7cc6f..3958f6ef6 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,10 @@ *XXXX-XX-XX* +**Enhancements** + +- 775_: disk_partitions() on Windows return mount points. + **Bug fixes** - 1193_: pids() may return False on OSX. diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index c840a2b7c..81d1b4a06 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -2489,6 +2489,7 @@ static char *psutil_get_drive_type(int type) { #define _ARRAYSIZE(a) (sizeof(a)/sizeof(a[0])) #endif + /* * Return disk partitions as a list of tuples such as * (drive_letter, drive_letter, type, "") @@ -2498,11 +2499,15 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { DWORD num_bytes; char drive_strings[255]; char *drive_letter = drive_strings; + char mp_buf[MAX_PATH]; + char mp_path[MAX_PATH]; int all; int type; int ret; unsigned int old_mode = 0; char opts[20]; + HANDLE mp_h; + BOOL mp_flag= TRUE; LPTSTR fs_type[MAX_PATH + 1] = { 0 }; DWORD pflags = 0; PyObject *py_all; @@ -2573,6 +2578,40 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { strcat_s(opts, _countof(opts), "rw"); if (pflags & FILE_VOLUME_IS_COMPRESSED) strcat_s(opts, _countof(opts), ",compressed"); + + // Check for mount points on this volume and add/get info + // (checks first to know if we can even have mount points) + if (pflags & FILE_SUPPORTS_REPARSE_POINTS) { + + mp_h = FindFirstVolumeMountPoint(drive_letter, mp_buf, MAX_PATH); + if (mp_h != INVALID_HANDLE_VALUE) { + while (mp_flag) { + + // Append full mount path with drive letter + strcpy_s(mp_path, _countof(mp_path), drive_letter); + strcat_s(mp_path, _countof(mp_path), mp_buf); + + py_tuple = Py_BuildValue( + "(ssss)", + drive_letter, + mp_path, + fs_type, // Typically NTFS + opts); + + if (!py_tuple || PyList_Append(py_retlist, py_tuple) == -1) { + FindVolumeMountPointClose(mp_h); + goto error; + } + + Py_DECREF(py_tuple); + + // Continue looking for more mount points + mp_flag = FindNextVolumeMountPoint(mp_h, mp_buf, MAX_PATH); + } + FindVolumeMountPointClose(mp_h); + } + + } } if (strlen(opts) > 0) From 384998d73bef61b9472f924b6abd6dc102a338b0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 Jan 2018 11:51:05 +0100 Subject: [PATCH 915/922] fix #1201: document that timeout kwarg is expressed in seconds --- docs/index.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 74e7d2323..0febaa558 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -917,8 +917,8 @@ Functions ``callback`` is a function which gets called when one of the processes being waited on is terminated and a :class:`Process` instance is passed as callback argument). - This tunction will return as soon as all processes terminate or when - *timeout* occurs, if specified. + This function will return as soon as all processes terminate or when + *timeout* (seconds) occurs. Differently from :meth:`Process.wait` it will not raise :class:`TimeoutExpired` if timeout occurs. A typical use case may be: @@ -1941,14 +1941,15 @@ Process class .. method:: wait(timeout=None) - Wait for process termination and if the process is a children of the - current one also return the exit code, else ``None``. On Windows there's + Wait for process termination and if the process is a child of the current + one also return the exit code, else ``None``. On Windows there's no such limitation (exit code is always returned). If the process is already terminated immediately return ``None`` instead of raising - :class:`NoSuchProcess`. If *timeout* is specified and process is still - alive raise :class:`TimeoutExpired` exception. It can also be used in a - non-blocking fashion by specifying ``timeout=0`` in which case it will - either return immediately or raise :class:`TimeoutExpired`. + :class:`NoSuchProcess`. + *timeout* is expressed in seconds. If specified and the process is still + alive raise :class:`TimeoutExpired` exception. + ``timeout=0`` can be used in non-blocking apps: it will either return + immediately or raise :class:`TimeoutExpired`. To wait for multiple processes use :func:`psutil.wait_procs()`. >>> import psutil From df3fa28142c6db03f503018cff6341ddaf5bb954 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 Jan 2018 21:24:14 +0100 Subject: [PATCH 916/922] pre-release --- HISTORY.rst | 2 +- Makefile | 8 ++++---- docs/index.rst | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 3958f6ef6..dd813c5b5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,7 +3,7 @@ 5.4.3 ===== -*XXXX-XX-XX* +*2018-01-01* **Enhancements** diff --git a/Makefile b/Makefile index 080e334cf..9ca92bb95 100644 --- a/Makefile +++ b/Makefile @@ -217,6 +217,10 @@ upload-win-wheels: ## Upload wheels in dist/* directory on PYPI. pre-release: ## Check if we're ready to produce a new release. rm -rf dist ${MAKE} install + ${MAKE} generate-manifest + git diff MANIFEST.in > /dev/null # ...otherwise 'git diff-index HEAD' will complain + ${MAKE} win-download-wheels + ${MAKE} sdist $(PYTHON) -c \ "from psutil import __version__ as ver; \ doc = open('docs/index.rst').read(); \ @@ -224,11 +228,7 @@ pre-release: ## Check if we're ready to produce a new release. assert ver in doc, '%r not in docs/index.rst' % ver; \ assert ver in history, '%r not in HISTORY.rst' % ver; \ assert 'XXXX' not in history, 'XXXX in HISTORY.rst';" - ${MAKE} generate-manifest - git diff MANIFEST.in > /dev/null # ...otherwise 'git diff-index HEAD' will complain $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else sys.exit(0);" - ${MAKE} win-download-wheels - ${MAKE} sdist release: ## Create a release (down/uploads tar.gz, wheels, git tag release). ${MAKE} pre-release diff --git a/docs/index.rst b/docs/index.rst index 0febaa558..9e3aa8ef1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2590,6 +2590,10 @@ take a look at the Timeline ======== +- 2018-01-01: + `5.4.3 `__ - + `what's new `__ - + `diff `__ - 2017-12-07: `5.4.2 `__ - `what's new `__ - From 958b61849b48ecbad74dbb978c976a5c34c52293 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 Jan 2018 21:31:11 +0100 Subject: [PATCH 917/922] pre release --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9ca92bb95..e3e735daf 100644 --- a/Makefile +++ b/Makefile @@ -228,7 +228,7 @@ pre-release: ## Check if we're ready to produce a new release. assert ver in doc, '%r not in docs/index.rst' % ver; \ assert ver in history, '%r not in HISTORY.rst' % ver; \ assert 'XXXX' not in history, 'XXXX in HISTORY.rst';" - $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else sys.exit(0);" + $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else 0;" release: ## Create a release (down/uploads tar.gz, wheels, git tag release). ${MAKE} pre-release From 238634831db13c167591450b40e901cbd11d0dc0 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 Jan 2018 21:31:53 +0100 Subject: [PATCH 918/922] pre release --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index e3e735daf..ebf88b5ec 100644 --- a/Makefile +++ b/Makefile @@ -228,7 +228,6 @@ pre-release: ## Check if we're ready to produce a new release. assert ver in doc, '%r not in docs/index.rst' % ver; \ assert ver in history, '%r not in HISTORY.rst' % ver; \ assert 'XXXX' not in history, 'XXXX in HISTORY.rst';" - $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff-index HEAD --', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else 0;" release: ## Create a release (down/uploads tar.gz, wheels, git tag release). ${MAKE} pre-release From ad4acae5489f86fc3bef645505b3873f156b4867 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 Jan 2018 21:32:56 +0100 Subject: [PATCH 919/922] pre release --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index ebf88b5ec..5081a4edd 100644 --- a/Makefile +++ b/Makefile @@ -228,6 +228,7 @@ pre-release: ## Check if we're ready to produce a new release. assert ver in doc, '%r not in docs/index.rst' % ver; \ assert ver in history, '%r not in HISTORY.rst' % ver; \ assert 'XXXX' not in history, 'XXXX in HISTORY.rst';" + $(PYTHON) -c "import subprocess, sys; out = subprocess.check_output('git diff --quiet && git diff --cached --quiet', shell=True).strip(); sys.exit('there are uncommitted changes:\n%s' % out) if out else 0 ;" release: ## Create a release (down/uploads tar.gz, wheels, git tag release). ${MAKE} pre-release From a86c6f65c123442802c44d27e45b5e014a62fe3b Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Mon, 1 Jan 2018 21:58:59 +0100 Subject: [PATCH 920/922] #1152: fix doc to mention CLI command necessary to enable disk_io_counters() on win --- docs/index.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index 9e3aa8ef1..d58e1c19d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -419,6 +419,8 @@ Disks numbers will always be increasing or remain the same, but never decrease. ``disk_io_counters.cache_clear()`` can be used to invalidate the *nowrap* cache. + On Windows it may be ncessary to issue ``diskperf -y`` command from cmd.exe + first in order to enable IO counters. >>> import psutil >>> psutil.disk_io_counters() From f7171c45d7cfc1ce68baa7cd0afdaa94e28305a5 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Thu, 11 Jan 2018 12:34:18 -0800 Subject: [PATCH 921/922] Pass python_requires argument to setuptools (#1208) Helps pip decide what version of the library to install. https://packaging.python.org/tutorials/distributing-packages/#python-requires > If your project only runs on certain Python versions, setting the > python_requires argument to the appropriate PEP 440 version specifier > string will prevent pip from installing the project on other Python > versions. https://setuptools.readthedocs.io/en/latest/setuptools.html#new-and-changed-setup-keywords > python_requires > > A string corresponding to a version specifier (as defined in PEP 440) > for the Python version, used to specify the Requires-Python defined in > PEP 345. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1625a3eb4..d8db694eb 100755 --- a/setup.py +++ b/setup.py @@ -338,6 +338,7 @@ def main(): ) if setuptools is not None: kwargs.update( + python_requires=">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", test_suite="psutil.tests.get_suite", tests_require=tests_require, extras_require=extras_require, From 133bde7de297f61f412c5e0cd049e64cc8f537f4 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Tue, 16 Jan 2018 05:21:39 -0800 Subject: [PATCH 922/922] Update INSTALL.rst --- INSTALL.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/INSTALL.rst b/INSTALL.rst index f2a80eede..8d54d6ae1 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -83,7 +83,10 @@ exe/wheel installers hosted on C:\Python27\python.exe -m pip install psutil If you want to compile psutil from sources you'll need **Visual Studio** -(Mingw32 is no longer supported): +(Mingw32 is no longer supported), which really is a mess. +The VS versions are the onle listed below. +This `blog post `__ +provides numerous info on how to properly set up VS (good luck with that). * Python 2.6, 2.7: `VS-2008 `__ * Python 3.3, 3.4: `VS-2010 `__