From a840dba69cbb68d814c413664a7327e82b2bedfb Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 30 Nov 2017 15:09:30 +0100 Subject: [PATCH 1/7] update HISTORY --- HISTORY.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index bfaa2568c..f8a7d3015 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -17,8 +17,8 @@ - 1169_: [Linux] users() "hostname" returns username instead. (patch by janderbrain) - 1172_: [Windows] `make test` does not work. -- 1179_: [Linux] Process.cmdline() correctly splits cmdline args for - misbehaving processes who overwrite /proc/pid/cmdline by using spaces +- 1179_: [Linux] Process.cmdline() is now able to splits cmdline args for + misbehaving processes which overwrite /proc/pid/cmdline and use spaces instead of null bytes as args separator. - 1181_: [OSX] Process.memory_maps() may raise ENOENT. From 8f8b4d79b1b2f0d6a94107f0038796efcc098278 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 30 Nov 2017 15:17:35 +0100 Subject: [PATCH 2/7] update doc --- docs/index.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index d34a1c457..0119b4233 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1716,9 +1716,8 @@ Process class .. method:: children(recursive=False) - Return the children of this process as a list of :Class:`Process` objects, - preemptively checking whether PID has been reused. If recursive is `True` - return all the parent descendants. + Return the children of this process as a list of :Class:`Process` objects. + If recursive is `True` return all the parent descendants. Pseudo code example assuming *A == this process*: :: @@ -1738,7 +1737,7 @@ Process class Note that in the example above if process X disappears process Y won't be returned either as the reference to process A is lost. This concept is well summaried by this - `unit test `__. + `unit test `__. See also how to `kill a process tree <#kill-process-tree>`__ and `terminate my children <#terminate-my-children>`__. From d23895023ef6c737beb5f50c464972a15efe5218 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 30 Nov 2017 18:18:37 +0100 Subject: [PATCH 3/7] #1183: speedup Process.children() by 2.2x --- HISTORY.rst | 1 + psutil/__init__.py | 81 +++++++++++++++++++++++----------------------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index f8a7d3015..870f351b2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,7 @@ - 1173_: introduced PSUTIL_DEBUG environment variable which can be set in order to print useful debug messages on stderr (useful in case of nasty errors). - 1177_: added support for sensors_battery() on OSX. (patch by Arnon Yaari) +- 1183_: Process.children() is around 2.2x faster on UNIX. **Bug fixes** diff --git a/psutil/__init__.py b/psutil/__init__.py index c8da0a3dc..41516e585 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -250,10 +250,30 @@ # ===================================================================== -# --- Process class +# --- Utils # ===================================================================== +if hasattr(_psplatform, 'ppid_map'): + # Windows only (C). + _ppid_map = _psplatform.ppid_map() +else: + def _ppid_map(): + """Obtain a {pid: ppid, ...} dict for all running processes in + one shot. Used to speed up Process.children(). + """ + ret = {} + for pid in pids(): + try: + proc = _psplatform.Process(pid) + ppid = proc.ppid() + except (NoSuchProcess, AccessDenied): + pass + else: + ret[pid] = ppid + return ret + + def _assert_pid_not_reused(fun): """Decorator which raises NoSuchProcess in case a process is no longer running or its PID has been reused. @@ -266,6 +286,11 @@ def wrapper(self, *args, **kwargs): return wrapper +# ===================================================================== +# --- Process class +# ===================================================================== + + class Process(object): """Represents an OS process with the given PID. If PID is omitted current process PID (os.getpid()) is used. @@ -848,55 +873,29 @@ def children(self, recursive=False): process Y won't be listed as the reference to process A is lost. """ - if hasattr(_psplatform, 'ppid_map'): - # Windows only: obtain a {pid:ppid, ...} dict for all running - # processes in one shot (faster). - ppid_map = _psplatform.ppid_map() - else: - ppid_map = None - + ppid_map = _ppid_map() ret = [] if not recursive: - if ppid_map is None: - # 'slow' version, common to all platforms except Windows - for p in process_iter(): + for pid, ppid in ppid_map.items(): + if ppid == self.pid: try: - if p.ppid() == self.pid: - # if child happens to be older than its parent - # (self) it means child's PID has been reused - if self.create_time() <= p.create_time(): - ret.append(p) + child = Process(pid) + # if child happens to be older than its parent + # (self) it means child's PID has been reused + if self.create_time() <= child.create_time(): + ret.append(child) except (NoSuchProcess, ZombieProcess): pass - else: # pragma: no cover - # Windows only (faster) - for pid, ppid in ppid_map.items(): - if ppid == self.pid: - try: - child = Process(pid) - # if child happens to be older than its parent - # (self) it means child's PID has been reused - if self.create_time() <= child.create_time(): - ret.append(child) - except (NoSuchProcess, ZombieProcess): - pass else: # construct a dict where 'values' are all the processes # having 'key' as their parent table = collections.defaultdict(list) - if ppid_map is None: - for p in process_iter(): - try: - table[p.ppid()].append(p) - except (NoSuchProcess, ZombieProcess): - pass - else: # pragma: no cover - for pid, ppid in ppid_map.items(): - try: - p = Process(pid) - table[ppid].append(p) - except (NoSuchProcess, ZombieProcess): - pass + for pid, ppid in ppid_map.items(): + try: + p = Process(pid) + table[ppid].append(p) + except (NoSuchProcess, ZombieProcess): + pass # At this point we have a mapping table where table[self.pid] # are the current process' children. # Below, we look for all descendants recursively, similarly From fe85bdf0e87227ba41db7510a150ad9fc51623c7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Thu, 30 Nov 2017 23:57:32 +0100 Subject: [PATCH 4/7] fix windows err --- psutil/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 41516e585..605bd7687 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -256,7 +256,7 @@ if hasattr(_psplatform, 'ppid_map'): # Windows only (C). - _ppid_map = _psplatform.ppid_map() + _ppid_map = _psplatform.ppid_map else: def _ppid_map(): """Obtain a {pid: ppid, ...} dict for all running processes in From 78f8cbcf0a93b2b075803ad9578bc317ce599fd7 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 1 Dec 2017 00:25:22 +0100 Subject: [PATCH 5/7] #1083 / #1084: implement linux-specific ppid_map() function speending things up from 2x to 2.4x --- HISTORY.rst | 2 +- psutil/__init__.py | 4 ++-- psutil/_pslinux.py | 21 +++++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 870f351b2..8bc3f7847 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,7 +10,7 @@ - 1173_: introduced PSUTIL_DEBUG environment variable which can be set in order to print useful debug messages on stderr (useful in case of nasty errors). - 1177_: added support for sensors_battery() on OSX. (patch by Arnon Yaari) -- 1183_: Process.children() is around 2.2x faster on UNIX. +- 1183_: Process.children() is 2x faster on UNIX and 2.4x faster on Linux. **Bug fixes** diff --git a/psutil/__init__.py b/psutil/__init__.py index 605bd7687..98e608708 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -255,11 +255,11 @@ if hasattr(_psplatform, 'ppid_map'): - # Windows only (C). + # Faster version (Windows and Linux). _ppid_map = _psplatform.ppid_map else: def _ppid_map(): - """Obtain a {pid: ppid, ...} dict for all running processes in + """Return a {pid: ppid, ...} dict for all running processes in one shot. Used to speed up Process.children(). """ ret = {} diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index a5a3fd891..5076b4b7a 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1356,6 +1356,27 @@ def pid_exists(pid): return pid in pids() +def ppid_map(): + """Obtain a {pid: ppid, ...} dict for all running processes in + one shot. Used to speed up Process.children(). + """ + ret = {} + procfs_path = get_procfs_path() + for pid in pids(): + try: + with open_binary("%s/%s/stat" % (procfs_path, pid)) as f: + data = f.read() + except EnvironmentError as err: + if err.errno not in (errno.ENOENT, errno.EPERM, errno.EACCES): + raise + else: + rpar = data.rfind(b')') + dset = data[rpar + 2:].split() + ppid = int(dset[1]) + ret[pid] = ppid + return ret + + def wrap_exceptions(fun): """Decorator which translates bare OSError and IOError exceptions into NoSuchProcess and AccessDenied. From 9afa91dd04357c3e6ad561ee1ea25b5dd6cf1715 Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 1 Dec 2017 12:45:03 +0100 Subject: [PATCH 6/7] add ESRCH to err handling --- psutil/_pslinux.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 5076b4b7a..ecf116f31 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1367,7 +1367,8 @@ def ppid_map(): with open_binary("%s/%s/stat" % (procfs_path, pid)) as f: data = f.read() except EnvironmentError as err: - if err.errno not in (errno.ENOENT, errno.EPERM, errno.EACCES): + if err.errno not in (errno.ENOENT, errno.ESRCH, + errno.EPERM, errno.EACCES): raise else: rpar = data.rfind(b')') From f114a6e1da74643a89e7ae5d32456ac955e8ae5a Mon Sep 17 00:00:00 2001 From: Giampaolo Rodola Date: Fri, 1 Dec 2017 12:54:44 +0100 Subject: [PATCH 7/7] update doc --- docs/index.rst | 3 ++- psutil/__init__.py | 1 + psutil/_pslinux.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 0119b4233..ea2384bb3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1716,7 +1716,8 @@ Process class .. method:: children(recursive=False) - Return the children of this process as a list of :Class:`Process` objects. + Return the children of this process as a list of :class:`Process` + instances. If recursive is `True` return all the parent descendants. Pseudo code example assuming *A == this process*: :: diff --git a/psutil/__init__.py b/psutil/__init__.py index 98e608708..a84479738 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -268,6 +268,7 @@ def _ppid_map(): proc = _psplatform.Process(pid) ppid = proc.ppid() except (NoSuchProcess, AccessDenied): + # Note: AccessDenied is unlikely to happen. pass else: ret[pid] = ppid diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index ecf116f31..b57adb34e 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -1367,6 +1367,8 @@ def ppid_map(): with open_binary("%s/%s/stat" % (procfs_path, pid)) as f: data = f.read() except EnvironmentError as err: + # Note: we should be able to access /stat for all processes + # so we won't bump into EPERM, which is good. if err.errno not in (errno.ENOENT, errno.ESRCH, errno.EPERM, errno.EACCES): raise