Fix py 3.9 [WinError 998] Invalid access to memory location (#1866)

README.rst

@@ -225,7 +225,7 @@ Network
     {'eth0': netio(bytes_sent=485291293, bytes_recv=6004858642, packets_sent=3251564, packets_recv=4787798, errin=0, errout=0, dropin=0, dropout=0),
      'lo': netio(bytes_sent=2838627, bytes_recv=2838627, packets_sent=30567, packets_recv=30567, errin=0, errout=0, dropin=0, dropout=0)}
     >>>
-    >>> psutil.net_connections()
+    >>> psutil.net_connections(kind='tcp')
     [sconn(fd=115, family=<AddressFamily.AF_INET: 2>, type=<SocketType.SOCK_STREAM: 1>, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED', pid=1254),
      sconn(fd=117, family=<AddressFamily.AF_INET: 2>, type=<SocketType.SOCK_STREAM: 1>, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status='CLOSING', pid=2987),
      ...]
@@ -359,7 +359,7 @@ Process management
     [popenfile(path='/home/giampaolo/monit.py', fd=3, position=0, mode='r', flags=32768),
      popenfile(path='/var/log/monit.log', fd=4, position=235542, mode='a', flags=33793)]
     >>>
-    >>> p.connections()
+    >>> p.connections(kind='tcp')
     [pconn(fd=115, family=<AddressFamily.AF_INET: 2>, type=<SocketType.SOCK_STREAM: 1>, laddr=addr(ip='10.0.0.1', port=48776), raddr=addr(ip='93.186.135.91', port=80), status='ESTABLISHED'),
      pconn(fd=117, family=<AddressFamily.AF_INET: 2>, type=<SocketType.SOCK_STREAM: 1>, laddr=addr(ip='10.0.0.1', port=43761), raddr=addr(ip='72.14.234.100', port=80), status='CLOSING')]
     >>>

appveyor.yml

@@ -15,10 +15,6 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "32"
 
-    - PYTHON: "C:\\Python35"
-      PYTHON_VERSION: "3.5.x"
-      PYTHON_ARCH: "32"
-
     - PYTHON: "C:\\Python36"
       PYTHON_VERSION: "3.6.x"
       PYTHON_ARCH: "32"
@@ -31,16 +27,17 @@ environment:
       PYTHON_VERSION: "3.8.x"
       PYTHON_ARCH: "32"
 
+    - PYTHON: "C:\\Python39"
+      PYTHON_VERSION: "3.9.x"
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+      PYTHON_ARCH: "32"
+
     # 64 bits
 
     - PYTHON: "C:\\Python27-x64"
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "64"
 
-    - PYTHON: "C:\\Python35-x64"
-      PYTHON_VERSION: "3.5.x"
-      PYTHON_ARCH: "64"
-
     - PYTHON: "C:\\Python36-x64"
       PYTHON_VERSION: "3.6.x"
       PYTHON_ARCH: "64"
@@ -53,6 +50,11 @@ environment:
       PYTHON_VERSION: "3.8.x"
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:\\Python39-x64"
+      PYTHON_VERSION: "3.9.x"
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+      PYTHON_ARCH: "64"
+
 init:
   - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
 
@@ -103,3 +105,4 @@ only_commits:
     - psutil/tests/*
     - scripts/*
     - setup.py
+

psutil/_psutil_common.c

@@ -199,13 +199,6 @@ int PSUTIL_WINVER;
 SYSTEM_INFO          PSUTIL_SYSTEM_INFO;
 CRITICAL_SECTION     PSUTIL_CRITICAL_SECTION;
 
-#define NT_FACILITY_MASK 0xfff
-#define NT_FACILITY_SHIFT 16
-#define NT_FACILITY(Status) \
-    ((((ULONG)(Status)) >> NT_FACILITY_SHIFT) & NT_FACILITY_MASK)
-#define NT_NTWIN32(status) (NT_FACILITY(Status) == FACILITY_WIN32)
-#define WIN32_FROM_NTSTATUS(Status) (((ULONG)(Status)) & 0xffff)
-
 
 // A wrapper around GetModuleHandle and GetProcAddress.
 PVOID

psutil/_psutil_common.h

@@ -134,6 +134,14 @@ void convert_kvm_err(const char *syscall, char *errbuf);
     #define MALLOC_ZERO(x) HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, (x))
     #define FREE(x) HeapFree(GetProcessHeap(), 0, (x))
 
+    #define _NT_FACILITY_MASK 0xfff
+    #define _NT_FACILITY_SHIFT 16
+    #define _NT_FACILITY(status) \
+        ((((ULONG)(status)) >> _NT_FACILITY_SHIFT) & _NT_FACILITY_MASK)
+
+    #define NT_NTWIN32(status) (_NT_FACILITY(status) == FACILITY_WIN32)
+    #define WIN32_FROM_NTSTATUS(status) (((ULONG)(status)) & 0xffff)
+
     #define LO_T 1e-7
     #define HI_T 429.4967296
 

psutil/arch/windows/process_info.c

@@ -50,6 +50,36 @@ enum psutil_process_data_kind {
 };
 
 
+static void
+psutil_convert_winerr(ULONG err, char* syscall) {
+    char fullmsg[8192];
+
+    if (err == ERROR_NOACCESS)  {
+        sprintf(
+            fullmsg,
+            "(originated from %s -> ERROR_NOACCESS; converted to AccessDenied)",
+            syscall);
+        psutil_debug(fullmsg);
+        AccessDenied(fullmsg);
+    }
+    else {
+        PyErr_SetFromOSErrnoWithSyscall(syscall);
+    }
+}
+
+
+static void
+psutil_convert_ntstatus_err(NTSTATUS status, char* syscall) {
+    ULONG err;
+
+    if (NT_NTWIN32(status))
+        err = WIN32_FROM_NTSTATUS(status);
+    else
+        err = RtlNtStatusToDosErrorNoTeb(status);
+    psutil_convert_winerr(err, syscall);
+}
+
+
 /*
  * Get data from the process with the given pid.  The data is returned
  * in the pdata output member as a nul terminated string which must be
@@ -87,16 +117,14 @@ psutil_get_process_data(DWORD pid,
          http://www.drdobbs.com/embracing-64-bit-windows/184401966
      */
     SIZE_T size = 0;
-#ifndef _WIN64
-    static __NtQueryInformationProcess NtWow64QueryInformationProcess64 = NULL;
-    static _NtWow64ReadVirtualMemory64 NtWow64ReadVirtualMemory64 = NULL;
-#endif
     HANDLE hProcess = NULL;
     LPCVOID src;
     WCHAR *buffer = NULL;
 #ifdef _WIN64
     LPVOID ppeb32 = NULL;
 #else
+    static __NtQueryInformationProcess NtWow64QueryInformationProcess64 = NULL;
+    static _NtWow64ReadVirtualMemory64 NtWow64ReadVirtualMemory64 = NULL;
     PVOID64 src64;
     BOOL weAreWow64;
     BOOL theyAreWow64;
@@ -133,7 +161,7 @@ psutil_get_process_data(DWORD pid,
         if (!ReadProcessMemory(hProcess, ppeb32, &peb32, sizeof(peb32), NULL)) {
             // May fail with ERROR_PARTIAL_COPY, see:
             // https://github.com/giampaolo/psutil/issues/875
-            PyErr_SetFromWindowsErr(0);
+            psutil_convert_winerr(GetLastError(), "ReadProcessMemory");
             goto error;
         }
 
@@ -146,7 +174,7 @@ psutil_get_process_data(DWORD pid,
         {
             // May fail with ERROR_PARTIAL_COPY, see:
             // https://github.com/giampaolo/psutil/issues/875
-            PyErr_SetFromWindowsErr(0);
+            psutil_convert_winerr(GetLastError(), "ReadProcessMemory");
             goto error;
         }
 
@@ -164,7 +192,7 @@ psutil_get_process_data(DWORD pid,
                 break;
         }
     } else
-#else
+#else  // #ifdef _WIN64
     /* 32 bit case.  Check if the target is also 32 bit. */
     if (!IsWow64Process(GetCurrentProcess(), &weAreWow64) ||
             !IsWow64Process(hProcess, &theyAreWow64)) {
@@ -206,10 +234,9 @@ psutil_get_process_data(DWORD pid,
                 sizeof(pbi64),
                 NULL);
         if (!NT_SUCCESS(status)) {
-            psutil_SetFromNTStatusErr(
-                    status,
-                    "NtWow64QueryInformationProcess64(ProcessBasicInformation)"
-            );
+            psutil_convert_ntstatus_err(
+                status,
+                "NtWow64QueryInformationProcess64(ProcessBasicInformation)");
             goto error;
         }
 
@@ -221,7 +248,8 @@ psutil_get_process_data(DWORD pid,
                 sizeof(peb64),
                 NULL);
         if (!NT_SUCCESS(status)) {
-            psutil_SetFromNTStatusErr(status, "NtWow64ReadVirtualMemory64");
+            psutil_convert_ntstatus_err(
+                status, "NtWow64ReadVirtualMemory64(pbi64.PebBaseAddress)");
             goto error;
         }
 
@@ -233,10 +261,8 @@ psutil_get_process_data(DWORD pid,
                 sizeof(procParameters64),
                 NULL);
         if (!NT_SUCCESS(status)) {
-            psutil_SetFromNTStatusErr(
-                    status,
-                    "NtWow64ReadVirtualMemory64(ProcessParameters)"
-            );
+            psutil_convert_ntstatus_err(
+                status, "NtWow64ReadVirtualMemory64(peb64.ProcessParameters)");
             goto error;
         }
 
@@ -284,7 +310,7 @@ psutil_get_process_data(DWORD pid,
         {
             // May fail with ERROR_PARTIAL_COPY, see:
             // https://github.com/giampaolo/psutil/issues/875
-            PyErr_SetFromWindowsErr(0);
+            psutil_convert_winerr(GetLastError(), "ReadProcessMemory");
             goto error;
         }
 
@@ -297,7 +323,7 @@ psutil_get_process_data(DWORD pid,
         {
             // May fail with ERROR_PARTIAL_COPY, see:
             // https://github.com/giampaolo/psutil/issues/875
-            PyErr_SetFromWindowsErr(0);
+            psutil_convert_winerr(GetLastError(), "ReadProcessMemory");
             goto error;
         }
 
@@ -343,15 +369,15 @@ psutil_get_process_data(DWORD pid,
                 size,
                 NULL);
         if (!NT_SUCCESS(status)) {
-            psutil_SetFromNTStatusErr(status, "NtWow64ReadVirtualMemory64");
+            psutil_convert_ntstatus_err(status, "NtWow64ReadVirtualMemory64");
             goto error;
         }
     } else
 #endif
     if (!ReadProcessMemory(hProcess, src, buffer, size, NULL)) {
         // May fail with ERROR_PARTIAL_COPY, see:
         // https://github.com/giampaolo/psutil/issues/875
-        PyErr_SetFromWindowsErr(0);
+        psutil_convert_winerr(GetLastError(), "ReadProcessMemory");
         goto error;
     }
 
@@ -698,16 +724,16 @@ psutil_proc_info(PyObject *self, PyObject *args) {
 
     py_retlist = Py_BuildValue(
 #if defined(_WIN64)
-        "kkdddiKKKKKK" "kKKKKKKKKK",
+        "kkdddkKKKKKK" "kKKKKKKKKK",
 #else
-        "kkdddiKKKKKK" "kIIIIIIIII",
+        "kkdddkKKKKKK" "kIIIIIIIII",
 #endif
         process->HandleCount,                   // num handles
         ctx_switches,                           // num ctx switches
         user_time,                              // cpu user time
         kernel_time,                            // cpu kernel time
         create_time,                            // create time
-        (int)process->NumberOfThreads,          // num threads
+        process->NumberOfThreads,               // num threads
         // IO counters
         process->ReadOperationCount.QuadPart,   // io rcount
         process->WriteOperationCount.QuadPart,  // io wcount

psutil/tests/test_contracts.py

@@ -32,6 +32,7 @@
 from psutil._compat import FileNotFoundError
 from psutil._compat import long
 from psutil._compat import range
+from psutil.tests import APPVEYOR
 from psutil.tests import create_sockets
 from psutil.tests import enum
 from psutil.tests import GITHUB_WHEELS
@@ -451,6 +452,8 @@ def ppid(self, ret, info):
 
     def name(self, ret, info):
         self.assertIsInstance(ret, str)
+        if APPVEYOR and not ret and info['status'] == 'stopped':
+            return
         # on AIX, "<exiting>" processes don't have names
         if not AIX:
             assert ret
@@ -521,6 +524,8 @@ def ionice(self, ret, info):
 
     def num_threads(self, ret, info):
         self.assertIsInstance(ret, int)
+        if APPVEYOR and not ret and info['status'] == 'stopped':
+            return
         self.assertGreaterEqual(ret, 1)
 
     def threads(self, ret, info):

psutil/tests/test_misc.py

@@ -28,7 +28,6 @@
 from psutil._compat import PY3
 from psutil.tests import APPVEYOR
 from psutil.tests import CI_TESTING
-from psutil.tests import DEVNULL
 from psutil.tests import HAS_BATTERY
 from psutil.tests import HAS_MEMORY_MAPS
 from psutil.tests import HAS_NET_IO_COUNTERS
@@ -719,7 +718,7 @@ def test_pmap(self):
     def test_procsmem(self):
         if 'uss' not in psutil.Process().memory_full_info()._fields:
             raise self.skipTest("not supported")
-        self.assert_stdout('procsmem.py', stderr=DEVNULL)
+        self.assert_stdout('procsmem.py')
 
     def test_killall(self):
         self.assert_syntax('killall.py')

scripts/internal/download_wheels_appveyor.py

@@ -25,7 +25,7 @@
 
 
 BASE_URL = 'https://ci.appveyor.com/api'
-PY_VERSIONS = ['2.7', '3.5', '3.6', '3.7', '3.8']
+PY_VERSIONS = ['2.7', '3.6', '3.7', '3.8', '3.9']
 TIMEOUT = 30
 
 

scripts/internal/winmake.py

@@ -525,16 +525,38 @@ def print_api_speed():
     sh("%s -Wa scripts\\internal\\print_api_speed.py" % PYTHON)
 
 
+def download_appveyor_wheels():
+    """Download appveyor wheels."""
+    sh("%s -Wa scripts\\internal\\download_wheels_appveyor.py "
+       "--user giampaolo --project psutil" % PYTHON)
+
+
 def get_python(path):
     if not path:
         return sys.executable
     if os.path.isabs(path):
         return path
     # try to look for a python installation given a shortcut name
     path = path.replace('.', '')
-    vers = ('26', '27', '36', '37', '38',
-            '26-64', '27-64', '36-64', '37-64', '38-64'
-            '26-32', '27-32', '36-32', '37-32', '38-32')
+    vers = (
+        '26',
+        '26-32',
+        '26-64',
+        '27',
+        '27-32',
+        '27-64',
+        '36',
+        '36-32',
+        '36-64',
+        '37',
+        '37-32',
+        '37-64',
+        '38',
+        '38-32',
+        '38-64',
+        '39-32',
+        '39-64',
+    )
     for v in vers:
         pypath = r'C:\\python%s\python.exe' % v
         if path in pypath and os.path.isfile(pypath):
@@ -554,6 +576,7 @@ def main():
     sp.add_parser('build', help="build")
     sp.add_parser('clean', help="deletes dev files")
     sp.add_parser('coverage', help="run coverage tests.")
+    sp.add_parser('download-appveyor-wheels', help="download wheels.")
     sp.add_parser('help', help="print this help")
     sp.add_parser('install', help="build + install in develop/edit mode")
     sp.add_parser('install-git-hooks', help="install GIT pre-commit hook")

scripts/netstat.py

@@ -48,13 +48,14 @@ def main():
         raddr = ""
         if c.raddr:
             raddr = "%s:%s" % (c.raddr)
+        name = proc_names.get(c.pid, '?') or ''
         print(templ % (
             proto_map[(c.family, c.type)],
             laddr,
             raddr or AD,
             c.status,
             c.pid or AD,
-            proc_names.get(c.pid, '?')[:15],
+            name[:15],
         ))