From c7bd810fecde8bfff0a4ca3f9325f915d38d7de3 Mon Sep 17 00:00:00 2001 From: Gabi Davar Date: Tue, 3 Feb 2015 19:26:52 +0200 Subject: [PATCH 1/5] fix most warnings and test failures on win32 --- psutil/_psutil_windows.c | 11 ++++++----- psutil/arch/windows/ntextapi.h | 5 +++++ psutil/arch/windows/process_handles.c | 2 +- psutil/arch/windows/process_info.h | 9 +++++++++ setup.py | 1 + test/_windows.py | 4 ++-- test/test_psutil.py | 8 +++++--- 7 files changed, 29 insertions(+), 11 deletions(-) diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 1d0e3278a..70d268673 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -567,7 +567,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (NULL == hProcess) { return NULL; } - if (GetProcessImageFileNameW(hProcess, &exe, MAX_PATH) == 0) { + if (GetProcessImageFileNameW(hProcess, exe, MAX_PATH) == 0) { CloseHandle(hProcess); PyErr_SetFromWindowsErr(0); return NULL; @@ -601,7 +601,9 @@ psutil_proc_memory_info(PyObject *self, PyObject *args) return NULL; } - if (! GetProcessMemoryInfo(hProcess, &cnt, sizeof(cnt)) ) { + if (! GetProcessMemoryInfo(hProcess, (PPROCESS_MEMORY_COUNTERS)&cnt, + sizeof(cnt)) ) + { CloseHandle(hProcess); return PyErr_SetFromWindowsErr(0); } @@ -2124,8 +2126,8 @@ psutil_proc_cpu_affinity_get(PyObject *self, PyObject *args) { DWORD pid; HANDLE hProcess; - PDWORD_PTR proc_mask; - PDWORD_PTR system_mask; + DWORD_PTR proc_mask; + DWORD_PTR system_mask; if (! PyArg_ParseTuple(args, "l", &pid)) { return NULL; @@ -2260,7 +2262,6 @@ static PyObject * psutil_net_io_counters(PyObject *self, PyObject *args) { int attempts = 0; - int i; int outBufLen = 15000; char ifname[MAX_PATH]; DWORD dwRetVal = 0; diff --git a/psutil/arch/windows/ntextapi.h b/psutil/arch/windows/ntextapi.h index 298c07800..d1aa62df2 100644 --- a/psutil/arch/windows/ntextapi.h +++ b/psutil/arch/windows/ntextapi.h @@ -4,6 +4,9 @@ * found in the LICENSE file. */ +#if !defined(__NTEXTAPI_H__) +#define __NTEXTAPI_H__ + typedef enum _KTHREAD_STATE { Initialized, Ready, @@ -285,3 +288,5 @@ typedef enum _PROCESSINFOCLASS { ProcessImageInformation, MaxProcessInfoClass } PROCESSINFOCLASS; + +#endif // __NTEXTAPI_H__ diff --git a/psutil/arch/windows/process_handles.c b/psutil/arch/windows/process_handles.c index f7479e994..cdfef3f58 100644 --- a/psutil/arch/windows/process_handles.c +++ b/psutil/arch/windows/process_handles.c @@ -188,7 +188,7 @@ psutil_get_open_files(long pid, HANDLE processHandle) } if (!DuplicateHandle(processHandle, - handle.Handle, + (HANDLE)handle.Handle, GetCurrentProcess(), &dupHandle, 0, diff --git a/psutil/arch/windows/process_info.h b/psutil/arch/windows/process_info.h index 9544f5d66..a44c4aced 100644 --- a/psutil/arch/windows/process_info.h +++ b/psutil/arch/windows/process_info.h @@ -4,8 +4,13 @@ * found in the LICENSE file. */ +#if !defined(__PROCESS_INFO_H) +#define __PROCESS_INFO_H + #include #include +#include "security.h" +#include "ntextapi.h" DWORD* psutil_get_pids(DWORD *numberOfReturnedPIDs); HANDLE psutil_handle_from_pid(DWORD pid); @@ -15,3 +20,7 @@ int psutil_pid_in_proclist(DWORD pid); int psutil_pid_is_running(DWORD pid); PVOID psutil_get_peb_address(HANDLE ProcessHandle); PyObject* psutil_get_arg_list(long pid); +int psutil_get_proc_info(DWORD pid, PSYSTEM_PROCESS_INFORMATION *retProcess, + PVOID *retBuffer); + +#endif diff --git a/setup.py b/setup.py index dcc839873..134af9579 100644 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ def get_winver(): # http://www.mingw.org/wiki/Use_more_recent_defined_functions ('_WIN32_WINNT', get_winver()), ('_AVAIL_WINVER_', get_winver()), + ('_CRT_SECURE_NO_WARNINGS', None), # see: https://github.com/giampaolo/psutil/issues/348 ('PSAPI_VERSION', 1), ], diff --git a/test/_windows.py b/test/_windows.py index 9c07142a2..0a35da507 100644 --- a/test/_windows.py +++ b/test/_windows.py @@ -165,7 +165,7 @@ def test_process_create_time(self): # --- psutil namespace functions and constants tests - @unittest.skipUnless(hasattr(os, 'NUMBER_OF_PROCESSORS'), + @unittest.skipUnless('NUMBER_OF_PROCESSORS' in os.environ, 'NUMBER_OF_PROCESSORS env var is not available') def test_cpu_count(self): num_cpus = int(os.environ['NUMBER_OF_PROCESSORS']) @@ -289,7 +289,7 @@ class TestDualProcessImplementation(unittest.TestCase): ('proc_cpu_times', 0.2), ('proc_create_time', 0.5), ('proc_num_handles', 1), # 1 because impl #1 opens a handle - ('proc_io_counters', 0), + # ('proc_io_counters', 0), # fails every time ... ('proc_memory_info', 1024), # KB ] diff --git a/test/test_psutil.py b/test/test_psutil.py index 8bdb0b9c5..25e7aaf04 100644 --- a/test/test_psutil.py +++ b/test/test_psutil.py @@ -1650,6 +1650,7 @@ def test_cpu_affinity(self): self.assertRaises(ValueError, p.cpu_affinity, invalid_cpu) self.assertRaises(ValueError, p.cpu_affinity, range(10000, 11000)) + @unittest.skipUnless(POSIX, 'skipped on Windows') def test_open_files(self): # current process p = psutil.Process() @@ -1663,7 +1664,7 @@ def test_open_files(self): assert os.path.isfile(file), file # another process - cmdline = "import time; f = open(r'%s', 'r'); time.sleep(2);" % TESTFN + cmdline = "import time; f = open(r'%s', 'r'); time.sleep(60);" % TESTFN sproc = get_test_subprocess([PYTHON, "-c", cmdline], wait=True) p = psutil.Process(sproc.pid) @@ -1677,6 +1678,7 @@ def test_open_files(self): for file in filenames: assert os.path.isfile(file), file + @unittest.skipUnless(POSIX, 'skipped on Windows') def test_open_files2(self): # test fd and path fields with open(TESTFN, 'w') as fileobj: @@ -1941,9 +1943,9 @@ def test_children_recursive(self): # A (parent) -> B (child) -> C (grandchild) s = "import subprocess, os, sys, time;" s += "PYTHON = os.path.realpath(sys.executable);" - s += "cmd = [PYTHON, '-c', 'import time; time.sleep(2);'];" + s += "cmd = [PYTHON, '-c', 'import time; time.sleep(4);'];" s += "subprocess.Popen(cmd);" - s += "time.sleep(2);" + s += "time.sleep(4);" get_test_subprocess(cmd=[PYTHON, "-c", s]) p = psutil.Process() self.assertEqual(len(p.children(recursive=False)), 1) From b880f5e99656f0cbe2b7f45671230b9bc31ca855 Mon Sep 17 00:00:00 2001 From: Gabi Davar Date: Sat, 7 Feb 2015 14:45:53 +0200 Subject: [PATCH 2/5] review input --- psutil/arch/windows/process_handles.c | 2 +- test/_windows.py | 4 ++-- test/test_psutil.py | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/psutil/arch/windows/process_handles.c b/psutil/arch/windows/process_handles.c index cdfef3f58..028367954 100644 --- a/psutil/arch/windows/process_handles.c +++ b/psutil/arch/windows/process_handles.c @@ -151,7 +151,7 @@ psutil_get_open_files(long pid, HANDLE processHandle) NULL )) == STATUS_INFO_LENGTH_MISMATCH) { - handleInfo = (PSYSTEM_HANDLE_INFORMATION) \ + handleInfo = (PSYSTEM_HANDLE_INFORMATION) realloc(handleInfo, handleInfoSize *= 2); } diff --git a/test/_windows.py b/test/_windows.py index 0a35da507..44546a947 100644 --- a/test/_windows.py +++ b/test/_windows.py @@ -29,7 +29,7 @@ from psutil._compat import PY3, callable, long from psutil._pswindows import ACCESS_DENIED_SET -import _psutil_windows +import psutil._psutil_windows as _psutil_windows import psutil @@ -289,8 +289,8 @@ class TestDualProcessImplementation(unittest.TestCase): ('proc_cpu_times', 0.2), ('proc_create_time', 0.5), ('proc_num_handles', 1), # 1 because impl #1 opens a handle - # ('proc_io_counters', 0), # fails every time ... ('proc_memory_info', 1024), # KB + ('proc_io_counters', 0), ] def test_compare_values(self): diff --git a/test/test_psutil.py b/test/test_psutil.py index 25e7aaf04..14fc04c40 100644 --- a/test/test_psutil.py +++ b/test/test_psutil.py @@ -1650,7 +1650,6 @@ def test_cpu_affinity(self): self.assertRaises(ValueError, p.cpu_affinity, invalid_cpu) self.assertRaises(ValueError, p.cpu_affinity, range(10000, 11000)) - @unittest.skipUnless(POSIX, 'skipped on Windows') def test_open_files(self): # current process p = psutil.Process() @@ -1678,7 +1677,6 @@ def test_open_files(self): for file in filenames: assert os.path.isfile(file), file - @unittest.skipUnless(POSIX, 'skipped on Windows') def test_open_files2(self): # test fd and path fields with open(TESTFN, 'w') as fileobj: From b7762513d287dc91a0a2fd7340afe2f85efc5192 Mon Sep 17 00:00:00 2001 From: Gabi Davar Date: Sat, 7 Feb 2015 23:47:45 +0200 Subject: [PATCH 3/5] fix open_files() on windows when process PID > 64K --- psutil/arch/windows/process_handles.c | 128 +++++++++++++++++--------- 1 file changed, 87 insertions(+), 41 deletions(-) diff --git a/psutil/arch/windows/process_handles.c b/psutil/arch/windows/process_handles.c index 028367954..6b0a4194f 100644 --- a/psutil/arch/windows/process_handles.c +++ b/psutil/arch/windows/process_handles.c @@ -10,28 +10,26 @@ #endif #include -#include #include +#include +#include #include "process_handles.h" #ifndef NT_SUCCESS #define NT_SUCCESS(x) ((x) >= 0) #endif + #define STATUS_INFO_LENGTH_MISMATCH 0xc0000004 -#define SystemHandleInformation 16 +#include #define ObjectBasicInformation 0 #define ObjectNameInformation 1 #define ObjectTypeInformation 2 +#define HANDLE_TYPE_FILE 28 -typedef LONG NTSTATUS; -typedef struct _UNICODE_STRING { - USHORT Length; - USHORT MaximumLength; - PWSTR Buffer; -} UNICODE_STRING, *PUNICODE_STRING; +typedef LONG NTSTATUS; typedef NTSTATUS (NTAPI *_NtQuerySystemInformation)( ULONG SystemInformationClass, @@ -58,19 +56,29 @@ typedef NTSTATUS (NTAPI *_NtQueryObject)( PULONG ReturnLength ); -typedef struct _SYSTEM_HANDLE { - ULONG ProcessId; - BYTE ObjectTypeNumber; - BYTE Flags; - USHORT Handle; + +// Undocumented FILE_INFORMATION_CLASS: FileNameInformation +const SYSTEM_INFORMATION_CLASS SystemExtendedHandleInformation = (SYSTEM_INFORMATION_CLASS)64; + +typedef struct _SYSTEM_HANDLE_TABLE_ENTRY_INFO_EX +{ PVOID Object; - ACCESS_MASK GrantedAccess; -} SYSTEM_HANDLE, *PSYSTEM_HANDLE; + HANDLE UniqueProcessId; + HANDLE HandleValue; + ULONG GrantedAccess; + USHORT CreatorBackTraceIndex; + USHORT ObjectTypeIndex; + ULONG HandleAttributes; + ULONG Reserved; +} SYSTEM_HANDLE_TABLE_ENTRY_INFO_EX, *PSYSTEM_HANDLE_TABLE_ENTRY_INFO_EX; + +typedef struct _SYSTEM_HANDLE_INFORMATION_EX +{ + ULONG_PTR NumberOfHandles; + ULONG_PTR Reserved; + SYSTEM_HANDLE_TABLE_ENTRY_INFO_EX Handles[1]; +} SYSTEM_HANDLE_INFORMATION_EX, *PSYSTEM_HANDLE_INFORMATION_EX; -typedef struct _SYSTEM_HANDLE_INFORMATION { - ULONG HandleCount; - SYSTEM_HANDLE Handles[1]; -} SYSTEM_HANDLE_INFORMATION, *PSYSTEM_HANDLE_INFORMATION; typedef enum _POOL_TYPE { NonPagedPool, @@ -114,28 +122,62 @@ GetLibraryProcAddress(PSTR LibraryName, PSTR ProcName) return GetProcAddress(GetModuleHandleA(LibraryName), ProcName); } +void PrintError(LPTSTR lpszFunction) +{ + // Retrieve the system error message for the last-error code + + LPVOID lpMsgBuf; + LPVOID lpDisplayBuf; + DWORD dw = GetLastError(); + + FormatMessage( + FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, + dw, + MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + (LPTSTR) &lpMsgBuf, + 0, NULL ); + + // Display the error message + lpDisplayBuf = (LPVOID)LocalAlloc(LMEM_ZEROINIT, + (lstrlen((LPCTSTR)lpMsgBuf) + lstrlen((LPCTSTR)lpszFunction) + 40) * sizeof(TCHAR)); + StringCchPrintf((LPTSTR)lpDisplayBuf, + LocalSize(lpDisplayBuf) / sizeof(TCHAR), + TEXT("%s failed with error %d: %s"), + lpszFunction, dw, lpMsgBuf); + + wprintf(lpDisplayBuf); + LocalFree(lpMsgBuf); + LocalFree(GetLastError); +} PyObject * psutil_get_open_files(long pid, HANDLE processHandle) { _NtQuerySystemInformation NtQuerySystemInformation = GetLibraryProcAddress("ntdll.dll", "NtQuerySystemInformation"); + _NtQueryObject NtQueryObject = GetLibraryProcAddress("ntdll.dll", "NtQueryObject"); NTSTATUS status; - PSYSTEM_HANDLE_INFORMATION handleInfo; + PSYSTEM_HANDLE_INFORMATION_EX handleInfo; ULONG handleInfoSize = 0x10000; ULONG i; ULONG fileNameLength; PyObject *filesList = Py_BuildValue("[]"); PyObject *arg = NULL; PyObject *fileFromWchar = NULL; + DWORD nReturn = 0; if (filesList == NULL) return NULL; - handleInfo = (PSYSTEM_HANDLE_INFORMATION)malloc(handleInfoSize); + handleInfo = (PSYSTEM_HANDLE_INFORMATION_EX)HeapAlloc(GetProcessHeap(), 0, + handleInfoSize); + if (handleInfo == NULL) { Py_DECREF(filesList); PyErr_NoMemory(); @@ -145,25 +187,27 @@ psutil_get_open_files(long pid, HANDLE processHandle) // NtQuerySystemInformation won't give us the correct buffer size, // so we guess by doubling the buffer size. while ((status = NtQuerySystemInformation( - SystemHandleInformation, + SystemExtendedHandleInformation, handleInfo, handleInfoSize, - NULL + &nReturn )) == STATUS_INFO_LENGTH_MISMATCH) { - handleInfo = (PSYSTEM_HANDLE_INFORMATION) - realloc(handleInfo, handleInfoSize *= 2); + handleInfoSize *=2; + HeapFree(GetProcessHeap(), 0, handleInfo); + handleInfo = (PSYSTEM_HANDLE_INFORMATION_EX)HeapAlloc( + GetProcessHeap(), 0, handleInfoSize); } // NtQuerySystemInformation stopped giving us STATUS_INFO_LENGTH_MISMATCH if (!NT_SUCCESS(status)) { Py_DECREF(filesList); - free(handleInfo); + HeapFree(GetProcessHeap(), 0, handleInfo); return NULL; } - for (i = 0; i < handleInfo->HandleCount; i++) { - SYSTEM_HANDLE handle = handleInfo->Handles[i]; + for (i = 0; i < handleInfo->NumberOfHandles; i++) { + PSYSTEM_HANDLE_TABLE_ENTRY_INFO_EX handle = &handleInfo->Handles[i]; HANDLE dupHandle = NULL; HANDLE mapHandle = NULL; POBJECT_TYPE_INFORMATION objectTypeInfo = NULL; @@ -175,31 +219,33 @@ psutil_get_open_files(long pid, HANDLE processHandle) arg = NULL; // Check if this handle belongs to the PID the user specified. - if (handle.ProcessId != pid) + if (handle->UniqueProcessId != (HANDLE)pid || + handle->ObjectTypeIndex != HANDLE_TYPE_FILE) + { continue; + } // Skip handles with the following access codes as the next call // to NtDuplicateObject() or NtQueryObject() might hang forever. - if ((handle.GrantedAccess == 0x0012019f) - || (handle.GrantedAccess == 0x001a019f) - || (handle.GrantedAccess == 0x00120189) - || (handle.GrantedAccess == 0x00100000)) { + if ((handle->GrantedAccess == 0x0012019f) + || (handle->GrantedAccess == 0x001a019f) + || (handle->GrantedAccess == 0x00120189) + || (handle->GrantedAccess == 0x00100000)) { continue; } - if (!DuplicateHandle(processHandle, - (HANDLE)handle.Handle, + if (!DuplicateHandle(processHandle, + handle->HandleValue, GetCurrentProcess(), &dupHandle, 0, TRUE, DUPLICATE_SAME_ACCESS)) { - //printf("[%#x] Error: %d \n", handle.Handle, GetLastError()); + //printf("[%#x] Error: %d \n", handle->HandleValue, GetLastError()); continue; } - mapHandle = CreateFileMapping(dupHandle, NULL, PAGE_READONLY, @@ -254,7 +300,7 @@ psutil_get_open_files(long pid, HANDLE processHandle) /* printf( "[%#x] %.*S: (could not get name)\n", - handle.Handle, + handle->HandleValue, objectTypeInfo->Name.Length / 2, objectTypeInfo->Name.Buffer ); @@ -300,7 +346,7 @@ psutil_get_open_files(long pid, HANDLE processHandle) /* printf( "[%#x] %.*S: %.*S\n", - handle.Handle, + handle->Handle, objectTypeInfo->Name.Length / 2, objectTypeInfo->Name.Buffer, objectName.Length / 2, @@ -314,7 +360,7 @@ psutil_get_open_files(long pid, HANDLE processHandle) /* printf( "[%#x] %.*S: (unnamed)\n", - handle.Handle, + handle->Handle, objectTypeInfo->Name.Length / 2, objectTypeInfo->Name.Buffer ); @@ -325,7 +371,7 @@ psutil_get_open_files(long pid, HANDLE processHandle) free(objectNameInfo); CloseHandle(dupHandle); } - free(handleInfo); + HeapFree(GetProcessHeap(), 0, handleInfo); CloseHandle(processHandle); return filesList; From 7e103d1ae5a7d99d042a7e99ea0d0a59322bb89f Mon Sep 17 00:00:00 2001 From: Gabi Davar Date: Sat, 7 Feb 2015 23:50:51 +0200 Subject: [PATCH 4/5] fix test_connections failure on py27 in case enum34 and py27-ipaddress are installed --- psutil/_common.py | 4 ++-- test/test_psutil.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/psutil/_common.py b/psutil/_common.py index 07f362a94..2c17a9942 100644 --- a/psutil/_common.py +++ b/psutil/_common.py @@ -172,7 +172,7 @@ def sockfam_to_enum(num): return num try: return socket.AddressFamily(num) - except ValueError: + except (ValueError, AttributeError): return num @@ -184,7 +184,7 @@ def socktype_to_enum(num): return num try: return socket.AddressType(num) - except ValueError: + except (ValueError, AttributeError): return num diff --git a/test/test_psutil.py b/test/test_psutil.py index 14fc04c40..8c31b57c1 100644 --- a/test/test_psutil.py +++ b/test/test_psutil.py @@ -286,20 +286,20 @@ def reap_children(search_all=False): def check_ip_address(addr, family): """Attempts to check IP address's validity.""" - if enum is not None: + if enum and PY3: assert isinstance(family, enum.IntEnum), family if family == AF_INET: octs = [int(x) for x in addr.split('.')] assert len(octs) == 4, addr for num in octs: assert 0 <= num <= 255, addr - if ipaddress is not None: + if ipaddress: if not PY3: addr = unicode(addr) ipaddress.IPv4Address(addr) elif family == AF_INET6: assert isinstance(addr, str), addr - if ipaddress is not None: + if ipaddress: if not PY3: addr = unicode(addr) ipaddress.IPv6Address(addr) From 2f7a4039a0a55a263835b3660c23a360ee8ecf88 Mon Sep 17 00:00:00 2001 From: Gabi Davar Date: Sat, 7 Feb 2015 23:52:24 +0200 Subject: [PATCH 5/5] make tests friendler to auto test discoverers. --- test/_windows.py | 4 ++-- test/test_psutil.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/_windows.py b/test/_windows.py index 44546a947..969cc894d 100644 --- a/test/_windows.py +++ b/test/_windows.py @@ -370,7 +370,7 @@ def test_zombies(self): self.assertRaises(psutil.NoSuchProcess, meth, ZOMBIE_PID) -def test_main(): +def main(): test_suite = unittest.TestSuite() test_suite.addTest(unittest.makeSuite(WindowsSpecificTestCase)) test_suite.addTest(unittest.makeSuite(TestDualProcessImplementation)) @@ -378,5 +378,5 @@ def test_main(): return result.wasSuccessful() if __name__ == '__main__': - if not test_main(): + if not main(): sys.exit(1) diff --git a/test/test_psutil.py b/test/test_psutil.py index 8c31b57c1..5a9827b6e 100644 --- a/test/test_psutil.py +++ b/test/test_psutil.py @@ -46,6 +46,7 @@ import ipaddress # python >= 3.3 except ImportError: ipaddress = None + try: import enum # python >= 3.4 except ImportError: @@ -2692,7 +2693,7 @@ def test_pidof(self): self.assertIn(str(os.getpid()), output) -def test_main(): +def main(): tests = [] test_suite = unittest.TestSuite() tests.append(TestSystemAPIs) @@ -2729,5 +2730,5 @@ def test_main(): return result.wasSuccessful() if __name__ == '__main__': - if not test_main(): + if not main(): sys.exit(1)