From 3796a4ccc661cd286e428bd84d3e1471c5e92bcc Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 26 Sep 2019 12:06:32 +0100 Subject: [PATCH 001/148] bpo-16575: Add checks for unions passed by value to functions. --- Lib/ctypes/test/test_structures.py | 47 ++++++++++++++++++++++++++++++ Modules/_ctypes/_ctypes.c | 19 ++++++++++++ Modules/_ctypes/_ctypes_test.c | 38 ++++++++++++++++++++++++ Modules/_ctypes/ctypes.h | 2 ++ Modules/_ctypes/stgdict.c | 8 +++++ 5 files changed, 114 insertions(+) diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 0c01c799042cc5..9eb58ed1ddb45e 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -525,6 +525,53 @@ class Test3(Structure): self.assertEqual(s.data[0], 3.14159) self.assertEqual(s.data[1], 2.71828) + def test_union_by_value(self): + # See bpo-16575 + + # These should mirror the structures in Modules/_ctypes/_ctypes_test.c + + class Nested1(Structure): + _fields = [ + ('an_int', c_int), + ('another_int', c_int), + ] + + class Test4(Union): + _fields_ = [ + ('a_long', c_long), + ('a_struct', Nested1), + ] + + class Nested2(Structure): + _fields = [ + ('an_int', c_int), + ('a_union', Test4), + ] + + class Test5(Structure): + _fields = [ + ('an_int', c_int), + ('nested', Nested2), + ] + + test4 = Test4() + dll = CDLL(_ctypes_test.__file__) + with self.assertRaises(TypeError) as ctx: + func = dll._testfunc_union_by_value1 + func.restype = c_long + func.argtypes = (Test4,) + result = func(test4) + self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' + 'a union by value, which is unsupported.') + test5 = Test5() + with self.assertRaises(TypeError) as ctx: + func = dll._testfunc_union_by_value2 + func.restype = c_long + func.argtypes = (Test5,) + result = func(test5) + self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' + 'a union by value, which is unsupported.') + class PointerMemberTestCase(unittest.TestCase): def test(self): diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 16a0cfe8dd4dc7..986e3b1a861e41 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2383,6 +2383,25 @@ converters_from_argtypes(PyObject *ob) for (i = 0; i < nArgs; ++i) { PyObject *cnv; PyObject *tp = PyTuple_GET_ITEM(ob, i); + StgDictObject *stgdict = PyType_stgdict(tp); + + if (stgdict != NULL) { + if (stgdict->flags & TYPEFLAG_HASUNION) { + Py_DECREF(converters); + Py_DECREF(ob); + if (!PyErr_Occurred()) { + PyErr_Format(PyExc_TypeError, + "item %zd in _argtypes_ passes a union by " + "value, which is unsupported.", + i + 1); + } + return NULL; + } + if (stgdict->flags & TYPEFLAG_HASBITFIELD) { + printf("found stgdict with bitfield\n"); + } + } + if (_PyObject_LookupAttrId(tp, &PyId_from_param, &cnv) <= 0) { Py_DECREF(converters); Py_DECREF(ob); diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 10f6591d24cd71..f495f2d58b853a 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -114,6 +114,44 @@ _testfunc_array_in_struct2(Test3 in) return result; } +typedef union { + long a_long; + struct { + int an_int; + int another_int; + } a_struct; +} Test4; + +typedef struct { + int an_int; + struct { + int an_int; + Test4 a_union; + } nested; +} Test5; + +EXPORT(long) +_testfunc_union_by_value1(Test4 in) { + long result = in.a_long + in.a_struct.an_int + in.a_struct.another_int; + + /* As the union/struct are passed by value, changes to them shouldn't be + * reflected in the caller. + */ + memset(&in, 0, sizeof(in)); + return result; +} + +EXPORT(long) +_testfunc_union_by_value2(Test5 in) { + long result = in.an_int + in.nested.an_int; + + /* As the union/struct are passed by value, changes to them shouldn't be + * reflected in the caller. + */ + memset(&in, 0, sizeof(in)); + return result; +} + EXPORT(void)testfunc_array(int values[4]) { printf("testfunc_array %d %d %d %d\n", diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 5d3b9663385f59..e58f85233cb71e 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -288,6 +288,8 @@ PyObject *_ctypes_callproc(PPROC pProc, #define TYPEFLAG_ISPOINTER 0x100 #define TYPEFLAG_HASPOINTER 0x200 +#define TYPEFLAG_HASUNION 0x400 +#define TYPEFLAG_HASBITFIELD 0x800 #define DICTFLAG_FINAL 0x1000 diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 947e9c25592fbd..6166345fb779b5 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -440,6 +440,13 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct PyMem_Free(stgdict->ffi_type_pointer.elements); basedict = PyType_stgdict((PyObject *)((PyTypeObject *)type)->tp_base); + if (basedict) { + stgdict->flags |= (basedict->flags & + (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD)); + } + if (!isStruct) { + stgdict->flags |= TYPEFLAG_HASUNION; + } if (basedict && !use_broken_old_ctypes_semantics) { size = offset = basedict->size; align = basedict->align; @@ -517,6 +524,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct stgdict->flags |= TYPEFLAG_HASPOINTER; dict->flags |= DICTFLAG_FINAL; /* mark field type final */ if (PyTuple_Size(pair) == 3) { /* bits specified */ + stgdict->flags |= TYPEFLAG_HASBITFIELD; switch(dict->ffi_type_pointer.type) { case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: From f6f496a2a4a2a61949917346aee9def31ccdae11 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 26 Sep 2019 16:57:32 +0100 Subject: [PATCH 002/148] Updated the logic for propagation of flags. --- Modules/_ctypes/_ctypes.c | 3 +++ Modules/_ctypes/stgdict.c | 1 + 2 files changed, 4 insertions(+) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 986e3b1a861e41..1290daa6cdfc52 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -504,6 +504,9 @@ StructUnionType_new(PyTypeObject *type, PyObject *args, PyObject *kwds, int isSt Py_DECREF(result); return NULL; } + if (!isStruct) { + dict->flags |= TYPEFLAG_HASUNION; + } /* replace the class dict by our updated stgdict, which holds info about storage requirements of the instances */ if (-1 == PyDict_Update((PyObject *)dict, result->tp_dict)) { diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 6166345fb779b5..e85fcdf54f63ab 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -522,6 +522,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct stgdict->ffi_type_pointer.elements[ffi_ofs + i] = &dict->ffi_type_pointer; if (dict->flags & (TYPEFLAG_ISPOINTER | TYPEFLAG_HASPOINTER)) stgdict->flags |= TYPEFLAG_HASPOINTER; + stgdict->flags |= dict->flags & (TYPEFLAG_HASUNION | TYPEFLAG_HASBITFIELD); dict->flags |= DICTFLAG_FINAL; /* mark field type final */ if (PyTuple_Size(pair) == 3) { /* bits specified */ stgdict->flags |= TYPEFLAG_HASBITFIELD; From dc31b8ff145196336aafa686eae39694a314cbf9 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 26 Sep 2019 16:58:47 +0100 Subject: [PATCH 003/148] Corrected typos in test. --- Lib/ctypes/test/test_structures.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 9eb58ed1ddb45e..ce6ff51330a932 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -531,7 +531,7 @@ def test_union_by_value(self): # These should mirror the structures in Modules/_ctypes/_ctypes_test.c class Nested1(Structure): - _fields = [ + _fields_ = [ ('an_int', c_int), ('another_int', c_int), ] @@ -543,13 +543,13 @@ class Test4(Union): ] class Nested2(Structure): - _fields = [ + _fields_ = [ ('an_int', c_int), ('a_union', Test4), ] class Test5(Structure): - _fields = [ + _fields_ = [ ('an_int', c_int), ('nested', Nested2), ] From b92f55ac69de1c075b2338133d34e793c6353197 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 26 Sep 2019 17:09:18 +0100 Subject: [PATCH 004/148] Removed debugging code. --- Modules/_ctypes/_ctypes.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 1290daa6cdfc52..876fc85f63bc32 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -2400,9 +2400,6 @@ converters_from_argtypes(PyObject *ob) } return NULL; } - if (stgdict->flags & TYPEFLAG_HASBITFIELD) { - printf("found stgdict with bitfield\n"); - } } if (_PyObject_LookupAttrId(tp, &PyId_from_param, &cnv) <= 0) { From 01ba08eb9050e6895688db35b6e6a34e5e7abae0 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 26 Sep 2019 17:40:24 +0100 Subject: [PATCH 005/148] Improved tests. --- Lib/ctypes/test/test_structures.py | 33 ++++++++++++++++++++++++++++++ Modules/_ctypes/_ctypes_test.c | 25 ++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index ce6ff51330a932..a411abaf57c76a 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -552,6 +552,7 @@ class Test5(Structure): _fields_ = [ ('an_int', c_int), ('nested', Nested2), + ('another_int', c_int), ] test4 = Test4() @@ -572,6 +573,38 @@ class Test5(Structure): self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' 'a union by value, which is unsupported.') + # passing by reference should be OK + test4.a_long = 12345; + func = dll._testfunc_union_by_reference1 + func.restype = c_long + func.argtypes = (POINTER(Test4),) + result = func(byref(test4)) + self.assertEqual(result, 12345) + self.assertEqual(test4.a_long, 0) + self.assertEqual(test4.a_struct.an_int, 0) + self.assertEqual(test4.a_struct.another_int, 0) + test4.a_struct.an_int = 0x12340000 + test4.a_struct.another_int = 0x5678 + func = dll._testfunc_union_by_reference2 + func.restype = c_long + func.argtypes = (POINTER(Test4),) + result = func(byref(test4)) + self.assertEqual(result, 0x12345678) + self.assertEqual(test4.a_long, 0) + self.assertEqual(test4.a_struct.an_int, 0) + self.assertEqual(test4.a_struct.another_int, 0) + test5.an_int = 0x12000000 + test5.nested.an_int = 0x345600 + test5.another_int = 0x78 + func = dll._testfunc_union_by_reference3 + func.restype = c_long + func.argtypes = (POINTER(Test5),) + result = func(byref(test5)) + self.assertEqual(result, 0x12345678) + self.assertEqual(test5.an_int, 0) + self.assertEqual(test5.nested.an_int, 0) + self.assertEqual(test5.another_int, 0) + class PointerMemberTestCase(unittest.TestCase): def test(self): diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index f495f2d58b853a..75ebc25e151b29 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -128,6 +128,7 @@ typedef struct { int an_int; Test4 a_union; } nested; + int another_int; } Test5; EXPORT(long) @@ -152,6 +153,30 @@ _testfunc_union_by_value2(Test5 in) { return result; } +EXPORT(long) +_testfunc_union_by_reference1(Test4 *in) { + long result = in->a_long; + + memset(in, 0, sizeof(Test4)); + return result; +} + +EXPORT(long) +_testfunc_union_by_reference2(Test4 *in) { + long result = in->a_struct.an_int + in->a_struct.another_int; + + memset(in, 0, sizeof(Test4)); + return result; +} + +EXPORT(long) +_testfunc_union_by_reference3(Test5 *in) { + long result = in->an_int + in->nested.an_int + in->another_int; + + memset(in, 0, sizeof(Test5)); + return result; +} + EXPORT(void)testfunc_array(int values[4]) { printf("testfunc_array %d %d %d %d\n", From 0bcbfa43d55d9558cdcb256d8998366281322080 Mon Sep 17 00:00:00 2001 From: Michael Felt Date: Thu, 26 Sep 2019 20:43:15 +0100 Subject: [PATCH 006/148] bpo-28009: Fix uuid.uuid1() and uuid.get_node() on AIX (GH-8672) --- Lib/test/test_uuid.py | 71 +++++-- Lib/uuid.py | 195 +++++++++++------- .../2018-08-04-12-26-11.bpo-28009.4JcHZb.rst | 4 + 3 files changed, 177 insertions(+), 93 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 92642d239b92cd..ddf7e6dc1b8e45 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,4 +1,4 @@ -import unittest.mock +import unittest from test import support import builtins import contextlib @@ -15,7 +15,6 @@ py_uuid = support.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = support.import_fresh_module('uuid', fresh=['_uuid']) - def importable(name): try: __import__(name) @@ -459,7 +458,7 @@ def test_uuid1_eui64(self): # uuid.getnode to fall back on uuid._random_getnode, which will # generate a valid value. too_large_getter = lambda: 1 << 48 - with unittest.mock.patch.multiple( + with mock.patch.multiple( self.uuid, _node=None, # Ignore any cached node value. _GETTERS=[too_large_getter], @@ -538,8 +537,8 @@ def mock_generate_time_safe(self, safe_value): f = self.uuid._generate_time_safe if f is None: self.skipTest('need uuid._generate_time_safe') - with unittest.mock.patch.object(self.uuid, '_generate_time_safe', - lambda: (f()[0], safe_value)): + with mock.patch.object(self.uuid, '_generate_time_safe', + lambda: (f()[0], safe_value)): yield @unittest.skipUnless(os.name == 'posix', 'POSIX-only test') @@ -674,27 +673,57 @@ class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): class BaseTestInternals: _uuid = py_uuid - @unittest.skipUnless(os.name == 'posix', 'requires Posix') - def test_find_mac(self): + + def test_find_under_heading(self): + data = '''\ +Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll +en0 1500 link#2 fe.ad.c.1.23.4 1714807956 0 711348489 0 0 + 01:00:5e:00:00:01 +en0 1500 192.168.129 x071 1714807956 0 711348489 0 0 + 224.0.0.1 +en0 1500 192.168.90 x071 1714807956 0 711348489 0 0 + 224.0.0.1 +''' + + def mock_get_command_stdout(command, args): + return io.BytesIO(data.encode()) + + # The above data is from AIX - with '.' as _MAC_DELIM and strings + # shorter than 17 bytes (no leading 0). (_MAC_OMITS_LEADING_ZEROES=True) + with mock.patch.multiple(self.uuid, + _MAC_DELIM=b'.', + _MAC_OMITS_LEADING_ZEROES=True, + _get_command_stdout=mock_get_command_stdout): + mac = self.uuid._find_mac_under_heading( + command='netstat', + args='-ian', + heading=b'Address', + ) + + self.assertEqual(mac, 0xfead0c012304) + + def test_find_mac_near_keyword(self): + # key and value are on the same line data = ''' -fake hwaddr +fake Link encap:UNSPEC hwaddr 00-00 cscotun0 Link encap:UNSPEC HWaddr 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00 eth0 Link encap:Ethernet HWaddr 12:34:56:78:90:ab ''' - popen = unittest.mock.MagicMock() - popen.stdout = io.BytesIO(data.encode()) - - with unittest.mock.patch.object(shutil, 'which', - return_value='/sbin/ifconfig'): - with unittest.mock.patch.object(subprocess, 'Popen', - return_value=popen): - mac = self.uuid._find_mac( - command='ifconfig', - args='', - hw_identifiers=[b'hwaddr'], - get_index=lambda x: x + 1, - ) + def mock_get_command_stdout(command, args): + return io.BytesIO(data.encode()) + + # The above data will only be parsed properly on non-AIX unixes. + with mock.patch.multiple(self.uuid, + _MAC_DELIM=b':', + _MAC_OMITS_LEADING_ZEROES=False, + _get_command_stdout=mock_get_command_stdout): + mac = self.uuid._find_mac_near_keyword( + command='ifconfig', + args='', + keywords=[b'hwaddr'], + get_word_index=lambda x: x + 1, + ) self.assertEqual(mac, 0x1234567890ab) diff --git a/Lib/uuid.py b/Lib/uuid.py index 5f3bc9e8de44c6..6a436d371a3fd2 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -59,6 +59,12 @@ _LINUX = platform.system() == 'Linux' _WINDOWS = platform.system() == 'Windows' +_MAC_DELIM = b':' +_MAC_OMITS_LEADING_ZEROES = False +if _AIX: + _MAC_DELIM = b'.' + _MAC_OMITS_LEADING_ZEROES = True + RESERVED_NCS, RFC_4122, RESERVED_MICROSOFT, RESERVED_FUTURE = [ 'reserved for NCS compatibility', 'specified in RFC 4122', 'reserved for Microsoft compatibility', 'reserved for future definition'] @@ -347,24 +353,32 @@ def version(self): if self.variant == RFC_4122: return int((self.int >> 76) & 0xf) -def _popen(command, *args): - import os, shutil, subprocess - executable = shutil.which(command) - if executable is None: - path = os.pathsep.join(('/sbin', '/usr/sbin')) - executable = shutil.which(command, path=path) + +def _get_command_stdout(command, *args): + import io, os, shutil, subprocess + + try: + path_dirs = os.environ.get('PATH', os.defpath).split(os.pathsep) + path_dirs.extend(['/sbin', '/usr/sbin']) + executable = shutil.which(command, path=os.pathsep.join(path_dirs)) if executable is None: return None - # LC_ALL=C to ensure English output, stderr=DEVNULL to prevent output - # on stderr (Note: we don't have an example where the words we search - # for are actually localized, but in theory some system could do so.) - env = dict(os.environ) - env['LC_ALL'] = 'C' - proc = subprocess.Popen((executable,) + args, - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - env=env) - return proc + # LC_ALL=C to ensure English output, stderr=DEVNULL to prevent output + # on stderr (Note: we don't have an example where the words we search + # for are actually localized, but in theory some system could do so.) + env = dict(os.environ) + env['LC_ALL'] = 'C' + proc = subprocess.Popen((executable,) + args, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + env=env) + if not proc: + return None + stdout, stderr = proc.communicate() + return io.BytesIO(stdout) + except (OSError, subprocess.SubprocessError): + return None + # For MAC (a.k.a. IEEE 802, or EUI-48) addresses, the second least significant # bit of the first octet signifies whether the MAC address is universally (0) @@ -384,40 +398,101 @@ def _popen(command, *args): def _is_universal(mac): return not (mac & (1 << 41)) -def _find_mac(command, args, hw_identifiers, get_index): + +def _find_mac_near_keyword(command, args, keywords, get_word_index): + """Searches a command's output for a MAC address near a keyword. + + Each line of words in the output is case-insensitively searched for + any of the given keywords. Upon a match, get_word_index is invoked + to pick a word from the line, given the index of the match. For + example, lambda i: 0 would get the first word on the line, while + lambda i: i - 1 would get the word preceding the keyword. + """ + stdout = _get_command_stdout(command, args) + if stdout is None: + return None + first_local_mac = None + for line in stdout: + words = line.lower().rstrip().split() + for i in range(len(words)): + if words[i] in keywords: + try: + word = words[get_word_index(i)] + mac = int(word.replace(_MAC_DELIM, b''), 16) + except (ValueError, IndexError): + # Virtual interfaces, such as those provided by + # VPNs, do not have a colon-delimited MAC address + # as expected, but a 16-byte HWAddr separated by + # dashes. These should be ignored in favor of a + # real MAC address + pass + else: + if _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac + return first_local_mac or None + + +def _find_mac_under_heading(command, args, heading): + """Looks for a MAC address under a heading in a command's output. + + The first line of words in the output is searched for the given + heading. Words at the same word index as the heading in subsequent + lines are then examined to see if they look like MAC addresses. + """ + stdout = _get_command_stdout(command, args) + if stdout is None: + return None + + keywords = stdout.readline().rstrip().split() try: - proc = _popen(command, *args.split()) - if not proc: - return None - with proc: - for line in proc.stdout: - words = line.lower().rstrip().split() - for i in range(len(words)): - if words[i] in hw_identifiers: - try: - word = words[get_index(i)] - mac = int(word.replace(b':', b''), 16) - if _is_universal(mac): - return mac - first_local_mac = first_local_mac or mac - except (ValueError, IndexError): - # Virtual interfaces, such as those provided by - # VPNs, do not have a colon-delimited MAC address - # as expected, but a 16-byte HWAddr separated by - # dashes. These should be ignored in favor of a - # real MAC address - pass - except OSError: - pass + column_index = keywords.index(heading) + except ValueError: + return None + + first_local_mac = None + for line in stdout: + try: + words = line.rstrip().split() + word = words[column_index] + if len(word) == 17: + mac = int(word.replace(_MAC_DELIM, b''), 16) + elif _MAC_OMITS_LEADING_ZEROES: + # (Only) on AIX the macaddr value given is not prefixed by 0, e.g. + # en0 1500 link#2 fa.bc.de.f7.62.4 110854824 0 160133733 0 0 + # not + # en0 1500 link#2 fa.bc.de.f7.62.04 110854824 0 160133733 0 0 + parts = word.split(_MAC_DELIM) + if len(parts) == 6 and all(0 < len(p) <= 2 for p in parts): + hexstr = b''.join(p.rjust(2, b'0') for p in parts) + mac = int(hexstr, 16) + else: + continue + else: + continue + except (ValueError, IndexError): + # Virtual interfaces, such as those provided by + # VPNs, do not have a colon-delimited MAC address + # as expected, but a 16-byte HWAddr separated by + # dashes. These should be ignored in favor of a + # real MAC address + pass + else: + if _is_universal(mac): + return mac + first_local_mac = first_local_mac or mac return first_local_mac or None + +# The following functions call external programs to 'get' a macaddr value to +# be used as basis for an uuid def _ifconfig_getnode(): """Get the hardware address on Unix by running ifconfig.""" # This works on Linux ('' or '-a'), Tru64 ('-av'), but not all Unixes. keywords = (b'hwaddr', b'ether', b'address:', b'lladdr') for args in ('', '-a', '-av'): - mac = _find_mac('ifconfig', args, keywords, lambda i: i+1) + mac = _find_mac_near_keyword('ifconfig', args, keywords, lambda i: i+1) if mac: return mac return None @@ -425,7 +500,7 @@ def _ifconfig_getnode(): def _ip_getnode(): """Get the hardware address on Unix by running ip.""" # This works on Linux with iproute2. - mac = _find_mac('ip', 'link', [b'link/ether'], lambda i: i+1) + mac = _find_mac_near_keyword('ip', 'link', [b'link/ether'], lambda i: i+1) if mac: return mac return None @@ -439,17 +514,17 @@ def _arp_getnode(): return None # Try getting the MAC addr from arp based on our IP address (Solaris). - mac = _find_mac('arp', '-an', [os.fsencode(ip_addr)], lambda i: -1) + mac = _find_mac_near_keyword('arp', '-an', [os.fsencode(ip_addr)], lambda i: -1) if mac: return mac # This works on OpenBSD - mac = _find_mac('arp', '-an', [os.fsencode(ip_addr)], lambda i: i+1) + mac = _find_mac_near_keyword('arp', '-an', [os.fsencode(ip_addr)], lambda i: i+1) if mac: return mac # This works on Linux, FreeBSD and NetBSD - mac = _find_mac('arp', '-an', [os.fsencode('(%s)' % ip_addr)], + mac = _find_mac_near_keyword('arp', '-an', [os.fsencode('(%s)' % ip_addr)], lambda i: i+2) # Return None instead of 0. if mac: @@ -459,36 +534,12 @@ def _arp_getnode(): def _lanscan_getnode(): """Get the hardware address on Unix by running lanscan.""" # This might work on HP-UX. - return _find_mac('lanscan', '-ai', [b'lan0'], lambda i: 0) + return _find_mac_near_keyword('lanscan', '-ai', [b'lan0'], lambda i: 0) def _netstat_getnode(): """Get the hardware address on Unix by running netstat.""" - # This might work on AIX, Tru64 UNIX. - first_local_mac = None - try: - proc = _popen('netstat', '-ia') - if not proc: - return None - with proc: - words = proc.stdout.readline().rstrip().split() - try: - i = words.index(b'Address') - except ValueError: - return None - for line in proc.stdout: - try: - words = line.rstrip().split() - word = words[i] - if len(word) == 17 and word.count(b':') == 5: - mac = int(word.replace(b':', b''), 16) - if _is_universal(mac): - return mac - first_local_mac = first_local_mac or mac - except (ValueError, IndexError): - pass - except OSError: - pass - return first_local_mac or None + # This works on AIX and might work on Tru64 UNIX. + return _find_mac_under_heading('netstat', '-ian', b'Address') def _ipconfig_getnode(): """Get the hardware address on Windows by running ipconfig.exe.""" diff --git a/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst new file mode 100644 index 00000000000000..40f638234a4586 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-08-04-12-26-11.bpo-28009.4JcHZb.rst @@ -0,0 +1,4 @@ +Fix uuid.getnode() on platforms with '.' as MAC Addr delimiter as well +fix for MAC Addr format that omits a leading 0 in MAC Addr values. +Currently, AIX is the only know platform with these settings. +Patch by Michael Felt. From 6ce03ec627680ce0829a5b3067fab7faed21b533 Mon Sep 17 00:00:00 2001 From: HongWeipeng <961365124@qq.com> Date: Fri, 27 Sep 2019 15:54:26 +0800 Subject: [PATCH 007/148] cleanup ababstractproperty in typing.py (GH-16432) --- Lib/typing.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index b1ac33e00e7058..2c75a76964873b 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -17,7 +17,7 @@ * Wrapper submodules for re and io related types. """ -from abc import abstractmethod, abstractproperty, ABCMeta +from abc import abstractmethod, ABCMeta import collections import collections.abc import contextlib @@ -1794,11 +1794,13 @@ class IO(Generic[AnyStr]): __slots__ = () - @abstractproperty + @property + @abstractmethod def mode(self) -> str: pass - @abstractproperty + @property + @abstractmethod def name(self) -> str: pass @@ -1894,23 +1896,28 @@ class TextIO(IO[str]): __slots__ = () - @abstractproperty + @property + @abstractmethod def buffer(self) -> BinaryIO: pass - @abstractproperty + @property + @abstractmethod def encoding(self) -> str: pass - @abstractproperty + @property + @abstractmethod def errors(self) -> Optional[str]: pass - @abstractproperty + @property + @abstractmethod def line_buffering(self) -> bool: pass - @abstractproperty + @property + @abstractmethod def newlines(self) -> Any: pass From 5faff977adbe089e1f91a5916ccb2160a22dd292 Mon Sep 17 00:00:00 2001 From: Ammar Askar Date: Fri, 27 Sep 2019 07:11:27 -0400 Subject: [PATCH 008/148] bpo-38206: Clarify tp_dealloc requirements for heap allocated types. (GH-16248) As mentioned in the bpo ticket, this mistake came up on two reviews: - https://github.com/python/cpython/pull/16127#pullrequestreview-288312751 - https://github.com/python/cpython/pull/16071#pullrequestreview-287819525 Would be nice to have it documented in a more permanent place than 3.8's whatsnew entry. https://bugs.python.org/issue38206 Automerge-Triggered-By: @encukou --- Doc/c-api/type.rst | 3 ++- Doc/c-api/typeobj.rst | 24 ++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 36309dd5254a3a..73311ec083a852 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -118,7 +118,8 @@ The following functions and structs are used to create .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) - Creates and returns a heap type object from the *spec*. + Creates and returns a heap type object from the *spec* + (:const:`Py_TPFLAGS_HEAPTYPE`). If *bases* is a tuple, the created heap type contains all types contained in it as base types. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 7c7a79129ccca2..bb767500743098 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -654,9 +654,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) the instance is still in existence, but there are no references to it. The destructor function should free all references which the instance owns, free all memory buffers owned by the instance (using the freeing function corresponding - to the allocation function used to allocate the buffer), and finally (as its - last action) call the type's :c:member:`~PyTypeObject.tp_free` function. If the type is not - subtypable (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is + to the allocation function used to allocate the buffer), and call the type's + :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable + (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is permissible to call the object deallocator directly instead of via :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the instance; this is normally :c:func:`PyObject_Del` if the instance was allocated @@ -664,6 +664,21 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:func:`PyObject_GC_Del` if the instance was allocated using :c:func:`PyObject_GC_New` or :c:func:`PyObject_GC_NewVar`. + Finally, if the type is heap allocated (:const:`Py_TPFLAGS_HEAPTYPE`), the + deallocator should decrement the reference count for its type object after + calling the type deallocator. In order to avoid dangling pointers, the + recommended way to achieve this is: + + .. code-block:: c + + static void foo_dealloc(foo_object *self) { + PyTypeObject *tp = Py_TYPE(self); + // free references and buffers here + tp->tp_free(self); + Py_DECREF(tp); + } + + **Inheritance:** This field is inherited by subtypes. @@ -1021,7 +1036,8 @@ and :c:type:`PyType_Type` effectively act as defaults.) .. data:: Py_TPFLAGS_HEAPTYPE - This bit is set when the type object itself is allocated on the heap. In this + This bit is set when the type object itself is allocated on the heap, for + example, types created dynamically using :c:func:`PyType_FromSpec`. In this case, the :attr:`ob_type` field of its instances is considered a reference to the type, and the type object is INCREF'ed when a new instance is created, and DECREF'ed when an instance is destroyed (this does not apply to instances of From 90558158093c0ad893102158fd3c2dd9f864e82e Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Fri, 27 Sep 2019 15:03:53 +0200 Subject: [PATCH 009/148] bpo-38270: More fixes for strict crypto policy (GH-16418) test_hmac and test_hashlib test built-in hashing implementations and OpenSSL-based hashing implementations. Add more checks to skip OpenSSL implementations when a strict crypto policy is active. Use EVP_DigestInit_ex() instead of EVP_DigestInit() to initialize the EVP context. The EVP_DigestInit() function clears alls flags and breaks usedforsecurity flag again. Signed-off-by: Christian Heimes https://bugs.python.org/issue38270 --- Lib/test/support/__init__.py | 20 ++++++++++++++++---- Lib/test/test_hashlib.py | 14 +++++++++++++- Lib/test/test_hmac.py | 12 ++++++------ Modules/_hashopenssl.c | 2 +- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index e401090c214681..d593fc18d96e17 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -69,6 +69,11 @@ except ImportError: resource = None +try: + import _hashlib +except ImportError: + _hashlib = None + __all__ = [ # globals "PIPE_MAX_SIZE", "verbose", "max_memuse", "use_resources", "failfast", @@ -86,8 +91,8 @@ "create_empty_file", "can_symlink", "fs_is_case_insensitive", # unittest "is_resource_enabled", "requires", "requires_freebsd_version", - "requires_linux_version", "requires_mac_ver", "check_syntax_error", - "check_syntax_warning", + "requires_linux_version", "requires_mac_ver", "requires_hashdigest", + "check_syntax_error", "check_syntax_warning", "TransientResource", "time_out", "socket_peer_reset", "ioerror_peer_reset", "transient_internet", "BasicTestRunner", "run_unittest", "run_doctest", "skip_unless_symlink", "requires_gzip", "requires_bz2", "requires_lzma", @@ -649,12 +654,16 @@ def wrapper(*args, **kw): return decorator -def requires_hashdigest(digestname): +def requires_hashdigest(digestname, openssl=None, usedforsecurity=True): """Decorator raising SkipTest if a hashing algorithm is not available The hashing algorithm could be missing or blocked by a strict crypto policy. + If 'openssl' is True, then the decorator checks that OpenSSL provides + the algorithm. Otherwise the check falls back to built-in + implementations. The usedforsecurity flag is passed to the constructor. + ValueError: [digital envelope routines: EVP_DigestInit_ex] disabled for FIPS ValueError: unsupported hash type md4 """ @@ -662,7 +671,10 @@ def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): try: - hashlib.new(digestname) + if openssl and _hashlib is not None: + _hashlib.new(digestname, usedforsecurity=usedforsecurity) + else: + hashlib.new(digestname, usedforsecurity=usedforsecurity) except ValueError: raise unittest.SkipTest( f"hash digest '{digestname}' is not available." diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index d55de02f91645d..0e30b2fb11f29c 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -8,6 +8,7 @@ import array from binascii import unhexlify +import functools import hashlib import importlib import itertools @@ -18,6 +19,7 @@ import warnings from test import support from test.support import _4G, bigmemtest, import_fresh_module +from test.support import requires_hashdigest from http.client import HTTPException # Were we compiled --with-pydebug or with #define Py_DEBUG? @@ -119,6 +121,7 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): constructors.add(_test_algorithm_via_hashlib_new) _hashlib = self._conditional_import_module('_hashlib') + self._hashlib = _hashlib if _hashlib: # These two algorithms should always be present when this module # is compiled. If not, something was compiled wrong. @@ -127,7 +130,13 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): for algorithm, constructors in self.constructors_to_test.items(): constructor = getattr(_hashlib, 'openssl_'+algorithm, None) if constructor: - constructors.add(constructor) + try: + constructor() + except ValueError: + # default constructor blocked by crypto policy + pass + else: + constructors.add(constructor) def add_builtin_constructor(name): constructor = getattr(hashlib, "__get_builtin_constructor")(name) @@ -193,6 +202,9 @@ def test_usedforsecurity(self): cons(b'', usedforsecurity=False) hashlib.new("sha256", usedforsecurity=True) hashlib.new("sha256", usedforsecurity=False) + if self._hashlib is not None: + self._hashlib.new("md5", usedforsecurity=False) + self._hashlib.openssl_md5(usedforsecurity=False) def test_unknown_hash(self): self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 2c09de8b26fa07..1bbf201727d7a8 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -21,7 +21,7 @@ def wrapper(*args, **kwargs): class TestVectorsTestCase(unittest.TestCase): - @requires_hashdigest('md5') + @requires_hashdigest('md5', openssl=True) def test_md5_vectors(self): # Test the HMAC module against test vectors from the RFC. @@ -79,7 +79,7 @@ def md5test(key, data, digest): b"and Larger Than One Block-Size Data"), "6f630fad67cda0ee1fb1f562db3aa53e") - @requires_hashdigest('sha1') + @requires_hashdigest('sha1', openssl=True) def test_sha_vectors(self): def shatest(key, data, digest): h = hmac.HMAC(key, data, digestmod=hashlib.sha1) @@ -272,19 +272,19 @@ def hmactest(key, data, hexdigests): '134676fb6de0446065c97440fa8c6a58', }) - @requires_hashdigest('sha224') + @requires_hashdigest('sha224', openssl=True) def test_sha224_rfc4231(self): self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64) - @requires_hashdigest('sha256') + @requires_hashdigest('sha256', openssl=True) def test_sha256_rfc4231(self): self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64) - @requires_hashdigest('sha384') + @requires_hashdigest('sha384', openssl=True) def test_sha384_rfc4231(self): self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128) - @requires_hashdigest('sha512') + @requires_hashdigest('sha512', openssl=True) def test_sha512_rfc4231(self): self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 48511f72bc6350..b147dbe8b3c55d 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -552,7 +552,7 @@ EVPnew(const EVP_MD *digest, } - if (!EVP_DigestInit(self->ctx, digest)) { + if (!EVP_DigestInit_ex(self->ctx, digest, NULL)) { _setException(PyExc_ValueError); Py_DECREF(self); return NULL; From 6693f730e0eb77d9453f73a3da33b78a97e996ee Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 27 Sep 2019 15:53:34 +0100 Subject: [PATCH 010/148] bpo-38187: Fix a refleak in Tools/c-analyzer. (gh-16304) The "Slot" helper (descriptor) is leaking references due to its caching mechanism. The change includes a partial fix to Slot, but also adds Variable.storage to replace the problematic use of Slot. https://bugs.python.org/issue38187 --- .../test_c_analyzer_common/__init__.py | 6 + .../test_c_analyzer_common/test_known.py | 3 - .../test_c_globals/__init__.py | 6 + .../test_c_globals/test_find.py | 5 +- .../test_c_analyzer/test_c_parser/__init__.py | 6 + .../test_c_parser/test_info.py | 108 ++++++++++++------ .../test_c_symbols/__init__.py | 6 + .../c-analyzer/c_analyzer_common/_generate.py | 4 +- Tools/c-analyzer/c_analyzer_common/known.py | 53 +++++---- Tools/c-analyzer/c_analyzer_common/util.py | 31 ++++- Tools/c-analyzer/c_parser/info.py | 57 ++++++--- Tools/c-analyzer/c_parser/naive.py | 4 +- Tools/c-analyzer/c_symbols/resolve.py | 8 +- 13 files changed, 208 insertions(+), 89 deletions(-) diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py index e69de29bb2d1d6..bc502ef32d2916 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/__init__.py @@ -0,0 +1,6 @@ +import os.path +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py index 93100e0438cd2b..215023da577b9c 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_analyzer_common/test_known.py @@ -15,9 +15,6 @@ class FromFileTests(unittest.TestCase): _return_read_tsv = () - def tearDown(self): - Variable._isglobal.instances.clear() - @property def calls(self): try: diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py index e69de29bb2d1d6..bc502ef32d2916 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/__init__.py @@ -0,0 +1,6 @@ +import os.path +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py index b29f966fd2ad0f..828899201b7cad 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_globals/test_find.py @@ -64,7 +64,9 @@ def test_typical(self): **self.kwargs)) self.assertEqual(found, [ + info.Variable.from_parts('dir1/spam.c', None, 'var1', 'int'), info.Variable.from_parts('dir1/spam.c', None, 'var2', 'static int'), + info.Variable.from_parts('dir1/spam.c', None, 'var3', 'char *'), info.Variable.from_parts('dir1/eggs.c', None, 'var1', 'static int'), info.Variable.from_parts('dir1/eggs.c', 'func1', 'var2', 'static char *'), ]) @@ -299,7 +301,7 @@ def test_typical(self): info.Variable.from_parts('src1/spam.c', None, 'var1', 'static const char *'), info.Variable.from_parts('src1/spam.c', None, 'var1b', 'const char *'), info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'static int'), - info.Variable.from_parts('src1/spam.c', 'ham', 'result', 'int'), + info.Variable.from_parts('src1/spam.c', 'ham', 'result', 'int'), # skipped info.Variable.from_parts('src1/spam.c', None, 'var2', 'static PyObject *'), info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'static int'), info.Variable.from_parts('src1/spam.c', None, 'freelist', 'static (PyTupleObject *)[10]'), @@ -318,6 +320,7 @@ def test_typical(self): self.assertEqual(found, [ info.Variable.from_parts('src1/spam.c', None, 'var1', 'static const char *'), + info.Variable.from_parts('src1/spam.c', None, 'var1b', 'const char *'), info.Variable.from_parts('src1/spam.c', 'ham', 'initialized', 'static int'), info.Variable.from_parts('src1/spam.c', None, 'var2', 'static PyObject *'), info.Variable.from_parts('src1/eggs.c', 'tofu', 'ready', 'static int'), diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py index e69de29bb2d1d6..bc502ef32d2916 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_parser/__init__.py @@ -0,0 +1,6 @@ +import os.path +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py index 1dfe5d066a3311..d1a966c58904db 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_parser/test_info.py @@ -4,7 +4,7 @@ from ..util import PseudoStr, StrProxy, Object from .. import tool_imports_for_tests with tool_imports_for_tests(): - from c_analyzer_common.info import ID + from c_analyzer_common.info import ID, UNKNOWN from c_parser.info import ( normalize_vartype, Variable, ) @@ -31,38 +31,47 @@ class VariableTests(unittest.TestCase): VALID_ARGS = ( ('x/y/z/spam.c', 'func', 'eggs'), + 'static', 'int', ) VALID_KWARGS = dict(zip(Variable._fields, VALID_ARGS)) VALID_EXPECTED = VALID_ARGS def test_init_typical_global(self): - static = Variable( - id=ID( - filename='x/y/z/spam.c', - funcname=None, - name='eggs', - ), - vartype='int', - ) + for storage in ('static', 'extern', 'implicit'): + with self.subTest(storage): + static = Variable( + id=ID( + filename='x/y/z/spam.c', + funcname=None, + name='eggs', + ), + storage=storage, + vartype='int', + ) - self.assertEqual(static, ( - ('x/y/z/spam.c', None, 'eggs'), - 'int', - )) + self.assertEqual(static, ( + ('x/y/z/spam.c', None, 'eggs'), + storage, + 'int', + )) def test_init_typical_local(self): - static = Variable( - id=ID( - filename='x/y/z/spam.c', - funcname='func', - name='eggs', - ), - vartype='int', - ) + for storage in ('static', 'local'): + with self.subTest(storage): + static = Variable( + id=ID( + filename='x/y/z/spam.c', + funcname='func', + name='eggs', + ), + storage=storage, + vartype='int', + ) self.assertEqual(static, ( ('x/y/z/spam.c', 'func', 'eggs'), + storage, 'int', )) @@ -71,10 +80,12 @@ def test_init_all_missing(self): with self.subTest(repr(value)): static = Variable( id=value, + storage=value, vartype=value, ) self.assertEqual(static, ( + None, None, None, )) @@ -89,34 +100,42 @@ def test_init_all_coerced(self): PseudoStr('func'), PseudoStr('spam'), ), + storage=PseudoStr('static'), vartype=PseudoStr('int'), ), (id, + 'static', 'int', )), ('non-str 1', dict( id=id, + storage=Object(), vartype=Object(), ), (id, + '', '', )), ('non-str 2', dict( id=id, + storage=StrProxy('static'), vartype=StrProxy('variable'), ), (id, + 'static', 'variable', )), ('non-str', dict( id=id, - vartype=('a', 'b', 'c'), + storage=('a', 'b', 'c'), + vartype=('x', 'y', 'z'), ), (id, "('a', 'b', 'c')", + "('x', 'y', 'z')", )), ] for summary, kwargs, expected in tests: @@ -134,36 +153,43 @@ def test_init_all_coerced(self): def test_iterable(self): static = Variable(**self.VALID_KWARGS) - id, vartype = static + id, storage, vartype = static - values = (id, vartype) + values = (id, storage, vartype) for value, expected in zip(values, self.VALID_EXPECTED): self.assertEqual(value, expected) def test_fields(self): - static = Variable(('a', 'b', 'z'), 'x') + static = Variable(('a', 'b', 'z'), 'x', 'y') self.assertEqual(static.id, ('a', 'b', 'z')) - self.assertEqual(static.vartype, 'x') + self.assertEqual(static.storage, 'x') + self.assertEqual(static.vartype, 'y') def test___getattr__(self): - static = Variable(('a', 'b', 'z'), 'x') + static = Variable(('a', 'b', 'z'), 'x', 'y') self.assertEqual(static.filename, 'a') self.assertEqual(static.funcname, 'b') self.assertEqual(static.name, 'z') def test_validate_typical(self): - static = Variable( - id=ID( - filename='x/y/z/spam.c', - funcname='func', - name='eggs', - ), - vartype='int', - ) + validstorage = ('static', 'extern', 'implicit', 'local') + self.assertEqual(set(validstorage), set(Variable.STORAGE)) + + for storage in validstorage: + with self.subTest(storage): + static = Variable( + id=ID( + filename='x/y/z/spam.c', + funcname='func', + name='eggs', + ), + storage=storage, + vartype='int', + ) - static.validate() # This does not fail. + static.validate() # This does not fail. def test_validate_missing_field(self): for field in Variable._fields: @@ -171,6 +197,13 @@ def test_validate_missing_field(self): static = Variable(**self.VALID_KWARGS) static = static._replace(**{field: None}) + with self.assertRaises(TypeError): + static.validate() + for field in ('storage', 'vartype'): + with self.subTest(field): + static = Variable(**self.VALID_KWARGS) + static = static._replace(**{field: UNKNOWN}) + with self.assertRaises(TypeError): static.validate() @@ -185,6 +218,7 @@ def test_validate_bad_field(self): ) + badch tests = [ ('id', ()), # Any non-empty str is okay. + ('storage', ('external', 'global') + notnames), ('vartype', ()), # Any non-empty str is okay. ] seen = set() @@ -199,6 +233,8 @@ def test_validate_bad_field(self): static.validate() for field, invalid in tests: + if field == 'id': + continue valid = seen - set(invalid) for value in valid: with self.subTest(f'{field}={value!r}'): diff --git a/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py b/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py index e69de29bb2d1d6..bc502ef32d2916 100644 --- a/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py +++ b/Lib/test/test_tools/test_c_analyzer/test_c_symbols/__init__.py @@ -0,0 +1,6 @@ +import os.path +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Tools/c-analyzer/c_analyzer_common/_generate.py b/Tools/c-analyzer/c_analyzer_common/_generate.py index 1629aa6b5210fa..9b2fc9edb5c824 100644 --- a/Tools/c-analyzer/c_analyzer_common/_generate.py +++ b/Tools/c-analyzer/c_analyzer_common/_generate.py @@ -262,7 +262,7 @@ def _known(symbol): raise if symbol.name not in decl: decl = decl + symbol.name - return Variable(varid, decl) + return Variable(varid, 'static', decl) def known_row(varid, decl): @@ -291,7 +291,7 @@ def known_rows(symbols, *, except KeyError: found = _find_match(symbol, cache, filenames) if found is None: - found = Variable(symbol.id, UNKNOWN) + found = Variable(symbol.id, UNKNOWN, UNKNOWN) yield _as_known(found.id, found.vartype) else: raise NotImplementedError # XXX incorporate KNOWN diff --git a/Tools/c-analyzer/c_analyzer_common/known.py b/Tools/c-analyzer/c_analyzer_common/known.py index a0c6dfa5aa47d7..dec1e1d2e09273 100644 --- a/Tools/c-analyzer/c_analyzer_common/known.py +++ b/Tools/c-analyzer/c_analyzer_common/known.py @@ -34,34 +34,41 @@ def from_file(infile, *, id = ID(filename, funcname, name) if kind == 'variable': values = known['variables'] - value = Variable(id, declaration) - value._isglobal = _is_global(declaration) or id.funcname is None + if funcname: + storage = _get_storage(declaration) or 'local' + else: + storage = _get_storage(declaration) or 'implicit' + value = Variable(id, storage, declaration) else: raise ValueError(f'unsupported kind in row {row}') - if value.name == 'id' and declaration == UNKNOWN: - # None of these are variables. - declaration = 'int id'; - else: - value.validate() + value.validate() +# if value.name == 'id' and declaration == UNKNOWN: +# # None of these are variables. +# declaration = 'int id'; +# else: +# value.validate() values[id] = value return known -def _is_global(vartype): +def _get_storage(decl): # statics - if vartype.startswith('static '): - return True - if vartype.startswith(('Py_LOCAL(', 'Py_LOCAL_INLINE(')): - return True - if vartype.startswith(('_Py_IDENTIFIER(', '_Py_static_string(')): - return True - if vartype.startswith('PyDoc_VAR('): - return True - if vartype.startswith(('SLOT1BINFULL(', 'SLOT1BIN(')): - return True - if vartype.startswith('WRAP_METHOD('): - return True + if decl.startswith('static '): + return 'static' + if decl.startswith(('Py_LOCAL(', 'Py_LOCAL_INLINE(')): + return 'static' + if decl.startswith(('_Py_IDENTIFIER(', '_Py_static_string(')): + return 'static' + if decl.startswith('PyDoc_VAR('): + return 'static' + if decl.startswith(('SLOT1BINFULL(', 'SLOT1BIN(')): + return 'static' + if decl.startswith('WRAP_METHOD('): + return 'static' # public extern - if vartype.startswith('PyAPI_DATA('): - return True - return False + if decl.startswith('extern '): + return 'extern' + if decl.startswith('PyAPI_DATA('): + return 'extern' + # implicit or local + return None diff --git a/Tools/c-analyzer/c_analyzer_common/util.py b/Tools/c-analyzer/c_analyzer_common/util.py index 511c54f1783450..43d0bb6e66565a 100644 --- a/Tools/c-analyzer/c_analyzer_common/util.py +++ b/Tools/c-analyzer/c_analyzer_common/util.py @@ -82,6 +82,13 @@ def __init__(self, initial=_NOT_SET, *, self.default = default self.readonly = readonly + # The instance cache is not inherently tied to the normal + # lifetime of the instances. So must do something in order to + # avoid keeping the instances alive by holding a reference here. + # Ideally we would use weakref.WeakValueDictionary to do this. + # However, most builtin types do not support weakrefs. So + # instead we monkey-patch __del__ on the attached class to clear + # the instance. self.instances = {} self.name = None @@ -89,6 +96,12 @@ def __set_name__(self, cls, name): if self.name is not None: raise TypeError('already used') self.name = name + try: + slotnames = cls.__slot_names__ + except AttributeError: + slotnames = cls.__slot_names__ = [] + slotnames.append(name) + self._ensure___del__(cls, slotnames) def __get__(self, obj, cls): if obj is None: # called on the class @@ -115,7 +128,23 @@ def __set__(self, obj, value): def __delete__(self, obj): if self.readonly: raise AttributeError(f'{self.name} is readonly') - self.instances[id(obj)] = self.default + self.instances[id(obj)] = self.default # XXX refleak? + + def _ensure___del__(self, cls, slotnames): # See the comment in __init__(). + try: + old___del__ = cls.__del__ + except AttributeError: + old___del__ = (lambda s: None) + else: + if getattr(old___del__, '_slotted', False): + return + + def __del__(_self): + for name in slotnames: + delattr(_self, name) + old___del__(_self) + __del__._slotted = True + cls.__del__ = __del__ def set(self, obj, value): """Update the cached value for an object. diff --git a/Tools/c-analyzer/c_parser/info.py b/Tools/c-analyzer/c_parser/info.py index d7368b48cde355..a4e32d75eed73f 100644 --- a/Tools/c-analyzer/c_parser/info.py +++ b/Tools/c-analyzer/c_parser/info.py @@ -1,4 +1,5 @@ from collections import namedtuple +import re from c_analyzer_common import info, util from c_analyzer_common.util import classonly, _NTBase @@ -15,28 +16,53 @@ def normalize_vartype(vartype): return str(vartype) +def extract_storage(decl, *, isfunc=False): + """Return (storage, vartype) based on the given declaration. + + The default storage is "implicit" or "local". + """ + if decl == info.UNKNOWN: + return decl, decl + if decl.startswith('static '): + return 'static', decl + #return 'static', decl.partition(' ')[2].strip() + elif decl.startswith('extern '): + return 'extern', decl + #return 'extern', decl.partition(' ')[2].strip() + elif re.match('.*\b(static|extern)\b', decl): + raise NotImplementedError + elif isfunc: + return 'local', decl + else: + return 'implicit', decl + + class Variable(_NTBase, - namedtuple('Variable', 'id vartype')): + namedtuple('Variable', 'id storage vartype')): """Information about a single variable declaration.""" __slots__ = () - _isglobal = util.Slot() - def __del__(self): - del self._isglobal + STORAGE = ( + 'static', + 'extern', + 'implicit', + 'local', + ) @classonly - def from_parts(cls, filename, funcname, name, vartype, isglobal=False): + def from_parts(cls, filename, funcname, name, decl, storage=None): + if storage is None: + storage, decl = extract_storage(decl, isfunc=funcname) id = info.ID(filename, funcname, name) - self = cls(id, vartype) - if isglobal: - self._isglobal = True + self = cls(id, storage, decl) return self - def __new__(cls, id, vartype): + def __new__(cls, id, storage, vartype): self = super().__new__( cls, id=info.ID.from_raw(id), + storage=str(storage) if storage else None, vartype=normalize_vartype(vartype) if vartype else None, ) return self @@ -63,18 +89,17 @@ def validate(self): """Fail if the object is invalid (i.e. init with bad data).""" self._validate_id() + if self.storage is None or self.storage == info.UNKNOWN: + raise TypeError('missing storage') + elif self.storage not in self.STORAGE: + raise ValueError(f'unsupported storage {self.storage:r}') + if self.vartype is None or self.vartype == info.UNKNOWN: raise TypeError('missing vartype') @property def isglobal(self): - try: - return self._isglobal - except AttributeError: - # XXX Include extern variables. - # XXX Ignore functions. - self._isglobal = ('static' in self.vartype.split()) - return self._isglobal + return self.storage != 'local' @property def isconst(self): diff --git a/Tools/c-analyzer/c_parser/naive.py b/Tools/c-analyzer/c_parser/naive.py index e0370cc3d1d439..160f96c279e261 100644 --- a/Tools/c-analyzer/c_parser/naive.py +++ b/Tools/c-analyzer/c_parser/naive.py @@ -163,7 +163,7 @@ def find_variables(varids, filenames=None, *, srcfiles = [varid.filename] else: if not filenames: - yield Variable(varid, UNKNOWN) + yield Variable(varid, UNKNOWN, UNKNOWN) continue srcfiles = filenames for filename in srcfiles: @@ -177,4 +177,4 @@ def find_variables(varids, filenames=None, *, used.add(found) break else: - yield Variable(varid, UNKNOWN) + yield Variable(varid, UNKNOWN, UNKNOWN) diff --git a/Tools/c-analyzer/c_symbols/resolve.py b/Tools/c-analyzer/c_symbols/resolve.py index dc876ae0b75966..56210cefeb8269 100644 --- a/Tools/c-analyzer/c_symbols/resolve.py +++ b/Tools/c-analyzer/c_symbols/resolve.py @@ -68,14 +68,11 @@ def find_in_source(symbol, dirnames, *, if symbol.funcname and symbol.funcname != UNKNOWN: raise NotImplementedError - (filename, funcname, vartype + (filename, funcname, decl ) = _find_symbol(symbol.name, filenames, _perfilecache) if filename == UNKNOWN: return None - return info.Variable( - id=(filename, funcname, symbol.name), - vartype=vartype, - ) + return info.Variable.from_parts(filename, funcname, symbol.name, decl) def get_resolver(knownvars=None, dirnames=None, *, @@ -144,6 +141,7 @@ def symbols_to_variables(symbols, *, #raise NotImplementedError(symbol) resolved = info.Variable( id=symbol.id, + storage=UNKNOWN, vartype=UNKNOWN, ) yield resolved From 5d6f5b629394066a5249af25cc01f1a1f0edc138 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 27 Sep 2019 20:02:58 +0300 Subject: [PATCH 011/148] bpo-32820: Simplify __format__ implementation for ipaddress. (GH-16378) Also cache the compiled RE for parsing the format specifier. --- Lib/ipaddress.py | 59 ++++++++++++++++-------------------------------- 1 file changed, 19 insertions(+), 40 deletions(-) diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index c389f0528823e8..7d80a52c158a9e 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -560,6 +560,8 @@ def __reduce__(self): return self.__class__, (str(self),) +_address_fmt_re = None + @functools.total_ordering class _BaseAddress(_IPAddressBase): @@ -623,72 +625,49 @@ def __format__(self, fmt): Supported presentation types are: 's': returns the IP address as a string (default) - 'b' or 'n': converts to binary and returns a zero-padded string + 'b': converts to binary and returns a zero-padded string 'X' or 'x': converts to upper- or lower-case hex and returns a zero-padded string + 'n': the same as 'b' for IPv4 and 'x' for IPv6 For binary and hex presentation types, the alternate form specifier '#' and the grouping option '_' are supported. """ - # Support string formatting if not fmt or fmt[-1] == 's': - # let format() handle it return format(str(self), fmt) # From here on down, support for 'bnXx' + global _address_fmt_re + if _address_fmt_re is None: + import re + _address_fmt_re = re.compile('(#?)(_?)([xbnX])') - import re - fmt_re = '^(?P#?)(?P_?)(?P[xbnX]){1}$' - m = re.match(fmt_re, fmt) + m = _address_fmt_re.fullmatch(fmt) if not m: return super().__format__(fmt) - groupdict = m.groupdict() - alternate = groupdict['alternate'] - grouping = groupdict['grouping'] - fmt_base = groupdict['fmt_base'] + alternate, grouping, fmt_base = m.groups() # Set some defaults if fmt_base == 'n': if self._version == 4: fmt_base = 'b' # Binary is default for ipv4 - if self._version == 6: + else: fmt_base = 'x' # Hex is default for ipv6 - # Handle binary formatting if fmt_base == 'b': - if self._version == 4: - # resulting string is '0b' + 32 bits - # plus 7 _ if needed - padlen = IPV4LENGTH+2 + (7*len(grouping)) - elif self._version == 6: - # resulting string is '0b' + 128 bits - # plus 31 _ if needed - padlen = IPV6LENGTH+2 + (31*len(grouping)) - - # Handle hex formatting - elif fmt_base in 'Xx': - if self._version == 4: - # resulting string is '0x' + 8 hex digits - # plus a single _ if needed - padlen = int(IPV4LENGTH/4)+2 + len(grouping) - elif self._version == 6: - # resulting string is '0x' + 32 hex digits - # plus 7 _ if needed - padlen = int(IPV6LENGTH/4)+2 + (7*len(grouping)) - - retstr = f'{int(self):#0{padlen}{grouping}{fmt_base}}' + padlen = self._max_prefixlen + else: + padlen = self._max_prefixlen // 4 - if fmt_base == 'X': - retstr = retstr.upper() + if grouping: + padlen += padlen // 4 - 1 - # If alternate is not set, strip the two leftmost - # characters ('0b') - if not alternate: - retstr = retstr[2:] + if alternate: + padlen += 2 # 0b or 0x - return retstr + return format(int(self), f'{alternate}0{padlen}{grouping}{fmt_base}') @functools.total_ordering From dd6117c6d7859fee57751593cd56f0862131de8b Mon Sep 17 00:00:00 2001 From: bariod <35639254+bariod@users.noreply.github.com> Date: Fri, 27 Sep 2019 20:01:33 +0200 Subject: [PATCH 012/148] Fix typo in the "Porting to Python 3.8" section. (GH-16435) --- Doc/whatsnew/3.8.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 0995cb3b91196e..4cab2f7af118dd 100755 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -1531,7 +1531,7 @@ Changes in Python behavior :class:`int`, :class:`float`, :class:`complex` and few classes from the standard library. They now inherit ``__str__()`` from :class:`object`. As result, defining the ``__repr__()`` method in the subclass of these - classes will affect they string representation. + classes will affect their string representation. (Contributed by Serhiy Storchaka in :issue:`36793`.) * On AIX, :attr:`sys.platform` doesn't contain the major version anymore. From e8650a4f8c7fb76f570d4ca9c1fbe44e91c8dfaa Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sat, 28 Sep 2019 04:59:37 +0900 Subject: [PATCH 013/148] bpo-38243, xmlrpc.server: Escape the server_title (GH-16373) Escape the server title of xmlrpc.server.DocXMLRPCServer when rendering the document page as HTML. --- Lib/test/test_docxmlrpc.py | 16 ++++++++++++++++ Lib/xmlrpc/server.py | 3 ++- .../2019-09-25-13-21-09.bpo-38243.1pfz24.rst | 3 +++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2019-09-25-13-21-09.bpo-38243.1pfz24.rst diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py index 116e626740df85..7d3e30cbee964a 100644 --- a/Lib/test/test_docxmlrpc.py +++ b/Lib/test/test_docxmlrpc.py @@ -1,5 +1,6 @@ from xmlrpc.server import DocXMLRPCServer import http.client +import re import sys import threading import unittest @@ -192,6 +193,21 @@ def test_annotations(self): b'method_annotation(x: bytes)'), response.read()) + def test_server_title_escape(self): + # bpo-38243: Ensure that the server title and documentation + # are escaped for HTML. + self.serv.set_server_title('test_title