From a8c7223f097167f67b6a6f62064d59371434727b Mon Sep 17 00:00:00 2001
From: Pauli Virtanen <pav@iki.fi>
Date: Mon, 28 Aug 2017 14:08:49 +0200
Subject: [PATCH 1/2] [2.7] bpo-10746: Fix ctypes PEP 3118 type codes for
 c_long, c_bool, c_int (GH-31)

Ctypes currently produces wrong pep3118 type codes for several types.
E.g. memoryview(ctypes.c_long()).format gives "<l" on 64-bit platforms,
but it should be "<q" instead for sizeof(c_long) == 8

The problem is that the '<>' endian specification in the struct syntax
also turns on the "standard size" mode, which makes type characters have
a platform-independent meaning, which does not match with the codes used
internally in ctypes.  The struct module format syntax also does not
allow specifying native-size non-native-endian items.

This commit adds a converter function that maps the internal ctypes
codes to appropriate struct module standard-size codes in the pep3118
format strings. The tests are modified to check for this..
(cherry picked from commit 07f1658aa09f6798793c473c72b2951b7fefe220)
---
 Lib/ctypes/test/test_pep3118.py               | 83 ++++++++++++-------
 .../2017-08-28-13-01-05.bpo-10746.nmAvfu.rst  |  1 +
 Modules/_ctypes/_ctypes.c                     | 69 ++++++++++++++-
 3 files changed, 123 insertions(+), 30 deletions(-)
 create mode 100644 Misc/NEWS.d/next/Library/2017-08-28-13-01-05.bpo-10746.nmAvfu.rst

diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/ctypes/test/test_pep3118.py
index 3e007e1b41d5b6..49a1356a2c4424 100644
--- a/Lib/ctypes/test/test_pep3118.py
+++ b/Lib/ctypes/test/test_pep3118.py
@@ -109,6 +109,34 @@ class Complete(Structure):
 # This table contains format strings as they look on little endian
 # machines.  The test replaces '<' with '>' on big endian machines.
 #
+
+# Platform-specific type codes
+s_bool = {1: '?', 2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_bool)]
+s_short = {2: 'h', 4: 'l', 8: 'q'}[sizeof(c_short)]
+s_ushort = {2: 'H', 4: 'L', 8: 'Q'}[sizeof(c_ushort)]
+s_int = {2: 'h', 4: 'i', 8: 'q'}[sizeof(c_int)]
+s_uint = {2: 'H', 4: 'I', 8: 'Q'}[sizeof(c_uint)]
+s_long = {4: 'l', 8: 'q'}[sizeof(c_long)]
+s_ulong = {4: 'L', 8: 'Q'}[sizeof(c_ulong)]
+s_longlong = "q"
+s_ulonglong = "Q"
+s_float = "f"
+s_double = "d"
+s_longdouble = "g"
+
+# Alias definitions in ctypes/__init__.py
+if c_int is c_long:
+    s_int = s_long
+if c_uint is c_ulong:
+    s_uint = s_ulong
+if c_longlong is c_long:
+    s_longlong = s_long
+if c_ulonglong is c_ulong:
+    s_ulonglong = s_ulong
+if c_longdouble is c_double:
+    s_longdouble = s_double
+
+
 native_types = [
     # type                      format                  shape           calc itemsize
 
@@ -117,52 +145,51 @@ class Complete(Structure):
     (c_char,                    "<c",                   None,           c_char),
     (c_byte,                    "<b",                   None,           c_byte),
     (c_ubyte,                   "<B",                   None,           c_ubyte),
-    (c_short,                   "<h",                   None,           c_short),
-    (c_ushort,                  "<H",                   None,           c_ushort),
+    (c_short,                   "<" + s_short,          None,           c_short),
+    (c_ushort,                  "<" + s_ushort,         None,           c_ushort),
 
-    # c_int and c_uint may be aliases to c_long
-    #(c_int,                     "<i",                   None,           c_int),
-    #(c_uint,                    "<I",                   None,           c_uint),
+    (c_int,                     "<" + s_int,            None,           c_int),
+    (c_uint,                    "<" + s_uint,           None,           c_uint),
 
-    (c_long,                    "<l",                   None,           c_long),
-    (c_ulong,                   "<L",                   None,           c_ulong),
+    (c_long,                    "<" + s_long,           None,           c_long),
+    (c_ulong,                   "<" + s_ulong,          None,           c_ulong),
 
-    # c_longlong and c_ulonglong are aliases on 64-bit platforms
-    #(c_longlong,                "<q",                   None,           c_longlong),
-    #(c_ulonglong,               "<Q",                   None,           c_ulonglong),
+    (c_longlong,                "<" + s_longlong,       None,           c_longlong),
+    (c_ulonglong,               "<" + s_ulonglong,      None,           c_ulonglong),
 
     (c_float,                   "<f",                   None,           c_float),
     (c_double,                  "<d",                   None,           c_double),
-    # c_longdouble may be an alias to c_double
 
-    (c_bool,                    "<?",                   None,           c_bool),
+    (c_longdouble,              "<" + s_longdouble,     None,           c_longdouble),
+
+    (c_bool,                    "<" + s_bool,           None,           c_bool),
     (py_object,                 "<O",                   None,           py_object),
 
     ## pointers
 
     (POINTER(c_byte),           "&<b",                  None,           POINTER(c_byte)),
-    (POINTER(POINTER(c_long)),  "&&<l",                 None,           POINTER(POINTER(c_long))),
+    (POINTER(POINTER(c_long)),  "&&<" + s_long,         None,           POINTER(POINTER(c_long))),
 
     ## arrays and pointers
 
     (c_double * 4,              "<d",                   (4,),           c_double),
     (c_float * 4 * 3 * 2,       "<f",                   (2,3,4),        c_float),
-    (POINTER(c_short) * 2,      "&<h",                  (2,),           POINTER(c_short)),
-    (POINTER(c_short) * 2 * 3,  "&<h",                  (3,2,),         POINTER(c_short)),
-    (POINTER(c_short * 2),      "&(2)<h",               None,           POINTER(c_short)),
+    (POINTER(c_short) * 2,      "&<" + s_short,         (2,),           POINTER(c_short)),
+    (POINTER(c_short) * 2 * 3,  "&<" + s_short,         (3,2,),         POINTER(c_short)),
+    (POINTER(c_short * 2),      "&(2)<" + s_short,      None,             POINTER(c_short)),
 
     ## structures and unions
 
-    (Point,                     "T{<l:x:<l:y:}",        None,           Point),
+    (Point,                     "T{<l:x:<l:y:}".replace('l', s_long),  None,  Point),
     # packed structures do not implement the pep
-    (PackedPoint,               "B",                    None,           PackedPoint),
-    (Point2,                    "T{<l:x:<l:y:}",        None,           Point2),
-    (EmptyStruct,               "T{}",                  None,           EmptyStruct),
+    (PackedPoint,               "B",                                   None,  PackedPoint),
+    (Point2,                    "T{<l:x:<l:y:}".replace('l', s_long),  None,  Point2),
+    (EmptyStruct,               "T{}",                                 None,  EmptyStruct),
     # the pep does't support unions
-    (aUnion,                    "B",                    None,           aUnion),
+    (aUnion,                    "B",                                   None,  aUnion),
     # structure with sub-arrays
-    (StructWithArrays,          "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}", None,  StructWithArrays),
-    (StructWithArrays * 3,      "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}", (3,),  StructWithArrays),
+    (StructWithArrays, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), None, StructWithArrays),
+    (StructWithArrays * 3, "T{(2,3)<l:x:(4)T{<l:x:<l:y:}:y:}".replace('l', s_long), (3,), StructWithArrays),
 
     ## pointer to incomplete structure
     (Incomplete,                "B",                    None,           Incomplete),
@@ -170,7 +197,7 @@ class Complete(Structure):
 
     # 'Complete' is a structure that starts incomplete, but is completed after the
     # pointer type to it has been created.
-    (Complete,                  "T{<l:a:}",             None,           Complete),
+    (Complete,                  "T{<l:a:}".replace('l', s_long), None, Complete),
     # Unfortunately the pointer format string is not fixed...
     (POINTER(Complete),         "&B",                   None,           POINTER(Complete)),
 
@@ -193,10 +220,10 @@ class LEPoint(LittleEndianStructure):
 # and little endian machines.
 #
 endian_types = [
-    (BEPoint,                   "T{>l:x:>l:y:}",        None,           BEPoint),
-    (LEPoint,                   "T{<l:x:<l:y:}",        None,           LEPoint),
-    (POINTER(BEPoint),          "&T{>l:x:>l:y:}",       None,           POINTER(BEPoint)),
-    (POINTER(LEPoint),          "&T{<l:x:<l:y:}",       None,           POINTER(LEPoint)),
+    (BEPoint, "T{>l:x:>l:y:}".replace('l', s_long), None, BEPoint),
+    (LEPoint, "T{<l:x:<l:y:}".replace('l', s_long), None, LEPoint),
+    (POINTER(BEPoint), "&T{>l:x:>l:y:}".replace('l', s_long), None, POINTER(BEPoint)),
+    (POINTER(LEPoint), "&T{<l:x:<l:y:}".replace('l', s_long), None, POINTER(LEPoint)),
     ]
 
 if __name__ == "__main__":
diff --git a/Misc/NEWS.d/next/Library/2017-08-28-13-01-05.bpo-10746.nmAvfu.rst b/Misc/NEWS.d/next/Library/2017-08-28-13-01-05.bpo-10746.nmAvfu.rst
new file mode 100644
index 00000000000000..e7625631002044
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2017-08-28-13-01-05.bpo-10746.nmAvfu.rst
@@ -0,0 +1 @@
+Fix ctypes producing wrong PEP 3118 type codes for integer types.
diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index 4e192fb3ad2818..1babbc7ad5dab3 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -292,6 +292,71 @@ PyDict_GetItemProxy(PyObject *dict, PyObject *key)
 }
 
 /******************************************************************/
+
+/*
+  Allocate a memory block for a pep3118 format string, filled with
+  a suitable PEP 3118 type code corresponding to the given ctypes
+  type. Returns NULL on failure, with the error indicator set.
+
+  This produces type codes in the standard size mode (cf. struct module),
+  since the endianness may need to be swapped to a non-native one
+  later on.
+ */
+static char *
+_ctypes_alloc_format_string_for_type(char code, int big_endian)
+{
+    char *result;
+    char pep_code = '\0';
+
+    switch (code) {
+#if SIZEOF_INT == 2
+    case 'i': pep_code = 'h'; break;
+    case 'I': pep_code = 'H'; break;
+#elif SIZEOF_INT == 4
+    case 'i': pep_code = 'i'; break;
+    case 'I': pep_code = 'I'; break;
+#elif SIZEOF_INT == 8
+    case 'i': pep_code = 'q'; break;
+    case 'I': pep_code = 'Q'; break;
+#else
+# error SIZEOF_INT has an unexpected value
+#endif /* SIZEOF_INT */
+#if SIZEOF_LONG == 4
+    case 'l': pep_code = 'l'; break;
+    case 'L': pep_code = 'L'; break;
+#elif SIZEOF_LONG == 8
+    case 'l': pep_code = 'q'; break;
+    case 'L': pep_code = 'Q'; break;
+#else
+# error SIZEOF_LONG has an unexpected value
+#endif /* SIZEOF_LONG */
+#if SIZEOF__BOOL == 1
+    case '?': pep_code = '?'; break;
+#elif SIZEOF__BOOL == 2
+    case '?': pep_code = 'H'; break;
+#elif SIZEOF__BOOL == 4
+    case '?': pep_code = 'L'; break;
+#elif SIZEOF__BOOL == 8
+    case '?': pep_code = 'Q'; break;
+#else
+# error SIZEOF__BOOL has an unexpected value
+#endif /* SIZEOF__BOOL */
+    default:
+        /* The standard-size code is the same as the ctypes one */
+        pep_code = code;
+        break;
+    }
+
+    result = PyMem_Malloc(3);
+    if (result == NULL)
+        return NULL;
+
+    result[0] = big_endian ? '>' : '<';
+    result[1] = pep_code;
+    result[2] = '\0';
+    return result;
+}
+
 /*
   Allocate a memory block for a pep3118 format string, copy prefix (if
   non-null) and suffix into it.  Returns NULL on failure, with the error
@@ -1999,9 +2064,9 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     stgdict->setfunc = fmt->setfunc;
     stgdict->getfunc = fmt->getfunc;
 #ifdef WORDS_BIGENDIAN
-    stgdict->format = _ctypes_alloc_format_string(">", proto_str);
+    stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 1);
 #else
-    stgdict->format = _ctypes_alloc_format_string("<", proto_str);
+    stgdict->format = _ctypes_alloc_format_string_for_type(proto_str[0], 0);
 #endif
     if (stgdict->format == NULL) {
         Py_DECREF(result);

From 33b7e42c1c5ed031cc11b8e2e1f5095cd5daf3e5 Mon Sep 17 00:00:00 2001
From: Pauli Virtanen <pav@iki.fi>
Date: Sat, 2 Sep 2017 16:31:11 +0300
Subject: [PATCH 2/2] [2.7] bpo-10746: ensure SIZEOF__BOOL is defined

---
 Modules/_ctypes/_ctypes.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c
index 1babbc7ad5dab3..025d3026d877cc 100644
--- a/Modules/_ctypes/_ctypes.c
+++ b/Modules/_ctypes/_ctypes.c
@@ -128,6 +128,12 @@ bytes(cdata)
 #endif
 #include "ctypes.h"
 
+/* Definition matching cfield.c:724 */
+#ifndef HAVE_C99_BOOL
+#undef SIZEOF__BOOL
+#define SIZEOF__BOOL 1
+#endif
+
 PyObject *PyExc_ArgError;
 
 /* This dict maps ctypes types to POINTER types */