From 79225852c16588e3c52027f12dd5c6b0e0482c60 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sat, 1 Oct 2022 17:47:43 +0800 Subject: [PATCH 01/91] gh-97588: Fix ctypes structs --- Lib/test/test_ctypes/test_bitfields.py | 66 ++++++++++- Modules/_ctypes/cfield.c | 148 ++++++++++++++++++++++++- Modules/_ctypes/stgdict.c | 3 +- 3 files changed, 211 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index dad71a0ba7ee4a..823f186a373883 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -3,6 +3,7 @@ from test import support import unittest import os +import sys import _ctypes_test @@ -40,8 +41,6 @@ def test_ints(self): setattr(b, name, i) self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) - # bpo-46913: _ctypes/cfield.c h_get() has an undefined behavior - @support.skip_if_sanitizer(ub=True) def test_shorts(self): b = BITS() name = "M" @@ -236,6 +235,69 @@ class X(Structure): else: self.assertEqual(sizeof(X), sizeof(c_int) * 2) + def test_mixed_5(self): + class X(Structure): + _fields_ = [ + ('A', c_uint, 1), + ('B', c_ushort, 16)] + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + def test_mixed_6(self): + class X(Structure): + _fields_ = [ + ('A', c_ulonglong, 1), + ('B', c_uint, 32)] + a = X() + a.A = 0 + a.B = 1 + self.assertEqual(1, a.B) + + def test_mixed_7(self): + class X(Structure): + _fields_ = [ + ("A", c_uint), + ('B', c_uint, 20), + ('C', c_ulonglong, 24)] + self.assertEqual(16, sizeof(X)) + + def test_mixed_8(self): + class Foo(Structure): + _fields_ = [ + ("A", c_uint), + ("B", c_uint, 32), + ("C", c_ulonglong, 1), + ] + + class Bar(Structure): + _fields_ = [ + ("A", c_uint), + ("B", c_uint), + ("C", c_ulonglong, 1), + ] + self.assertEqual(sizeof(Foo), sizeof(Bar)) + + @unittest.skipIf(sys.platform == 'win32', "Doesn't work on Windows, yet") + def test_mixed_9(self): + class X(Structure): + _fields_ = [ + ("A", c_uint8), + ("B", c_uint, 1), + ] + self.assertEqual(4, sizeof(X)) + + @unittest.skipIf(sys.platform == 'win32', "Doesn't work on Windows, yet") + def test_mixed_10(self): + class X(Structure): + _fields_ = [ + ("A", c_uint32, 1), + ("B", c_uint64, 1), + ] + self.assertEqual(8, alignment(X)) + self.assertEqual(8, sizeof(X)) + def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message class X(Structure): diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 13ed8b7eda6555..f946addfdb3348 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -38,6 +38,29 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)obj; } +static inline +Py_ssize_t round_down(Py_ssize_t numToRound, Py_ssize_t multiple) +{ + assert(numToRound >= 0); + if (multiple == 0) + return numToRound; + return (numToRound / multiple) * multiple; +} + +static inline +Py_ssize_t round_up(Py_ssize_t numToRound, Py_ssize_t multiple) +{ + assert(numToRound >= 0); + if (multiple == 0) + return numToRound; + return ((numToRound + multiple - 1) / multiple) * multiple; +} + +static inline +Py_ssize_t LOW_BIT(Py_ssize_t x); +static inline +Py_ssize_t NUM_BITS(Py_ssize_t x); + /* * Expects the size, index and offset for the current field in *psize and * *poffset, stores the total size so far in *psize, the offset for the next @@ -51,7 +74,7 @@ PyCField_new(PyTypeObject *type, PyObject *args, PyObject *kwds) * prev_desc points to the type of the previous bitfield, if any. */ PyObject * -PyCField_FromDesc(PyObject *desc, Py_ssize_t index, +PyCField_FromDesc_old(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, int *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian) @@ -205,6 +228,119 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, return (PyObject *)self; } +PyObject * +PyCField_FromDesc(PyObject *desc, Py_ssize_t index, + Py_ssize_t *pfield_size, int bitsize, int *pbitofs, + Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, + int pack, int big_endian) +{ + #ifndef MS_WIN32 + if(big_endian || pack || *poffset || *pfield_size) + #endif + { + // Fall back to old behaviour for cases that I don't understand well + // enough. + // TODO(Matthias): Learn enough so we don't need to fall back. + return PyCField_FromDesc_old(desc, index, + pfield_size, bitsize, pbitofs, + psize, poffset, palign, + pack, big_endian); + } + + // Change: + // * pbitofs is now relative to the start of the struct, not the start of + // the current field + // * we don't need pfield_size anymore + // * same for poffset, unless it's in use by our caller? + // poffset doesn't seem to be used after this function returns. + // Though we might have to honour it's starting point? + // I guess fall back, if it ain't zero? + + CFieldObject* self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + if (self == NULL) + return NULL; + StgDictObject* dict = PyType_stgdict(desc); + if (!dict) { + PyErr_SetString(PyExc_TypeError, + "has no _stginfo_"); + Py_DECREF(self); + return NULL; + } + + int is_bitfield = !!bitsize; + if(!is_bitfield) { + bitsize = 8 * dict->size; // might still be 0 afterwards. + } + + if ((bitsize > 0) + && (round_down(*pbitofs, 8 * dict->align) + < round_down(*pbitofs + bitsize - 1, 8 * dict->align))) { + // We would be straddling alignment units. + *pbitofs = round_up(*pbitofs, 8*dict->align); + } + + PyObject* proto = desc; + + /* Field descriptors for 'c_char * n' are be scpecial cased to + return a Python string instead of an Array object instance... + */ + SETFUNC setfunc = NULL; + GETFUNC getfunc = NULL; + if (PyCArrayTypeObject_Check(proto)) { + StgDictObject *adict = PyType_stgdict(proto); + StgDictObject *idict; + if (adict && adict->proto) { + idict = PyType_stgdict(adict->proto); + if (!idict) { + PyErr_SetString(PyExc_TypeError, + "has no _stginfo_"); + Py_DECREF(self); + return NULL; + } + if (idict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + struct fielddesc *fd = _ctypes_get_fielddesc("s"); + getfunc = fd->getfunc; + setfunc = fd->setfunc; + } + if (idict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + struct fielddesc *fd = _ctypes_get_fielddesc("U"); + getfunc = fd->getfunc; + setfunc = fd->setfunc; + } + } + } + + self->setfunc = setfunc; + self->getfunc = getfunc; + self->index = index; + + Py_INCREF(proto); + self->proto = proto; + + assert(bitsize <= dict->size * 8); + assert(*poffset == 0); + + // We need to fit within alignment and within size. + // But we only really care about size, when we have a bitfield. + if(is_bitfield) { + self->offset = round_down(*pbitofs, 8*dict->size) / 8; + Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; + self->size = (bitsize << 16 ) + effective_bitsof; + assert(dict->size == dict->align); + assert(effective_bitsof <= dict->size * 8); + } else { + self->offset = round_down(*pbitofs, 8*dict->align) / 8; + self->size = dict->size; + } + + *pbitofs += bitsize; + *psize = round_up(*pbitofs, 8) / 8; + *palign = dict->align; + + assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); + return (PyObject *)self; +} + static int PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) { @@ -408,8 +544,14 @@ get_ulonglong(PyObject *v, unsigned long long *p) */ /* how to decode the size field, for integer get/set functions */ -#define LOW_BIT(x) ((x) & 0xFFFF) -#define NUM_BITS(x) ((x) >> 16) +static inline +Py_ssize_t LOW_BIT(Py_ssize_t x) { + return x & 0xFFFF; +} +static inline +Py_ssize_t NUM_BITS(Py_ssize_t x) { + return x >> 16; +} /* Doesn't work if NUM_BITS(size) == 0, but it never happens in SET() call. */ #define BIT_MASK(type, size) (((((type)1 << (NUM_BITS(size) - 1)) - 1) << 1) + 1) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index a819ce910d4b52..c017323d5a754e 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -354,7 +354,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct Py_ssize_t len, offset, size, align, i; Py_ssize_t union_size, total_align; Py_ssize_t field_size = 0; - int bitofs; + int bitofs = 0; PyObject *tmp; int isPacked; int pack; @@ -613,6 +613,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct pack, big_endian); } else /* union */ { size = 0; + bitofs = 0; offset = 0; align = 0; prop = PyCField_FromDesc(desc, i, From 23079321a6ff7093b6f34a3df7a5451e06c9f2ca Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 1 Oct 2022 09:56:28 +0000 Subject: [PATCH 02/91] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst diff --git a/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst b/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst new file mode 100644 index 00000000000000..09569ef5e9235d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst @@ -0,0 +1 @@ +Fix ctypes construction of structs from description. From 47f826c396b412ed4194338aae0dc9e85d0d20c9 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sat, 1 Oct 2022 18:05:16 +0800 Subject: [PATCH 03/91] Handling pack as well --- Modules/_ctypes/cfield.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index f946addfdb3348..4356d8aff5c42c 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -235,7 +235,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, int pack, int big_endian) { #ifndef MS_WIN32 - if(big_endian || pack || *poffset || *pfield_size) + if(big_endian || *poffset || *pfield_size) #endif { // Fall back to old behaviour for cases that I don't understand well @@ -272,11 +272,17 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, bitsize = 8 * dict->size; // might still be 0 afterwards. } + Py_ssize_t align; + if (pack) + align = min(pack, dict->align); + else + align = dict->align; + if ((bitsize > 0) - && (round_down(*pbitofs, 8 * dict->align) - < round_down(*pbitofs + bitsize - 1, 8 * dict->align))) { + && (round_down(*pbitofs, 8 * align) + < round_down(*pbitofs + bitsize - 1, 8 * align))) { // We would be straddling alignment units. - *pbitofs = round_up(*pbitofs, 8*dict->align); + *pbitofs = round_up(*pbitofs, 8*align); } PyObject* proto = desc; @@ -329,13 +335,13 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, assert(dict->size == dict->align); assert(effective_bitsof <= dict->size * 8); } else { - self->offset = round_down(*pbitofs, 8*dict->align) / 8; + self->offset = round_down(*pbitofs, 8*align) / 8; self->size = dict->size; } *pbitofs += bitsize; *psize = round_up(*pbitofs, 8) / 8; - *palign = dict->align; + *palign = align; assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); return (PyObject *)self; From 5a322112d78b0c6a25670ee4811279aaba4bbc9e Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sat, 1 Oct 2022 18:14:25 +0800 Subject: [PATCH 04/91] Handle basedict, too --- Modules/_ctypes/cfield.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 4356d8aff5c42c..f73dd0706bfa19 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -235,7 +235,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, int pack, int big_endian) { #ifndef MS_WIN32 - if(big_endian || *poffset || *pfield_size) + if(big_endian) #endif { // Fall back to old behaviour for cases that I don't understand well @@ -247,6 +247,10 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, pack, big_endian); } + assert(*pfield_size == 0); + *pbitofs += *poffset * 8; + *poffset = 0; + // Change: // * pbitofs is now relative to the start of the struct, not the start of // the current field From e5ed9ac3a50bad6c399f83457884a3e7c1064f94 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 5 Oct 2022 13:33:48 +0800 Subject: [PATCH 05/91] Split windows and linux --- Modules/_ctypes/cfield.c | 154 +++++++++++++++++++++++++++++++++++---- 1 file changed, 140 insertions(+), 14 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index f73dd0706bfa19..5dfe6e3ee0deca 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -74,7 +74,7 @@ Py_ssize_t NUM_BITS(Py_ssize_t x); * prev_desc points to the type of the previous bitfield, if any. */ PyObject * -PyCField_FromDesc_old(PyObject *desc, Py_ssize_t index, +PyCField_FromDesc_big_endian(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, int *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian) @@ -229,24 +229,121 @@ PyCField_FromDesc_old(PyObject *desc, Py_ssize_t index, } PyObject * -PyCField_FromDesc(PyObject *desc, Py_ssize_t index, +PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, int *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian) { - #ifndef MS_WIN32 - if(big_endian) - #endif - { - // Fall back to old behaviour for cases that I don't understand well - // enough. - // TODO(Matthias): Learn enough so we don't need to fall back. - return PyCField_FromDesc_old(desc, index, - pfield_size, bitsize, pbitofs, - psize, poffset, palign, - pack, big_endian); + assert(*pfield_size == 0); + *pbitofs += *poffset * 8; + *poffset = 0; + + // Change: + // * pbitofs is now relative to the start of the struct, not the start of + // the current field + // * we don't need pfield_size anymore + // * same for poffset, unless it's in use by our caller? + // poffset doesn't seem to be used after this function returns. + // Though we might have to honour it's starting point? + // I guess fall back, if it ain't zero? + + CFieldObject* self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + if (self == NULL) + return NULL; + StgDictObject* dict = PyType_stgdict(desc); + if (!dict) { + PyErr_SetString(PyExc_TypeError, + "has no _stginfo_"); + Py_DECREF(self); + return NULL; + } + + int is_bitfield = !!bitsize; + if(!is_bitfield) { + bitsize = 8 * dict->size; // might still be 0 afterwards. } + Py_ssize_t align; + if (pack) + align = min(pack, dict->align); + else + align = dict->align; + + if ((bitsize > 0) + && (round_down(*pbitofs, 8 * align) + < round_down(*pbitofs + bitsize - 1, 8 * align))) { + // We would be straddling alignment units. + *pbitofs = round_up(*pbitofs, 8*align); + } + + PyObject* proto = desc; + + /* Field descriptors for 'c_char * n' are be scpecial cased to + return a Python string instead of an Array object instance... + */ + SETFUNC setfunc = NULL; + GETFUNC getfunc = NULL; + if (PyCArrayTypeObject_Check(proto)) { + StgDictObject *adict = PyType_stgdict(proto); + StgDictObject *idict; + if (adict && adict->proto) { + idict = PyType_stgdict(adict->proto); + if (!idict) { + PyErr_SetString(PyExc_TypeError, + "has no _stginfo_"); + Py_DECREF(self); + return NULL; + } + if (idict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + struct fielddesc *fd = _ctypes_get_fielddesc("s"); + getfunc = fd->getfunc; + setfunc = fd->setfunc; + } + if (idict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + struct fielddesc *fd = _ctypes_get_fielddesc("U"); + getfunc = fd->getfunc; + setfunc = fd->setfunc; + } + } + } + + self->setfunc = setfunc; + self->getfunc = getfunc; + self->index = index; + + Py_INCREF(proto); + self->proto = proto; + + assert(bitsize <= dict->size * 8); + assert(*poffset == 0); + + // We need to fit within alignment and within size. + // But we only really care about size, when we have a bitfield. + if(is_bitfield) { + self->offset = round_down(*pbitofs, 8*dict->size) / 8; + Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; + self->size = (bitsize << 16 ) + effective_bitsof; + assert(dict->size == dict->align); + assert(effective_bitsof <= dict->size * 8); + } else { + self->offset = round_down(*pbitofs, 8*align) / 8; + self->size = dict->size; + } + + *pbitofs += bitsize; + *psize = round_up(*pbitofs, 8) / 8; + *palign = align; + + assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); + return (PyObject *)self; +} + +PyObject * +PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, + Py_ssize_t *pfield_size, int bitsize, int *pbitofs, + Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, + int pack, int big_endian) +{ assert(*pfield_size == 0); *pbitofs += *poffset * 8; *poffset = 0; @@ -351,6 +448,36 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, return (PyObject *)self; } +PyObject * +PyCField_FromDesc(PyObject *desc, Py_ssize_t index, + Py_ssize_t *pfield_size, int bitsize, int *pbitofs, + Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, + int pack, int big_endian) +{ + if(big_endian) { + return PyCField_FromDesc_big_endian(desc, index, + pfield_size, bitsize, pbitofs, + psize, poffset, palign, + pack, big_endian); + } else if ( + #ifdef MS_WIN32 + true + #else + pack != 0 + #endif + ) { + return PyCField_FromDesc_windows(desc, index, + pfield_size, bitsize, pbitofs, + psize, poffset, palign, + pack, big_endian); + } else { + return PyCField_FromDesc_linux(desc, index, + pfield_size, bitsize, pbitofs, + psize, poffset, palign, + pack, big_endian); + } +} + static int PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) { @@ -370,7 +497,6 @@ PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) } return PyCData_set(inst, self->proto, self->setfunc, value, self->index, self->size, ptr); -} static PyObject * PyCField_get(CFieldObject *self, PyObject *inst, PyTypeObject *type) From b0f9819b12f15b26b3e66e8f59d0499fbebcfd6f Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 5 Oct 2022 13:40:50 +0800 Subject: [PATCH 06/91] Compiles --- Modules/_ctypes/cfield.c | 84 +++++++++++++++------------------------- 1 file changed, 31 insertions(+), 53 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 5dfe6e3ee0deca..00ecd5dd2bb11f 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -77,30 +77,20 @@ PyObject * PyCField_FromDesc_big_endian(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, int *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian) + int pack, int big_endian, + CFieldObject* self, StgDictObject* dict + ) { - CFieldObject *self; PyObject *proto; Py_ssize_t size, align; SETFUNC setfunc = NULL; GETFUNC getfunc = NULL; - StgDictObject *dict; int fieldtype; #define NO_BITFIELD 0 #define NEW_BITFIELD 1 #define CONT_BITFIELD 2 #define EXPAND_BITFIELD 3 - self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); - if (self == NULL) - return NULL; - dict = PyType_stgdict(desc); - if (!dict) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - Py_DECREF(self); - return NULL; - } if (bitsize /* this is a bitfield request */ && *pfield_size /* we have a bitfield open */ #ifdef MS_WIN32 @@ -232,42 +222,21 @@ PyObject * PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, int *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian) + int pack, int big_endian, + CFieldObject* self, StgDictObject* dict + ) { assert(*pfield_size == 0); *pbitofs += *poffset * 8; *poffset = 0; - // Change: - // * pbitofs is now relative to the start of the struct, not the start of - // the current field - // * we don't need pfield_size anymore - // * same for poffset, unless it's in use by our caller? - // poffset doesn't seem to be used after this function returns. - // Though we might have to honour it's starting point? - // I guess fall back, if it ain't zero? - - CFieldObject* self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); - if (self == NULL) - return NULL; - StgDictObject* dict = PyType_stgdict(desc); - if (!dict) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - Py_DECREF(self); - return NULL; - } int is_bitfield = !!bitsize; if(!is_bitfield) { bitsize = 8 * dict->size; // might still be 0 afterwards. } - Py_ssize_t align; - if (pack) - align = min(pack, dict->align); - else - align = dict->align; + Py_ssize_t align = dict->align;; if ((bitsize > 0) && (round_down(*pbitofs, 8 * align) @@ -342,7 +311,9 @@ PyObject * PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, int *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian) + int pack, int big_endian, + CFieldObject* self, StgDictObject* dict + ) { assert(*pfield_size == 0); *pbitofs += *poffset * 8; @@ -357,17 +328,6 @@ PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, // Though we might have to honour it's starting point? // I guess fall back, if it ain't zero? - CFieldObject* self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); - if (self == NULL) - return NULL; - StgDictObject* dict = PyType_stgdict(desc); - if (!dict) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - Py_DECREF(self); - return NULL; - } - int is_bitfield = !!bitsize; if(!is_bitfield) { bitsize = 8 * dict->size; // might still be 0 afterwards. @@ -454,11 +414,24 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian) { + CFieldObject* self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); + if (self == NULL) + return NULL; + StgDictObject* dict = PyType_stgdict(desc); + if (!dict) { + PyErr_SetString(PyExc_TypeError, + "has no _stginfo_"); + Py_DECREF(self); + return NULL; + } + if(big_endian) { return PyCField_FromDesc_big_endian(desc, index, pfield_size, bitsize, pbitofs, psize, poffset, palign, - pack, big_endian); + pack, big_endian, + self, dict + ); } else if ( #ifdef MS_WIN32 true @@ -469,12 +442,16 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, return PyCField_FromDesc_windows(desc, index, pfield_size, bitsize, pbitofs, psize, poffset, palign, - pack, big_endian); + pack, big_endian, + self, dict + ); } else { return PyCField_FromDesc_linux(desc, index, pfield_size, bitsize, pbitofs, psize, poffset, palign, - pack, big_endian); + pack, big_endian, + self, dict + ); } } @@ -497,6 +474,7 @@ PyCField_set(CFieldObject *self, PyObject *inst, PyObject *value) } return PyCData_set(inst, self->proto, self->setfunc, value, self->index, self->size, ptr); +} static PyObject * PyCField_get(CFieldObject *self, PyObject *inst, PyTypeObject *type) From 2dee0e3234bc701af36dc8b2c64c73bd0ce6c953 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 5 Oct 2022 13:47:31 +0800 Subject: [PATCH 07/91] Abstract out proto stuff --- Modules/_ctypes/cfield.c | 152 ++++++++++----------------------------- 1 file changed, 38 insertions(+), 114 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 00ecd5dd2bb11f..0e772a6bac034e 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -81,10 +81,7 @@ PyCField_FromDesc_big_endian(PyObject *desc, Py_ssize_t index, CFieldObject* self, StgDictObject* dict ) { - PyObject *proto; Py_ssize_t size, align; - SETFUNC setfunc = NULL; - GETFUNC getfunc = NULL; int fieldtype; #define NO_BITFIELD 0 #define NEW_BITFIELD 1 @@ -124,41 +121,6 @@ PyCField_FromDesc_big_endian(PyObject *desc, Py_ssize_t index, } size = dict->size; - proto = desc; - - /* Field descriptors for 'c_char * n' are be scpecial cased to - return a Python string instead of an Array object instance... - */ - if (PyCArrayTypeObject_Check(proto)) { - StgDictObject *adict = PyType_stgdict(proto); - StgDictObject *idict; - if (adict && adict->proto) { - idict = PyType_stgdict(adict->proto); - if (!idict) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - Py_DECREF(self); - return NULL; - } - if (idict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { - struct fielddesc *fd = _ctypes_get_fielddesc("s"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; - } - if (idict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { - struct fielddesc *fd = _ctypes_get_fielddesc("U"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; - } - } - } - - self->setfunc = setfunc; - self->getfunc = getfunc; - self->index = index; - - Py_INCREF(proto); - self->proto = proto; switch (fieldtype) { case NEW_BITFIELD: @@ -245,44 +207,6 @@ PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, *pbitofs = round_up(*pbitofs, 8*align); } - PyObject* proto = desc; - - /* Field descriptors for 'c_char * n' are be scpecial cased to - return a Python string instead of an Array object instance... - */ - SETFUNC setfunc = NULL; - GETFUNC getfunc = NULL; - if (PyCArrayTypeObject_Check(proto)) { - StgDictObject *adict = PyType_stgdict(proto); - StgDictObject *idict; - if (adict && adict->proto) { - idict = PyType_stgdict(adict->proto); - if (!idict) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - Py_DECREF(self); - return NULL; - } - if (idict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { - struct fielddesc *fd = _ctypes_get_fielddesc("s"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; - } - if (idict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { - struct fielddesc *fd = _ctypes_get_fielddesc("U"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; - } - } - } - - self->setfunc = setfunc; - self->getfunc = getfunc; - self->index = index; - - Py_INCREF(proto); - self->proto = proto; - assert(bitsize <= dict->size * 8); assert(*poffset == 0); @@ -346,44 +270,6 @@ PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, *pbitofs = round_up(*pbitofs, 8*align); } - PyObject* proto = desc; - - /* Field descriptors for 'c_char * n' are be scpecial cased to - return a Python string instead of an Array object instance... - */ - SETFUNC setfunc = NULL; - GETFUNC getfunc = NULL; - if (PyCArrayTypeObject_Check(proto)) { - StgDictObject *adict = PyType_stgdict(proto); - StgDictObject *idict; - if (adict && adict->proto) { - idict = PyType_stgdict(adict->proto); - if (!idict) { - PyErr_SetString(PyExc_TypeError, - "has no _stginfo_"); - Py_DECREF(self); - return NULL; - } - if (idict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { - struct fielddesc *fd = _ctypes_get_fielddesc("s"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; - } - if (idict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { - struct fielddesc *fd = _ctypes_get_fielddesc("U"); - getfunc = fd->getfunc; - setfunc = fd->setfunc; - } - } - } - - self->setfunc = setfunc; - self->getfunc = getfunc; - self->index = index; - - Py_INCREF(proto); - self->proto = proto; - assert(bitsize <= dict->size * 8); assert(*poffset == 0); @@ -425,6 +311,44 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, return NULL; } + PyObject* proto = desc; + + /* Field descriptors for 'c_char * n' are be scpecial cased to + return a Python string instead of an Array object instance... + */ + SETFUNC setfunc = NULL; + GETFUNC getfunc = NULL; + if (PyCArrayTypeObject_Check(proto)) { + StgDictObject *adict = PyType_stgdict(proto); + StgDictObject *idict; + if (adict && adict->proto) { + idict = PyType_stgdict(adict->proto); + if (!idict) { + PyErr_SetString(PyExc_TypeError, + "has no _stginfo_"); + Py_DECREF(self); + return NULL; + } + if (idict->getfunc == _ctypes_get_fielddesc("c")->getfunc) { + struct fielddesc *fd = _ctypes_get_fielddesc("s"); + getfunc = fd->getfunc; + setfunc = fd->setfunc; + } + if (idict->getfunc == _ctypes_get_fielddesc("u")->getfunc) { + struct fielddesc *fd = _ctypes_get_fielddesc("U"); + getfunc = fd->getfunc; + setfunc = fd->setfunc; + } + } + } + + self->setfunc = setfunc; + self->getfunc = getfunc; + self->index = index; + + Py_INCREF(proto); + self->proto = proto; + if(big_endian) { return PyCField_FromDesc_big_endian(desc, index, pfield_size, bitsize, pbitofs, From 79ef347384bc3615e3e84824e0a60ee631348a08 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 5 Oct 2022 13:49:38 +0800 Subject: [PATCH 08/91] Clean align --- Modules/_ctypes/cfield.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 0e772a6bac034e..929e347eaf6533 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -198,13 +198,13 @@ PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, bitsize = 8 * dict->size; // might still be 0 afterwards. } - Py_ssize_t align = dict->align;; + *palign = dict->align; if ((bitsize > 0) - && (round_down(*pbitofs, 8 * align) - < round_down(*pbitofs + bitsize - 1, 8 * align))) { + && (round_down(*pbitofs, 8 * dict->align) + < round_down(*pbitofs + bitsize - 1, 8 * dict->align))) { // We would be straddling alignment units. - *pbitofs = round_up(*pbitofs, 8*align); + *pbitofs = round_up(*pbitofs, 8*dict->align); } assert(bitsize <= dict->size * 8); @@ -219,13 +219,12 @@ PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, assert(dict->size == dict->align); assert(effective_bitsof <= dict->size * 8); } else { - self->offset = round_down(*pbitofs, 8*align) / 8; + self->offset = round_down(*pbitofs, 8*dict->align) / 8; self->size = dict->size; } *pbitofs += bitsize; *psize = round_up(*pbitofs, 8) / 8; - *palign = align; assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); return (PyObject *)self; From 6170dad206e3253621bed7eafe9b1aae0c94cd90 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 5 Oct 2022 15:12:05 +0800 Subject: [PATCH 09/91] Hypothesis tests pass --- Modules/_ctypes/cfield.c | 41 ++++++++++++++++++++++++--------------- Modules/_ctypes/stgdict.c | 1 + 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 929e347eaf6533..d778c4e9f7ac20 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -238,9 +238,12 @@ PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, CFieldObject* self, StgDictObject* dict ) { - assert(*pfield_size == 0); - *pbitofs += *poffset * 8; - *poffset = 0; + /* + Now: pbitofs is relative to poffset; + we assume that poffset == 0 is aligned as much as we need it. + */ + // *pbitofs += *poffset * 8; + // *poffset = 0; // Change: // * pbitofs is now relative to the start of the struct, not the start of @@ -262,31 +265,37 @@ PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, else align = dict->align; - if ((bitsize > 0) - && (round_down(*pbitofs, 8 * align) - < round_down(*pbitofs + bitsize - 1, 8 * align))) { - // We would be straddling alignment units. - *pbitofs = round_up(*pbitofs, 8*align); + assert(bitsize <= dict->size * 8); + + // New thing: poffset points to end of bitfield. + // And we work with negative *pbitofs; + if (0 < *pbitofs + bitsize || 8 * dict->size != *pfield_size) { + // Close bitfield, ... + // ... align, + *poffset = round_up(*poffset, align); + + // ... and re-open. + *poffset += dict->size; + + *pfield_size = dict->size * 8; + *pbitofs = - *pfield_size; } - assert(bitsize <= dict->size * 8); - assert(*poffset == 0); + assert(8 * dict->size == *pfield_size); // We need to fit within alignment and within size. // But we only really care about size, when we have a bitfield. + self->offset = *poffset - (*pfield_size) / 8; if(is_bitfield) { - self->offset = round_down(*pbitofs, 8*dict->size) / 8; - Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; - self->size = (bitsize << 16 ) + effective_bitsof; + self->size = (bitsize << 16 ) + (*pfield_size + *pbitofs); assert(dict->size == dict->align); - assert(effective_bitsof <= dict->size * 8); } else { - self->offset = round_down(*pbitofs, 8*align) / 8; self->size = dict->size; } + assert(*pfield_size + *pbitofs <= dict->size * 8); *pbitofs += bitsize; - *psize = round_up(*pbitofs, 8) / 8; + *psize = *poffset; *palign = align; assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index c017323d5a754e..f0dc367a7a1850 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -612,6 +612,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct &size, &offset, &align, pack, big_endian); } else /* union */ { + field_size = 0; size = 0; bitofs = 0; offset = 0; From 871ca1afcbb8c44ba3824ac383be4d22010c96fa Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 5 Oct 2022 15:13:01 +0800 Subject: [PATCH 10/91] Formatting --- Modules/_ctypes/cfield.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index d778c4e9f7ac20..8533e6bff23c7c 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -273,7 +273,7 @@ PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, // Close bitfield, ... // ... align, *poffset = round_up(*poffset, align); - + // ... and re-open. *poffset += dict->size; From c162144f7485decd14467c31bcdde650174a5e10 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 5 Oct 2022 15:14:33 +0800 Subject: [PATCH 11/91] Clean up --- Modules/_ctypes/cfield.c | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 8533e6bff23c7c..0962225f5409a6 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -238,22 +238,6 @@ PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, CFieldObject* self, StgDictObject* dict ) { - /* - Now: pbitofs is relative to poffset; - we assume that poffset == 0 is aligned as much as we need it. - */ - // *pbitofs += *poffset * 8; - // *poffset = 0; - - // Change: - // * pbitofs is now relative to the start of the struct, not the start of - // the current field - // * we don't need pfield_size anymore - // * same for poffset, unless it's in use by our caller? - // poffset doesn't seem to be used after this function returns. - // Though we might have to honour it's starting point? - // I guess fall back, if it ain't zero? - int is_bitfield = !!bitsize; if(!is_bitfield) { bitsize = 8 * dict->size; // might still be 0 afterwards. @@ -287,8 +271,10 @@ PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, // But we only really care about size, when we have a bitfield. self->offset = *poffset - (*pfield_size) / 8; if(is_bitfield) { - self->size = (bitsize << 16 ) + (*pfield_size + *pbitofs); assert(dict->size == dict->align); + assert(0 <= (*pfield_size + *pbitofs)); + assert((*pfield_size + *pbitofs) < dict->size * 8); + self->size = (bitsize << 16 ) + (*pfield_size + *pbitofs); } else { self->size = dict->size; } From a57cf2cd6cd8cb2fd5471093bba71544a0f927ea Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 5 Oct 2022 15:20:32 +0800 Subject: [PATCH 12/91] Adapt tests for Windows --- Lib/test/test_ctypes/test_bitfields.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 823f186a373883..2692ddb3ab3ba8 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -279,24 +279,29 @@ class Bar(Structure): ] self.assertEqual(sizeof(Foo), sizeof(Bar)) - @unittest.skipIf(sys.platform == 'win32', "Doesn't work on Windows, yet") def test_mixed_9(self): class X(Structure): _fields_ = [ ("A", c_uint8), ("B", c_uint, 1), ] - self.assertEqual(4, sizeof(X)) + if sys.platform == 'win32': + self.assertEqual(sizeof(X), 8) + else: + self.assertEqual(4, sizeof(X)) - @unittest.skipIf(sys.platform == 'win32', "Doesn't work on Windows, yet") def test_mixed_10(self): class X(Structure): _fields_ = [ ("A", c_uint32, 1), ("B", c_uint64, 1), ] - self.assertEqual(8, alignment(X)) - self.assertEqual(8, sizeof(X)) + if sys.platform == 'win32': + self.assertEqual(16, alignment(X)) + self.assertEqual(16, sizeof(X)) + else: + self.assertEqual(8, alignment(X)) + self.assertEqual(8, sizeof(X)) def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message From 3f7c4ccfc7eba649596f887467aa942b1b376752 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 5 Oct 2022 15:21:21 +0800 Subject: [PATCH 13/91] Fix order --- Lib/test/test_ctypes/test_bitfields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 2692ddb3ab3ba8..cf12b97ebeca68 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -286,7 +286,7 @@ class X(Structure): ("B", c_uint, 1), ] if sys.platform == 'win32': - self.assertEqual(sizeof(X), 8) + self.assertEqual(8, sizeof(X)) else: self.assertEqual(4, sizeof(X)) From e39a271632e9c0fc80558b5c082cc48d37d44bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20G=C3=B6rgens?= Date: Wed, 5 Oct 2022 16:08:37 +0800 Subject: [PATCH 14/91] Fix alignment test --- Lib/test/test_ctypes/test_bitfields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index cf12b97ebeca68..95c965f02c28b1 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -297,7 +297,7 @@ class X(Structure): ("B", c_uint64, 1), ] if sys.platform == 'win32': - self.assertEqual(16, alignment(X)) + self.assertEqual(8, alignment(X)) self.assertEqual(16, sizeof(X)) else: self.assertEqual(8, alignment(X)) From 52ef8d29c37336e6a5b2141fbbd7b4da608434ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20G=C3=B6rgens?= Date: Wed, 5 Oct 2022 16:42:50 +0800 Subject: [PATCH 15/91] Avoid casting --- Modules/_ctypes/cfield.c | 8 ++++---- Modules/_ctypes/stgdict.c | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 0962225f5409a6..522f8189cb5b1b 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -75,7 +75,7 @@ Py_ssize_t NUM_BITS(Py_ssize_t x); */ PyObject * PyCField_FromDesc_big_endian(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, + Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, CFieldObject* self, StgDictObject* dict @@ -182,7 +182,7 @@ PyCField_FromDesc_big_endian(PyObject *desc, Py_ssize_t index, PyObject * PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, + Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, CFieldObject* self, StgDictObject* dict @@ -232,7 +232,7 @@ PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, PyObject * PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, + Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, CFieldObject* self, StgDictObject* dict @@ -290,7 +290,7 @@ PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, PyObject * PyCField_FromDesc(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, + Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian) { diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index f0dc367a7a1850..194a511034f7fa 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -354,7 +354,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct Py_ssize_t len, offset, size, align, i; Py_ssize_t union_size, total_align; Py_ssize_t field_size = 0; - int bitofs = 0; + Py_ssize_t bitofs = 0; PyObject *tmp; int isPacked; int pack; From e8102c41abcc7831c1218ebc2e8989aaf6af613a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20G=C3=B6rgens?= Date: Wed, 5 Oct 2022 17:02:55 +0800 Subject: [PATCH 16/91] Fixup --- Modules/_ctypes/ctypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 88eb9f59922a04..eeb36d11eba311 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -141,7 +141,7 @@ extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); extern PyObject * PyCField_FromDesc(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, int *pbitofs, + Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int is_big_endian); From 6b6fa8adf071f00edc52bc5225cd617be92bad3b Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sat, 8 Oct 2022 16:15:43 +0800 Subject: [PATCH 17/91] Add ability to force msvc compatibility even when not doing any packing --- Modules/_ctypes/cfield.c | 2 +- Modules/_ctypes/ctypes.h | 2 +- Modules/_ctypes/stgdict.c | 20 ++++++++++++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 522f8189cb5b1b..4edbbdad6b6edd 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -292,7 +292,7 @@ PyObject * PyCField_FromDesc(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian) + int pack, int big_endian, int ms_struct) { CFieldObject* self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); if (self == NULL) diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index eeb36d11eba311..3bb19ad59c6a45 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -143,7 +143,7 @@ extern PyObject * PyCField_FromDesc(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int is_big_endian); + int pack, int is_big_endian, int ms_struct); extern PyObject *PyCData_AtAddress(PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(PyObject *type, char *data, Py_ssize_t length); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 194a511034f7fa..e8c51447511dcc 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -350,6 +350,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct _Py_IDENTIFIER(_swappedbytes_); _Py_IDENTIFIER(_use_broken_old_ctypes_structure_semantics_); _Py_IDENTIFIER(_pack_); + _Py_IDENTIFIER(_ms_struct_); StgDictObject *stgdict, *basedict; Py_ssize_t len, offset, size, align, i; Py_ssize_t union_size, total_align; @@ -360,6 +361,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct int pack; Py_ssize_t ffi_ofs; int big_endian; + int ms_struct; int arrays_seen = 0; /* HACK Alert: I cannot be bothered to fix ctypes.com, so there has to @@ -384,6 +386,20 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct big_endian = PY_BIG_ENDIAN; } + #ifdef MS_WIN32 + ms_struct = 1; + #else + ms_struct = 0; + #endif + + if (_PyObject_LookupAttrId(type, &PyId__ms_struct_, &tmp) < 0) { + return -1; + } + if (tmp) { + ms_struct = _PyLong_AsInt(tmp); + Py_DECREF(tmp); + } + if (_PyObject_LookupAttrId(type, &PyId__use_broken_old_ctypes_structure_semantics_, &tmp) < 0) { @@ -610,7 +626,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct prop = PyCField_FromDesc(desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian); + pack, big_endian, ms_struct); } else /* union */ { field_size = 0; size = 0; @@ -620,7 +636,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct prop = PyCField_FromDesc(desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian); + pack, big_endian, ms_struct); union_size = max(size, union_size); } total_align = max(align, total_align); From ca9d580dde68ca42e211b0bc4946094c807f7224 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sat, 8 Oct 2022 21:36:18 +0800 Subject: [PATCH 18/91] More tests --- Lib/test/test_ctypes/test_bitfields.py | 71 ++++++++++++++++++++++++++ Modules/_ctypes/cfield.c | 8 +-- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 95c965f02c28b1..40bc2f7639c244 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -303,6 +303,77 @@ class X(Structure): self.assertEqual(8, alignment(X)) self.assertEqual(8, sizeof(X)) + def test_gh_95496(self): + for field_width in range(1, 33): + class TestStruct(Structure): + _fields_ = [ + ("Field1", c_uint32, field_width), + ("Field2", c_uint8, 8) + ] + + cmd = TestStruct() + cmd.Field2 = 1 + self.assertEqual(1, cmd.Field2) + + def test_gh_84039(self): + class Bad(Structure): + _pack_ = 1 + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + + class GoodA(Structure): + _pack_ = 1 + _fields_ = [ + ("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ] + + + class Good(Structure): + _pack_ = 1 + _fields_ = [ + ("a", GoodA), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12), + ] + + self.assertEqual(3, sizeof(Bad)) + self.assertEqual(3, sizeof(Good)) + + def test_gh_73939(self): + class MyStructure(Structure): + _pack_ = 1 + _fields_ = [ + ("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2) + ] + self.assertEqual(8, sizeof(MyStructure)) + def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message class X(Structure): diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 4edbbdad6b6edd..8e61ccd9ae3825 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -350,13 +350,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, pack, big_endian, self, dict ); - } else if ( - #ifdef MS_WIN32 - true - #else - pack != 0 - #endif - ) { + } else if (ms_struct || pack != 0) { return PyCField_FromDesc_windows(desc, index, pfield_size, bitsize, pbitofs, psize, poffset, palign, From 235fa68b9ddb8c6e51527d52fa13ba2bcc5b7a02 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 12:57:46 +0800 Subject: [PATCH 19/91] Big endian works --- Modules/_ctypes/cfield.c | 195 ++++++++++----------------------------- 1 file changed, 50 insertions(+), 145 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 8e61ccd9ae3825..969d85155d2ac5 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -57,9 +57,11 @@ Py_ssize_t round_up(Py_ssize_t numToRound, Py_ssize_t multiple) } static inline -Py_ssize_t LOW_BIT(Py_ssize_t x); +Py_ssize_t NUM_BITS(Py_ssize_t bitsize); static inline -Py_ssize_t NUM_BITS(Py_ssize_t x); +Py_ssize_t LOW_BIT(Py_ssize_t offset); +static inline +Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset); /* * Expects the size, index and offset for the current field in *psize and @@ -73,131 +75,20 @@ Py_ssize_t NUM_BITS(Py_ssize_t x); * pbitofs points to the current bit offset, this will be updated. * prev_desc points to the type of the previous bitfield, if any. */ -PyObject * -PyCField_FromDesc_big_endian(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian, - CFieldObject* self, StgDictObject* dict - ) -{ - Py_ssize_t size, align; - int fieldtype; -#define NO_BITFIELD 0 -#define NEW_BITFIELD 1 -#define CONT_BITFIELD 2 -#define EXPAND_BITFIELD 3 - if (bitsize /* this is a bitfield request */ - && *pfield_size /* we have a bitfield open */ -#ifdef MS_WIN32 - /* MSVC, GCC with -mms-bitfields */ - && dict->size * 8 == *pfield_size -#else - /* GCC */ - && dict->size * 8 <= *pfield_size -#endif - && (*pbitofs + bitsize) <= *pfield_size) { - /* continue bit field */ - fieldtype = CONT_BITFIELD; -#ifndef MS_WIN32 - } else if (bitsize /* this is a bitfield request */ - && *pfield_size /* we have a bitfield open */ - && dict->size * 8 >= *pfield_size - && (*pbitofs + bitsize) <= dict->size * 8) { - /* expand bit field */ - fieldtype = EXPAND_BITFIELD; -#endif - } else if (bitsize) { - /* start new bitfield */ - fieldtype = NEW_BITFIELD; - *pbitofs = 0; - *pfield_size = dict->size * 8; - } else { - /* not a bit field */ - fieldtype = NO_BITFIELD; - *pbitofs = 0; - *pfield_size = 0; - } - - size = dict->size; - - switch (fieldtype) { - case NEW_BITFIELD: - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - *pbitofs = bitsize; - /* fall through */ - case NO_BITFIELD: - if (pack) - align = min(pack, dict->align); - else - align = dict->align; - if (align && *poffset % align) { - Py_ssize_t delta = align - (*poffset % align); - *psize += delta; - *poffset += delta; - } - - if (bitsize == 0) - self->size = size; - *psize += size; - - self->offset = *poffset; - *poffset += size; - - *palign = align; - break; - - case EXPAND_BITFIELD: - *poffset += dict->size - *pfield_size/8; - *psize += dict->size - *pfield_size/8; - - *pfield_size = dict->size * 8; - - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - - self->offset = *poffset - size; /* poffset is already updated for the NEXT field */ - *pbitofs += bitsize; - break; - - case CONT_BITFIELD: - if (big_endian) - self->size = (bitsize << 16) + *pfield_size - *pbitofs - bitsize; - else - self->size = (bitsize << 16) + *pbitofs; - - self->offset = *poffset - size; /* poffset is already updated for the NEXT field */ - *pbitofs += bitsize; - break; - } - - return (PyObject *)self; -} - -PyObject * +void PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, - CFieldObject* self, StgDictObject* dict + CFieldObject* self, StgDictObject* dict, + int is_bitfield ) { assert(*pfield_size == 0); *pbitofs += *poffset * 8; *poffset = 0; - - int is_bitfield = !!bitsize; - if(!is_bitfield) { - bitsize = 8 * dict->size; // might still be 0 afterwards. - } - *palign = dict->align; if ((bitsize > 0) @@ -215,7 +106,7 @@ PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, if(is_bitfield) { self->offset = round_down(*pbitofs, 8*dict->size) / 8; Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; - self->size = (bitsize << 16 ) + effective_bitsof; + self->size = BUILD_SIZE(bitsize, effective_bitsof); assert(dict->size == dict->align); assert(effective_bitsof <= dict->size * 8); } else { @@ -227,22 +118,17 @@ PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, *psize = round_up(*pbitofs, 8) / 8; assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); - return (PyObject *)self; } -PyObject * +void PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, - CFieldObject* self, StgDictObject* dict + CFieldObject* self, StgDictObject* dict, + int is_bitfield ) { - int is_bitfield = !!bitsize; - if(!is_bitfield) { - bitsize = 8 * dict->size; // might still be 0 afterwards. - } - Py_ssize_t align; if (pack) align = min(pack, dict->align); @@ -274,7 +160,7 @@ PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, assert(dict->size == dict->align); assert(0 <= (*pfield_size + *pbitofs)); assert((*pfield_size + *pbitofs) < dict->size * 8); - self->size = (bitsize << 16 ) + (*pfield_size + *pbitofs); + self->size = BUILD_SIZE(bitsize, *pfield_size + *pbitofs); } else { self->size = dict->size; } @@ -285,7 +171,6 @@ PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, *palign = align; assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); - return (PyObject *)self; } PyObject * @@ -343,28 +228,33 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, Py_INCREF(proto); self->proto = proto; - if(big_endian) { - return PyCField_FromDesc_big_endian(desc, index, - pfield_size, bitsize, pbitofs, - psize, poffset, palign, - pack, big_endian, - self, dict - ); - } else if (ms_struct || pack != 0) { - return PyCField_FromDesc_windows(desc, index, + int is_bitfield = !!bitsize; + if(!is_bitfield) { + bitsize = 8 * dict->size; // might still be 0 afterwards. + } + + if (ms_struct || pack != 0) { + PyCField_FromDesc_windows(desc, index, pfield_size, bitsize, pbitofs, psize, poffset, palign, pack, big_endian, - self, dict + self, dict, + is_bitfield ); } else { - return PyCField_FromDesc_linux(desc, index, + PyCField_FromDesc_linux(desc, index, pfield_size, bitsize, pbitofs, psize, poffset, palign, pack, big_endian, - self, dict + self, dict, + is_bitfield ); } + + if(big_endian && is_bitfield) { + self->size = BUILD_SIZE(NUM_BITS(self->size), 8*dict->size - LOW_BIT(self->size) - bitsize); + } + return (PyObject *)self; } static int @@ -450,8 +340,9 @@ static PyObject * PyCField_repr(CFieldObject *self) { PyObject *result; - Py_ssize_t bits = self->size >> 16; - Py_ssize_t size = self->size & 0xFFFF; + // TODO: use function. + Py_ssize_t bits = NUM_BITS(self->size); + Py_ssize_t size = LOW_BIT(self->size); const char *name; name = ((PyTypeObject *)self->proto)->tp_name; @@ -571,12 +462,26 @@ get_ulonglong(PyObject *v, unsigned long long *p) /* how to decode the size field, for integer get/set functions */ static inline -Py_ssize_t LOW_BIT(Py_ssize_t x) { - return x & 0xFFFF; +Py_ssize_t LOW_BIT(Py_ssize_t offset) { + return offset & 0xFFFF; } static inline -Py_ssize_t NUM_BITS(Py_ssize_t x) { - return x >> 16; +Py_ssize_t NUM_BITS(Py_ssize_t bitsize) { + return bitsize >> 16; +} + +static inline +Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset) { + assert(0 <= offset); + assert(offset <= 0xFFFF); + // We don't support zero length bitfields. + // And GET_BITFIELD uses NUM_BITS(size)==0, + // to figure out whether we are handling a bitfield. + assert(0 < bitsize); + Py_ssize_t result = (bitsize << 16) + offset; + assert(bitsize == NUM_BITS(result)); + assert(offset == LOW_BIT(result)); + return result; } /* Doesn't work if NUM_BITS(size) == 0, but it never happens in SET() call. */ From 53db061cc9e6756da4633bade96f582b6b66897f Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 14:24:19 +0800 Subject: [PATCH 20/91] More tests --- Lib/test/test_ctypes/test_bitfields.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 40bc2f7639c244..46a82a0b0a77f2 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -374,6 +374,15 @@ class MyStructure(Structure): ] self.assertEqual(8, sizeof(MyStructure)) + def test_gh_86098(self): + class X(Structure): + _fields_ = [ + ("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16) + ] + self.assertEqual(4, sizeof(X)) + def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message class X(Structure): From 9c249c34d2f7ebf7aa0ae69a27716647bf5aebde Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 14:33:32 +0800 Subject: [PATCH 21/91] More asserts --- Modules/_ctypes/cfield.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 969d85155d2ac5..65824f209f336f 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -42,6 +42,7 @@ static inline Py_ssize_t round_down(Py_ssize_t numToRound, Py_ssize_t multiple) { assert(numToRound >= 0); + assert(multiple >= 0); if (multiple == 0) return numToRound; return (numToRound / multiple) * multiple; @@ -51,6 +52,7 @@ static inline Py_ssize_t round_up(Py_ssize_t numToRound, Py_ssize_t multiple) { assert(numToRound >= 0); + assert(multiple >= 0); if (multiple == 0) return numToRound; return ((numToRound + multiple - 1) / multiple) * multiple; From 3cf4747bbddd1f4857d211a2647119ddda92a2f6 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 14:39:05 +0800 Subject: [PATCH 22/91] Drop unneeded parameters --- Modules/_ctypes/cfield.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 65824f209f336f..fb5e7bacbe6398 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -79,15 +79,13 @@ Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset); */ void -PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, +PyCField_FromDesc_gcc(Py_ssize_t index, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, CFieldObject* self, StgDictObject* dict, int is_bitfield ) { - assert(*pfield_size == 0); *pbitofs += *poffset * 8; *poffset = 0; @@ -123,7 +121,7 @@ PyCField_FromDesc_linux(PyObject *desc, Py_ssize_t index, } void -PyCField_FromDesc_windows(PyObject *desc, Py_ssize_t index, +PyCField_FromDesc_msvc(Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, @@ -236,7 +234,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, } if (ms_struct || pack != 0) { - PyCField_FromDesc_windows(desc, index, + PyCField_FromDesc_msvc(index, pfield_size, bitsize, pbitofs, psize, poffset, palign, pack, big_endian, @@ -244,8 +242,8 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, is_bitfield ); } else { - PyCField_FromDesc_linux(desc, index, - pfield_size, bitsize, pbitofs, + PyCField_FromDesc_gcc(index, + bitsize, pbitofs, psize, poffset, palign, pack, big_endian, self, dict, From 8beddb912bb6177ae7d3378f3bc67e696b3fff8a Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 14:40:12 +0800 Subject: [PATCH 23/91] Drop unneeded parameters --- Modules/_ctypes/cfield.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index fb5e7bacbe6398..b4cfc2f5645bfc 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -81,7 +81,6 @@ Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset); void PyCField_FromDesc_gcc(Py_ssize_t index, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian, CFieldObject* self, StgDictObject* dict, int is_bitfield ) @@ -124,7 +123,7 @@ void PyCField_FromDesc_msvc(Py_ssize_t index, Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian, + int pack, CFieldObject* self, StgDictObject* dict, int is_bitfield ) @@ -237,7 +236,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, PyCField_FromDesc_msvc(index, pfield_size, bitsize, pbitofs, psize, poffset, palign, - pack, big_endian, + pack, self, dict, is_bitfield ); @@ -245,7 +244,6 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, PyCField_FromDesc_gcc(index, bitsize, pbitofs, psize, poffset, palign, - pack, big_endian, self, dict, is_bitfield ); From 9b64ff6ccb36798b79c21322c1b473ac89aa3058 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 14:59:43 +0800 Subject: [PATCH 24/91] Drop unneeded parameters --- Modules/_ctypes/cfield.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index b4cfc2f5645bfc..a7d53cec693b0c 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -79,7 +79,7 @@ Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset); */ void -PyCField_FromDesc_gcc(Py_ssize_t index, int bitsize, Py_ssize_t *pbitofs, +PyCField_FromDesc_gcc(int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgDictObject* dict, int is_bitfield @@ -120,7 +120,7 @@ PyCField_FromDesc_gcc(Py_ssize_t index, int bitsize, Py_ssize_t *pbitofs, } void -PyCField_FromDesc_msvc(Py_ssize_t index, +PyCField_FromDesc_msvc( Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, @@ -233,7 +233,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, } if (ms_struct || pack != 0) { - PyCField_FromDesc_msvc(index, + PyCField_FromDesc_msvc( pfield_size, bitsize, pbitofs, psize, poffset, palign, pack, @@ -241,7 +241,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, is_bitfield ); } else { - PyCField_FromDesc_gcc(index, + PyCField_FromDesc_gcc( bitsize, pbitofs, psize, poffset, palign, self, dict, From 4d0dab5c6649e2dde8a996adee55c775291f234b Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 15:15:49 +0800 Subject: [PATCH 25/91] Clean up --- Modules/_ctypes/cfield.c | 7 ++----- Modules/_ctypes/stgdict.c | 5 ++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index a7d53cec693b0c..2835d5c3116521 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -96,8 +96,6 @@ PyCField_FromDesc_gcc(int bitsize, Py_ssize_t *pbitofs, // We would be straddling alignment units. *pbitofs = round_up(*pbitofs, 8*dict->align); } - - assert(bitsize <= dict->size * 8); assert(*poffset == 0); // We need to fit within alignment and within size. @@ -134,10 +132,8 @@ PyCField_FromDesc_msvc( else align = dict->align; - assert(bitsize <= dict->size * 8); - // New thing: poffset points to end of bitfield. - // And we work with negative *pbitofs; + // And we work with negative *pbitofs. if (0 < *pbitofs + bitsize || 8 * dict->size != *pfield_size) { // Close bitfield, ... // ... align, @@ -231,6 +227,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, if(!is_bitfield) { bitsize = 8 * dict->size; // might still be 0 afterwards. } + assert(bitsize <= dict->size * 8); if (ms_struct || pack != 0) { PyCField_FromDesc_msvc( diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index e8c51447511dcc..851fbf86ab5364 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -361,7 +361,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct int pack; Py_ssize_t ffi_ofs; int big_endian; - int ms_struct; int arrays_seen = 0; /* HACK Alert: I cannot be bothered to fix ctypes.com, so there has to @@ -387,9 +386,9 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct } #ifdef MS_WIN32 - ms_struct = 1; + int ms_struct = 1; #else - ms_struct = 0; + int ms_struct = 0; #endif if (_PyObject_LookupAttrId(type, &PyId__ms_struct_, &tmp) < 0) { From bf9866707e3b6b8846dbe08cf3472ea76210fb57 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 15:17:16 +0800 Subject: [PATCH 26/91] Explain --- Modules/_ctypes/cfield.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 2835d5c3116521..a6e8264951da24 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -229,6 +229,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, } assert(bitsize <= dict->size * 8); + // `pack` only makes sense in msvc compatibility mode. if (ms_struct || pack != 0) { PyCField_FromDesc_msvc( pfield_size, bitsize, pbitofs, From 8223063c1853cb4e1d472741ff12790cf910a927 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 15:28:06 +0800 Subject: [PATCH 27/91] More cleanup --- Modules/_ctypes/cfield.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index a6e8264951da24..dd30f078ed6ddf 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -85,6 +85,7 @@ PyCField_FromDesc_gcc(int bitsize, Py_ssize_t *pbitofs, int is_bitfield ) { + // We don't use poffset here, so clear it, if it has been set. *pbitofs += *poffset * 8; *poffset = 0; @@ -126,30 +127,31 @@ PyCField_FromDesc_msvc( int is_bitfield ) { - Py_ssize_t align; if (pack) - align = min(pack, dict->align); + *palign = min(pack, dict->align); else - align = dict->align; + *palign = dict->align; - // New thing: poffset points to end of bitfield. - // And we work with negative *pbitofs. + // *poffset points to end of current bitfield. + // *pbitofs is generally non-positive, + // and 8 * (*poffset) + *pbitofs points just behind + // the end of the last field we placed. if (0 < *pbitofs + bitsize || 8 * dict->size != *pfield_size) { - // Close bitfield, ... - // ... align, - *poffset = round_up(*poffset, align); + // Close the previous bitfield (if any). + // and start a new bitfield: + *poffset = round_up(*poffset, *palign); - // ... and re-open. *poffset += dict->size; *pfield_size = dict->size * 8; + // Reminder: 8 * (*poffset) + *pbitofs points to where we would start a + // new field. Ie just behind where we placed the last field plus an + // allowance for alignment. *pbitofs = - *pfield_size; } assert(8 * dict->size == *pfield_size); - // We need to fit within alignment and within size. - // But we only really care about size, when we have a bitfield. self->offset = *poffset - (*pfield_size) / 8; if(is_bitfield) { assert(dict->size == dict->align); @@ -163,9 +165,8 @@ PyCField_FromDesc_msvc( *pbitofs += bitsize; *psize = *poffset; - *palign = align; - assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); + assert(!is_bitfield || LOW_BIT(self->size) <= self->size * 8); } PyObject * From e80a09a268f572e1d26de59f43371fdcb1110afd Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 15:52:34 +0800 Subject: [PATCH 28/91] More cleanup --- Modules/_ctypes/cfield.c | 45 ++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index dd30f078ed6ddf..2abc3116bf98b4 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -65,20 +65,34 @@ Py_ssize_t LOW_BIT(Py_ssize_t offset); static inline Py_ssize_t BUILD_SIZE(Py_ssize_t bitsize, Py_ssize_t offset); -/* - * Expects the size, index and offset for the current field in *psize and - * *poffset, stores the total size so far in *psize, the offset for the next - * field in *poffset, the alignment requirements for the current field in - * *palign, and returns a field descriptor for this field. - */ -/* - * bitfields extension: - * bitsize != 0: this is a bit field. - * pbitofs points to the current bit offset, this will be updated. - * prev_desc points to the type of the previous bitfield, if any. - */ +/* PyCField_FromDesc creates and returns a struct/union field descriptor. -void +The function expects to be called repeatedly for all fields in a struct or +union. It uses helper functions PyCField_FromDesc_gcc and +PyCField_FromDesc_msvc to simulate the corresponding compilers. + +GCC mode places fields one after another, bit by bit. But when a field would straddle an alignment boundary for its type, we insert a few bits of padding to +avoid that. + +MSVC mode works similar expect for bitfield packing. Adjacent bit-fields are +packed into the same 1-, 2-, or 4-byte allocation unit if the integral types +are the same size and if the next bit-field fits into the current allocation +unit without crossing the boundary imposed by the common alignment requirements +of the bit-fields. + +See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields for details. + +We do not support zero length bitfields. In fact we use bitsize != 0 to +indicate a bitfield. + +PyCField_FromDesc manages: +- *psize: the size of the structure / union so far. +- *poffset, *pbitofs: 8* (*poffset) + *pbitofs points to where the next field + would start. +- *palign: the alignment requirements of the last field we placed. +*/ + +static void PyCField_FromDesc_gcc(int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgDictObject* dict, @@ -118,7 +132,7 @@ PyCField_FromDesc_gcc(int bitsize, Py_ssize_t *pbitofs, assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); } -void +static void PyCField_FromDesc_msvc( Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, @@ -226,7 +240,8 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, int is_bitfield = !!bitsize; if(!is_bitfield) { - bitsize = 8 * dict->size; // might still be 0 afterwards. + bitsize = 8 * dict->size; + // Caution: bitsize might still be 0 now. } assert(bitsize <= dict->size * 8); From 33a10fb36a86865dbf0ec208e14a93e18b522128 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 15:53:01 +0800 Subject: [PATCH 29/91] More cleanup --- Modules/_ctypes/cfield.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 2abc3116bf98b4..f479631378168f 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -113,8 +113,6 @@ PyCField_FromDesc_gcc(int bitsize, Py_ssize_t *pbitofs, } assert(*poffset == 0); - // We need to fit within alignment and within size. - // But we only really care about size, when we have a bitfield. if(is_bitfield) { self->offset = round_down(*pbitofs, 8*dict->size) / 8; Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; From 0f3c5a30488b8eadb2e2967f98f925449c71fa19 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 15:53:54 +0800 Subject: [PATCH 30/91] Move common assert --- Modules/_ctypes/cfield.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index f479631378168f..a1fa665e2f9416 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -126,8 +126,6 @@ PyCField_FromDesc_gcc(int bitsize, Py_ssize_t *pbitofs, *pbitofs += bitsize; *psize = round_up(*pbitofs, 8) / 8; - - assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); } static void @@ -177,8 +175,6 @@ PyCField_FromDesc_msvc( *pbitofs += bitsize; *psize = *poffset; - - assert(!is_bitfield || LOW_BIT(self->size) <= self->size * 8); } PyObject * @@ -260,7 +256,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, is_bitfield ); } - + assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); if(big_endian && is_bitfield) { self->size = BUILD_SIZE(NUM_BITS(self->size), 8*dict->size - LOW_BIT(self->size) - bitsize); } From 47eca3c913d4b533de9cc86742766bc0a6bb9b70 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 15:55:19 +0800 Subject: [PATCH 31/91] Break line --- Modules/_ctypes/cfield.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index a1fa665e2f9416..2595391f2e8e3e 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -71,7 +71,8 @@ The function expects to be called repeatedly for all fields in a struct or union. It uses helper functions PyCField_FromDesc_gcc and PyCField_FromDesc_msvc to simulate the corresponding compilers. -GCC mode places fields one after another, bit by bit. But when a field would straddle an alignment boundary for its type, we insert a few bits of padding to +GCC mode places fields one after another, bit by bit. But when a field would +straddle an alignment boundary for its type, we insert a few bits of padding to avoid that. MSVC mode works similar expect for bitfield packing. Adjacent bit-fields are From 456afd030c25b51f2787583f9305b3aa09b30505 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 15:56:23 +0800 Subject: [PATCH 32/91] Clean TODO --- Modules/_ctypes/cfield.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 2595391f2e8e3e..6b93e6eafaf89d 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -347,7 +347,6 @@ static PyObject * PyCField_repr(CFieldObject *self) { PyObject *result; - // TODO: use function. Py_ssize_t bits = NUM_BITS(self->size); Py_ssize_t size = LOW_BIT(self->size); const char *name; From 212cf13228dabc2a1fc544d076eaa7bf68feeec6 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 20:27:53 +0800 Subject: [PATCH 33/91] Fix test for Windows --- Lib/test/test_ctypes/test_bitfields.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 46a82a0b0a77f2..82277713134a33 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -381,7 +381,10 @@ class X(Structure): ("b", c_uint8, 8), ("c", c_uint32, 16) ] - self.assertEqual(4, sizeof(X)) + if sys.platform == 'win32': + self.assertEqual(8, sizeof(X)) + else: + self.assertEqual(4, sizeof(X)) def test_anon_bitfields(self): # anonymous bit-fields gave a strange error message From 45e26ecfbfc73e6d46a550de3b2d2689e6047f30 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 21:05:32 +0800 Subject: [PATCH 34/91] Remove warning --- Modules/_ctypes/cfield.c | 15 +++++++++------ Modules/_ctypes/ctypes.h | 5 +++-- Modules/_ctypes/stgdict.c | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 6b93e6eafaf89d..7878dc60ee1c16 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -94,7 +94,7 @@ PyCField_FromDesc manages: */ static void -PyCField_FromDesc_gcc(int bitsize, Py_ssize_t *pbitofs, +PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgDictObject* dict, int is_bitfield @@ -131,9 +131,9 @@ PyCField_FromDesc_gcc(int bitsize, Py_ssize_t *pbitofs, static void PyCField_FromDesc_msvc( - Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, + Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, + Py_ssize_t *palign, int pack, CFieldObject* self, StgDictObject* dict, int is_bitfield ) @@ -180,8 +180,8 @@ PyCField_FromDesc_msvc( PyObject * PyCField_FromDesc(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, + Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, int big_endian, int ms_struct) { CFieldObject* self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); @@ -235,6 +235,9 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, int is_bitfield = !!bitsize; if(!is_bitfield) { + assert(dict->size >= 0); + // assert: no overflow; + assert(dict->size < (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8); bitsize = 8 * dict->size; // Caution: bitsize might still be 0 now. } diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 3bb19ad59c6a45..8851d0d0d264d6 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -141,8 +141,9 @@ extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); extern PyObject * PyCField_FromDesc(PyObject *desc, Py_ssize_t index, - Py_ssize_t *pfield_size, int bitsize, Py_ssize_t *pbitofs, - Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, + Py_ssize_t *pfield_size, Py_ssize_t bitsize, + Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, + Py_ssize_t *palign, int pack, int is_big_endian, int ms_struct); extern PyObject *PyCData_AtAddress(PyObject *type, void *buf); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 851fbf86ab5364..6fd26de424c7ef 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -525,7 +525,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct PyObject *pair = PySequence_GetItem(fields, i); PyObject *prop; StgDictObject *dict; - int bitsize = 0; + Py_ssize_t bitsize = 0; if (!pair || !PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { PyErr_SetString(PyExc_TypeError, From ab42187f73dc6d855317cae4587aa6a146b3b1de Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 21:20:53 +0800 Subject: [PATCH 35/91] Port test from PR-19850 --- Lib/test/test_ctypes/test_bitfields.py | 63 ++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 82277713134a33..0d39d1f166edcb 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -303,6 +303,69 @@ class X(Structure): self.assertEqual(8, alignment(X)) self.assertEqual(8, sizeof(X)) + @unittest.skipIf(os.name == 'nt', reason='Posix only') + def test_packed_posix(self): + test_cases = { + ( + ("a", c_uint8, 4), + ("b", c_uint8, 4), + ): 1, + ( + ("a", c_uint8, 1), + ("b", c_uint16, 1), + ("c", c_uint32, 1), + ("d", c_uint64, 1), + ): 1, + ( + ("a", c_uint8, 8), + ("b", c_uint16, 1), + ("c", c_uint32, 1), + ("d", c_uint64, 1), + ): 2, + ( + ("a", c_uint32, 9), + ("b", c_uint16, 10), + ("c", c_uint32, 25), + ("d", c_uint64, 1), + ): 6, + ( + ("a", c_uint32, 9), + ("b", c_uint16, 10), + ("c", c_uint32, 25), + ("d", c_uint64, 5), + ): 7, + ( + ("a", c_uint16), + ("b", c_uint16, 9), + ("c", c_uint16, 1), + ("d", c_uint16, 1), + ("e", c_uint16, 1), + ("f", c_uint16, 1), + ("g", c_uint16, 3), + ("h", c_uint32, 10), + ("i", c_uint32, 20), + ("j", c_uint32, 2), + ): 8, + ( + ("a", c_uint16, 9), + ("b", c_uint16, 10), + ("d", c_uint16), + ("c", c_uint8, 8), + ): 6, + ( + ("a", c_uint32, 9), + ("b", c_uint32), + ("c", c_uint32, 8), + ): 7, + } + + for fields, size in test_cases.items(): + with self.subTest(fields=fields): + class X(Structure): + _pack_ = 1 + _fields_ = list(fields) + self.assertEqual(sizeof(X), size) + def test_gh_95496(self): for field_width in range(1, 33): class TestStruct(Structure): From 640c0628cf0a326d7f7e28b131c1c8cc8455dc10 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 21:32:48 +0800 Subject: [PATCH 36/91] Fix assert --- Modules/_ctypes/cfield.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 7878dc60ee1c16..f616ae13bc7685 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -237,7 +237,8 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, if(!is_bitfield) { assert(dict->size >= 0); // assert: no overflow; - assert(dict->size < (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8); + assert((unsigned long long int) dict->size + < (1ULL << (8*sizeof(Py_ssize_t)-1)) / 8); bitsize = 8 * dict->size; // Caution: bitsize might still be 0 now. } From f3e04afacc6578d119d4ab87f1baec9a7e8cf011 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 21:33:56 +0800 Subject: [PATCH 37/91] Revert "Port test from PR-19850" This reverts commit ab42187f73dc6d855317cae4587aa6a146b3b1de. --- Lib/test/test_ctypes/test_bitfields.py | 63 -------------------------- 1 file changed, 63 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 0d39d1f166edcb..82277713134a33 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -303,69 +303,6 @@ class X(Structure): self.assertEqual(8, alignment(X)) self.assertEqual(8, sizeof(X)) - @unittest.skipIf(os.name == 'nt', reason='Posix only') - def test_packed_posix(self): - test_cases = { - ( - ("a", c_uint8, 4), - ("b", c_uint8, 4), - ): 1, - ( - ("a", c_uint8, 1), - ("b", c_uint16, 1), - ("c", c_uint32, 1), - ("d", c_uint64, 1), - ): 1, - ( - ("a", c_uint8, 8), - ("b", c_uint16, 1), - ("c", c_uint32, 1), - ("d", c_uint64, 1), - ): 2, - ( - ("a", c_uint32, 9), - ("b", c_uint16, 10), - ("c", c_uint32, 25), - ("d", c_uint64, 1), - ): 6, - ( - ("a", c_uint32, 9), - ("b", c_uint16, 10), - ("c", c_uint32, 25), - ("d", c_uint64, 5), - ): 7, - ( - ("a", c_uint16), - ("b", c_uint16, 9), - ("c", c_uint16, 1), - ("d", c_uint16, 1), - ("e", c_uint16, 1), - ("f", c_uint16, 1), - ("g", c_uint16, 3), - ("h", c_uint32, 10), - ("i", c_uint32, 20), - ("j", c_uint32, 2), - ): 8, - ( - ("a", c_uint16, 9), - ("b", c_uint16, 10), - ("d", c_uint16), - ("c", c_uint8, 8), - ): 6, - ( - ("a", c_uint32, 9), - ("b", c_uint32), - ("c", c_uint32, 8), - ): 7, - } - - for fields, size in test_cases.items(): - with self.subTest(fields=fields): - class X(Structure): - _pack_ = 1 - _fields_ = list(fields) - self.assertEqual(sizeof(X), size) - def test_gh_95496(self): for field_width in range(1, 33): class TestStruct(Structure): From bff34a13b83b8a396aac9350fb438cc13457e0fc Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 22:08:45 +0800 Subject: [PATCH 38/91] Support gcc's packed structs --- Modules/_ctypes/cfield.c | 9 ++--- Modules/_ctypes/ctypes.h | 2 +- Modules/_ctypes/stgdict.c | 69 ++++++++++++++++++++++++++++++--------- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index f616ae13bc7685..bacdc92f1a9d5e 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -97,7 +97,7 @@ static void PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgDictObject* dict, - int is_bitfield + int is_bitfield, int gcc_packed ) { // We don't use poffset here, so clear it, if it has been set. @@ -107,7 +107,8 @@ PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, *palign = dict->align; if ((bitsize > 0) - && (round_down(*pbitofs, 8 * dict->align) + && !gcc_packed + && (round_down(*pbitofs, 8 * dict->align) < round_down(*pbitofs + bitsize - 1, 8 * dict->align))) { // We would be straddling alignment units. *pbitofs = round_up(*pbitofs, 8*dict->align); @@ -182,7 +183,7 @@ PyObject * PyCField_FromDesc(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian, int ms_struct) + int pack, int big_endian, int ms_struct, int gcc_packed) { CFieldObject* self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); if (self == NULL) @@ -258,7 +259,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, bitsize, pbitofs, psize, poffset, palign, self, dict, - is_bitfield + is_bitfield, gcc_packed ); } assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 8851d0d0d264d6..b173babc554c38 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -144,7 +144,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int is_big_endian, int ms_struct); + int pack, int is_big_endian, int ms_struct, int gcc_packed); extern PyObject *PyCData_AtAddress(PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(PyObject *type, char *data, Py_ssize_t length); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 6fd26de424c7ef..88551ad5726f06 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -351,6 +351,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct _Py_IDENTIFIER(_use_broken_old_ctypes_structure_semantics_); _Py_IDENTIFIER(_pack_); _Py_IDENTIFIER(_ms_struct_); + _Py_IDENTIFIER(_gcc_packed_); StgDictObject *stgdict, *basedict; Py_ssize_t len, offset, size, align, i; Py_ssize_t union_size, total_align; @@ -385,20 +386,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct big_endian = PY_BIG_ENDIAN; } - #ifdef MS_WIN32 - int ms_struct = 1; - #else - int ms_struct = 0; - #endif - - if (_PyObject_LookupAttrId(type, &PyId__ms_struct_, &tmp) < 0) { - return -1; - } - if (tmp) { - ms_struct = _PyLong_AsInt(tmp); - Py_DECREF(tmp); - } - if (_PyObject_LookupAttrId(type, &PyId__use_broken_old_ctypes_structure_semantics_, &tmp) < 0) { @@ -419,6 +406,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct isPacked = 1; pack = _PyLong_AsInt(tmp); Py_DECREF(tmp); + // TODO(Matthias): It looks like pack == 0 triggers a bug. if (pack < 0) { if (!PyErr_Occurred() || PyErr_ExceptionMatches(PyExc_TypeError) || @@ -435,6 +423,55 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct pack = 0; } + #ifdef MS_WIN32 + int ms_struct = 1; + #else + int ms_struct = 0; + #endif + + if (_PyObject_LookupAttrId(type, &PyId__ms_struct_, &tmp) < 0) { + return -1; + } + if (tmp) { + ms_struct = _PyLong_AsInt(tmp); + Py_DECREF(tmp); + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_TypeError) || + PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_SetString(PyExc_ValueError, + "_pack_ must be a bool or integer"); + } + return -1; + } + + if(!ms_struct && isPacked) { + PyErr_SetString(PyExc_ValueError, + "_ms_struct_ == 0 is not compatible with _pack_ != 0"); + return -1; + } + } else { + ms_struct = isPacked; + } + + int gcc_packed = 0; + if (_PyObject_LookupAttrId(type, &PyId__gcc_packed_, &tmp) < 0) { + return -1; + } + if (tmp) { + Py_DECREF(tmp); + gcc_packed = 1; + if(isPacked) { + PyErr_SetString(PyExc_ValueError, + "_gcc_packed_ is not compatible with _pack_"); + return -1; + } + if(ms_struct) { + PyErr_SetString(PyExc_ValueError, + "_gcc_packed_ is not compatible with _ms_struct_ != 0"); + return -1; + } + } + len = PySequence_Size(fields); if (len == -1) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { @@ -625,7 +662,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct prop = PyCField_FromDesc(desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian, ms_struct); + pack, big_endian, ms_struct, gcc_packed); } else /* union */ { field_size = 0; size = 0; @@ -635,7 +672,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct prop = PyCField_FromDesc(desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian, ms_struct); + pack, big_endian, ms_struct, gcc_packed); union_size = max(size, union_size); } total_align = max(align, total_align); From 5d6f0f75a80a332219d5d264aa586fca8d648543 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 9 Oct 2022 22:25:26 +0800 Subject: [PATCH 39/91] Remove gcc_packed experiment --- Modules/_ctypes/cfield.c | 7 +++---- Modules/_ctypes/ctypes.h | 2 +- Modules/_ctypes/stgdict.c | 24 ++---------------------- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index bacdc92f1a9d5e..373701d800fb45 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -97,7 +97,7 @@ static void PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgDictObject* dict, - int is_bitfield, int gcc_packed + int is_bitfield ) { // We don't use poffset here, so clear it, if it has been set. @@ -107,7 +107,6 @@ PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, *palign = dict->align; if ((bitsize > 0) - && !gcc_packed && (round_down(*pbitofs, 8 * dict->align) < round_down(*pbitofs + bitsize - 1, 8 * dict->align))) { // We would be straddling alignment units. @@ -183,7 +182,7 @@ PyObject * PyCField_FromDesc(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian, int ms_struct, int gcc_packed) + int pack, int big_endian, int ms_struct) { CFieldObject* self = (CFieldObject *)_PyObject_CallNoArgs((PyObject *)&PyCField_Type); if (self == NULL) @@ -259,7 +258,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, bitsize, pbitofs, psize, poffset, palign, self, dict, - is_bitfield, gcc_packed + is_bitfield ); } assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index b173babc554c38..8851d0d0d264d6 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -144,7 +144,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int is_big_endian, int ms_struct, int gcc_packed); + int pack, int is_big_endian, int ms_struct); extern PyObject *PyCData_AtAddress(PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(PyObject *type, char *data, Py_ssize_t length); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 88551ad5726f06..6e3e65a1d9d9ac 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -351,7 +351,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct _Py_IDENTIFIER(_use_broken_old_ctypes_structure_semantics_); _Py_IDENTIFIER(_pack_); _Py_IDENTIFIER(_ms_struct_); - _Py_IDENTIFIER(_gcc_packed_); StgDictObject *stgdict, *basedict; Py_ssize_t len, offset, size, align, i; Py_ssize_t union_size, total_align; @@ -453,25 +452,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct ms_struct = isPacked; } - int gcc_packed = 0; - if (_PyObject_LookupAttrId(type, &PyId__gcc_packed_, &tmp) < 0) { - return -1; - } - if (tmp) { - Py_DECREF(tmp); - gcc_packed = 1; - if(isPacked) { - PyErr_SetString(PyExc_ValueError, - "_gcc_packed_ is not compatible with _pack_"); - return -1; - } - if(ms_struct) { - PyErr_SetString(PyExc_ValueError, - "_gcc_packed_ is not compatible with _ms_struct_ != 0"); - return -1; - } - } - len = PySequence_Size(fields); if (len == -1) { if (PyErr_ExceptionMatches(PyExc_TypeError)) { @@ -662,7 +642,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct prop = PyCField_FromDesc(desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian, ms_struct, gcc_packed); + pack, big_endian, ms_struct); } else /* union */ { field_size = 0; size = 0; @@ -672,7 +652,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct prop = PyCField_FromDesc(desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian, ms_struct, gcc_packed); + pack, big_endian, ms_struct); union_size = max(size, union_size); } total_align = max(align, total_align); From d9c1fca36854dd9a256af58a9002b7a053856a8a Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Mon, 10 Oct 2022 09:37:19 +0800 Subject: [PATCH 40/91] Fix test --- Lib/test/test_ctypes/test_bitfields.py | 43 +++++++++++++++++++++++- Modules/_ctypes/_ctypes_test.c | 45 +++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 82277713134a33..628c4bbd4db710 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -29,6 +29,30 @@ class BITS(Structure): func = CDLL(_ctypes_test.__file__).unpack_bitfields func.argtypes = POINTER(BITS), c_char + +class BITS_msvc(Structure): + _ms_struct_ = 1 + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9), + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +func_msvc = CDLL(_ctypes_test.__file__).unpack_bitfields_msvc +func_msvc.argtypes = POINTER(BITS_msvc), c_char + ##for n in "ABCDEFGHIMNOPQRS": ## print n, hex(getattr(BITS, n).size), getattr(BITS, n).offset @@ -50,7 +74,24 @@ def test_shorts(self): for name in "MNOPQRS": b = BITS() setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + self.assertEqual( + getattr(b, name), + func(byref(b), + (name.encode('ascii'), i))) + + def test_shorts_msvc_mode(self): + b = BITS_msvc() + name = "M" + if func_msvc(byref(b), name.encode('ascii')) == 999: + self.skipTest("Compiler does not support signed short bitfields") + for i in range(256): + for name in "MNOPQRS": + b = BITS_msvc() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func_msvc(byref(b), name.encode('ascii')), + (name, i)) signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index e1f91b476a49fd..4cdafd08577773 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -592,7 +592,7 @@ struct BITS { */ #ifndef __xlc__ #define SIGNED_SHORT_BITFIELDS - short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; + signed short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; #endif }; @@ -622,6 +622,49 @@ EXPORT(int) unpack_bitfields(struct BITS *bits, char name) return 999; } +struct +#ifndef MS_WIN32 +__attribute__ ((ms_struct)) +#endif +BITS_msvc +{ + signed int A: 1, B:2, C:3, D:4, E: 5, F: 6, G: 7, H: 8, I: 9; +/* + * The test case needs/uses "signed short" bitfields, but the + * IBM XLC compiler does not support this + */ +#ifndef __xlc__ +#define SIGNED_SHORT_BITFIELDS + signed short M: 1, N: 2, O: 3, P: 4, Q: 5, R: 6, S: 7; +#endif +}; + +EXPORT(int) unpack_bitfields_msvc(struct BITS_msvc *bits, char name) +{ + switch (name) { + case 'A': return bits->A; + case 'B': return bits->B; + case 'C': return bits->C; + case 'D': return bits->D; + case 'E': return bits->E; + case 'F': return bits->F; + case 'G': return bits->G; + case 'H': return bits->H; + case 'I': return bits->I; + +#ifdef SIGNED_SHORT_BITFIELDS + case 'M': return bits->M; + case 'N': return bits->N; + case 'O': return bits->O; + case 'P': return bits->P; + case 'Q': return bits->Q; + case 'R': return bits->R; + case 'S': return bits->S; +#endif + } + return 999; +} + static PyMethodDef module_methods[] = { /* {"get_last_tf_arg_s", get_last_tf_arg_s, METH_NOARGS}, {"get_last_tf_arg_u", get_last_tf_arg_u, METH_NOARGS}, From cc9ea8a514c3beee1636eb358ca6fd215418b2a1 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Mon, 10 Oct 2022 09:40:52 +0800 Subject: [PATCH 41/91] Fix ms_struct --- Modules/_ctypes/stgdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 6e3e65a1d9d9ac..8353da3828cfde 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -449,7 +449,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct return -1; } } else { - ms_struct = isPacked; + ms_struct ||= isPacked; } len = PySequence_Size(fields); From 15540833e3eca5c58472de21c2718ac3ce73e7f9 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Mon, 10 Oct 2022 09:41:49 +0800 Subject: [PATCH 42/91] Fix ms_struct --- Modules/_ctypes/stgdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 8353da3828cfde..3807897d2566de 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -449,7 +449,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct return -1; } } else { - ms_struct ||= isPacked; + ms_struct = ms_struct || isPacked; } len = PySequence_Size(fields); From a86eadc0b0e2cb8d7a9dc672ba06a5af2d07248f Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Mon, 10 Oct 2022 09:44:06 +0800 Subject: [PATCH 43/91] Indenting --- Modules/_ctypes/stgdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 3807897d2566de..71e2aaee928205 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -445,7 +445,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct if(!ms_struct && isPacked) { PyErr_SetString(PyExc_ValueError, - "_ms_struct_ == 0 is not compatible with _pack_ != 0"); + "_ms_struct_ == 0 is not compatible with _pack_ != 0"); return -1; } } else { From f0a92b0de4b5aeddf50f3c49b8130dc4935a5d99 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Mon, 10 Oct 2022 10:40:29 +0800 Subject: [PATCH 44/91] Fix miscounted parens --- Lib/test/test_ctypes/test_bitfields.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 628c4bbd4db710..2fd7fe4ff33704 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -76,8 +76,8 @@ def test_shorts(self): setattr(b, name, i) self.assertEqual( getattr(b, name), - func(byref(b), - (name.encode('ascii'), i))) + func(byref(b), (name.encode('ascii'))), + (name, i)) def test_shorts_msvc_mode(self): b = BITS_msvc() From 5a7ea093333704a70cef7544439ef651f2aec56d Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Tue, 14 Mar 2023 11:42:08 +0800 Subject: [PATCH 45/91] make regen-all --- Include/internal/pycore_global_objects_fini_generated.h | 2 +- Include/internal/pycore_global_strings.h | 2 +- Include/internal/pycore_runtime_init_generated.h | 2 +- Include/internal/pycore_unicodeobject_generated.h | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 7eff607e374567..3602317ca4f98b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -737,7 +737,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_ms_struct_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_asyncio_future_blocking)); @@ -760,6 +759,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_loop)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_ms_struct_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_needs_com_addref_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_pack_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_restype_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 0a06502a5be790..cecfc25df58f1d 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -223,7 +223,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_active) STRUCT_FOR_ID(_annotation) STRUCT_FOR_ID(_anonymous_) - STRUCT_FOR_ID(_ms_struct_) STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) STRUCT_FOR_ID(_asyncio_future_blocking) @@ -246,6 +245,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) STRUCT_FOR_ID(_loop) + STRUCT_FOR_ID(_ms_struct_) STRUCT_FOR_ID(_needs_com_addref_) STRUCT_FOR_ID(_pack_) STRUCT_FOR_ID(_restype_) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 52a9024d382d68..f0226de0a316ad 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -729,7 +729,6 @@ extern "C" { INIT_ID(_active), \ INIT_ID(_annotation), \ INIT_ID(_anonymous_), \ - INIT_ID(_ms_struct_), \ INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ INIT_ID(_asyncio_future_blocking), \ @@ -752,6 +751,7 @@ extern "C" { INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ INIT_ID(_loop), \ + INIT_ID(_ms_struct_), \ INIT_ID(_needs_com_addref_), \ INIT_ID(_pack_), \ INIT_ID(_restype_), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index d20ab34baa7b9c..50b322b406654d 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -352,8 +352,6 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_anonymous_); PyUnicode_InternInPlace(&string); - string = &_Py_ID(_ms_struct_); - PyUnicode_InternInPlace(&string); string = &_Py_ID(_argtypes_); PyUnicode_InternInPlace(&string); string = &_Py_ID(_as_parameter_); @@ -398,6 +396,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(_loop); PyUnicode_InternInPlace(&string); + string = &_Py_ID(_ms_struct_); + PyUnicode_InternInPlace(&string); string = &_Py_ID(_needs_com_addref_); PyUnicode_InternInPlace(&string); string = &_Py_ID(_pack_); From 0f7391980dc677945beb67989c0f80d5a24cff04 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Tue, 14 Mar 2023 11:47:03 +0800 Subject: [PATCH 46/91] clean up --- Modules/_ctypes/stgdict.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 322f85ab97651c..375e375e870f90 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -421,7 +421,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct int ms_struct = 0; #endif - // figure out how this pyid replacement thing works. if (_PyObject_LookupAttr(type, &_Py_ID(_ms_struct_), &tmp) < 0) { return -1; } From 6226e55c220c87cfe7ff2b66b79ccff815504b2a Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 15 Mar 2023 19:13:39 +0800 Subject: [PATCH 47/91] make regen-all --- Include/internal/pycore_unicodeobject_generated.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index a1eeea0b10824a..084a476f48f663 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -589,6 +589,7 @@ _PyUnicode_InitStaticStrings(void) { assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_ms_struct_); + assert(_PyUnicode_CheckConsistency(string, 1)); PyUnicode_InternInPlace(&string); string = &_Py_ID(_needs_com_addref_); assert(_PyUnicode_CheckConsistency(string, 1)); From eb502d7de8bd539429dbff110bae927e1d03750f Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Fri, 17 Mar 2023 09:19:56 +0800 Subject: [PATCH 48/91] Explain where 999 comes from --- Lib/test/test_ctypes/test_bitfields.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 2fd7fe4ff33704..4cf7877812e26d 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -68,7 +68,12 @@ def test_ints(self): def test_shorts(self): b = BITS() name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. if func(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. self.skipTest("Compiler does not support signed short bitfields") for i in range(256): for name in "MNOPQRS": From 0cf5f4eb3d5cdaa7440646b726c84fea97e92f41 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Fri, 17 Mar 2023 09:20:33 +0800 Subject: [PATCH 49/91] fix typo --- Modules/_ctypes/cfield.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 9047d21246d025..dede8623aa5e49 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -68,7 +68,7 @@ GCC mode places fields one after another, bit by bit. But when a field would straddle an alignment boundary for its type, we insert a few bits of padding to avoid that. -MSVC mode works similar expect for bitfield packing. Adjacent bit-fields are +MSVC mode works similar except for bitfield packing. Adjacent bit-fields are packed into the same 1-, 2-, or 4-byte allocation unit if the integral types are the same size and if the next bit-field fits into the current allocation unit without crossing the boundary imposed by the common alignment requirements From fd3cd0be8d1918d0008c1193310e4b857b4a87e5 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Fri, 17 Mar 2023 12:20:29 +0800 Subject: [PATCH 50/91] Explain magic 999 --- Lib/test/test_ctypes/test_bitfields.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 4cf7877812e26d..fbfc5954cb92b9 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -87,7 +87,12 @@ def test_shorts(self): def test_shorts_msvc_mode(self): b = BITS_msvc() name = "M" + # See Modules/_ctypes/_ctypes_test.c for where the magic 999 comes from. if func_msvc(byref(b), name.encode('ascii')) == 999: + # unpack_bitfields and unpack_bitfields_msvc in + # Modules/_ctypes/_ctypes_test.c return 999 to indicate + # an invalid name. 'M' is only valid, if signed short bitfields + # are supported by the C compiler. self.skipTest("Compiler does not support signed short bitfields") for i in range(256): for name in "MNOPQRS": From 2d8492ec0d5259d1e1ea5402b5adfe1cc0766bfb Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Mon, 27 Mar 2023 13:59:18 +0800 Subject: [PATCH 51/91] Incorporate Sam's tests From https://github.com/python/cpython/pull/103052 --- Lib/test/test_ctypes/test_structures.py | 24 ++++++++++++++++++++++++ Modules/_ctypes/_ctypes_test.c | 25 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index df39dc7f50d3f7..9c948fb2b066a2 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -738,6 +738,30 @@ class Test8(Union): self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' 'a union by value, which is unsupported.') + def test_bitfield_matches_c(self): + class Test9(Structure): + _pack_ = 1 + _fields_ = (('A', c_uint16), # 2 bytes + ('B', c_uint16, 9), + ('C', c_uint16, 1), + ('D', c_uint16, 1), + ('E', c_uint16, 1), + ('F', c_uint16, 1), + ('G', c_uint16, 3), # 4 bytes + ('H', c_uint32, 10), + ('I', c_uint32, 20), + ('J', c_uint32, 2)) # 8 bytes + dll = CDLL(_ctypes_test.__file__) + func = dll._testfunc_bitfield_by_reference3 + func.restype = c_long + func.argtypes = (POINTER(Test9),c_long,) + ind = 0 + for field in Test9._fields_: + test9 = Test9() + setattr(test9,field[0], 1) + self.assertEqual(func(test9, ind), 1) + ind += 1 + class PointerMemberTestCase(unittest.TestCase): def test(self): diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 3906ad0f903baf..d099a926c89927 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -231,6 +231,31 @@ _testfunc_bitfield_by_reference2(Test7 *in) { return result; } +typedef struct{ + uint16_t A ; + uint16_t B : 9; + uint16_t C : 1; + uint16_t D : 1; + uint16_t E : 1; + uint16_t F : 1; + uint16_t G : 3; + uint32_t H : 10; + uint32_t I : 20; + uint32_t J : 2; +} Test9; + +EXPORT(long) +_testfunc_bitfield_by_reference3(Test9 *in, long pos) { + long data[] = {in->A , in->B , in->C , in->D , in->E , in->F , in->G , in->H , in->I , in->J}; + long data_length = (long) (sizeof(data)/sizeof(data[0])); + if(pos < 0) + return -1; + if(pos >= data_length) + return -1; + + return data[pos]; +} + typedef union { signed int A: 1, B:2, C:3, D:2; } Test8; From bee6c531be5405f892bee9cc1efd716f22c59865 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Wed, 24 May 2023 10:38:25 +0800 Subject: [PATCH 52/91] Tickle CI/CD From 98767dae8d1f29832077bc9f8335fae097da27ef Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 14 Apr 2024 10:46:21 +0800 Subject: [PATCH 53/91] Fix compile --- Modules/_ctypes/stgdict.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index f28800ef21016b..57200742cc9926 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -431,11 +431,11 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct int ms_struct = 0; #endif - if (_PyObject_LookupAttr(type, &_Py_ID(_ms_struct_), &tmp) < 0) { + if (PyObject_GetOptionalAttr(type, &_Py_ID(_ms_struct_), &tmp) < 0) { return -1; } if (tmp) { - ms_struct = _PyLong_AsInt(tmp); + ms_struct = PyLong_AsInt(tmp); Py_DECREF(tmp); if (PyErr_Occurred()) { if (PyErr_ExceptionMatches(PyExc_TypeError) || From 0369d0d6cb220282927f44be5809679d1c7f8fc9 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 14 Apr 2024 11:18:46 +0800 Subject: [PATCH 54/91] Fix merge --- Lib/test/test_ctypes/test_structures.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 2bb73d9b6d2b9b..b094f2af557208 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -881,7 +881,6 @@ class Test8(Union): self.assertEqual(ctx.exception.args[0], 'item 1 in _argtypes_ passes ' 'a union by value, which is unsupported.') -<<<<<<< HEAD def test_bitfield_matches_c(self): class Test9(Structure): _pack_ = 1 @@ -905,8 +904,6 @@ class Test9(Structure): setattr(test9,field[0], 1) self.assertEqual(func(test9, ind), 1) ind += 1 -======= ->>>>>>> 698a0da7d440856a90b45964e9082b5a55387b80 class PointerMemberTestCase(unittest.TestCase): From d4dc2c0c6bd0926e11f606d81a7c82fe94ff3ab0 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 14 Apr 2024 11:28:41 +0800 Subject: [PATCH 55/91] Clean up --- Lib/test/test_ctypes/test_bitfields.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 4e160347a95ee0..6eea7224c5ac9d 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -1,7 +1,6 @@ import os import sys import unittest -from ctypes import * from ctypes import (CDLL, Structure, sizeof, POINTER, byref, alignment, LittleEndianStructure, BigEndianStructure, c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, @@ -58,8 +57,6 @@ class BITS_msvc(Structure): func_msvc = CDLL(_ctypes_test.__file__).unpack_bitfields_msvc func_msvc.argtypes = POINTER(BITS_msvc), c_char -##for n in "ABCDEFGHIMNOPQRS": -## print n, hex(getattr(BITS, n).size), getattr(BITS, n).offset class C_Test(unittest.TestCase): From f75d7d66396436b3c2886950513befa938c7c4b0 Mon Sep 17 00:00:00 2001 From: Matthias Goergens Date: Sun, 14 Apr 2024 12:13:55 +0800 Subject: [PATCH 56/91] Fix tests --- Lib/test/test_ctypes/test_bitfields.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 6eea7224c5ac9d..87da6585833d4b 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -4,7 +4,7 @@ from ctypes import (CDLL, Structure, sizeof, POINTER, byref, alignment, LittleEndianStructure, BigEndianStructure, c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, - c_uint32, c_uint64, + c_uint8, c_uint16, c_uint32, c_uint64, c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong) from test import support from test.support import import_helper From 0da36ad5fc960bcd31cdf4d4a729133f03d96a2f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 22 Apr 2024 18:07:19 +0200 Subject: [PATCH 57/91] Add Hypothesis test --- .gitignore | 1 + .../test_ctypes/test_bitfields_hypothesis.py | 184 ++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 Lib/test/test_ctypes/test_bitfields_hypothesis.py diff --git a/.gitignore b/.gitignore index 8872e9d5508ff1..24366a1b5bc1d5 100644 --- a/.gitignore +++ b/.gitignore @@ -153,6 +153,7 @@ Tools/unicode/data/ /coverage/ /externals/ /htmlcov/ +/.hypothesis/ Tools/msi/obj Tools/ssl/amd64 Tools/ssl/win32 diff --git a/Lib/test/test_ctypes/test_bitfields_hypothesis.py b/Lib/test/test_ctypes/test_bitfields_hypothesis.py new file mode 100644 index 00000000000000..02cbef5ef8736e --- /dev/null +++ b/Lib/test/test_ctypes/test_bitfields_hypothesis.py @@ -0,0 +1,184 @@ +import dataclasses +import tempfile +from pathlib import Path +import subprocess + +import ctypes +from ctypes import ( + Structure, + alignment, + sizeof, + pointer, + POINTER, +) + +from hypothesis import assume, given, note, settings +from hypothesis import strategies + +C_TYPES = { + # Note: for Hypothesis minimization to work, "simpler" types should + # generally go first. + #'char': ctypes.c_char, + 'signed char': ctypes.c_byte, + 'unsigned char': ctypes.c_ubyte, + 'int': ctypes.c_int, + 'unsigned int': ctypes.c_uint, + **{f'{u}int{n}_t': getattr(ctypes, f'c_{u}int{n}') + for u in ('', 'u') + for n in ('8', '16', '32', '64') + }, + **{f'{"unsigned" if u else "signed"} {t}': getattr(ctypes, f'c_{u}{t}') + for u in ('', 'u') + for t in ('short', 'long', 'longlong') + }, + #'float': ctypes.c_float, + #'double': ctypes.c_double, +} + +def struct_c_decl(attrdict, name='s'): + lines = [] + pack = attrdict.get('_pack_') + if pack is not None: + lines.append(f'#pragma pack({pack})') + attributes = [] + endian_name = attrdict['endian_name'] + if endian_name != 'native': + attributes.append(f'scalar_storage_order("{endian_name}-endian")') + if attrdict.get('_ms_struct_'): + attributes.append('ms_struct') + lines.append(f'#pragma ms_struct on') + lines.append(f'struct') + if attributes: + lines.append(f'__attribute__ (( {", ".join(attributes)} ))') + lines.append(f' {name} {{') + for field, ctype_name in zip(attrdict['_fields_'], attrdict['ctype_names']): + try: + name, tp, bitsize = field + except ValueError: + name, tp = field + bitfieldspec = '' + else: + bitfieldspec = f':{bitsize}' + lines.append(f' {ctype_name} {name}{bitfieldspec};') + lines.append('};') + lines.append('') + return '\n'.join(lines) + + +@strategies.composite +def structure_args( + draw, + types_strategy=strategies.lists( + strategies.sampled_from(list(C_TYPES.items())), + min_size=1, max_size=200, + ), +): + fields = [] + ctype_names = [] + for i, (ctype_name, ctype) in enumerate(draw(types_strategy)): + name = f'field_{i}' + bit_size = draw(strategies.integers(1, sizeof(ctype) * 8)) + is_bitfield = draw(strategies.booleans()) + if is_bitfield: + fields.append((name, ctype, bit_size)) + else: + fields.append((name, ctype)) + ctype_names.append(ctype_name) + endian_name, cls = draw(strategies.sampled_from(( + ('native', ctypes.Structure), + ('little', ctypes.LittleEndianStructure), + ('big', ctypes.BigEndianStructure), + ))) + + # XXX: Handle Anonymouses + + attrdict = { + '_fields_': fields, + 'endian_name': endian_name, + 'ctype_names': ctype_names, + } + if draw(strategies.booleans()): + attrdict['_ms_struct_'] = True + pack = 2 ** draw(strategies.integers(0, 4)) + if pack: + attrdict['_pack_'] = pack + attrdict['c_decl'] = struct_c_decl(attrdict) + result = ( + 'RandomTestStruct', + (cls,), + attrdict, + ) + return result + + +@given(structure_args()) +def test_structure(s_args): + structure = type(*s_args) + with tempfile.TemporaryDirectory() as tmpdirname: + temppath = Path(tmpdirname) + setloop_lines = [] + for field, ctype_name in zip(structure._fields_, structure.ctype_names): + setloop_lines.append(f'value.{field[0]} = ({ctype_name})-1;') + setloop_lines.append(f'dump_bytes(&value, sizeof(value));') + program = ( + r""" + #include + #include + #include + #include + + #define longlong long long + + %STRUCTDEF% + + void dump_bytes(void *ptr, size_t sz) { + unsigned char *cptr = ptr; + for (size_t i=0; i Date: Tue, 30 Apr 2024 07:59:27 +0200 Subject: [PATCH 58/91] dict -> info --- Modules/_ctypes/cfield.c | 43 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index f586fecd989feb..cd65966ac9f70b 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -89,7 +89,7 @@ PyCField_FromDesc manages: static void PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - CFieldObject* self, StgInfo* dict, + CFieldObject* self, StgInfo* info, int is_bitfield ) { @@ -97,25 +97,26 @@ PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, *pbitofs += *poffset * 8; *poffset = 0; - *palign = dict->align; + *palign = info->align; if ((bitsize > 0) - && (round_down(*pbitofs, 8 * dict->align) - < round_down(*pbitofs + bitsize - 1, 8 * dict->align))) { + && (round_down(*pbitofs, 8 * info->align) + < round_down(*pbitofs + bitsize - 1, 8 * info->align))) { // We would be straddling alignment units. - *pbitofs = round_up(*pbitofs, 8*dict->align); + *pbitofs = round_up(*pbitofs, 8*info->align); } assert(*poffset == 0); if(is_bitfield) { - self->offset = round_down(*pbitofs, 8*dict->size) / 8; + self->offset = round_down(*pbitofs, 8*info->size) / 8; Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; self->size = BUILD_SIZE(bitsize, effective_bitsof); - assert(dict->size == dict->align); - assert(effective_bitsof <= dict->size * 8); + assert(info->size == info->align); + assert(effective_bitsof <= info->size * 8); } else { - self->offset = round_down(*pbitofs, 8*dict->align) / 8; - self->size = dict->size; + assert(bitsize == 0); + self->offset = round_down(*pbitofs, 8*info->align) / 8; + self->size = info->size; } *pbitofs += bitsize; @@ -127,45 +128,45 @@ PyCField_FromDesc_msvc( Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, int pack, - CFieldObject* self, StgInfo* dict, + CFieldObject* self, StgInfo* info, int is_bitfield ) { if (pack) - *palign = min(pack, dict->align); + *palign = min(pack, info->align); else - *palign = dict->align; + *palign = info->align; // *poffset points to end of current bitfield. // *pbitofs is generally non-positive, // and 8 * (*poffset) + *pbitofs points just behind // the end of the last field we placed. - if (0 < *pbitofs + bitsize || 8 * dict->size != *pfield_size) { + if (0 < *pbitofs + bitsize || 8 * info->size != *pfield_size) { // Close the previous bitfield (if any). // and start a new bitfield: *poffset = round_up(*poffset, *palign); - *poffset += dict->size; + *poffset += info->size; - *pfield_size = dict->size * 8; + *pfield_size = info->size * 8; // Reminder: 8 * (*poffset) + *pbitofs points to where we would start a // new field. Ie just behind where we placed the last field plus an // allowance for alignment. *pbitofs = - *pfield_size; } - assert(8 * dict->size == *pfield_size); + assert(8 * info->size == *pfield_size); self->offset = *poffset - (*pfield_size) / 8; if(is_bitfield) { - assert(dict->size == dict->align); + assert(info->size == info->align); assert(0 <= (*pfield_size + *pbitofs)); - assert((*pfield_size + *pbitofs) < dict->size * 8); + assert((*pfield_size + *pbitofs) < info->size * 8); self->size = BUILD_SIZE(bitsize, *pfield_size + *pbitofs); } else { - self->size = dict->size; + self->size = info->size; } - assert(*pfield_size + *pbitofs <= dict->size * 8); + assert(*pfield_size + *pbitofs <= info->size * 8); *pbitofs += bitsize; *psize = *poffset; From de22b397b724c0d47e6308aa82fe4465799875f0 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 30 Apr 2024 08:00:16 +0200 Subject: [PATCH 59/91] Conditionalize ms_struct --- Lib/test/test_ctypes/test_bitfields.py | 14 ++++++++++++-- Modules/_ctypes/_ctypes_test.c | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 87da6585833d4b..52082aba9b8cd6 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -54,8 +54,16 @@ class BITS_msvc(Structure): ("R", c_short, 6), ("S", c_short, 7)] -func_msvc = CDLL(_ctypes_test.__file__).unpack_bitfields_msvc -func_msvc.argtypes = POINTER(BITS_msvc), c_char + +try: + func_msvc = CDLL(_ctypes_test.__file__).unpack_bitfields_msvc +except AttributeError as err: + # The MSVC struct must be available on Windows; it's optional elsewhere + if support.MS_WINDOWS: + raise err + func_msvc = None +else: + func_msvc.argtypes = POINTER(BITS_msvc), c_char class C_Test(unittest.TestCase): @@ -86,6 +94,8 @@ def test_shorts(self): func(byref(b), (name.encode('ascii'))), (name, i)) + + @unittest.skipUnless(func_msvc, "need MSVC or __attribute__((ms_struct))") def test_shorts_msvc_mode(self): b = BITS_msvc() name = "M" diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 0db9d789bfdacc..ae9a5d25ade40d 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -759,6 +759,7 @@ EXPORT(int) unpack_bitfields(struct BITS *bits, char name) return 999; } +#if defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__) struct #ifndef MS_WIN32 __attribute__ ((ms_struct)) @@ -801,6 +802,7 @@ EXPORT(int) unpack_bitfields_msvc(struct BITS_msvc *bits, char name) } return 999; } +#endif static PyMethodDef module_methods[] = { /* {"get_last_tf_arg_s", get_last_tf_arg_s, METH_NOARGS}, From cdd1860da3fb6d2ad521b63ff6c902639a4bf170 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 30 Apr 2024 08:00:48 +0200 Subject: [PATCH 60/91] Use fixed-width types in tests --- Lib/test/test_ctypes/test_bitfields.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 52082aba9b8cd6..f5988481f08084 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -319,23 +319,23 @@ class X(Structure): def test_mixed_7(self): class X(Structure): _fields_ = [ - ("A", c_uint), - ('B', c_uint, 20), - ('C', c_ulonglong, 24)] + ("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] self.assertEqual(16, sizeof(X)) def test_mixed_8(self): class Foo(Structure): _fields_ = [ - ("A", c_uint), - ("B", c_uint, 32), + ("A", c_uint32), + ("B", c_uint32, 32), ("C", c_ulonglong, 1), ] class Bar(Structure): _fields_ = [ - ("A", c_uint), - ("B", c_uint), + ("A", c_uint32), + ("B", c_uint32), ("C", c_ulonglong, 1), ] self.assertEqual(sizeof(Foo), sizeof(Bar)) @@ -344,7 +344,7 @@ def test_mixed_9(self): class X(Structure): _fields_ = [ ("A", c_uint8), - ("B", c_uint, 1), + ("B", c_uint32, 1), ] if sys.platform == 'win32': self.assertEqual(8, sizeof(X)) From dd84ac3db54e7ce0931c07b1e7dc71e9c895c57a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 30 Apr 2024 08:00:29 +0200 Subject: [PATCH 61/91] Use subTest --- Lib/test/test_ctypes/test_bitfields.py | 34 ++++++++++++++------------ 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index f5988481f08084..10277563f3065a 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -71,9 +71,12 @@ class C_Test(unittest.TestCase): def test_ints(self): for i in range(512): for name in "ABCDEFGHI": - b = BITS() - setattr(b, name, i) - self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) def test_shorts(self): b = BITS() @@ -87,13 +90,12 @@ def test_shorts(self): self.skipTest("Compiler does not support signed short bitfields") for i in range(256): for name in "MNOPQRS": - b = BITS() - setattr(b, name, i) - self.assertEqual( - getattr(b, name), - func(byref(b), (name.encode('ascii'))), - (name, i)) - + with self.subTest(i=i, name=name): + b = BITS() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func(byref(b), (name.encode('ascii')))) @unittest.skipUnless(func_msvc, "need MSVC or __attribute__((ms_struct))") def test_shorts_msvc_mode(self): @@ -108,12 +110,12 @@ def test_shorts_msvc_mode(self): self.skipTest("Compiler does not support signed short bitfields") for i in range(256): for name in "MNOPQRS": - b = BITS_msvc() - setattr(b, name, i) - self.assertEqual( - getattr(b, name), - func_msvc(byref(b), name.encode('ascii')), - (name, i)) + with self.subTest(i=i, name=name): + b = BITS_msvc() + setattr(b, name, i) + self.assertEqual( + getattr(b, name), + func_msvc(byref(b), name.encode('ascii'))) signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) From 8a6fb671bef8777021f7b734c5c39cdc0d5ccf0f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 30 Apr 2024 08:22:28 +0200 Subject: [PATCH 62/91] fixup! dict --- Modules/_ctypes/cfield.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index cd65966ac9f70b..11c97fd46fee9d 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -182,9 +182,6 @@ PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); if (self == NULL) return NULL; - // Note(Matthias): We get most of what used to be `dict` out of `info` now. - // StgDictObject* dict = PyType_stgdict(desc); - // if (!dict) { StgInfo *info; if (PyStgInfo_FromType(st, desc, &info) < 0) { Py_DECREF(self); From 07ea42def08b306b5ffef41f7ef45760d1cd9818 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 30 Apr 2024 08:34:04 +0200 Subject: [PATCH 63/91] Turn `info->size != info->align` assertion into a TypeError --- Modules/_ctypes/cfield.c | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 11c97fd46fee9d..f9a0325d476563 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -86,7 +86,7 @@ PyCField_FromDesc manages: - *palign: the alignment requirements of the last field we placed. */ -static void +static int PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, CFieldObject* self, StgInfo* info, @@ -111,8 +111,13 @@ PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, self->offset = round_down(*pbitofs, 8*info->size) / 8; Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; self->size = BUILD_SIZE(bitsize, effective_bitsof); - assert(info->size == info->align); assert(effective_bitsof <= info->size * 8); + if (info->size != info->align) { + PyErr_SetString( + PyExc_TypeError, + "bitfield's base type size differs from alignment"); + return -1; + } } else { assert(bitsize == 0); self->offset = round_down(*pbitofs, 8*info->align) / 8; @@ -121,9 +126,11 @@ PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, *pbitofs += bitsize; *psize = round_up(*pbitofs, 8) / 8; + + return 0; } -static void +static int PyCField_FromDesc_msvc( Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, @@ -159,7 +166,12 @@ PyCField_FromDesc_msvc( self->offset = *poffset - (*pfield_size) / 8; if(is_bitfield) { - assert(info->size == info->align); + if (info->size != info->align) { + PyErr_SetString( + PyExc_TypeError, + "bitfield's base type size differs from alignment"); + return -1; + } assert(0 <= (*pfield_size + *pbitofs)); assert((*pfield_size + *pbitofs) < info->size * 8); self->size = BUILD_SIZE(bitsize, *pfield_size + *pbitofs); @@ -170,6 +182,8 @@ PyCField_FromDesc_msvc( *pbitofs += bitsize; *psize = *poffset; + + return 0; } PyObject * @@ -251,8 +265,9 @@ PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, assert(bitsize <= info->size * 8); // `pack` only makes sense in msvc compatibility mode. + int result; if (ms_struct || pack != 0) { - PyCField_FromDesc_msvc( + result = PyCField_FromDesc_msvc( pfield_size, bitsize, pbitofs, psize, poffset, palign, pack, @@ -260,13 +275,17 @@ PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, is_bitfield ); } else { - PyCField_FromDesc_gcc( + result = PyCField_FromDesc_gcc( bitsize, pbitofs, psize, poffset, palign, self, info, is_bitfield ); } + if (result < 0) { + Py_DECREF(self); + return NULL; + } assert(!is_bitfield || (LOW_BIT(self->size) <= self->size * 8)); if(big_endian && is_bitfield) { self->size = BUILD_SIZE(NUM_BITS(self->size), 8*info->size - LOW_BIT(self->size) - bitsize); From 3fa7f5527bd7047d7075af029c97f425ea61bef0 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 30 Apr 2024 08:34:51 +0200 Subject: [PATCH 64/91] PEP 7 --- Modules/_ctypes/cfield.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index f9a0325d476563..6698560d68fe6b 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -139,10 +139,11 @@ PyCField_FromDesc_msvc( int is_bitfield ) { - if (pack) - *palign = min(pack, info->align); - else + if (pack) { + *palign = Py_MIN(pack, info->align); + } else { *palign = info->align; + } // *poffset points to end of current bitfield. // *pbitofs is generally non-positive, From eb2c4fbc47b7eadb3c9d51db5572439a5907820c Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 30 Apr 2024 09:22:37 +0200 Subject: [PATCH 65/91] Change _ms_struct_ to _layout_ --- Doc/library/ctypes.rst | 36 +++++++++++++-- Include/internal/pycore_global_strings.h | 2 +- .../internal/pycore_runtime_init_generated.h | 2 +- .../internal/pycore_unicodeobject_generated.h | 6 +-- Lib/test/test_ctypes/test_bitfields.py | 2 +- .../test_ctypes/test_bitfields_hypothesis.py | 4 +- Modules/_ctypes/cfield.c | 6 +-- Modules/_ctypes/ctypes.h | 6 ++- Modules/_ctypes/stgdict.c | 45 ++++++++++++++----- 9 files changed, 82 insertions(+), 27 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 1deaa3a0f3899f..a9081fca3147f9 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -661,11 +661,11 @@ for debugging because they can provide useful information:: guaranteed by the library to work in the general case. Unions and structures with bit-fields should always be passed to functions by pointer. -Structure/union alignment and byte order -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Structure/union layout, alignment and byte order +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By default, Structure and Union fields are aligned in the same way the C -compiler does it. It is possible to override this behavior by specifying a +By default, Structure and Union fields are laid out in the same way the C +compiler does it. It is possible to override this behavior by specifying a :attr:`~Structure._pack_` class attribute in the subclass definition. This must be set to a positive integer and specifies the maximum alignment for the fields. This is what ``#pragma pack(n)`` also does in MSVC. @@ -674,6 +674,8 @@ same way ``#pragma align(n)`` works in MSVC. This can be achieved by specifying a ::attr:`~Structure._align_` class attribute in the subclass definition. +To override the, the compiler ::attr:`~Structure._layout_` class attribute. + :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the :class:`BigEndianStructure`, :class:`LittleEndianStructure`, @@ -2540,6 +2542,32 @@ fields, or any other data types containing pointer type fields. the structure when being packed or unpacked to/from memory. Setting this attribute to 0 is the same as not setting it at all. + .. attribute:: _layout_ + + An optional string naming the struct and union layout. It can currently + be set to: + + - ``"ms"``: the layout used by the Microsoft compiler (MSVC). + GCC and Clang, this layout can be selected with + ``__attribute__((ms_struct))``. + - ``"gcc-sysv"``: the layout used by GCC with the System V or “SysV-like” + data model, as used on Linux and macOS. TThis is generally compatible + with clang's defaults. + With this layout, :attr:`~Structure._pack_` must be unset or zero. + + If not set explicitly, :attr:`!_layout_` will be set to a default that + matches the platform's conventions. This default may change in future + Python releases (for example, when a new platform gains official support, + or when a difference between similar platforms is found). + Currently the default will be: + + - On Windows: ``"ms"`` + - When :attr:`~Structure._pack_` is specified: ``"ms"`` + - Otherwise: ``"gcc-sysv"`` + + :attr:`!_layout_` must already be defined when + :attr:`~Structure._fields_` is assigned, otherwise it will have no effect. + .. attribute:: _anonymous_ An optional sequence that lists the names of unnamed (anonymous) fields. diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 807037c6b47d26..f879d49cd5bd68 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -254,11 +254,11 @@ struct _Py_global_strings { STRUCT_FOR_ID(_initializing) STRUCT_FOR_ID(_io) STRUCT_FOR_ID(_is_text_encoding) + STRUCT_FOR_ID(_layout_) STRUCT_FOR_ID(_length_) STRUCT_FOR_ID(_limbo) STRUCT_FOR_ID(_lock_unlock_module) STRUCT_FOR_ID(_loop) - STRUCT_FOR_ID(_ms_struct_) STRUCT_FOR_ID(_needs_com_addref_) STRUCT_FOR_ID(_pack_) STRUCT_FOR_ID(_restype_) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 6d36dea8c91dd0..5a572a87fdb97a 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -763,11 +763,11 @@ extern "C" { INIT_ID(_initializing), \ INIT_ID(_io), \ INIT_ID(_is_text_encoding), \ + INIT_ID(_layout_), \ INIT_ID(_length_), \ INIT_ID(_limbo), \ INIT_ID(_lock_unlock_module), \ INIT_ID(_loop), \ - INIT_ID(_ms_struct_), \ INIT_ID(_needs_com_addref_), \ INIT_ID(_pack_), \ INIT_ID(_restype_), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index ef56503dc429d4..41f87b24976f5f 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -603,6 +603,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_is_text_encoding); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(_layout_); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_length_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -615,9 +618,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_loop); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_ms_struct_); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_needs_com_addref_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 10277563f3065a..eac30af70055ca 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -35,7 +35,7 @@ class BITS(Structure): class BITS_msvc(Structure): - _ms_struct_ = 1 + _layout_ = "ms" _fields_ = [("A", c_int, 1), ("B", c_int, 2), ("C", c_int, 3), diff --git a/Lib/test/test_ctypes/test_bitfields_hypothesis.py b/Lib/test/test_ctypes/test_bitfields_hypothesis.py index 02cbef5ef8736e..b518f6d5da8c5f 100644 --- a/Lib/test/test_ctypes/test_bitfields_hypothesis.py +++ b/Lib/test/test_ctypes/test_bitfields_hypothesis.py @@ -44,7 +44,7 @@ def struct_c_decl(attrdict, name='s'): endian_name = attrdict['endian_name'] if endian_name != 'native': attributes.append(f'scalar_storage_order("{endian_name}-endian")') - if attrdict.get('_ms_struct_'): + if attrdict.get('_layout_' == 'ms'): attributes.append('ms_struct') lines.append(f'#pragma ms_struct on') lines.append(f'struct') @@ -98,7 +98,7 @@ def structure_args( 'ctype_names': ctype_names, } if draw(strategies.booleans()): - attrdict['_ms_struct_'] = True + attrdict['_layout_'] = 'ms' pack = 2 ** draw(strategies.integers(0, 4)) if pack: attrdict['_pack_'] = pack diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 6698560d68fe6b..2b8bc2cba6c04b 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -191,7 +191,7 @@ PyObject * PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int big_endian, int ms_struct) + int pack, int big_endian, LayoutMode layout_mode) { PyTypeObject *tp = st->PyCField_Type; CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); @@ -265,9 +265,8 @@ PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, } assert(bitsize <= info->size * 8); - // `pack` only makes sense in msvc compatibility mode. int result; - if (ms_struct || pack != 0) { + if (layout_mode == LAYOUT_MODE_MS) { result = PyCField_FromDesc_msvc( pfield_size, bitsize, pbitofs, psize, poffset, palign, @@ -276,6 +275,7 @@ PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, is_bitfield ); } else { + assert(pack == 0); result = PyCField_FromDesc_gcc( bitsize, pbitofs, psize, poffset, palign, diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7b551dfc541066..2f878dd209fb73 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -224,13 +224,17 @@ extern int PyObject_stginfo(PyObject *self, Py_ssize_t *psize, Py_ssize_t *palig extern struct fielddesc *_ctypes_get_fielddesc(const char *fmt); +typedef enum { + LAYOUT_MODE_MS, + LAYOUT_MODE_GCC_SYSV, +} LayoutMode; extern PyObject * PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, Py_ssize_t *pfield_size, Py_ssize_t bitsize, Py_ssize_t *pbitofs, Py_ssize_t *psize, Py_ssize_t *poffset, Py_ssize_t *palign, - int pack, int is_big_endian, int ms_struct); + int pack, int is_big_endian, LayoutMode layout_mode); extern PyObject *PyCData_AtAddress(ctypes_state *st, PyObject *type, void *buf); extern PyObject *PyCData_FromBytes(ctypes_state *st, PyObject *type, char *data, Py_ssize_t length); diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index fc85d58ab9dcd5..790e1f5d0b8c32 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -289,23 +289,46 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct } #ifdef MS_WIN32 - int ms_struct = 1; + LayoutMode layout_mode = LAYOUT_MODE_MS; #else - int ms_struct = 0; + LayoutMode layout_mode = (pack > 0) ? LAYOUT_MODE_MS : LAYOUT_MODE_GCC_SYSV; #endif - if (PyObject_GetOptionalAttr(type, &_Py_ID(_ms_struct_), &tmp) < 0) { + if (PyObject_GetOptionalAttr(type, &_Py_ID(_layout_), &tmp) < 0) { return -1; } if (tmp) { - ms_struct = PyLong_AsInt(tmp); - Py_DECREF(tmp); - if (PyErr_Occurred()) { - if (PyErr_ExceptionMatches(PyExc_TypeError) || - PyErr_ExceptionMatches(PyExc_OverflowError)) { + if (!PyUnicode_Check(tmp)) { + PyErr_SetString(PyExc_TypeError, + "_layout_ must be a string"); + return -1; + } + if (PyUnicode_CompareWithASCIIString(tmp, "ms") == 0) { + layout_mode = LAYOUT_MODE_MS; + } + else if (PyUnicode_CompareWithASCIIString(tmp, "gcc-sysv") == 0) { + layout_mode = LAYOUT_MODE_GCC_SYSV; + if (pack > 0) { PyErr_SetString(PyExc_ValueError, - "_pack_ must be a bool or integer"); + "_pack_ is not compatible with _layout_=\"gcc-sysv\""); + return -1; } + } + else { + PyErr_Format(PyExc_ValueError, + "unknown _layout_ %R", tmp); + return -1; + } + } + else { + // Set the default + const char *name = (layout_mode == LAYOUT_MODE_MS) ? "ms" + : "gcc-sysv"; + PyObject *name_obj = PyUnicode_FromString(name); + if (!name) { + return -1; + } + if (PyObject_SetAttr(type, &_Py_ID(_layout_), name_obj) < 0) { return -1; } } @@ -515,7 +538,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct prop = PyCField_FromDesc(st, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian, ms_struct); + pack, big_endian, layout_mode); if (prop == NULL) { Py_DECREF(pair); return -1; @@ -571,7 +594,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct prop = PyCField_FromDesc(st, desc, i, &field_size, bitsize, &bitofs, &size, &offset, &align, - pack, big_endian, ms_struct); + pack, big_endian, layout_mode); if (prop == NULL) { Py_DECREF(pair); return -1; From 637b961a9ca22b299c01c855ac396ddcdbb35058 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 30 Apr 2024 14:30:46 +0200 Subject: [PATCH 66/91] PEP 7 --- Modules/_ctypes/cfield.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 2b8bc2cba6c04b..01517ba3e03046 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -195,8 +195,9 @@ PyCField_FromDesc(ctypes_state *st, PyObject *desc, Py_ssize_t index, { PyTypeObject *tp = st->PyCField_Type; CFieldObject* self = (CFieldObject *)tp->tp_alloc(tp, 0); - if (self == NULL) + if (self == NULL) { return NULL; + } StgInfo *info; if (PyStgInfo_FromType(st, desc, &info) < 0) { Py_DECREF(self); From 5309b7e5d0ddff21c1b669fe2b6138a9e75ca81f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 30 Apr 2024 16:40:32 +0200 Subject: [PATCH 67/91] TMP --- Doc/library/ctypes.rst | 18 +- Doc/whatsnew/3.13.rst | 9 + .../pycore_global_objects_fini_generated.h | 3 +- Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/test/test_ctypes/_dump_ctype.py | 240 ++++++++++++++++++ .../test_ctypes/test_bitfields_hypothesis.py | 26 +- ...2-10-01-09-56-27.gh-issue-97588.Gvg54o.rst | 3 +- Modules/_ctypes/cfield.c | 5 +- Modules/_ctypes/stgdict.c | 1 - Python/generated_cases.c.h | 2 +- 12 files changed, 286 insertions(+), 26 deletions(-) create mode 100644 Lib/test/test_ctypes/_dump_ctype.py diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index a9081fca3147f9..624482f6d1a8ec 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -665,17 +665,19 @@ Structure/union layout, alignment and byte order ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, Structure and Union fields are laid out in the same way the C -compiler does it. It is possible to override this behavior by specifying a -:attr:`~Structure._pack_` class attribute in the subclass definition. -This must be set to a positive integer and specifies the maximum alignment for the fields. -This is what ``#pragma pack(n)`` also does in MSVC. +compiler does it. It is possible to override this behavior entirely by specifying a +:attr:`~Structure._layout_` class attribute in the subclass definition; see +the attribute documentation for details. + +It is possible to specify the maximum alignment for the fields by setting +the :attr:`~Structure._pack_` class attribute to a positive integer. +This matches what ``#pragma pack(n)`` does in MSVC. + It is also possible to set a minimum alignment for how the subclass itself is packed in the same way ``#pragma align(n)`` works in MSVC. This can be achieved by specifying a ::attr:`~Structure._align_` class attribute in the subclass definition. -To override the, the compiler ::attr:`~Structure._layout_` class attribute. - :mod:`ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the :class:`BigEndianStructure`, :class:`LittleEndianStructure`, @@ -2551,8 +2553,8 @@ fields, or any other data types containing pointer type fields. GCC and Clang, this layout can be selected with ``__attribute__((ms_struct))``. - ``"gcc-sysv"``: the layout used by GCC with the System V or “SysV-like” - data model, as used on Linux and macOS. TThis is generally compatible - with clang's defaults. + data model, as used on Linux and macOS. This is generally compatible + with clang. With this layout, :attr:`~Structure._pack_` must be unset or zero. If not set explicitly, :attr:`!_layout_` will be set to a default that diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 98349a5984bb7e..f55ddf191b5b5d 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -412,6 +412,15 @@ copy any user classes which define the :meth:`!__replace__` method. (Contributed by Serhiy Storchaka in :gh:`108751`.) +ctypes +------ + +* The layout of :ref:`bit fields ` in + :class:`~ctypes.Structure` and :class:`~ctypes.Union` was improved to better + match platform defaults (GCC/Clang or MSC). In particular, fields no longer + overlap. + (Contributed by Matthias Görgens in :gh:`97702`.) + dbm --- diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 8546029eb5b9fc..0dfc38a9e3e34b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -765,11 +765,11 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_initializing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_io)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_is_text_encoding)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_layout_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_length_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_limbo)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_lock_unlock_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_loop)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_ms_struct_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_needs_com_addref_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_pack_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_restype_)); @@ -989,6 +989,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index f879d49cd5bd68..414292512d1e1e 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -478,6 +478,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(hour) + STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 5a572a87fdb97a..4bc836df2fc0c9 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -987,6 +987,7 @@ extern "C" { INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(hour), \ + INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 41f87b24976f5f..03ed1ee278e31f 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1275,6 +1275,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(hour); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(id); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(ident); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/test/test_ctypes/_dump_ctype.py b/Lib/test/test_ctypes/_dump_ctype.py new file mode 100644 index 00000000000000..a2a9cccbf19662 --- /dev/null +++ b/Lib/test/test_ctypes/_dump_ctype.py @@ -0,0 +1,240 @@ +from textwrap import dedent +from operator import attrgetter +from dataclasses import dataclass +from functools import cached_property + +import ctypes +import _ctypes + +print((ctypes.c_bool).mro()) +print(dir(ctypes.POINTER(ctypes.c_int8))) +print(ctypes.POINTER(ctypes.c_int8)._type_) + +def dump_ctype(tp, name='%'): + length = getattr(tp, '_length_', None) + if length is not None: + return f'{dump_ctype(tp._type_, name)}[{length}]' + if tp == ctypes.c_char_p: + return f'char *{name}' + if tp == ctypes.c_wchar_p: + return f'wchar *{name}' + if tp == ctypes.c_wchar: + return f'wchar {name}' + if tp == ctypes.c_void_p: + return f'void *{name}' + if tp == ctypes.c_bool: + _space = ' ' if name else '' + return f'bool{_space}{name}' + if tp == ctypes.py_object: + return f'PyObject *{name}' + if issubclass(tp, _ctypes._Pointer): + return f'{dump_ctype(tp._type_, '*' + name)}' + if issubclass(tp, (ctypes.Structure, ctypes.Union)): + if issubclass(tp, ctypes.Structure): + lines = ['struct {'] + else: + lines = ['union {'] + for fielddesc in tp._fields_: + f_name, f_tp, f_bits = unpack_field_desc(fielddesc) + if f_name in getattr(tp, '_anonymous_', ()): + f_name = '' + for line in dump_ctype(f_tp, f_name).splitlines(): + lines.append(' ' + line) + if f_bits is None: + lines[-1] += ';' + else: + lines[-1] += f' :{f_bits};' + _space = ' ' if name else '' + lines.append(f'}}{_space}{name}') + return '\n'.join(lines) + if issubclass(tp, ctypes._SimpleCData): + if tp(-1).value == -1: + sign = '' + else: + sign = 'u' + size = ctypes.sizeof(tp) + _space = ' ' if name else '' + return f'{sign}int{size * 8}_t{_space}{name}' + restype = getattr(tp, '_restype_', None) + if restype is not None: + return f'{dump_ctype(restype, "")} (*{name})({", ".join(dump_ctype(t, "") for t in tp._argtypes_)})' + +def unpack_field_desc(fielddesc): + try: + f_name, f_tp, f_bits = fielddesc + return f_name, f_tp, f_bits + except ValueError: + f_name, f_tp = fielddesc + return f_name, f_tp, None + +@dataclass +class FieldInfo: + name: str + tp: type + bits: int | None + parent_type: type + parent: 'FieldInfo' #| None + + @cached_property + def field_descriptor(self): + return getattr(self.parent_type, self.name) + + @cached_property + def offset(self): + offset = self.field_descriptor.offset + if self.parent: + offset += self.parent.offset + return offset + + @cached_property + def field_path(self): + if self.parent: + return (*self.parent.field_path, self.name) + return (self.name,) + + @cached_property + def attr_path(self): + if self.name in getattr(self.parent_type, '_anonymous_', ()): + selfpath = () + else: + selfpath = (self.name,) + if self.parent: + return (*self.parent.attr_path, *selfpath) + else: + return selfpath + + @cached_property + def full_name(self): + return '.'.join(self.attr_path) + + @cached_property + def is_in_union(self): + if self.parent and self.parent.is_in_union: + return True + return issubclass(self.parent_type, ctypes.Union) + + +def iterfields(tp, parent=None): + try: + fields = tp._fields_ + except AttributeError: + yield parent + else: + for fielddesc in fields: + f_name, f_tp, f_bits = unpack_field_desc(fielddesc) + sub = FieldInfo(f_name, f_tp, f_bits, tp, parent) + yield from iterfields(f_tp, sub) + + +class Uni(ctypes.Union): + _fields_ = [ + ('u', ctypes.c_int8), + ('v', ctypes.c_int16, 9), + ('w', ctypes.c_int16, 3), + ] + +class Sub(ctypes.Structure): + _fields_ = [ + ('x', ctypes.c_int8), + ('y', ctypes.c_int8, 3), + ('z', ctypes.c_int16, 9), + ] + +class S(ctypes.Structure): + _fields_ = [ + ('a', ctypes.c_int8), + ('b', ctypes.c_int8, 3), + ('c', Sub), + ('anon', Sub), + ('d', Uni), + ] + _anonymous_ = ['anon'] + +TESTCASES = ( + (ctypes.c_int8, 'int8_t %'), + (ctypes.c_uint8, 'uint8_t %'), + (ctypes.c_int16, 'int16_t %'), + (ctypes.c_uint16, 'uint16_t %'), + (ctypes.c_int32, 'int32_t %'), + (ctypes.c_uint32, 'uint32_t %'), + (ctypes.c_int64, 'int64_t %'), + (ctypes.c_uint64, 'uint64_t %'), + (ctypes.c_int8 * 8, 'int8_t %[8]'), + (ctypes.CFUNCTYPE(ctypes.c_int8, ctypes.c_uint16, ctypes.c_uint32), 'int8_t (*%)(uint16_t, uint32_t)'), + (ctypes.c_char_p, 'char *%'), + (ctypes.POINTER(ctypes.c_int8), 'int8_t *%'), + (ctypes.POINTER(ctypes.POINTER(ctypes.c_int8)), 'int8_t **%'), + (ctypes.c_void_p, 'void *%'), + (ctypes.c_wchar_p, 'wchar *%'), + (ctypes.c_wchar, 'wchar %'), + (ctypes.c_bool, 'bool %'), + (ctypes.py_object, 'PyObject *%'), + (S, dedent(""" + struct { + int8_t a; + int8_t b :3; + struct { + int8_t x; + int8_t y :3; + int16_t z :9; + } c; + struct { + int8_t x; + int8_t y :3; + int16_t z :9; + }; + union { + int8_t u; + int16_t v :9; + int16_t w :3; + } d; + } % + """).strip()), +) + +for ctype, expected_name in TESTCASES: + print(ctype, expected_name, '...') + got_name = dump_ctype(ctype) + print(ctype, got_name, '!!!') + if expected_name != got_name: + raise AssertionError() + +def getfieldtype(tp, attrnames): + match attrnames: + case [name]: + return getattr(tp, name) + case [name, *rest]: + return getfieldtype(getattr(tp, name).type, rest) + case _: + raise AttributeError(attrnames) + +print(S.x) +print(dir(S.x)) +s = S() +expected_names = ['a', 'b', 'c.x', 'c.y', 'c.z', 'x', 'y', 'z', 'd.u', 'd.v', 'd.w'] +for got_info, expected_name in zip(iterfields(S), expected_names, strict=True): + print(got_info, expected_name, got_info.offset) + assert attrgetter(expected_name)(s) == 0 + print(got_info.full_name) + print(got_info.attr_path) + print(got_info.field_path) + print(got_info.field_descriptor) + print(got_info.offset) + print(got_info.is_in_union) + assert got_info.full_name == expected_name + +print(dir(ctypes)) +print(type(ctypes.c_int)) +print(dir(type(ctypes.c_int))) +print(ctypes.sizeof(ctypes.c_int)) +print(dir(ctypes.c_int)) +print(dir(ctypes.c_int*8)) +print((ctypes.c_int*8)._length_) +print((ctypes.c_int*8)._type_) +print(dir(ctypes.CFUNCTYPE(ctypes.c_int8, ctypes.c_uint16, ctypes.c_uint32))) +print(dir(ctypes.PYFUNCTYPE(ctypes.c_int8, ctypes.c_uint16, ctypes.c_uint32))) +print(dir(ctypes.c_char_p)) +print(ctypes.c_char_p) +print(ctypes.POINTER(ctypes.c_int8)) +print(dir(ctypes.POINTER(ctypes.c_int8))) +print(ctypes.POINTER(ctypes.c_int8)._type_) diff --git a/Lib/test/test_ctypes/test_bitfields_hypothesis.py b/Lib/test/test_ctypes/test_bitfields_hypothesis.py index b518f6d5da8c5f..61e158b5fecc19 100644 --- a/Lib/test/test_ctypes/test_bitfields_hypothesis.py +++ b/Lib/test/test_ctypes/test_bitfields_hypothesis.py @@ -12,8 +12,11 @@ POINTER, ) -from hypothesis import assume, given, note, settings -from hypothesis import strategies +from test.support.hypothesis_helper import hypothesis + +assume = hypothesis.assume +given = hypothesis.given +strategies = hypothesis.strategies C_TYPES = { # Note: for Hypothesis minimization to work, "simpler" types should @@ -64,18 +67,18 @@ def struct_c_decl(attrdict, name='s'): lines.append('') return '\n'.join(lines) - @strategies.composite -def structure_args( - draw, - types_strategy=strategies.lists( - strategies.sampled_from(list(C_TYPES.items())), - min_size=1, max_size=200, - ), -): +def structure_args(draw): + num_fields = draw(strategies.integers(1, 50)) fields = [] ctype_names = [] - for i, (ctype_name, ctype) in enumerate(draw(types_strategy)): + for i in range(num_fields): + if draw(strategies.booleans()) and 1: + ctype_name, ctype = draw(strategies.sampled_from(list(C_TYPES.items()))) + else: + sub_args = draw(structure_args()) + ctype_name = f's{id(sub_args)}' + ctype = type(sub_args) name = f'field_{i}' bit_size = draw(strategies.integers(1, sizeof(ctype) * 8)) is_bitfield = draw(strategies.booleans()) @@ -103,6 +106,7 @@ def structure_args( if pack: attrdict['_pack_'] = pack attrdict['c_decl'] = struct_c_decl(attrdict) + attrdict['c_name'] = f's{id(attrdict)}' result = ( 'RandomTestStruct', (cls,), diff --git a/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst b/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst index 09569ef5e9235d..0bb0f5bcd501ef 100644 --- a/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst +++ b/Misc/NEWS.d/next/C API/2022-10-01-09-56-27.gh-issue-97588.Gvg54o.rst @@ -1 +1,2 @@ -Fix ctypes construction of structs from description. +Fix creating bitfields in :mod:`ctypes` structures and unions. Fields +no longer overlap. diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 01517ba3e03046..03a8b2f8e440ee 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -76,8 +76,8 @@ of the bit-fields. See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html#index-mms-bitfields for details. -We do not support zero length bitfields. In fact we use bitsize != 0 to -indicate a bitfield. +We do not support zero length bitfields. In fact we use bitsize != 0 elsewhere +to indicate a bitfield. Here, non-bitfields need bitsize set to size*8. PyCField_FromDesc manages: - *psize: the size of the structure / union so far. @@ -119,7 +119,6 @@ PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, return -1; } } else { - assert(bitsize == 0); self->offset = round_down(*pbitofs, 8*info->align) / 8; self->size = info->size; } diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 790e1f5d0b8c32..e1fdb5320f9c2d 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -271,7 +271,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (tmp) { pack = PyLong_AsInt(tmp); Py_DECREF(tmp); - // TODO(Matthias): It looks like pack == 0 triggers a bug. if (pack < 0) { if (!PyErr_Occurred() || PyErr_ExceptionMatches(PyExc_TypeError) || diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7c1cc147de4e1a..397c9a40646a5d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6035,7 +6035,7 @@ gen_frame->previous = NULL; /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE + #if 0 assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || From 489c5c8300ed7226b92c8bcdc0ecbd48fc63d2ac Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 2 May 2024 15:18:18 +0200 Subject: [PATCH 68/91] Add generated tests --- .../test_ctypes/test_generated_structs.py | 641 +++++ Makefile.pre.in | 1 + Modules/_ctypes/_ctypes_test.c | 6 +- Modules/_ctypes/_ctypes_test_generated.c.h | 2245 +++++++++++++++++ PCbuild/_ctypes_test.vcxproj | 3 +- PCbuild/_ctypes_test.vcxproj.filters | 5 +- 6 files changed, 2898 insertions(+), 3 deletions(-) create mode 100644 Lib/test/test_ctypes/test_generated_structs.py create mode 100644 Modules/_ctypes/_ctypes_test_generated.c.h diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py new file mode 100644 index 00000000000000..747035a29a4059 --- /dev/null +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -0,0 +1,641 @@ +"""Test CTypes structs, unions, bitfields against C equivalents. + +The types here are auto-converted to C source at +`Modules/_ctypes/_ctypes_test_generated.c.h`, which is compiled into +_ctypes_test. + +The generated tests return a list of raw things to check, +or [None, skip_reason] to skip the test. + +Run this module to regenerate the files: + +./python Lib/test/test_ctypes/test_generated_structs.py > Modules/_ctypes/_ctypes_test_generated.c.h +""" + +import unittest +from test.support import import_helper +import re +from dataclasses import dataclass +from functools import cached_property +import sys + +import ctypes +from ctypes import Structure, Union, _SimpleCData +from ctypes import sizeof, alignment, pointer, string_at +_ctypes_test = import_helper.import_module("_ctypes_test") + +KNOWN_COMPILERS = 'defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)' + +# ctypes erases the difference between `c_int` and e.g.`c_int16`. +# To keep it, we'll use custom subclasses with the C name in `_c_name`. + +for c_name, ctypes_name in { + 'signed char': 'c_byte', + 'short': 'c_short', + 'int': 'c_int', + 'long': 'c_long', + 'long long': 'c_longlong', + 'unsigned char': 'c_ubyte', + 'unsigned short': 'c_ushort', + 'unsigned int': 'c_uint', + 'unsigned long': 'c_ulong', + 'unsigned long long': 'c_ulonglong', + **{f'{u}int{n}_t': f'c_{u}int{n}' + for u in ('', 'u') + for n in (8, 16, 32, 64)} +}.items(): + ctype = getattr(ctypes, ctypes_name) + newtype = type(ctypes_name, (ctype,), {'_c_name': c_name}) + globals()[ctypes_name] = newtype + +TESTCASES = {} +def register(name=None): + def decorator(cls, name=name): + if name is None: + name = cls.__name__ + assert name.isascii() # will be used in _PyUnicode_EqualToASCIIString + assert name.isidentifier() # will be used as a C identifier + assert name not in TESTCASES + TESTCASES[name] = cls + return cls + return decorator + +@register() +class SingleInt(Structure): + _fields_ = [('a', c_int)] + +@register() +class SingleInt_Union(Union): + _fields_ = [('a', c_int)] + + +@register() +class SingleU32(Structure): + _fields_ = [('a', c_uint32)] + + +@register() +class SimpleStruct(Structure): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class SimpleUnion(Union): + _fields_ = [('x', c_int32), ('y', c_int8), ('z', c_uint16)] + + +@register() +class ManyTypes(Structure): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class ManyTypesU(Union): + _fields_ = [ + ('i8', c_int8), ('u8', c_uint8), + ('i16', c_int16), ('u16', c_uint16), + ('i32', c_int32), ('u32', c_uint32), + ('i64', c_int64), ('u64', c_uint64), + ] + + +@register() +class Nested(Union): + _fields_ = [ + ('a', SimpleStruct), ('b', SimpleUnion), ('anon', SimpleStruct), + ] + _anonymous_ = ['anon'] + + +@register() +class Packed1(Union): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 1 + + +@register() +class Packed2(Union): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 2 + + +@register() +class Packed3(Union): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 4 + + +@register() +class Packed4(Union): + _fields_ = [('a', c_int8), ('b', c_int64)] + _pack_ = 8 + +@register() +class IntBits(Structure): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits(Structure): + _fields_ = [*IntBits._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class IntBits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class Bits_MSVC(Structure): + _layout_ = "ms" + _fields_ = [*IntBits_MSVC._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class IntBits_Union(Union): + _fields_ = [("A", c_int, 1), + ("B", c_int, 2), + ("C", c_int, 3), + ("D", c_int, 4), + ("E", c_int, 5), + ("F", c_int, 6), + ("G", c_int, 7), + ("H", c_int, 8), + ("I", c_int, 9)] + +@register() +class BitsUnion(Union): + _fields_ = [*IntBits_Union._fields_, + + ("M", c_short, 1), + ("N", c_short, 2), + ("O", c_short, 3), + ("P", c_short, 4), + ("Q", c_short, 5), + ("R", c_short, 6), + ("S", c_short, 7)] + +@register() +class I64Bits(Structure): + _fields_ = [("a", c_int64, 1), + ("b", c_int64, 62), + ("c", c_int64, 1)] + +@register() +class U64Bits(Structure): + _fields_ = [("a", c_uint64, 1), + ("b", c_uint64, 62), + ("c", c_uint64, 1)] + +for n in 8, 16, 32, 64: + for signedness in '', 'u': + ctype = globals()[f'c_{signedness}int{n}'] + + @register(f'Struct331_{signedness}{n}') + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, 3), + ("c", ctype, 1)] + + @register(f'Struct1x1_{signedness}{n}') + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct1nx1_{signedness}{n}') + class _cls(Structure): + _fields_ = [("a", ctype, 1), + ("full", ctype), + ("b", ctype, n-2), + ("c", ctype, 1)] + + @register(f'Struct3xx_{signedness}{n}') + class _cls(Structure): + _fields_ = [("a", ctype, 3), + ("b", ctype, n-2), + ("c", ctype, n-2)] + +@register() +class Mixed1(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int, 4)] + +@register() +class Mixed2(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_int32, 32)] + +@register() +class Mixed3(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + +@register() +class Mixed4(Structure): + _fields_ = [("a", c_short, 4), + ("b", c_short, 4), + ("c", c_int, 24), + ("d", c_short, 4), + ("e", c_short, 4), + ("f", c_int, 24)] + +@register() +class Mixed5(Structure): + _fields_ = [('A', c_uint, 1), + ('B', c_ushort, 16)] + +@register() +class Mixed6(Structure): + _fields_ = [('A', c_ulonglong, 1), + ('B', c_uint, 32)] + +@register() +class Mixed7(Structure): + _fields_ = [("A", c_uint32), + ('B', c_uint32, 20), + ('C', c_uint64, 24)] + +@register() +class Mixed8_a(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32, 32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed8_b(Structure): + _fields_ = [("A", c_uint32), + ("B", c_uint32), + ("C", c_ulonglong, 1)] + +@register() +class Mixed9(Structure): + _fields_ = [("A", c_uint8), + ("B", c_uint32, 1)] + +@register() +class Mixed10(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_95496(Structure): + _fields_ = [("A", c_uint32, 1), + ("B", c_uint64, 1)] + +@register() +class Example_gh_84039_bad(Structure): + _pack_ = 1 + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_84039_good_a(Structure): + _pack_ = 1 + _fields_ = [("a0", c_uint8, 1), + ("a1", c_uint8, 1), + ("a2", c_uint8, 1), + ("a3", c_uint8, 1), + ("a4", c_uint8, 1), + ("a5", c_uint8, 1), + ("a6", c_uint8, 1), + ("a7", c_uint8, 1)] + +@register() +class Example_gh_84039_good(Structure): + _pack_ = 1 + _fields_ = [("a", Example_gh_84039_good_a), + ("b0", c_uint16, 4), + ("b1", c_uint16, 12)] + +@register() +class Example_gh_73939(Structure): + _pack_ = 1 + _fields_ = [("P", c_uint16), + ("L", c_uint16, 9), + ("Pro", c_uint16, 1), + ("G", c_uint16, 1), + ("IB", c_uint16, 1), + ("IR", c_uint16, 1), + ("R", c_uint16, 3), + ("T", c_uint32, 10), + ("C", c_uint32, 20), + ("R2", c_uint32, 2)] + +@register() +class Example_gh_86098(Structure): + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class Example_gh_86098_pack(Structure): + _pack_ = 1 + _fields_ = [("a", c_uint8, 8), + ("b", c_uint8, 8), + ("c", c_uint32, 16)] + +@register() +class AnonBitfields(Structure): + class X(Structure): + _fields_ = [("a", c_byte, 4), + ("b", c_ubyte, 4)] + _anonymous_ = ["_"] + _fields_ = [("_", X), ('y', c_byte)] + + +class GeneratedTest(unittest.TestCase): + def test_generated_data(self): + for name, cls in TESTCASES.items(): + with self.subTest(name=name): + expected = iter(_ctypes_test.get_generated_test_data(name)) + expected_name = next(expected) + if expected_name is None: + unittest.skipTest(next(expected)) + self.assertEqual(name, expected_name) + self.assertEqual(sizeof(cls), next(expected)) + with self.subTest('alignment'): + self.assertEqual(alignment(cls), next(expected)) + obj = cls() + ptr = pointer(obj) + for field in iterfields(cls): + for value in -1, 1, 0: + with self.subTest(field=field.full_name, value=value): + field.set_to(obj, value) + mem = string_at(ptr, sizeof(obj)) + self.assertEqual(mem, next(expected)) + + +# The rest of this file is generating C code from a ctypes type. +# This is only tested with the known inputs here. + +def c_str_repr(string): + return '"' + re.sub('([\"\'\\\\\n])', r'\\\1', string) + '"' + +def dump_simple_ctype(tp, variable_name='', semi=''): + length = getattr(tp, '_length_', None) + if length is not None: + return f'{dump_simple_ctype(tp._type_, variable_name)}[{length}]{semi}' + assert not issubclass(tp, (Structure, Union)) + return f'{tp._c_name}{maybe_space(variable_name)}{semi}' + + +def dump_ctype(tp, agg_name='', variable_name='', semi=''): + requires = set() + if issubclass(tp, (Structure, Union)): + attributes = [] + pushes = [] + pops = [] + pack = getattr(tp, '_pack_', None) + if pack is not None: + requires.add(KNOWN_COMPILERS) + pushes.append(f'#pragma pack(push, {pack})') + pops.append(f'#pragma pack(pop)') + layout = getattr(tp, '_layout_', None) + if layout == 'ms': + requires.add(KNOWN_COMPILERS) + attributes.append('ms_struct') + if attributes: + a = f' GCC_ATTR({", ".join(attributes)})' + else: + a = '' + lines = [f'{struct_or_union(tp)}{a}{maybe_space(agg_name)} ' +'{'] + for fielddesc in tp._fields_: + f_name, f_tp, f_bits = unpack_field_desc(fielddesc) + if f_name in getattr(tp, '_anonymous_', ()): + f_name = '' + if f_bits is None: + subsemi = ';' + else: + if f_tp not in (c_int, c_uint): + # XLC can reportedly only handle int & unsigned int + # bitfields (the only types required by C spec) + requires.add('!defined(__xlc__)') + subsemi = f' :{f_bits};' + sub_lines, sub_requires = dump_ctype( + f_tp, variable_name=f_name, semi=subsemi) + requires.update(sub_requires) + for line in sub_lines: + lines.append(' ' + line) + lines.append(f'}}{maybe_space(variable_name)}{semi}') + return [*pushes, *lines, *reversed(pops)], requires + else: + return [dump_simple_ctype(tp, variable_name, semi)], requires + +def struct_or_union(cls): + if issubclass(cls, Structure): + return 'struct' + if issubclass(cls, Union): + return 'union' + raise TypeError(cls) + +def maybe_space(string): + if string: + return ' ' + string + return string + +def unpack_field_desc(fielddesc): + try: + f_name, f_tp, f_bits = fielddesc + return f_name, f_tp, f_bits + except ValueError: + f_name, f_tp = fielddesc + return f_name, f_tp, None + +@dataclass +class FieldInfo: + name: str + tp: type + bits: int | None + parent_type: type + parent: 'FieldInfo' #| None + + @cached_property + def field_descriptor(self): + return getattr(self.parent_type, self.name) + + @cached_property + def field_path(self): + if self.parent: + return (*self.parent.field_path, self.name) + return (self.name,) + + @cached_property + def attr_path(self): + if self.name in getattr(self.parent_type, '_anonymous_', ()): + selfpath = () + else: + selfpath = (self.name,) + if self.parent: + return (*self.parent.attr_path, *selfpath) + else: + return selfpath + + @cached_property + def full_name(self): + return '.'.join(self.attr_path) + + def get_from(self, obj): + for attr_name in self.attr_path: + obj = getattr(obj, attr_name) + return obj + + def set_to(self, obj, new): + for attr_name in self.attr_path[:-1]: + obj = getattr(obj, attr_name) + setattr(obj, self.attr_path[-1], new) + + +def iterfields(tp, parent=None): + try: + fields = tp._fields_ + except AttributeError: + yield parent + else: + for fielddesc in fields: + f_name, f_tp, f_bits = unpack_field_desc(fielddesc) + sub = FieldInfo(f_name, f_tp, f_bits, tp, parent) + yield from iterfields(f_tp, sub) + + +if __name__ == '__main__': + print('/* Generated by Lib/test/test_ctypes/test_generated_structs.py */') + print(""" + // Append VALUE to the result. + #define APPEND(VAL) { \\ + if (!VAL) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + int rv = PyList_Append(result, VAL); \\ + Py_DECREF(VAL); \\ + if (rv < 0) { \\ + Py_DECREF(result); \\ + return NULL; \\ + } \\ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \\ + TYPE v = VAL; \\ + TARGET = v; \\ + APPEND(PyBytes_FromStringAndSize( \\ + (char*)&value, sizeof(value))); \\ + } + + // Set a field to -1, 1 and 0; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \\ + SET_AND_APPEND(TYPE, TARGET, -1) \\ + SET_AND_APPEND(TYPE, TARGET, 1) \\ + SET_AND_APPEND(TYPE, TARGET, 0) \\ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + """) + for name, cls in TESTCASES.items(): + print(""" + if (_PyUnicode_EqualToASCIIString(name, %s)) { + """ % c_str_repr(name)) + lines, requires = dump_ctype(cls, agg_name=name, semi=';') + if requires: + print(f""" + #if {" && ".join(f'({r})' for r in sorted(requires))} + """) + for line in lines: + print(' ', line, sep='') + typename = f'{struct_or_union(cls)} {name}' + print(f""" + {typename} value = {{0}}; + APPEND(PyUnicode_FromString({c_str_repr(name)})); + APPEND(PyLong_FromLong(sizeof({typename}))); + APPEND(PyLong_FromLong(_Alignof({typename}))); + """.rstrip()) + for field in iterfields(cls): + f_tp = dump_simple_ctype(field.tp) + print(f"""\ + TEST_FIELD({f_tp}, value.{field.full_name}); + """.rstrip()) + if requires: + print(f""" + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + """) + print(""" + return result; + } + """) + + print(""" + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + """) + diff --git a/Makefile.pre.in b/Makefile.pre.in index 0e52e10602cf85..b419890129efd7 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3094,6 +3094,7 @@ MODULE_PYEXPAT_DEPS=@LIBEXPAT_INTERNAL@ MODULE_UNICODEDATA_DEPS=$(srcdir)/Modules/unicodedata_db.h $(srcdir)/Modules/unicodename_db.h MODULE__BLAKE2_DEPS=$(srcdir)/Modules/_blake2/impl/blake2-config.h $(srcdir)/Modules/_blake2/impl/blake2-impl.h $(srcdir)/Modules/_blake2/impl/blake2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2b-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2b-ref.c $(srcdir)/Modules/_blake2/impl/blake2b-round.h $(srcdir)/Modules/_blake2/impl/blake2b.c $(srcdir)/Modules/_blake2/impl/blake2s-load-sse2.h $(srcdir)/Modules/_blake2/impl/blake2s-load-sse41.h $(srcdir)/Modules/_blake2/impl/blake2s-load-xop.h $(srcdir)/Modules/_blake2/impl/blake2s-ref.c $(srcdir)/Modules/_blake2/impl/blake2s-round.h $(srcdir)/Modules/_blake2/impl/blake2s.c $(srcdir)/Modules/_blake2/blake2module.h $(srcdir)/Modules/hashlib.h MODULE__CTYPES_DEPS=$(srcdir)/Modules/_ctypes/ctypes.h +MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index ae9a5d25ade40d..f1325fb7297e46 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,7 +1,7 @@ // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +//# define Py_LIMITED_API 0x030c0000 #endif // gh-85283: On Windows, Py_LIMITED_API requires Py_BUILD_CORE to not attempt @@ -12,6 +12,7 @@ #define Py_BUILD_CORE_MODULE #include +#include // _PyUnicode_EqualToASCIIString #include // printf() #include // qsort() @@ -22,6 +23,8 @@ #define EXPORT(x) Py_EXPORTED_SYMBOL x +#include "_ctypes_test_generated.c.h" + /* some functions handy for testing */ EXPORT(int) @@ -810,6 +813,7 @@ static PyMethodDef module_methods[] = { */ {"func_si", py_func_si, METH_VARARGS}, {"func", py_func, METH_NOARGS}, + {"get_generated_test_data", get_generated_test_data, METH_O}, { NULL, NULL, 0, NULL}, }; diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h new file mode 100644 index 00000000000000..2df39ae92c2f20 --- /dev/null +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -0,0 +1,2245 @@ +/* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + + // Append VALUE to the result. + #define APPEND(VAL) { \ + if (!VAL) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + int rv = PyList_Append(result, VAL); \ + Py_DECREF(VAL); \ + if (rv < 0) { \ + Py_DECREF(result); \ + return NULL; \ + } \ + } + + // Set TARGET, and append a snapshot of `value`'s + // memory to the result. + #define SET_AND_APPEND(TYPE, TARGET, VAL) { \ + TYPE v = VAL; \ + TARGET = v; \ + APPEND(PyBytes_FromStringAndSize( \ + (char*)&value, sizeof(value))); \ + } + + // Set a field to -1, 1 and 0; append a snapshot of the memory + // after each of the operations. + #define TEST_FIELD(TYPE, TARGET) { \ + SET_AND_APPEND(TYPE, TARGET, -1) \ + SET_AND_APPEND(TYPE, TARGET, 1) \ + SET_AND_APPEND(TYPE, TARGET, 0) \ + } + + #if defined(__GNUC__) || defined(__clang__) + #define GCC_ATTR(X) __attribute__((X)) + #else + #define GCC_ATTR(X) /* */ + #endif + + static PyObject * + get_generated_test_data(PyObject *self, PyObject *name) + { + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "need a string"); + return NULL; + } + PyObject *result = PyList_New(0); + if (!result) { + return NULL; + } + + + if (_PyUnicode_EqualToASCIIString(name, "SingleInt")) { + + struct SingleInt { + int a; + }; + + struct SingleInt value = {0}; + APPEND(PyUnicode_FromString("SingleInt")); + APPEND(PyLong_FromLong(sizeof(struct SingleInt))); + APPEND(PyLong_FromLong(_Alignof(struct SingleInt))); + TEST_FIELD(int, value.a); + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "SingleInt_Union")) { + + union SingleInt_Union { + int a; + }; + + union SingleInt_Union value = {0}; + APPEND(PyUnicode_FromString("SingleInt_Union")); + APPEND(PyLong_FromLong(sizeof(union SingleInt_Union))); + APPEND(PyLong_FromLong(_Alignof(union SingleInt_Union))); + TEST_FIELD(int, value.a); + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "SingleU32")) { + + struct SingleU32 { + uint32_t a; + }; + + struct SingleU32 value = {0}; + APPEND(PyUnicode_FromString("SingleU32")); + APPEND(PyLong_FromLong(sizeof(struct SingleU32))); + APPEND(PyLong_FromLong(_Alignof(struct SingleU32))); + TEST_FIELD(uint32_t, value.a); + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "SimpleStruct")) { + + struct SimpleStruct { + int32_t x; + int8_t y; + uint16_t z; + }; + + struct SimpleStruct value = {0}; + APPEND(PyUnicode_FromString("SimpleStruct")); + APPEND(PyLong_FromLong(sizeof(struct SimpleStruct))); + APPEND(PyLong_FromLong(_Alignof(struct SimpleStruct))); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "SimpleUnion")) { + + union SimpleUnion { + int32_t x; + int8_t y; + uint16_t z; + }; + + union SimpleUnion value = {0}; + APPEND(PyUnicode_FromString("SimpleUnion")); + APPEND(PyLong_FromLong(sizeof(union SimpleUnion))); + APPEND(PyLong_FromLong(_Alignof(union SimpleUnion))); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "ManyTypes")) { + + struct ManyTypes { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + }; + + struct ManyTypes value = {0}; + APPEND(PyUnicode_FromString("ManyTypes")); + APPEND(PyLong_FromLong(sizeof(struct ManyTypes))); + APPEND(PyLong_FromLong(_Alignof(struct ManyTypes))); + TEST_FIELD(int8_t, value.i8); + TEST_FIELD(uint8_t, value.u8); + TEST_FIELD(int16_t, value.i16); + TEST_FIELD(uint16_t, value.u16); + TEST_FIELD(int32_t, value.i32); + TEST_FIELD(uint32_t, value.u32); + TEST_FIELD(int64_t, value.i64); + TEST_FIELD(uint64_t, value.u64); + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "ManyTypesU")) { + + union ManyTypesU { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + }; + + union ManyTypesU value = {0}; + APPEND(PyUnicode_FromString("ManyTypesU")); + APPEND(PyLong_FromLong(sizeof(union ManyTypesU))); + APPEND(PyLong_FromLong(_Alignof(union ManyTypesU))); + TEST_FIELD(int8_t, value.i8); + TEST_FIELD(uint8_t, value.u8); + TEST_FIELD(int16_t, value.i16); + TEST_FIELD(uint16_t, value.u16); + TEST_FIELD(int32_t, value.i32); + TEST_FIELD(uint32_t, value.u32); + TEST_FIELD(int64_t, value.i64); + TEST_FIELD(uint64_t, value.u64); + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Nested")) { + + union Nested { + struct { + int32_t x; + int8_t y; + uint16_t z; + } a; + union { + int32_t x; + int8_t y; + uint16_t z; + } b; + struct { + int32_t x; + int8_t y; + uint16_t z; + }; + }; + + union Nested value = {0}; + APPEND(PyUnicode_FromString("Nested")); + APPEND(PyLong_FromLong(sizeof(union Nested))); + APPEND(PyLong_FromLong(_Alignof(union Nested))); + TEST_FIELD(int32_t, value.a.x); + TEST_FIELD(int8_t, value.a.y); + TEST_FIELD(uint16_t, value.a.z); + TEST_FIELD(int32_t, value.b.x); + TEST_FIELD(int8_t, value.b.y); + TEST_FIELD(uint16_t, value.b.z); + TEST_FIELD(int32_t, value.x); + TEST_FIELD(int8_t, value.y); + TEST_FIELD(uint16_t, value.z); + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Packed1")) { + + + #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + #pragma pack(push, 1) + union Packed1 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + + union Packed1 value = {0}; + APPEND(PyUnicode_FromString("Packed1")); + APPEND(PyLong_FromLong(sizeof(union Packed1))); + APPEND(PyLong_FromLong(_Alignof(union Packed1))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Packed2")) { + + + #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + #pragma pack(push, 2) + union Packed2 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + + union Packed2 value = {0}; + APPEND(PyUnicode_FromString("Packed2")); + APPEND(PyLong_FromLong(sizeof(union Packed2))); + APPEND(PyLong_FromLong(_Alignof(union Packed2))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Packed3")) { + + + #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + #pragma pack(push, 4) + union Packed3 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + + union Packed3 value = {0}; + APPEND(PyUnicode_FromString("Packed3")); + APPEND(PyLong_FromLong(sizeof(union Packed3))); + APPEND(PyLong_FromLong(_Alignof(union Packed3))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Packed4")) { + + + #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + #pragma pack(push, 8) + union GCC_ATTR(ms_struct) Packed4 { + int8_t a; + int64_t b; + }; + #pragma pack(pop) + + union Packed4 value = {0}; + APPEND(PyUnicode_FromString("Packed4")); + APPEND(PyLong_FromLong(sizeof(union Packed4))); + APPEND(PyLong_FromLong(_Alignof(union Packed4))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int64_t, value.b); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "IntBits")) { + + struct IntBits { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + + struct IntBits value = {0}; + APPEND(PyUnicode_FromString("IntBits")); + APPEND(PyLong_FromLong(sizeof(struct IntBits))); + APPEND(PyLong_FromLong(_Alignof(struct IntBits))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Bits")) { + + + #if (!defined(__xlc__)) + + struct Bits { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + + struct Bits value = {0}; + APPEND(PyUnicode_FromString("Bits")); + APPEND(PyLong_FromLong(sizeof(struct Bits))); + APPEND(PyLong_FromLong(_Alignof(struct Bits))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "IntBits_MSVC")) { + + + #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + struct GCC_ATTR(ms_struct) IntBits_MSVC { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + + struct IntBits_MSVC value = {0}; + APPEND(PyUnicode_FromString("IntBits_MSVC")); + APPEND(PyLong_FromLong(sizeof(struct IntBits_MSVC))); + APPEND(PyLong_FromLong(_Alignof(struct IntBits_MSVC))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Bits_MSVC")) { + + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + struct GCC_ATTR(ms_struct) Bits_MSVC { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + + struct Bits_MSVC value = {0}; + APPEND(PyUnicode_FromString("Bits_MSVC")); + APPEND(PyLong_FromLong(sizeof(struct Bits_MSVC))); + APPEND(PyLong_FromLong(_Alignof(struct Bits_MSVC))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "IntBits_Union")) { + + union IntBits_Union { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + }; + + union IntBits_Union value = {0}; + APPEND(PyUnicode_FromString("IntBits_Union")); + APPEND(PyLong_FromLong(sizeof(union IntBits_Union))); + APPEND(PyLong_FromLong(_Alignof(union IntBits_Union))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "BitsUnion")) { + + + #if (!defined(__xlc__)) + + union BitsUnion { + int A :1; + int B :2; + int C :3; + int D :4; + int E :5; + int F :6; + int G :7; + int H :8; + int I :9; + short M :1; + short N :2; + short O :3; + short P :4; + short Q :5; + short R :6; + short S :7; + }; + + union BitsUnion value = {0}; + APPEND(PyUnicode_FromString("BitsUnion")); + APPEND(PyLong_FromLong(sizeof(union BitsUnion))); + APPEND(PyLong_FromLong(_Alignof(union BitsUnion))); + TEST_FIELD(int, value.A); + TEST_FIELD(int, value.B); + TEST_FIELD(int, value.C); + TEST_FIELD(int, value.D); + TEST_FIELD(int, value.E); + TEST_FIELD(int, value.F); + TEST_FIELD(int, value.G); + TEST_FIELD(int, value.H); + TEST_FIELD(int, value.I); + TEST_FIELD(short, value.M); + TEST_FIELD(short, value.N); + TEST_FIELD(short, value.O); + TEST_FIELD(short, value.P); + TEST_FIELD(short, value.Q); + TEST_FIELD(short, value.R); + TEST_FIELD(short, value.S); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "I64Bits")) { + + + #if (!defined(__xlc__)) + + struct I64Bits { + int64_t a :1; + int64_t b :62; + int64_t c :1; + }; + + struct I64Bits value = {0}; + APPEND(PyUnicode_FromString("I64Bits")); + APPEND(PyLong_FromLong(sizeof(struct I64Bits))); + APPEND(PyLong_FromLong(_Alignof(struct I64Bits))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "U64Bits")) { + + + #if (!defined(__xlc__)) + + struct U64Bits { + uint64_t a :1; + uint64_t b :62; + uint64_t c :1; + }; + + struct U64Bits value = {0}; + APPEND(PyUnicode_FromString("U64Bits")); + APPEND(PyLong_FromLong(sizeof(struct U64Bits))); + APPEND(PyLong_FromLong(_Alignof(struct U64Bits))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct331_8")) { + + + #if (!defined(__xlc__)) + + struct Struct331_8 { + int8_t a :3; + int8_t b :3; + int8_t c :1; + }; + + struct Struct331_8 value = {0}; + APPEND(PyUnicode_FromString("Struct331_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_8")) { + + + #if (!defined(__xlc__)) + + struct Struct1x1_8 { + int8_t a :1; + int8_t b :6; + int8_t c :1; + }; + + struct Struct1x1_8 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_8")) { + + + #if (!defined(__xlc__)) + + struct Struct1nx1_8 { + int8_t a :1; + int8_t full; + int8_t b :6; + int8_t c :1; + }; + + struct Struct1nx1_8 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.full); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_8")) { + + + #if (!defined(__xlc__)) + + struct Struct3xx_8 { + int8_t a :3; + int8_t b :6; + int8_t c :6; + }; + + struct Struct3xx_8 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_8")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_8))); + TEST_FIELD(int8_t, value.a); + TEST_FIELD(int8_t, value.b); + TEST_FIELD(int8_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct331_u8")) { + + + #if (!defined(__xlc__)) + + struct Struct331_u8 { + uint8_t a :3; + uint8_t b :3; + uint8_t c :1; + }; + + struct Struct331_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u8")) { + + + #if (!defined(__xlc__)) + + struct Struct1x1_u8 { + uint8_t a :1; + uint8_t b :6; + uint8_t c :1; + }; + + struct Struct1x1_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u8")) { + + + #if (!defined(__xlc__)) + + struct Struct1nx1_u8 { + uint8_t a :1; + uint8_t full; + uint8_t b :6; + uint8_t c :1; + }; + + struct Struct1nx1_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.full); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u8")) { + + + #if (!defined(__xlc__)) + + struct Struct3xx_u8 { + uint8_t a :3; + uint8_t b :6; + uint8_t c :6; + }; + + struct Struct3xx_u8 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u8")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u8))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u8))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint8_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct331_16")) { + + + #if (!defined(__xlc__)) + + struct Struct331_16 { + int16_t a :3; + int16_t b :3; + int16_t c :1; + }; + + struct Struct331_16 value = {0}; + APPEND(PyUnicode_FromString("Struct331_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_16")) { + + + #if (!defined(__xlc__)) + + struct Struct1x1_16 { + int16_t a :1; + int16_t b :14; + int16_t c :1; + }; + + struct Struct1x1_16 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_16")) { + + + #if (!defined(__xlc__)) + + struct Struct1nx1_16 { + int16_t a :1; + int16_t full; + int16_t b :14; + int16_t c :1; + }; + + struct Struct1nx1_16 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.full); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_16")) { + + + #if (!defined(__xlc__)) + + struct Struct3xx_16 { + int16_t a :3; + int16_t b :14; + int16_t c :14; + }; + + struct Struct3xx_16 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_16")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_16))); + TEST_FIELD(int16_t, value.a); + TEST_FIELD(int16_t, value.b); + TEST_FIELD(int16_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct331_u16")) { + + + #if (!defined(__xlc__)) + + struct Struct331_u16 { + uint16_t a :3; + uint16_t b :3; + uint16_t c :1; + }; + + struct Struct331_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u16")) { + + + #if (!defined(__xlc__)) + + struct Struct1x1_u16 { + uint16_t a :1; + uint16_t b :14; + uint16_t c :1; + }; + + struct Struct1x1_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u16")) { + + + #if (!defined(__xlc__)) + + struct Struct1nx1_u16 { + uint16_t a :1; + uint16_t full; + uint16_t b :14; + uint16_t c :1; + }; + + struct Struct1nx1_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.full); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u16")) { + + + #if (!defined(__xlc__)) + + struct Struct3xx_u16 { + uint16_t a :3; + uint16_t b :14; + uint16_t c :14; + }; + + struct Struct3xx_u16 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u16")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u16))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u16))); + TEST_FIELD(uint16_t, value.a); + TEST_FIELD(uint16_t, value.b); + TEST_FIELD(uint16_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct331_32")) { + + + #if (!defined(__xlc__)) + + struct Struct331_32 { + int32_t a :3; + int32_t b :3; + int32_t c :1; + }; + + struct Struct331_32 value = {0}; + APPEND(PyUnicode_FromString("Struct331_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_32")) { + + + #if (!defined(__xlc__)) + + struct Struct1x1_32 { + int32_t a :1; + int32_t b :30; + int32_t c :1; + }; + + struct Struct1x1_32 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_32")) { + + + #if (!defined(__xlc__)) + + struct Struct1nx1_32 { + int32_t a :1; + int32_t full; + int32_t b :30; + int32_t c :1; + }; + + struct Struct1nx1_32 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.full); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_32")) { + + + #if (!defined(__xlc__)) + + struct Struct3xx_32 { + int32_t a :3; + int32_t b :30; + int32_t c :30; + }; + + struct Struct3xx_32 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_32")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_32))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int32_t, value.b); + TEST_FIELD(int32_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct331_u32")) { + + + #if (!defined(__xlc__)) + + struct Struct331_u32 { + uint32_t a :3; + uint32_t b :3; + uint32_t c :1; + }; + + struct Struct331_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u32")) { + + + #if (!defined(__xlc__)) + + struct Struct1x1_u32 { + uint32_t a :1; + uint32_t b :30; + uint32_t c :1; + }; + + struct Struct1x1_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u32")) { + + + #if (!defined(__xlc__)) + + struct Struct1nx1_u32 { + uint32_t a :1; + uint32_t full; + uint32_t b :30; + uint32_t c :1; + }; + + struct Struct1nx1_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.full); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u32")) { + + + #if (!defined(__xlc__)) + + struct Struct3xx_u32 { + uint32_t a :3; + uint32_t b :30; + uint32_t c :30; + }; + + struct Struct3xx_u32 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u32")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u32))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u32))); + TEST_FIELD(uint32_t, value.a); + TEST_FIELD(uint32_t, value.b); + TEST_FIELD(uint32_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct331_64")) { + + + #if (!defined(__xlc__)) + + struct Struct331_64 { + int64_t a :3; + int64_t b :3; + int64_t c :1; + }; + + struct Struct331_64 value = {0}; + APPEND(PyUnicode_FromString("Struct331_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_64")) { + + + #if (!defined(__xlc__)) + + struct Struct1x1_64 { + int64_t a :1; + int64_t b :62; + int64_t c :1; + }; + + struct Struct1x1_64 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_64")) { + + + #if (!defined(__xlc__)) + + struct Struct1nx1_64 { + int64_t a :1; + int64_t full; + int64_t b :62; + int64_t c :1; + }; + + struct Struct1nx1_64 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.full); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_64")) { + + + #if (!defined(__xlc__)) + + struct Struct3xx_64 { + int64_t a :3; + int64_t b :62; + int64_t c :62; + }; + + struct Struct3xx_64 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_64")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_64))); + TEST_FIELD(int64_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int64_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct331_u64")) { + + + #if (!defined(__xlc__)) + + struct Struct331_u64 { + uint64_t a :3; + uint64_t b :3; + uint64_t c :1; + }; + + struct Struct331_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct331_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct331_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct331_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u64")) { + + + #if (!defined(__xlc__)) + + struct Struct1x1_u64 { + uint64_t a :1; + uint64_t b :62; + uint64_t c :1; + }; + + struct Struct1x1_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct1x1_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1x1_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u64")) { + + + #if (!defined(__xlc__)) + + struct Struct1nx1_u64 { + uint64_t a :1; + uint64_t full; + uint64_t b :62; + uint64_t c :1; + }; + + struct Struct1nx1_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct1nx1_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct1nx1_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.full); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u64")) { + + + #if (!defined(__xlc__)) + + struct Struct3xx_u64 { + uint64_t a :3; + uint64_t b :62; + uint64_t c :62; + }; + + struct Struct3xx_u64 value = {0}; + APPEND(PyUnicode_FromString("Struct3xx_u64")); + APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u64))); + APPEND(PyLong_FromLong(_Alignof(struct Struct3xx_u64))); + TEST_FIELD(uint64_t, value.a); + TEST_FIELD(uint64_t, value.b); + TEST_FIELD(uint64_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed1")) { + + + #if (!defined(__xlc__)) + + struct Mixed1 { + signed char a :4; + int b :4; + }; + + struct Mixed1 value = {0}; + APPEND(PyUnicode_FromString("Mixed1")); + APPEND(PyLong_FromLong(sizeof(struct Mixed1))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed1))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(int, value.b); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed2")) { + + + #if (!defined(__xlc__)) + + struct Mixed2 { + signed char a :4; + int32_t b :32; + }; + + struct Mixed2 value = {0}; + APPEND(PyUnicode_FromString("Mixed2")); + APPEND(PyLong_FromLong(sizeof(struct Mixed2))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed2))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(int32_t, value.b); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed3")) { + + + #if (!defined(__xlc__)) + + struct Mixed3 { + signed char a :4; + unsigned char b :4; + }; + + struct Mixed3 value = {0}; + APPEND(PyUnicode_FromString("Mixed3")); + APPEND(PyLong_FromLong(sizeof(struct Mixed3))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed3))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(unsigned char, value.b); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed4")) { + + + #if (!defined(__xlc__)) + + struct Mixed4 { + short a :4; + short b :4; + int c :24; + short d :4; + short e :4; + int f :24; + }; + + struct Mixed4 value = {0}; + APPEND(PyUnicode_FromString("Mixed4")); + APPEND(PyLong_FromLong(sizeof(struct Mixed4))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed4))); + TEST_FIELD(short, value.a); + TEST_FIELD(short, value.b); + TEST_FIELD(int, value.c); + TEST_FIELD(short, value.d); + TEST_FIELD(short, value.e); + TEST_FIELD(int, value.f); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed5")) { + + + #if (!defined(__xlc__)) + + struct Mixed5 { + unsigned int A :1; + unsigned short B :16; + }; + + struct Mixed5 value = {0}; + APPEND(PyUnicode_FromString("Mixed5")); + APPEND(PyLong_FromLong(sizeof(struct Mixed5))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed5))); + TEST_FIELD(unsigned int, value.A); + TEST_FIELD(unsigned short, value.B); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed6")) { + + + #if (!defined(__xlc__)) + + struct Mixed6 { + unsigned long long A :1; + unsigned int B :32; + }; + + struct Mixed6 value = {0}; + APPEND(PyUnicode_FromString("Mixed6")); + APPEND(PyLong_FromLong(sizeof(struct Mixed6))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed6))); + TEST_FIELD(unsigned long long, value.A); + TEST_FIELD(unsigned int, value.B); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed7")) { + + + #if (!defined(__xlc__)) + + struct Mixed7 { + uint32_t A; + uint32_t B :20; + uint64_t C :24; + }; + + struct Mixed7 value = {0}; + APPEND(PyUnicode_FromString("Mixed7")); + APPEND(PyLong_FromLong(sizeof(struct Mixed7))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed7))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(uint64_t, value.C); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed8_a")) { + + + #if (!defined(__xlc__)) + + struct Mixed8_a { + uint32_t A; + uint32_t B :32; + unsigned long long C :1; + }; + + struct Mixed8_a value = {0}; + APPEND(PyUnicode_FromString("Mixed8_a")); + APPEND(PyLong_FromLong(sizeof(struct Mixed8_a))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed8_a))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(unsigned long long, value.C); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed8_b")) { + + + #if (!defined(__xlc__)) + + struct Mixed8_b { + uint32_t A; + uint32_t B; + unsigned long long C :1; + }; + + struct Mixed8_b value = {0}; + APPEND(PyUnicode_FromString("Mixed8_b")); + APPEND(PyLong_FromLong(sizeof(struct Mixed8_b))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed8_b))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint32_t, value.B); + TEST_FIELD(unsigned long long, value.C); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed9")) { + + + #if (!defined(__xlc__)) + + struct Mixed9 { + uint8_t A; + uint32_t B :1; + }; + + struct Mixed9 value = {0}; + APPEND(PyUnicode_FromString("Mixed9")); + APPEND(PyLong_FromLong(sizeof(struct Mixed9))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed9))); + TEST_FIELD(uint8_t, value.A); + TEST_FIELD(uint32_t, value.B); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Mixed10")) { + + + #if (!defined(__xlc__)) + + struct Mixed10 { + uint32_t A :1; + uint64_t B :1; + }; + + struct Mixed10 value = {0}; + APPEND(PyUnicode_FromString("Mixed10")); + APPEND(PyLong_FromLong(sizeof(struct Mixed10))); + APPEND(PyLong_FromLong(_Alignof(struct Mixed10))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint64_t, value.B); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Example_gh_95496")) { + + + #if (!defined(__xlc__)) + + struct Example_gh_95496 { + uint32_t A :1; + uint64_t B :1; + }; + + struct Example_gh_95496 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_95496")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_95496))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_95496))); + TEST_FIELD(uint32_t, value.A); + TEST_FIELD(uint64_t, value.B); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Example_gh_84039_bad")) { + + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_bad { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + uint16_t b0 :4; + uint16_t b1 :12; + }; + #pragma pack(pop) + + struct Example_gh_84039_bad value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_bad")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_bad))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_bad))); + TEST_FIELD(uint8_t, value.a0); + TEST_FIELD(uint8_t, value.a1); + TEST_FIELD(uint8_t, value.a2); + TEST_FIELD(uint8_t, value.a3); + TEST_FIELD(uint8_t, value.a4); + TEST_FIELD(uint8_t, value.a5); + TEST_FIELD(uint8_t, value.a6); + TEST_FIELD(uint8_t, value.a7); + TEST_FIELD(uint16_t, value.b0); + TEST_FIELD(uint16_t, value.b1); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Example_gh_84039_good_a")) { + + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_good_a { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + }; + #pragma pack(pop) + + struct Example_gh_84039_good_a value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_good_a")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good_a))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_good_a))); + TEST_FIELD(uint8_t, value.a0); + TEST_FIELD(uint8_t, value.a1); + TEST_FIELD(uint8_t, value.a2); + TEST_FIELD(uint8_t, value.a3); + TEST_FIELD(uint8_t, value.a4); + TEST_FIELD(uint8_t, value.a5); + TEST_FIELD(uint8_t, value.a6); + TEST_FIELD(uint8_t, value.a7); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Example_gh_84039_good")) { + + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_84039_good { + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) { + uint8_t a0 :1; + uint8_t a1 :1; + uint8_t a2 :1; + uint8_t a3 :1; + uint8_t a4 :1; + uint8_t a5 :1; + uint8_t a6 :1; + uint8_t a7 :1; + } a; + #pragma pack(pop) + uint16_t b0 :4; + uint16_t b1 :12; + }; + #pragma pack(pop) + + struct Example_gh_84039_good value = {0}; + APPEND(PyUnicode_FromString("Example_gh_84039_good")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_84039_good))); + TEST_FIELD(uint8_t, value.a.a0); + TEST_FIELD(uint8_t, value.a.a1); + TEST_FIELD(uint8_t, value.a.a2); + TEST_FIELD(uint8_t, value.a.a3); + TEST_FIELD(uint8_t, value.a.a4); + TEST_FIELD(uint8_t, value.a.a5); + TEST_FIELD(uint8_t, value.a.a6); + TEST_FIELD(uint8_t, value.a.a7); + TEST_FIELD(uint16_t, value.b0); + TEST_FIELD(uint16_t, value.b1); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Example_gh_73939")) { + + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_73939 { + uint16_t P; + uint16_t L :9; + uint16_t Pro :1; + uint16_t G :1; + uint16_t IB :1; + uint16_t IR :1; + uint16_t R :3; + uint32_t T :10; + uint32_t C :20; + uint32_t R2 :2; + }; + #pragma pack(pop) + + struct Example_gh_73939 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_73939")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_73939))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_73939))); + TEST_FIELD(uint16_t, value.P); + TEST_FIELD(uint16_t, value.L); + TEST_FIELD(uint16_t, value.Pro); + TEST_FIELD(uint16_t, value.G); + TEST_FIELD(uint16_t, value.IB); + TEST_FIELD(uint16_t, value.IR); + TEST_FIELD(uint16_t, value.R); + TEST_FIELD(uint32_t, value.T); + TEST_FIELD(uint32_t, value.C); + TEST_FIELD(uint32_t, value.R2); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Example_gh_86098")) { + + + #if (!defined(__xlc__)) + + struct Example_gh_86098 { + uint8_t a :8; + uint8_t b :8; + uint32_t c :16; + }; + + struct Example_gh_86098 value = {0}; + APPEND(PyUnicode_FromString("Example_gh_86098")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_86098))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint32_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "Example_gh_86098_pack")) { + + + #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + + #pragma pack(push, 1) + struct GCC_ATTR(ms_struct) Example_gh_86098_pack { + uint8_t a :8; + uint8_t b :8; + uint32_t c :16; + }; + #pragma pack(pop) + + struct Example_gh_86098_pack value = {0}; + APPEND(PyUnicode_FromString("Example_gh_86098_pack")); + APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098_pack))); + APPEND(PyLong_FromLong(_Alignof(struct Example_gh_86098_pack))); + TEST_FIELD(uint8_t, value.a); + TEST_FIELD(uint8_t, value.b); + TEST_FIELD(uint32_t, value.c); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + if (_PyUnicode_EqualToASCIIString(name, "AnonBitfields")) { + + + #if (!defined(__xlc__)) + + struct AnonBitfields { + struct { + signed char a :4; + unsigned char b :4; + }; + signed char y; + }; + + struct AnonBitfields value = {0}; + APPEND(PyUnicode_FromString("AnonBitfields")); + APPEND(PyLong_FromLong(sizeof(struct AnonBitfields))); + APPEND(PyLong_FromLong(_Alignof(struct AnonBitfields))); + TEST_FIELD(signed char, value.a); + TEST_FIELD(unsigned char, value.b); + TEST_FIELD(signed char, value.y); + + #else + APPEND(Py_NewRef(Py_None)); + APPEND(PyUnicode_FromString("skipped on this compiler")); + #endif + + + return result; + } + + + Py_DECREF(result); + PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); + return NULL; + } + + #undef GCC_ATTR + #undef TEST_FIELD + #undef SET_AND_APPEND + #undef APPEND + diff --git a/PCbuild/_ctypes_test.vcxproj b/PCbuild/_ctypes_test.vcxproj index 97354739c09834..50d8575ad7bda3 100644 --- a/PCbuild/_ctypes_test.vcxproj +++ b/PCbuild/_ctypes_test.vcxproj @@ -94,6 +94,7 @@ + @@ -109,4 +110,4 @@ - \ No newline at end of file + diff --git a/PCbuild/_ctypes_test.vcxproj.filters b/PCbuild/_ctypes_test.vcxproj.filters index 5174196c52e4d0..618cfb32115e99 100644 --- a/PCbuild/_ctypes_test.vcxproj.filters +++ b/PCbuild/_ctypes_test.vcxproj.filters @@ -15,6 +15,9 @@ Header Files + + Header Files + @@ -26,4 +29,4 @@ Resource Files - \ No newline at end of file + From f7804965be7aab2e688750ae48535b44124f9d64 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 2 May 2024 15:44:48 +0200 Subject: [PATCH 69/91] Remove ideas --- .gitignore | 1 - Lib/test/test_ctypes/_dump_ctype.py | 240 ------------------ .../test_ctypes/test_bitfields_hypothesis.py | 188 -------------- Lib/test/test_ctypes/test_structures.py | 23 -- Python/generated_cases.c.h | 2 +- 5 files changed, 1 insertion(+), 453 deletions(-) delete mode 100644 Lib/test/test_ctypes/_dump_ctype.py delete mode 100644 Lib/test/test_ctypes/test_bitfields_hypothesis.py diff --git a/.gitignore b/.gitignore index 24366a1b5bc1d5..8872e9d5508ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -153,7 +153,6 @@ Tools/unicode/data/ /coverage/ /externals/ /htmlcov/ -/.hypothesis/ Tools/msi/obj Tools/ssl/amd64 Tools/ssl/win32 diff --git a/Lib/test/test_ctypes/_dump_ctype.py b/Lib/test/test_ctypes/_dump_ctype.py deleted file mode 100644 index a2a9cccbf19662..00000000000000 --- a/Lib/test/test_ctypes/_dump_ctype.py +++ /dev/null @@ -1,240 +0,0 @@ -from textwrap import dedent -from operator import attrgetter -from dataclasses import dataclass -from functools import cached_property - -import ctypes -import _ctypes - -print((ctypes.c_bool).mro()) -print(dir(ctypes.POINTER(ctypes.c_int8))) -print(ctypes.POINTER(ctypes.c_int8)._type_) - -def dump_ctype(tp, name='%'): - length = getattr(tp, '_length_', None) - if length is not None: - return f'{dump_ctype(tp._type_, name)}[{length}]' - if tp == ctypes.c_char_p: - return f'char *{name}' - if tp == ctypes.c_wchar_p: - return f'wchar *{name}' - if tp == ctypes.c_wchar: - return f'wchar {name}' - if tp == ctypes.c_void_p: - return f'void *{name}' - if tp == ctypes.c_bool: - _space = ' ' if name else '' - return f'bool{_space}{name}' - if tp == ctypes.py_object: - return f'PyObject *{name}' - if issubclass(tp, _ctypes._Pointer): - return f'{dump_ctype(tp._type_, '*' + name)}' - if issubclass(tp, (ctypes.Structure, ctypes.Union)): - if issubclass(tp, ctypes.Structure): - lines = ['struct {'] - else: - lines = ['union {'] - for fielddesc in tp._fields_: - f_name, f_tp, f_bits = unpack_field_desc(fielddesc) - if f_name in getattr(tp, '_anonymous_', ()): - f_name = '' - for line in dump_ctype(f_tp, f_name).splitlines(): - lines.append(' ' + line) - if f_bits is None: - lines[-1] += ';' - else: - lines[-1] += f' :{f_bits};' - _space = ' ' if name else '' - lines.append(f'}}{_space}{name}') - return '\n'.join(lines) - if issubclass(tp, ctypes._SimpleCData): - if tp(-1).value == -1: - sign = '' - else: - sign = 'u' - size = ctypes.sizeof(tp) - _space = ' ' if name else '' - return f'{sign}int{size * 8}_t{_space}{name}' - restype = getattr(tp, '_restype_', None) - if restype is not None: - return f'{dump_ctype(restype, "")} (*{name})({", ".join(dump_ctype(t, "") for t in tp._argtypes_)})' - -def unpack_field_desc(fielddesc): - try: - f_name, f_tp, f_bits = fielddesc - return f_name, f_tp, f_bits - except ValueError: - f_name, f_tp = fielddesc - return f_name, f_tp, None - -@dataclass -class FieldInfo: - name: str - tp: type - bits: int | None - parent_type: type - parent: 'FieldInfo' #| None - - @cached_property - def field_descriptor(self): - return getattr(self.parent_type, self.name) - - @cached_property - def offset(self): - offset = self.field_descriptor.offset - if self.parent: - offset += self.parent.offset - return offset - - @cached_property - def field_path(self): - if self.parent: - return (*self.parent.field_path, self.name) - return (self.name,) - - @cached_property - def attr_path(self): - if self.name in getattr(self.parent_type, '_anonymous_', ()): - selfpath = () - else: - selfpath = (self.name,) - if self.parent: - return (*self.parent.attr_path, *selfpath) - else: - return selfpath - - @cached_property - def full_name(self): - return '.'.join(self.attr_path) - - @cached_property - def is_in_union(self): - if self.parent and self.parent.is_in_union: - return True - return issubclass(self.parent_type, ctypes.Union) - - -def iterfields(tp, parent=None): - try: - fields = tp._fields_ - except AttributeError: - yield parent - else: - for fielddesc in fields: - f_name, f_tp, f_bits = unpack_field_desc(fielddesc) - sub = FieldInfo(f_name, f_tp, f_bits, tp, parent) - yield from iterfields(f_tp, sub) - - -class Uni(ctypes.Union): - _fields_ = [ - ('u', ctypes.c_int8), - ('v', ctypes.c_int16, 9), - ('w', ctypes.c_int16, 3), - ] - -class Sub(ctypes.Structure): - _fields_ = [ - ('x', ctypes.c_int8), - ('y', ctypes.c_int8, 3), - ('z', ctypes.c_int16, 9), - ] - -class S(ctypes.Structure): - _fields_ = [ - ('a', ctypes.c_int8), - ('b', ctypes.c_int8, 3), - ('c', Sub), - ('anon', Sub), - ('d', Uni), - ] - _anonymous_ = ['anon'] - -TESTCASES = ( - (ctypes.c_int8, 'int8_t %'), - (ctypes.c_uint8, 'uint8_t %'), - (ctypes.c_int16, 'int16_t %'), - (ctypes.c_uint16, 'uint16_t %'), - (ctypes.c_int32, 'int32_t %'), - (ctypes.c_uint32, 'uint32_t %'), - (ctypes.c_int64, 'int64_t %'), - (ctypes.c_uint64, 'uint64_t %'), - (ctypes.c_int8 * 8, 'int8_t %[8]'), - (ctypes.CFUNCTYPE(ctypes.c_int8, ctypes.c_uint16, ctypes.c_uint32), 'int8_t (*%)(uint16_t, uint32_t)'), - (ctypes.c_char_p, 'char *%'), - (ctypes.POINTER(ctypes.c_int8), 'int8_t *%'), - (ctypes.POINTER(ctypes.POINTER(ctypes.c_int8)), 'int8_t **%'), - (ctypes.c_void_p, 'void *%'), - (ctypes.c_wchar_p, 'wchar *%'), - (ctypes.c_wchar, 'wchar %'), - (ctypes.c_bool, 'bool %'), - (ctypes.py_object, 'PyObject *%'), - (S, dedent(""" - struct { - int8_t a; - int8_t b :3; - struct { - int8_t x; - int8_t y :3; - int16_t z :9; - } c; - struct { - int8_t x; - int8_t y :3; - int16_t z :9; - }; - union { - int8_t u; - int16_t v :9; - int16_t w :3; - } d; - } % - """).strip()), -) - -for ctype, expected_name in TESTCASES: - print(ctype, expected_name, '...') - got_name = dump_ctype(ctype) - print(ctype, got_name, '!!!') - if expected_name != got_name: - raise AssertionError() - -def getfieldtype(tp, attrnames): - match attrnames: - case [name]: - return getattr(tp, name) - case [name, *rest]: - return getfieldtype(getattr(tp, name).type, rest) - case _: - raise AttributeError(attrnames) - -print(S.x) -print(dir(S.x)) -s = S() -expected_names = ['a', 'b', 'c.x', 'c.y', 'c.z', 'x', 'y', 'z', 'd.u', 'd.v', 'd.w'] -for got_info, expected_name in zip(iterfields(S), expected_names, strict=True): - print(got_info, expected_name, got_info.offset) - assert attrgetter(expected_name)(s) == 0 - print(got_info.full_name) - print(got_info.attr_path) - print(got_info.field_path) - print(got_info.field_descriptor) - print(got_info.offset) - print(got_info.is_in_union) - assert got_info.full_name == expected_name - -print(dir(ctypes)) -print(type(ctypes.c_int)) -print(dir(type(ctypes.c_int))) -print(ctypes.sizeof(ctypes.c_int)) -print(dir(ctypes.c_int)) -print(dir(ctypes.c_int*8)) -print((ctypes.c_int*8)._length_) -print((ctypes.c_int*8)._type_) -print(dir(ctypes.CFUNCTYPE(ctypes.c_int8, ctypes.c_uint16, ctypes.c_uint32))) -print(dir(ctypes.PYFUNCTYPE(ctypes.c_int8, ctypes.c_uint16, ctypes.c_uint32))) -print(dir(ctypes.c_char_p)) -print(ctypes.c_char_p) -print(ctypes.POINTER(ctypes.c_int8)) -print(dir(ctypes.POINTER(ctypes.c_int8))) -print(ctypes.POINTER(ctypes.c_int8)._type_) diff --git a/Lib/test/test_ctypes/test_bitfields_hypothesis.py b/Lib/test/test_ctypes/test_bitfields_hypothesis.py deleted file mode 100644 index 61e158b5fecc19..00000000000000 --- a/Lib/test/test_ctypes/test_bitfields_hypothesis.py +++ /dev/null @@ -1,188 +0,0 @@ -import dataclasses -import tempfile -from pathlib import Path -import subprocess - -import ctypes -from ctypes import ( - Structure, - alignment, - sizeof, - pointer, - POINTER, -) - -from test.support.hypothesis_helper import hypothesis - -assume = hypothesis.assume -given = hypothesis.given -strategies = hypothesis.strategies - -C_TYPES = { - # Note: for Hypothesis minimization to work, "simpler" types should - # generally go first. - #'char': ctypes.c_char, - 'signed char': ctypes.c_byte, - 'unsigned char': ctypes.c_ubyte, - 'int': ctypes.c_int, - 'unsigned int': ctypes.c_uint, - **{f'{u}int{n}_t': getattr(ctypes, f'c_{u}int{n}') - for u in ('', 'u') - for n in ('8', '16', '32', '64') - }, - **{f'{"unsigned" if u else "signed"} {t}': getattr(ctypes, f'c_{u}{t}') - for u in ('', 'u') - for t in ('short', 'long', 'longlong') - }, - #'float': ctypes.c_float, - #'double': ctypes.c_double, -} - -def struct_c_decl(attrdict, name='s'): - lines = [] - pack = attrdict.get('_pack_') - if pack is not None: - lines.append(f'#pragma pack({pack})') - attributes = [] - endian_name = attrdict['endian_name'] - if endian_name != 'native': - attributes.append(f'scalar_storage_order("{endian_name}-endian")') - if attrdict.get('_layout_' == 'ms'): - attributes.append('ms_struct') - lines.append(f'#pragma ms_struct on') - lines.append(f'struct') - if attributes: - lines.append(f'__attribute__ (( {", ".join(attributes)} ))') - lines.append(f' {name} {{') - for field, ctype_name in zip(attrdict['_fields_'], attrdict['ctype_names']): - try: - name, tp, bitsize = field - except ValueError: - name, tp = field - bitfieldspec = '' - else: - bitfieldspec = f':{bitsize}' - lines.append(f' {ctype_name} {name}{bitfieldspec};') - lines.append('};') - lines.append('') - return '\n'.join(lines) - -@strategies.composite -def structure_args(draw): - num_fields = draw(strategies.integers(1, 50)) - fields = [] - ctype_names = [] - for i in range(num_fields): - if draw(strategies.booleans()) and 1: - ctype_name, ctype = draw(strategies.sampled_from(list(C_TYPES.items()))) - else: - sub_args = draw(structure_args()) - ctype_name = f's{id(sub_args)}' - ctype = type(sub_args) - name = f'field_{i}' - bit_size = draw(strategies.integers(1, sizeof(ctype) * 8)) - is_bitfield = draw(strategies.booleans()) - if is_bitfield: - fields.append((name, ctype, bit_size)) - else: - fields.append((name, ctype)) - ctype_names.append(ctype_name) - endian_name, cls = draw(strategies.sampled_from(( - ('native', ctypes.Structure), - ('little', ctypes.LittleEndianStructure), - ('big', ctypes.BigEndianStructure), - ))) - - # XXX: Handle Anonymouses - - attrdict = { - '_fields_': fields, - 'endian_name': endian_name, - 'ctype_names': ctype_names, - } - if draw(strategies.booleans()): - attrdict['_layout_'] = 'ms' - pack = 2 ** draw(strategies.integers(0, 4)) - if pack: - attrdict['_pack_'] = pack - attrdict['c_decl'] = struct_c_decl(attrdict) - attrdict['c_name'] = f's{id(attrdict)}' - result = ( - 'RandomTestStruct', - (cls,), - attrdict, - ) - return result - - -@given(structure_args()) -def test_structure(s_args): - structure = type(*s_args) - with tempfile.TemporaryDirectory() as tmpdirname: - temppath = Path(tmpdirname) - setloop_lines = [] - for field, ctype_name in zip(structure._fields_, structure.ctype_names): - setloop_lines.append(f'value.{field[0]} = ({ctype_name})-1;') - setloop_lines.append(f'dump_bytes(&value, sizeof(value));') - program = ( - r""" - #include - #include - #include - #include - - #define longlong long long - - %STRUCTDEF% - - void dump_bytes(void *ptr, size_t sz) { - unsigned char *cptr = ptr; - for (size_t i=0; iprevious = NULL; /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if 0 + #if TIER_ONE assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || From 6ade02251794b0385b9d27cfc659612b35a31f13 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 2 May 2024 15:54:26 +0200 Subject: [PATCH 70/91] Regen --- Modules/_ctypes/_ctypes_test_generated.c.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index 2df39ae92c2f20..b9732b90323115 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -329,7 +329,7 @@ #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 8) - union GCC_ATTR(ms_struct) Packed4 { + union Packed4 { int8_t a; int64_t b; }; From cfa66475de7328377d4f6e230dc24167c578ad65 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 2 May 2024 16:01:59 +0200 Subject: [PATCH 71/91] Do not set _layout_ --- Doc/library/ctypes.rst | 11 +++++------ Modules/_ctypes/stgdict.c | 12 ------------ 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 624482f6d1a8ec..d14fb0f9cc3fe4 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -2546,19 +2546,18 @@ fields, or any other data types containing pointer type fields. .. attribute:: _layout_ - An optional string naming the struct and union layout. It can currently + An optional string naming the struct/union layout. It can currently be set to: - ``"ms"``: the layout used by the Microsoft compiler (MSVC). - GCC and Clang, this layout can be selected with + On GCC and Clang, this layout can be selected with ``__attribute__((ms_struct))``. - ``"gcc-sysv"``: the layout used by GCC with the System V or “SysV-like” - data model, as used on Linux and macOS. This is generally compatible - with clang. + data model, as used on Linux and macOS. With this layout, :attr:`~Structure._pack_` must be unset or zero. - If not set explicitly, :attr:`!_layout_` will be set to a default that - matches the platform's conventions. This default may change in future + If not set explicitly, ``ctypes`` will use a default that + matches the platform conventions. This default may change in future Python releases (for example, when a new platform gains official support, or when a difference between similar platforms is found). Currently the default will be: diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index e1fdb5320f9c2d..a3f74e99eac794 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -319,18 +319,6 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return -1; } } - else { - // Set the default - const char *name = (layout_mode == LAYOUT_MODE_MS) ? "ms" - : "gcc-sysv"; - PyObject *name_obj = PyUnicode_FromString(name); - if (!name) { - return -1; - } - if (PyObject_SetAttr(type, &_Py_ID(_layout_), name_obj) < 0) { - return -1; - } - } if (PyObject_GetOptionalAttr(type, &_Py_ID(_align_), &tmp) < 0) { return -1; } From df162b0e6be9e3fb49bfa4c90f955b7648e798cf Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 2 May 2024 16:10:32 +0200 Subject: [PATCH 72/91] We don't always get union alignment right --- .../test_ctypes/test_generated_structs.py | 16 +- Modules/_ctypes/_ctypes_test_generated.c.h | 139 ++++-------------- 2 files changed, 35 insertions(+), 120 deletions(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index 747035a29a4059..46abb8d58e0bae 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -105,7 +105,7 @@ class ManyTypesU(Union): @register() -class Nested(Union): +class Nested(Structure): _fields_ = [ ('a', SimpleStruct), ('b', SimpleUnion), ('anon', SimpleStruct), ] @@ -113,25 +113,25 @@ class Nested(Union): @register() -class Packed1(Union): +class Packed1(Structure): _fields_ = [('a', c_int8), ('b', c_int64)] _pack_ = 1 @register() -class Packed2(Union): +class Packed2(Structure): _fields_ = [('a', c_int8), ('b', c_int64)] _pack_ = 2 @register() -class Packed3(Union): +class Packed3(Structure): _fields_ = [('a', c_int8), ('b', c_int64)] _pack_ = 4 @register() -class Packed4(Union): +class Packed4(Structure): _fields_ = [('a', c_int8), ('b', c_int64)] _pack_ = 8 @@ -185,7 +185,8 @@ class Bits_MSVC(Structure): ("R", c_short, 6), ("S", c_short, 7)] -@register() +# Skipped for now -- we don't always match the alignment +#@register() class IntBits_Union(Union): _fields_ = [("A", c_int, 1), ("B", c_int, 2), @@ -197,7 +198,8 @@ class IntBits_Union(Union): ("H", c_int, 8), ("I", c_int, 9)] -@register() +# Skipped for now -- we don't always match the alignment +#@register() class BitsUnion(Union): _fields_ = [*IntBits_Union._fields_, diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index b9732b90323115..54ace559306937 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -200,7 +200,7 @@ if (_PyUnicode_EqualToASCIIString(name, "Nested")) { - union Nested { + struct Nested { struct { int32_t x; int8_t y; @@ -218,10 +218,10 @@ }; }; - union Nested value = {0}; + struct Nested value = {0}; APPEND(PyUnicode_FromString("Nested")); - APPEND(PyLong_FromLong(sizeof(union Nested))); - APPEND(PyLong_FromLong(_Alignof(union Nested))); + APPEND(PyLong_FromLong(sizeof(struct Nested))); + APPEND(PyLong_FromLong(_Alignof(struct Nested))); TEST_FIELD(int32_t, value.a.x); TEST_FIELD(int8_t, value.a.y); TEST_FIELD(uint16_t, value.a.z); @@ -242,16 +242,16 @@ #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - union Packed1 { + struct Packed1 { int8_t a; int64_t b; }; #pragma pack(pop) - union Packed1 value = {0}; + struct Packed1 value = {0}; APPEND(PyUnicode_FromString("Packed1")); - APPEND(PyLong_FromLong(sizeof(union Packed1))); - APPEND(PyLong_FromLong(_Alignof(union Packed1))); + APPEND(PyLong_FromLong(sizeof(struct Packed1))); + APPEND(PyLong_FromLong(_Alignof(struct Packed1))); TEST_FIELD(int8_t, value.a); TEST_FIELD(int64_t, value.b); @@ -271,16 +271,16 @@ #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 2) - union Packed2 { + struct Packed2 { int8_t a; int64_t b; }; #pragma pack(pop) - union Packed2 value = {0}; + struct Packed2 value = {0}; APPEND(PyUnicode_FromString("Packed2")); - APPEND(PyLong_FromLong(sizeof(union Packed2))); - APPEND(PyLong_FromLong(_Alignof(union Packed2))); + APPEND(PyLong_FromLong(sizeof(struct Packed2))); + APPEND(PyLong_FromLong(_Alignof(struct Packed2))); TEST_FIELD(int8_t, value.a); TEST_FIELD(int64_t, value.b); @@ -300,16 +300,16 @@ #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 4) - union Packed3 { + struct Packed3 { int8_t a; int64_t b; }; #pragma pack(pop) - union Packed3 value = {0}; + struct Packed3 value = {0}; APPEND(PyUnicode_FromString("Packed3")); - APPEND(PyLong_FromLong(sizeof(union Packed3))); - APPEND(PyLong_FromLong(_Alignof(union Packed3))); + APPEND(PyLong_FromLong(sizeof(struct Packed3))); + APPEND(PyLong_FromLong(_Alignof(struct Packed3))); TEST_FIELD(int8_t, value.a); TEST_FIELD(int64_t, value.b); @@ -329,16 +329,16 @@ #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 8) - union Packed4 { + struct Packed4 { int8_t a; int64_t b; }; #pragma pack(pop) - union Packed4 value = {0}; + struct Packed4 value = {0}; APPEND(PyUnicode_FromString("Packed4")); - APPEND(PyLong_FromLong(sizeof(union Packed4))); - APPEND(PyLong_FromLong(_Alignof(union Packed4))); + APPEND(PyLong_FromLong(sizeof(struct Packed4))); + APPEND(PyLong_FromLong(_Alignof(struct Packed4))); TEST_FIELD(int8_t, value.a); TEST_FIELD(int64_t, value.b); @@ -535,93 +535,6 @@ } - if (_PyUnicode_EqualToASCIIString(name, "IntBits_Union")) { - - union IntBits_Union { - int A :1; - int B :2; - int C :3; - int D :4; - int E :5; - int F :6; - int G :7; - int H :8; - int I :9; - }; - - union IntBits_Union value = {0}; - APPEND(PyUnicode_FromString("IntBits_Union")); - APPEND(PyLong_FromLong(sizeof(union IntBits_Union))); - APPEND(PyLong_FromLong(_Alignof(union IntBits_Union))); - TEST_FIELD(int, value.A); - TEST_FIELD(int, value.B); - TEST_FIELD(int, value.C); - TEST_FIELD(int, value.D); - TEST_FIELD(int, value.E); - TEST_FIELD(int, value.F); - TEST_FIELD(int, value.G); - TEST_FIELD(int, value.H); - TEST_FIELD(int, value.I); - - return result; - } - - - if (_PyUnicode_EqualToASCIIString(name, "BitsUnion")) { - - - #if (!defined(__xlc__)) - - union BitsUnion { - int A :1; - int B :2; - int C :3; - int D :4; - int E :5; - int F :6; - int G :7; - int H :8; - int I :9; - short M :1; - short N :2; - short O :3; - short P :4; - short Q :5; - short R :6; - short S :7; - }; - - union BitsUnion value = {0}; - APPEND(PyUnicode_FromString("BitsUnion")); - APPEND(PyLong_FromLong(sizeof(union BitsUnion))); - APPEND(PyLong_FromLong(_Alignof(union BitsUnion))); - TEST_FIELD(int, value.A); - TEST_FIELD(int, value.B); - TEST_FIELD(int, value.C); - TEST_FIELD(int, value.D); - TEST_FIELD(int, value.E); - TEST_FIELD(int, value.F); - TEST_FIELD(int, value.G); - TEST_FIELD(int, value.H); - TEST_FIELD(int, value.I); - TEST_FIELD(short, value.M); - TEST_FIELD(short, value.N); - TEST_FIELD(short, value.O); - TEST_FIELD(short, value.P); - TEST_FIELD(short, value.Q); - TEST_FIELD(short, value.R); - TEST_FIELD(short, value.S); - - #else - APPEND(Py_NewRef(Py_None)); - APPEND(PyUnicode_FromString("skipped on this compiler")); - #endif - - - return result; - } - - if (_PyUnicode_EqualToASCIIString(name, "I64Bits")) { @@ -1968,7 +1881,7 @@ #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct GCC_ATTR(ms_struct) Example_gh_84039_bad { + struct Example_gh_84039_bad { uint8_t a0 :1; uint8_t a1 :1; uint8_t a2 :1; @@ -2013,7 +1926,7 @@ #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct GCC_ATTR(ms_struct) Example_gh_84039_good_a { + struct Example_gh_84039_good_a { uint8_t a0 :1; uint8_t a1 :1; uint8_t a2 :1; @@ -2054,9 +1967,9 @@ #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct GCC_ATTR(ms_struct) Example_gh_84039_good { + struct Example_gh_84039_good { #pragma pack(push, 1) - struct GCC_ATTR(ms_struct) { + struct { uint8_t a0 :1; uint8_t a1 :1; uint8_t a2 :1; @@ -2103,7 +2016,7 @@ #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct GCC_ATTR(ms_struct) Example_gh_73939 { + struct Example_gh_73939 { uint16_t P; uint16_t L :9; uint16_t Pro :1; @@ -2177,7 +2090,7 @@ #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct GCC_ATTR(ms_struct) Example_gh_86098_pack { + struct Example_gh_86098_pack { uint8_t a :8; uint8_t b :8; uint32_t c :16; From 836a5cc323f4ab812cdc7d26f8a5628ebce2e080 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 2 May 2024 16:13:19 +0200 Subject: [PATCH 73/91] Remove unused stuff --- Lib/test/test_ctypes/test_generated_structs.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index 46abb8d58e0bae..e3cc113e95d6b5 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -494,16 +494,6 @@ class FieldInfo: parent_type: type parent: 'FieldInfo' #| None - @cached_property - def field_descriptor(self): - return getattr(self.parent_type, self.name) - - @cached_property - def field_path(self): - if self.parent: - return (*self.parent.field_path, self.name) - return (self.name,) - @cached_property def attr_path(self): if self.name in getattr(self.parent_type, '_anonymous_', ()): @@ -519,11 +509,6 @@ def attr_path(self): def full_name(self): return '.'.join(self.attr_path) - def get_from(self, obj): - for attr_name in self.attr_path: - obj = getattr(obj, attr_name) - return obj - def set_to(self, obj, new): for attr_name in self.attr_path[:-1]: obj = getattr(obj, attr_name) From b07ae33409bc1e21131cf370c71a89fbf5084e96 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 2 May 2024 16:14:35 +0200 Subject: [PATCH 74/91] Packing implies ms layout --- .../test_ctypes/test_generated_structs.py | 2 +- Modules/_ctypes/_ctypes_test_generated.c.h | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index e3cc113e95d6b5..ee676c39e17d48 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -436,7 +436,7 @@ def dump_ctype(tp, agg_name='', variable_name='', semi=''): pushes.append(f'#pragma pack(push, {pack})') pops.append(f'#pragma pack(pop)') layout = getattr(tp, '_layout_', None) - if layout == 'ms': + if layout == 'ms' or pack: requires.add(KNOWN_COMPILERS) attributes.append('ms_struct') if attributes: diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index 54ace559306937..feecf8c4474c1a 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -242,7 +242,7 @@ #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct Packed1 { + struct GCC_ATTR(ms_struct) Packed1 { int8_t a; int64_t b; }; @@ -271,7 +271,7 @@ #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 2) - struct Packed2 { + struct GCC_ATTR(ms_struct) Packed2 { int8_t a; int64_t b; }; @@ -300,7 +300,7 @@ #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 4) - struct Packed3 { + struct GCC_ATTR(ms_struct) Packed3 { int8_t a; int64_t b; }; @@ -329,7 +329,7 @@ #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 8) - struct Packed4 { + struct GCC_ATTR(ms_struct) Packed4 { int8_t a; int64_t b; }; @@ -1881,7 +1881,7 @@ #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct Example_gh_84039_bad { + struct GCC_ATTR(ms_struct) Example_gh_84039_bad { uint8_t a0 :1; uint8_t a1 :1; uint8_t a2 :1; @@ -1926,7 +1926,7 @@ #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct Example_gh_84039_good_a { + struct GCC_ATTR(ms_struct) Example_gh_84039_good_a { uint8_t a0 :1; uint8_t a1 :1; uint8_t a2 :1; @@ -1967,9 +1967,9 @@ #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct Example_gh_84039_good { + struct GCC_ATTR(ms_struct) Example_gh_84039_good { #pragma pack(push, 1) - struct { + struct GCC_ATTR(ms_struct) { uint8_t a0 :1; uint8_t a1 :1; uint8_t a2 :1; @@ -2016,7 +2016,7 @@ #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct Example_gh_73939 { + struct GCC_ATTR(ms_struct) Example_gh_73939 { uint16_t P; uint16_t L :9; uint16_t Pro :1; @@ -2090,7 +2090,7 @@ #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) #pragma pack(push, 1) - struct Example_gh_86098_pack { + struct GCC_ATTR(ms_struct) Example_gh_86098_pack { uint8_t a :8; uint8_t b :8; uint32_t c :16; From 6dd7a8d52af1221e1f0075857109acf448136bc3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 2 May 2024 16:47:30 +0200 Subject: [PATCH 75/91] Docs & whitespace --- Doc/whatsnew/3.13.rst | 3 + .../test_ctypes/test_generated_structs.py | 95 ++-- Modules/_ctypes/_ctypes_test.c | 5 +- Modules/_ctypes/_ctypes_test_generated.c.h | 471 +++--------------- 4 files changed, 143 insertions(+), 431 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 3cc8e96472cd64..f8d9e47806bf91 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -432,6 +432,9 @@ ctypes match platform defaults (GCC/Clang or MSC). In particular, fields no longer overlap. (Contributed by Matthias Görgens in :gh:`97702`.) +* A :attr:`ctypes.Structure._layout_`` class attribute can be set + to help match a non-default ABI. + (Contributed by Petr Viktorin in :gh:`97702`.) dbm --- diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index ee676c39e17d48..4ab5680956db89 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -4,9 +4,6 @@ `Modules/_ctypes/_ctypes_test_generated.c.h`, which is compiled into _ctypes_test. -The generated tests return a list of raw things to check, -or [None, skip_reason] to skip the test. - Run this module to regenerate the files: ./python Lib/test/test_ctypes/test_generated_structs.py > Modules/_ctypes/_ctypes_test_generated.c.h @@ -27,8 +24,11 @@ KNOWN_COMPILERS = 'defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)' # ctypes erases the difference between `c_int` and e.g.`c_int16`. -# To keep it, we'll use custom subclasses with the C name in `_c_name`. +# To keep it, we'll use custom subclasses with the C name stashed in `_c_name`: +class c_bool(ctypes.c_bool): + _c_name = '_Bool' +# To do it for all the other types, use some metaprogramming: for c_name, ctypes_name in { 'signed char': 'c_byte', 'short': 'c_short', @@ -48,6 +48,9 @@ newtype = type(ctypes_name, (ctype,), {'_c_name': c_name}) globals()[ctypes_name] = newtype + +# Register structs and unions to test + TESTCASES = {} def register(name=None): def decorator(cls, name=name): @@ -390,6 +393,24 @@ class X(Structure): class GeneratedTest(unittest.TestCase): def test_generated_data(self): + """Check that a ctypes struct/union matches its C equivalent. + + This compares with data from get_generated_test_data(), a list of: + - name (str) + - size (int) + - alignment (int) + - for each field, three snapshots of memory, as bytes: + - memory after the field is set to -1 + - memory after the field is set to 1 + - memory after the field is set to 0 + + or: + - None + - reason to skip the test (str) + + This does depend on the C compiler keeping padding bits zero. + Common compilers seem to do so. + """ for name, cls in TESTCASES.items(): with self.subTest(name=name): expected = iter(_ctypes_test.get_generated_test_data(name)) @@ -411,12 +432,18 @@ def test_generated_data(self): # The rest of this file is generating C code from a ctypes type. -# This is only tested with the known inputs here. +# This is only meant for (and tested with) the known inputs in this file! def c_str_repr(string): + """Return a string as a C literal""" return '"' + re.sub('([\"\'\\\\\n])', r'\\\1', string) + '"' def dump_simple_ctype(tp, variable_name='', semi=''): + """Get C type name or declaration of a scalar type + + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ length = getattr(tp, '_length_', None) if length is not None: return f'{dump_simple_ctype(tp._type_, variable_name)}[{length}]{semi}' @@ -424,7 +451,13 @@ def dump_simple_ctype(tp, variable_name='', semi=''): return f'{tp._c_name}{maybe_space(variable_name)}{semi}' -def dump_ctype(tp, agg_name='', variable_name='', semi=''): +def dump_ctype(tp, struct_or_union_tag='', variable_name='', semi=''): + """Get C type name or declaration of a ctype + + struct_or_union_tag: name of the struct or union + variable_name: if given, declare the given variable + semi: a semicolon, and/or bitfield specification to tack on to the end + """ requires = set() if issubclass(tp, (Structure, Union)): attributes = [] @@ -443,9 +476,9 @@ def dump_ctype(tp, agg_name='', variable_name='', semi=''): a = f' GCC_ATTR({", ".join(attributes)})' else: a = '' - lines = [f'{struct_or_union(tp)}{a}{maybe_space(agg_name)} ' +'{'] + lines = [f'{struct_or_union(tp)}{a}{maybe_space(struct_or_union_tag)} ' +'{'] for fielddesc in tp._fields_: - f_name, f_tp, f_bits = unpack_field_desc(fielddesc) + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) if f_name in getattr(tp, '_anonymous_', ()): f_name = '' if f_bits is None: @@ -478,24 +511,22 @@ def maybe_space(string): return ' ' + string return string -def unpack_field_desc(fielddesc): - try: - f_name, f_tp, f_bits = fielddesc - return f_name, f_tp, f_bits - except ValueError: - f_name, f_tp = fielddesc - return f_name, f_tp, None +def unpack_field_desc(f_name, f_tp, f_bits=None): + """Unpack a _fields_ entry into a (name, type, bits) triple""" + return f_name, f_tp, f_bits @dataclass class FieldInfo: + """Information about a (possibly nested) struct/union field""" name: str tp: type - bits: int | None + bits: int | None # number if this is a bit field parent_type: type parent: 'FieldInfo' #| None @cached_property def attr_path(self): + """Attribute names to get at the value of this field""" if self.name in getattr(self.parent_type, '_anonymous_', ()): selfpath = () else: @@ -507,29 +538,37 @@ def attr_path(self): @cached_property def full_name(self): + """Attribute names to get at the value of this field""" return '.'.join(self.attr_path) def set_to(self, obj, new): + """Set the field on a given Structure/Union instance""" for attr_name in self.attr_path[:-1]: obj = getattr(obj, attr_name) setattr(obj, self.attr_path[-1], new) def iterfields(tp, parent=None): + """Get *leaf* fields of a structure or union, as FieldInfo""" try: fields = tp._fields_ except AttributeError: yield parent else: for fielddesc in fields: - f_name, f_tp, f_bits = unpack_field_desc(fielddesc) + f_name, f_tp, f_bits = unpack_field_desc(*fielddesc) sub = FieldInfo(f_name, f_tp, f_bits, tp, parent) yield from iterfields(f_tp, sub) if __name__ == '__main__': - print('/* Generated by Lib/test/test_ctypes/test_generated_structs.py */') - print(""" + # Dump C source to stdout + def output(string): + print(re.compile(r'^ +$', re.MULTILINE).sub('', string).lstrip('\n')) + output(""" + /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + + // Append VALUE to the result. #define APPEND(VAL) { \\ if (!VAL) { \\ @@ -580,18 +619,18 @@ def iterfields(tp, parent=None): } """) for name, cls in TESTCASES.items(): - print(""" + output(""" if (_PyUnicode_EqualToASCIIString(name, %s)) { """ % c_str_repr(name)) - lines, requires = dump_ctype(cls, agg_name=name, semi=';') + lines, requires = dump_ctype(cls, struct_or_union_tag=name, semi=';') if requires: - print(f""" + output(f""" #if {" && ".join(f'({r})' for r in sorted(requires))} """) for line in lines: - print(' ', line, sep='') + output(' ' + line) typename = f'{struct_or_union(cls)} {name}' - print(f""" + output(f""" {typename} value = {{0}}; APPEND(PyUnicode_FromString({c_str_repr(name)})); APPEND(PyLong_FromLong(sizeof({typename}))); @@ -599,22 +638,22 @@ def iterfields(tp, parent=None): """.rstrip()) for field in iterfields(cls): f_tp = dump_simple_ctype(field.tp) - print(f"""\ + output(f"""\ TEST_FIELD({f_tp}, value.{field.full_name}); """.rstrip()) if requires: - print(f""" + output(f""" #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif """) - print(""" + output(""" return result; } """) - print(""" + output(""" Py_DECREF(result); PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); return NULL; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index f1325fb7297e46..4d63feb7b7df3e 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,8 +1,5 @@ // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -//# define Py_LIMITED_API 0x030c0000 -#endif +// Need internal API for _PyUnicode_EqualToASCIIString // gh-85283: On Windows, Py_LIMITED_API requires Py_BUILD_CORE to not attempt // linking the extension to python3.lib (which fails). Py_BUILD_CORE_MODULE is diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index feecf8c4474c1a..6acb7cd436bd91 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -1,4 +1,5 @@ -/* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + /* Generated by Lib/test/test_ctypes/test_generated_structs.py */ + // Append VALUE to the result. #define APPEND(VAL) { \ @@ -48,64 +49,53 @@ if (!result) { return NULL; } - if (_PyUnicode_EqualToASCIIString(name, "SingleInt")) { - + struct SingleInt { int a; }; - struct SingleInt value = {0}; APPEND(PyUnicode_FromString("SingleInt")); APPEND(PyLong_FromLong(sizeof(struct SingleInt))); APPEND(PyLong_FromLong(_Alignof(struct SingleInt))); TEST_FIELD(int, value.a); - return result; } - if (_PyUnicode_EqualToASCIIString(name, "SingleInt_Union")) { - + union SingleInt_Union { int a; }; - union SingleInt_Union value = {0}; APPEND(PyUnicode_FromString("SingleInt_Union")); APPEND(PyLong_FromLong(sizeof(union SingleInt_Union))); APPEND(PyLong_FromLong(_Alignof(union SingleInt_Union))); TEST_FIELD(int, value.a); - return result; } - if (_PyUnicode_EqualToASCIIString(name, "SingleU32")) { - + struct SingleU32 { uint32_t a; }; - struct SingleU32 value = {0}; APPEND(PyUnicode_FromString("SingleU32")); APPEND(PyLong_FromLong(sizeof(struct SingleU32))); APPEND(PyLong_FromLong(_Alignof(struct SingleU32))); TEST_FIELD(uint32_t, value.a); - return result; } - if (_PyUnicode_EqualToASCIIString(name, "SimpleStruct")) { - + struct SimpleStruct { int32_t x; int8_t y; uint16_t z; }; - struct SimpleStruct value = {0}; APPEND(PyUnicode_FromString("SimpleStruct")); APPEND(PyLong_FromLong(sizeof(struct SimpleStruct))); @@ -113,19 +103,16 @@ TEST_FIELD(int32_t, value.x); TEST_FIELD(int8_t, value.y); TEST_FIELD(uint16_t, value.z); - return result; } - if (_PyUnicode_EqualToASCIIString(name, "SimpleUnion")) { - + union SimpleUnion { int32_t x; int8_t y; uint16_t z; }; - union SimpleUnion value = {0}; APPEND(PyUnicode_FromString("SimpleUnion")); APPEND(PyLong_FromLong(sizeof(union SimpleUnion))); @@ -133,13 +120,11 @@ TEST_FIELD(int32_t, value.x); TEST_FIELD(int8_t, value.y); TEST_FIELD(uint16_t, value.z); - return result; } - if (_PyUnicode_EqualToASCIIString(name, "ManyTypes")) { - + struct ManyTypes { int8_t i8; uint8_t u8; @@ -150,7 +135,6 @@ int64_t i64; uint64_t u64; }; - struct ManyTypes value = {0}; APPEND(PyUnicode_FromString("ManyTypes")); APPEND(PyLong_FromLong(sizeof(struct ManyTypes))); @@ -163,13 +147,11 @@ TEST_FIELD(uint32_t, value.u32); TEST_FIELD(int64_t, value.i64); TEST_FIELD(uint64_t, value.u64); - return result; } - if (_PyUnicode_EqualToASCIIString(name, "ManyTypesU")) { - + union ManyTypesU { int8_t i8; uint8_t u8; @@ -180,7 +162,6 @@ int64_t i64; uint64_t u64; }; - union ManyTypesU value = {0}; APPEND(PyUnicode_FromString("ManyTypesU")); APPEND(PyLong_FromLong(sizeof(union ManyTypesU))); @@ -193,13 +174,11 @@ TEST_FIELD(uint32_t, value.u32); TEST_FIELD(int64_t, value.i64); TEST_FIELD(uint64_t, value.u64); - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Nested")) { - + struct Nested { struct { int32_t x; @@ -217,7 +196,6 @@ uint16_t z; }; }; - struct Nested value = {0}; APPEND(PyUnicode_FromString("Nested")); APPEND(PyLong_FromLong(sizeof(struct Nested))); @@ -231,129 +209,107 @@ TEST_FIELD(int32_t, value.x); TEST_FIELD(int8_t, value.y); TEST_FIELD(uint16_t, value.z); - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Packed1")) { - #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Packed1 { int8_t a; int64_t b; }; #pragma pack(pop) - struct Packed1 value = {0}; APPEND(PyUnicode_FromString("Packed1")); APPEND(PyLong_FromLong(sizeof(struct Packed1))); APPEND(PyLong_FromLong(_Alignof(struct Packed1))); TEST_FIELD(int8_t, value.a); TEST_FIELD(int64_t, value.b); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Packed2")) { - #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + #pragma pack(push, 2) struct GCC_ATTR(ms_struct) Packed2 { int8_t a; int64_t b; }; #pragma pack(pop) - struct Packed2 value = {0}; APPEND(PyUnicode_FromString("Packed2")); APPEND(PyLong_FromLong(sizeof(struct Packed2))); APPEND(PyLong_FromLong(_Alignof(struct Packed2))); TEST_FIELD(int8_t, value.a); TEST_FIELD(int64_t, value.b); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Packed3")) { - #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + #pragma pack(push, 4) struct GCC_ATTR(ms_struct) Packed3 { int8_t a; int64_t b; }; #pragma pack(pop) - struct Packed3 value = {0}; APPEND(PyUnicode_FromString("Packed3")); APPEND(PyLong_FromLong(sizeof(struct Packed3))); APPEND(PyLong_FromLong(_Alignof(struct Packed3))); TEST_FIELD(int8_t, value.a); TEST_FIELD(int64_t, value.b); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Packed4")) { - #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + #pragma pack(push, 8) struct GCC_ATTR(ms_struct) Packed4 { int8_t a; int64_t b; }; #pragma pack(pop) - struct Packed4 value = {0}; APPEND(PyUnicode_FromString("Packed4")); APPEND(PyLong_FromLong(sizeof(struct Packed4))); APPEND(PyLong_FromLong(_Alignof(struct Packed4))); TEST_FIELD(int8_t, value.a); TEST_FIELD(int64_t, value.b); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "IntBits")) { - + struct IntBits { int A :1; int B :2; @@ -365,7 +321,6 @@ int H :8; int I :9; }; - struct IntBits value = {0}; APPEND(PyUnicode_FromString("IntBits")); APPEND(PyLong_FromLong(sizeof(struct IntBits))); @@ -379,16 +334,13 @@ TEST_FIELD(int, value.G); TEST_FIELD(int, value.H); TEST_FIELD(int, value.I); - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Bits")) { - #if (!defined(__xlc__)) - + struct Bits { int A :1; int B :2; @@ -407,7 +359,6 @@ short R :6; short S :7; }; - struct Bits value = {0}; APPEND(PyUnicode_FromString("Bits")); APPEND(PyLong_FromLong(sizeof(struct Bits))); @@ -428,22 +379,18 @@ TEST_FIELD(short, value.Q); TEST_FIELD(short, value.R); TEST_FIELD(short, value.S); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "IntBits_MSVC")) { - #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + struct GCC_ATTR(ms_struct) IntBits_MSVC { int A :1; int B :2; @@ -455,7 +402,6 @@ int H :8; int I :9; }; - struct IntBits_MSVC value = {0}; APPEND(PyUnicode_FromString("IntBits_MSVC")); APPEND(PyLong_FromLong(sizeof(struct IntBits_MSVC))); @@ -469,22 +415,18 @@ TEST_FIELD(int, value.G); TEST_FIELD(int, value.H); TEST_FIELD(int, value.I); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Bits_MSVC")) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + struct GCC_ATTR(ms_struct) Bits_MSVC { int A :1; int B :2; @@ -503,7 +445,6 @@ short R :6; short S :7; }; - struct Bits_MSVC value = {0}; APPEND(PyUnicode_FromString("Bits_MSVC")); APPEND(PyLong_FromLong(sizeof(struct Bits_MSVC))); @@ -524,28 +465,23 @@ TEST_FIELD(short, value.Q); TEST_FIELD(short, value.R); TEST_FIELD(short, value.S); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "I64Bits")) { - #if (!defined(__xlc__)) - + struct I64Bits { int64_t a :1; int64_t b :62; int64_t c :1; }; - struct I64Bits value = {0}; APPEND(PyUnicode_FromString("I64Bits")); APPEND(PyLong_FromLong(sizeof(struct I64Bits))); @@ -553,28 +489,23 @@ TEST_FIELD(int64_t, value.a); TEST_FIELD(int64_t, value.b); TEST_FIELD(int64_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "U64Bits")) { - #if (!defined(__xlc__)) - + struct U64Bits { uint64_t a :1; uint64_t b :62; uint64_t c :1; }; - struct U64Bits value = {0}; APPEND(PyUnicode_FromString("U64Bits")); APPEND(PyLong_FromLong(sizeof(struct U64Bits))); @@ -582,28 +513,23 @@ TEST_FIELD(uint64_t, value.a); TEST_FIELD(uint64_t, value.b); TEST_FIELD(uint64_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_8")) { - #if (!defined(__xlc__)) - + struct Struct331_8 { int8_t a :3; int8_t b :3; int8_t c :1; }; - struct Struct331_8 value = {0}; APPEND(PyUnicode_FromString("Struct331_8")); APPEND(PyLong_FromLong(sizeof(struct Struct331_8))); @@ -611,28 +537,23 @@ TEST_FIELD(int8_t, value.a); TEST_FIELD(int8_t, value.b); TEST_FIELD(int8_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_8")) { - #if (!defined(__xlc__)) - + struct Struct1x1_8 { int8_t a :1; int8_t b :6; int8_t c :1; }; - struct Struct1x1_8 value = {0}; APPEND(PyUnicode_FromString("Struct1x1_8")); APPEND(PyLong_FromLong(sizeof(struct Struct1x1_8))); @@ -640,29 +561,24 @@ TEST_FIELD(int8_t, value.a); TEST_FIELD(int8_t, value.b); TEST_FIELD(int8_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_8")) { - #if (!defined(__xlc__)) - + struct Struct1nx1_8 { int8_t a :1; int8_t full; int8_t b :6; int8_t c :1; }; - struct Struct1nx1_8 value = {0}; APPEND(PyUnicode_FromString("Struct1nx1_8")); APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_8))); @@ -671,28 +587,23 @@ TEST_FIELD(int8_t, value.full); TEST_FIELD(int8_t, value.b); TEST_FIELD(int8_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_8")) { - #if (!defined(__xlc__)) - + struct Struct3xx_8 { int8_t a :3; int8_t b :6; int8_t c :6; }; - struct Struct3xx_8 value = {0}; APPEND(PyUnicode_FromString("Struct3xx_8")); APPEND(PyLong_FromLong(sizeof(struct Struct3xx_8))); @@ -700,28 +611,23 @@ TEST_FIELD(int8_t, value.a); TEST_FIELD(int8_t, value.b); TEST_FIELD(int8_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_u8")) { - #if (!defined(__xlc__)) - + struct Struct331_u8 { uint8_t a :3; uint8_t b :3; uint8_t c :1; }; - struct Struct331_u8 value = {0}; APPEND(PyUnicode_FromString("Struct331_u8")); APPEND(PyLong_FromLong(sizeof(struct Struct331_u8))); @@ -729,28 +635,23 @@ TEST_FIELD(uint8_t, value.a); TEST_FIELD(uint8_t, value.b); TEST_FIELD(uint8_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u8")) { - #if (!defined(__xlc__)) - + struct Struct1x1_u8 { uint8_t a :1; uint8_t b :6; uint8_t c :1; }; - struct Struct1x1_u8 value = {0}; APPEND(PyUnicode_FromString("Struct1x1_u8")); APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u8))); @@ -758,29 +659,24 @@ TEST_FIELD(uint8_t, value.a); TEST_FIELD(uint8_t, value.b); TEST_FIELD(uint8_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u8")) { - #if (!defined(__xlc__)) - + struct Struct1nx1_u8 { uint8_t a :1; uint8_t full; uint8_t b :6; uint8_t c :1; }; - struct Struct1nx1_u8 value = {0}; APPEND(PyUnicode_FromString("Struct1nx1_u8")); APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u8))); @@ -789,28 +685,23 @@ TEST_FIELD(uint8_t, value.full); TEST_FIELD(uint8_t, value.b); TEST_FIELD(uint8_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u8")) { - #if (!defined(__xlc__)) - + struct Struct3xx_u8 { uint8_t a :3; uint8_t b :6; uint8_t c :6; }; - struct Struct3xx_u8 value = {0}; APPEND(PyUnicode_FromString("Struct3xx_u8")); APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u8))); @@ -818,28 +709,23 @@ TEST_FIELD(uint8_t, value.a); TEST_FIELD(uint8_t, value.b); TEST_FIELD(uint8_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_16")) { - #if (!defined(__xlc__)) - + struct Struct331_16 { int16_t a :3; int16_t b :3; int16_t c :1; }; - struct Struct331_16 value = {0}; APPEND(PyUnicode_FromString("Struct331_16")); APPEND(PyLong_FromLong(sizeof(struct Struct331_16))); @@ -847,28 +733,23 @@ TEST_FIELD(int16_t, value.a); TEST_FIELD(int16_t, value.b); TEST_FIELD(int16_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_16")) { - #if (!defined(__xlc__)) - + struct Struct1x1_16 { int16_t a :1; int16_t b :14; int16_t c :1; }; - struct Struct1x1_16 value = {0}; APPEND(PyUnicode_FromString("Struct1x1_16")); APPEND(PyLong_FromLong(sizeof(struct Struct1x1_16))); @@ -876,29 +757,24 @@ TEST_FIELD(int16_t, value.a); TEST_FIELD(int16_t, value.b); TEST_FIELD(int16_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_16")) { - #if (!defined(__xlc__)) - + struct Struct1nx1_16 { int16_t a :1; int16_t full; int16_t b :14; int16_t c :1; }; - struct Struct1nx1_16 value = {0}; APPEND(PyUnicode_FromString("Struct1nx1_16")); APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_16))); @@ -907,28 +783,23 @@ TEST_FIELD(int16_t, value.full); TEST_FIELD(int16_t, value.b); TEST_FIELD(int16_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_16")) { - #if (!defined(__xlc__)) - + struct Struct3xx_16 { int16_t a :3; int16_t b :14; int16_t c :14; }; - struct Struct3xx_16 value = {0}; APPEND(PyUnicode_FromString("Struct3xx_16")); APPEND(PyLong_FromLong(sizeof(struct Struct3xx_16))); @@ -936,28 +807,23 @@ TEST_FIELD(int16_t, value.a); TEST_FIELD(int16_t, value.b); TEST_FIELD(int16_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_u16")) { - #if (!defined(__xlc__)) - + struct Struct331_u16 { uint16_t a :3; uint16_t b :3; uint16_t c :1; }; - struct Struct331_u16 value = {0}; APPEND(PyUnicode_FromString("Struct331_u16")); APPEND(PyLong_FromLong(sizeof(struct Struct331_u16))); @@ -965,28 +831,23 @@ TEST_FIELD(uint16_t, value.a); TEST_FIELD(uint16_t, value.b); TEST_FIELD(uint16_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u16")) { - #if (!defined(__xlc__)) - + struct Struct1x1_u16 { uint16_t a :1; uint16_t b :14; uint16_t c :1; }; - struct Struct1x1_u16 value = {0}; APPEND(PyUnicode_FromString("Struct1x1_u16")); APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u16))); @@ -994,29 +855,24 @@ TEST_FIELD(uint16_t, value.a); TEST_FIELD(uint16_t, value.b); TEST_FIELD(uint16_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u16")) { - #if (!defined(__xlc__)) - + struct Struct1nx1_u16 { uint16_t a :1; uint16_t full; uint16_t b :14; uint16_t c :1; }; - struct Struct1nx1_u16 value = {0}; APPEND(PyUnicode_FromString("Struct1nx1_u16")); APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u16))); @@ -1025,28 +881,23 @@ TEST_FIELD(uint16_t, value.full); TEST_FIELD(uint16_t, value.b); TEST_FIELD(uint16_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u16")) { - #if (!defined(__xlc__)) - + struct Struct3xx_u16 { uint16_t a :3; uint16_t b :14; uint16_t c :14; }; - struct Struct3xx_u16 value = {0}; APPEND(PyUnicode_FromString("Struct3xx_u16")); APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u16))); @@ -1054,28 +905,23 @@ TEST_FIELD(uint16_t, value.a); TEST_FIELD(uint16_t, value.b); TEST_FIELD(uint16_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_32")) { - #if (!defined(__xlc__)) - + struct Struct331_32 { int32_t a :3; int32_t b :3; int32_t c :1; }; - struct Struct331_32 value = {0}; APPEND(PyUnicode_FromString("Struct331_32")); APPEND(PyLong_FromLong(sizeof(struct Struct331_32))); @@ -1083,28 +929,23 @@ TEST_FIELD(int32_t, value.a); TEST_FIELD(int32_t, value.b); TEST_FIELD(int32_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_32")) { - #if (!defined(__xlc__)) - + struct Struct1x1_32 { int32_t a :1; int32_t b :30; int32_t c :1; }; - struct Struct1x1_32 value = {0}; APPEND(PyUnicode_FromString("Struct1x1_32")); APPEND(PyLong_FromLong(sizeof(struct Struct1x1_32))); @@ -1112,29 +953,24 @@ TEST_FIELD(int32_t, value.a); TEST_FIELD(int32_t, value.b); TEST_FIELD(int32_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_32")) { - #if (!defined(__xlc__)) - + struct Struct1nx1_32 { int32_t a :1; int32_t full; int32_t b :30; int32_t c :1; }; - struct Struct1nx1_32 value = {0}; APPEND(PyUnicode_FromString("Struct1nx1_32")); APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_32))); @@ -1143,28 +979,23 @@ TEST_FIELD(int32_t, value.full); TEST_FIELD(int32_t, value.b); TEST_FIELD(int32_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_32")) { - #if (!defined(__xlc__)) - + struct Struct3xx_32 { int32_t a :3; int32_t b :30; int32_t c :30; }; - struct Struct3xx_32 value = {0}; APPEND(PyUnicode_FromString("Struct3xx_32")); APPEND(PyLong_FromLong(sizeof(struct Struct3xx_32))); @@ -1172,28 +1003,23 @@ TEST_FIELD(int32_t, value.a); TEST_FIELD(int32_t, value.b); TEST_FIELD(int32_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_u32")) { - #if (!defined(__xlc__)) - + struct Struct331_u32 { uint32_t a :3; uint32_t b :3; uint32_t c :1; }; - struct Struct331_u32 value = {0}; APPEND(PyUnicode_FromString("Struct331_u32")); APPEND(PyLong_FromLong(sizeof(struct Struct331_u32))); @@ -1201,28 +1027,23 @@ TEST_FIELD(uint32_t, value.a); TEST_FIELD(uint32_t, value.b); TEST_FIELD(uint32_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u32")) { - #if (!defined(__xlc__)) - + struct Struct1x1_u32 { uint32_t a :1; uint32_t b :30; uint32_t c :1; }; - struct Struct1x1_u32 value = {0}; APPEND(PyUnicode_FromString("Struct1x1_u32")); APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u32))); @@ -1230,29 +1051,24 @@ TEST_FIELD(uint32_t, value.a); TEST_FIELD(uint32_t, value.b); TEST_FIELD(uint32_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u32")) { - #if (!defined(__xlc__)) - + struct Struct1nx1_u32 { uint32_t a :1; uint32_t full; uint32_t b :30; uint32_t c :1; }; - struct Struct1nx1_u32 value = {0}; APPEND(PyUnicode_FromString("Struct1nx1_u32")); APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u32))); @@ -1261,28 +1077,23 @@ TEST_FIELD(uint32_t, value.full); TEST_FIELD(uint32_t, value.b); TEST_FIELD(uint32_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u32")) { - #if (!defined(__xlc__)) - + struct Struct3xx_u32 { uint32_t a :3; uint32_t b :30; uint32_t c :30; }; - struct Struct3xx_u32 value = {0}; APPEND(PyUnicode_FromString("Struct3xx_u32")); APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u32))); @@ -1290,28 +1101,23 @@ TEST_FIELD(uint32_t, value.a); TEST_FIELD(uint32_t, value.b); TEST_FIELD(uint32_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_64")) { - #if (!defined(__xlc__)) - + struct Struct331_64 { int64_t a :3; int64_t b :3; int64_t c :1; }; - struct Struct331_64 value = {0}; APPEND(PyUnicode_FromString("Struct331_64")); APPEND(PyLong_FromLong(sizeof(struct Struct331_64))); @@ -1319,28 +1125,23 @@ TEST_FIELD(int64_t, value.a); TEST_FIELD(int64_t, value.b); TEST_FIELD(int64_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_64")) { - #if (!defined(__xlc__)) - + struct Struct1x1_64 { int64_t a :1; int64_t b :62; int64_t c :1; }; - struct Struct1x1_64 value = {0}; APPEND(PyUnicode_FromString("Struct1x1_64")); APPEND(PyLong_FromLong(sizeof(struct Struct1x1_64))); @@ -1348,29 +1149,24 @@ TEST_FIELD(int64_t, value.a); TEST_FIELD(int64_t, value.b); TEST_FIELD(int64_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_64")) { - #if (!defined(__xlc__)) - + struct Struct1nx1_64 { int64_t a :1; int64_t full; int64_t b :62; int64_t c :1; }; - struct Struct1nx1_64 value = {0}; APPEND(PyUnicode_FromString("Struct1nx1_64")); APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_64))); @@ -1379,28 +1175,23 @@ TEST_FIELD(int64_t, value.full); TEST_FIELD(int64_t, value.b); TEST_FIELD(int64_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_64")) { - #if (!defined(__xlc__)) - + struct Struct3xx_64 { int64_t a :3; int64_t b :62; int64_t c :62; }; - struct Struct3xx_64 value = {0}; APPEND(PyUnicode_FromString("Struct3xx_64")); APPEND(PyLong_FromLong(sizeof(struct Struct3xx_64))); @@ -1408,28 +1199,23 @@ TEST_FIELD(int64_t, value.a); TEST_FIELD(int64_t, value.b); TEST_FIELD(int64_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_u64")) { - #if (!defined(__xlc__)) - + struct Struct331_u64 { uint64_t a :3; uint64_t b :3; uint64_t c :1; }; - struct Struct331_u64 value = {0}; APPEND(PyUnicode_FromString("Struct331_u64")); APPEND(PyLong_FromLong(sizeof(struct Struct331_u64))); @@ -1437,28 +1223,23 @@ TEST_FIELD(uint64_t, value.a); TEST_FIELD(uint64_t, value.b); TEST_FIELD(uint64_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u64")) { - #if (!defined(__xlc__)) - + struct Struct1x1_u64 { uint64_t a :1; uint64_t b :62; uint64_t c :1; }; - struct Struct1x1_u64 value = {0}; APPEND(PyUnicode_FromString("Struct1x1_u64")); APPEND(PyLong_FromLong(sizeof(struct Struct1x1_u64))); @@ -1466,29 +1247,24 @@ TEST_FIELD(uint64_t, value.a); TEST_FIELD(uint64_t, value.b); TEST_FIELD(uint64_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u64")) { - #if (!defined(__xlc__)) - + struct Struct1nx1_u64 { uint64_t a :1; uint64_t full; uint64_t b :62; uint64_t c :1; }; - struct Struct1nx1_u64 value = {0}; APPEND(PyUnicode_FromString("Struct1nx1_u64")); APPEND(PyLong_FromLong(sizeof(struct Struct1nx1_u64))); @@ -1497,28 +1273,23 @@ TEST_FIELD(uint64_t, value.full); TEST_FIELD(uint64_t, value.b); TEST_FIELD(uint64_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u64")) { - #if (!defined(__xlc__)) - + struct Struct3xx_u64 { uint64_t a :3; uint64_t b :62; uint64_t c :62; }; - struct Struct3xx_u64 value = {0}; APPEND(PyUnicode_FromString("Struct3xx_u64")); APPEND(PyLong_FromLong(sizeof(struct Struct3xx_u64))); @@ -1526,103 +1297,84 @@ TEST_FIELD(uint64_t, value.a); TEST_FIELD(uint64_t, value.b); TEST_FIELD(uint64_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed1")) { - #if (!defined(__xlc__)) - + struct Mixed1 { signed char a :4; int b :4; }; - struct Mixed1 value = {0}; APPEND(PyUnicode_FromString("Mixed1")); APPEND(PyLong_FromLong(sizeof(struct Mixed1))); APPEND(PyLong_FromLong(_Alignof(struct Mixed1))); TEST_FIELD(signed char, value.a); TEST_FIELD(int, value.b); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed2")) { - #if (!defined(__xlc__)) - + struct Mixed2 { signed char a :4; int32_t b :32; }; - struct Mixed2 value = {0}; APPEND(PyUnicode_FromString("Mixed2")); APPEND(PyLong_FromLong(sizeof(struct Mixed2))); APPEND(PyLong_FromLong(_Alignof(struct Mixed2))); TEST_FIELD(signed char, value.a); TEST_FIELD(int32_t, value.b); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed3")) { - #if (!defined(__xlc__)) - + struct Mixed3 { signed char a :4; unsigned char b :4; }; - struct Mixed3 value = {0}; APPEND(PyUnicode_FromString("Mixed3")); APPEND(PyLong_FromLong(sizeof(struct Mixed3))); APPEND(PyLong_FromLong(_Alignof(struct Mixed3))); TEST_FIELD(signed char, value.a); TEST_FIELD(unsigned char, value.b); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed4")) { - #if (!defined(__xlc__)) - + struct Mixed4 { short a :4; short b :4; @@ -1631,7 +1383,6 @@ short e :4; int f :24; }; - struct Mixed4 value = {0}; APPEND(PyUnicode_FromString("Mixed4")); APPEND(PyLong_FromLong(sizeof(struct Mixed4))); @@ -1642,82 +1393,67 @@ TEST_FIELD(short, value.d); TEST_FIELD(short, value.e); TEST_FIELD(int, value.f); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed5")) { - #if (!defined(__xlc__)) - + struct Mixed5 { unsigned int A :1; unsigned short B :16; }; - struct Mixed5 value = {0}; APPEND(PyUnicode_FromString("Mixed5")); APPEND(PyLong_FromLong(sizeof(struct Mixed5))); APPEND(PyLong_FromLong(_Alignof(struct Mixed5))); TEST_FIELD(unsigned int, value.A); TEST_FIELD(unsigned short, value.B); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed6")) { - #if (!defined(__xlc__)) - + struct Mixed6 { unsigned long long A :1; unsigned int B :32; }; - struct Mixed6 value = {0}; APPEND(PyUnicode_FromString("Mixed6")); APPEND(PyLong_FromLong(sizeof(struct Mixed6))); APPEND(PyLong_FromLong(_Alignof(struct Mixed6))); TEST_FIELD(unsigned long long, value.A); TEST_FIELD(unsigned int, value.B); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed7")) { - #if (!defined(__xlc__)) - + struct Mixed7 { uint32_t A; uint32_t B :20; uint64_t C :24; }; - struct Mixed7 value = {0}; APPEND(PyUnicode_FromString("Mixed7")); APPEND(PyLong_FromLong(sizeof(struct Mixed7))); @@ -1725,28 +1461,23 @@ TEST_FIELD(uint32_t, value.A); TEST_FIELD(uint32_t, value.B); TEST_FIELD(uint64_t, value.C); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed8_a")) { - #if (!defined(__xlc__)) - + struct Mixed8_a { uint32_t A; uint32_t B :32; unsigned long long C :1; }; - struct Mixed8_a value = {0}; APPEND(PyUnicode_FromString("Mixed8_a")); APPEND(PyLong_FromLong(sizeof(struct Mixed8_a))); @@ -1754,28 +1485,23 @@ TEST_FIELD(uint32_t, value.A); TEST_FIELD(uint32_t, value.B); TEST_FIELD(unsigned long long, value.C); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed8_b")) { - #if (!defined(__xlc__)) - + struct Mixed8_b { uint32_t A; uint32_t B; unsigned long long C :1; }; - struct Mixed8_b value = {0}; APPEND(PyUnicode_FromString("Mixed8_b")); APPEND(PyLong_FromLong(sizeof(struct Mixed8_b))); @@ -1783,103 +1509,84 @@ TEST_FIELD(uint32_t, value.A); TEST_FIELD(uint32_t, value.B); TEST_FIELD(unsigned long long, value.C); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed9")) { - #if (!defined(__xlc__)) - + struct Mixed9 { uint8_t A; uint32_t B :1; }; - struct Mixed9 value = {0}; APPEND(PyUnicode_FromString("Mixed9")); APPEND(PyLong_FromLong(sizeof(struct Mixed9))); APPEND(PyLong_FromLong(_Alignof(struct Mixed9))); TEST_FIELD(uint8_t, value.A); TEST_FIELD(uint32_t, value.B); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed10")) { - #if (!defined(__xlc__)) - + struct Mixed10 { uint32_t A :1; uint64_t B :1; }; - struct Mixed10 value = {0}; APPEND(PyUnicode_FromString("Mixed10")); APPEND(PyLong_FromLong(sizeof(struct Mixed10))); APPEND(PyLong_FromLong(_Alignof(struct Mixed10))); TEST_FIELD(uint32_t, value.A); TEST_FIELD(uint64_t, value.B); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_95496")) { - #if (!defined(__xlc__)) - + struct Example_gh_95496 { uint32_t A :1; uint64_t B :1; }; - struct Example_gh_95496 value = {0}; APPEND(PyUnicode_FromString("Example_gh_95496")); APPEND(PyLong_FromLong(sizeof(struct Example_gh_95496))); APPEND(PyLong_FromLong(_Alignof(struct Example_gh_95496))); TEST_FIELD(uint32_t, value.A); TEST_FIELD(uint64_t, value.B); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_84039_bad")) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Example_gh_84039_bad { uint8_t a0 :1; @@ -1894,7 +1601,6 @@ uint16_t b1 :12; }; #pragma pack(pop) - struct Example_gh_84039_bad value = {0}; APPEND(PyUnicode_FromString("Example_gh_84039_bad")); APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_bad))); @@ -1909,22 +1615,18 @@ TEST_FIELD(uint8_t, value.a7); TEST_FIELD(uint16_t, value.b0); TEST_FIELD(uint16_t, value.b1); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_84039_good_a")) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Example_gh_84039_good_a { uint8_t a0 :1; @@ -1937,7 +1639,6 @@ uint8_t a7 :1; }; #pragma pack(pop) - struct Example_gh_84039_good_a value = {0}; APPEND(PyUnicode_FromString("Example_gh_84039_good_a")); APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good_a))); @@ -1950,22 +1651,18 @@ TEST_FIELD(uint8_t, value.a5); TEST_FIELD(uint8_t, value.a6); TEST_FIELD(uint8_t, value.a7); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_84039_good")) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Example_gh_84039_good { #pragma pack(push, 1) @@ -1984,7 +1681,6 @@ uint16_t b1 :12; }; #pragma pack(pop) - struct Example_gh_84039_good value = {0}; APPEND(PyUnicode_FromString("Example_gh_84039_good")); APPEND(PyLong_FromLong(sizeof(struct Example_gh_84039_good))); @@ -1999,22 +1695,18 @@ TEST_FIELD(uint8_t, value.a.a7); TEST_FIELD(uint16_t, value.b0); TEST_FIELD(uint16_t, value.b1); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_73939")) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Example_gh_73939 { uint16_t P; @@ -2029,7 +1721,6 @@ uint32_t R2 :2; }; #pragma pack(pop) - struct Example_gh_73939 value = {0}; APPEND(PyUnicode_FromString("Example_gh_73939")); APPEND(PyLong_FromLong(sizeof(struct Example_gh_73939))); @@ -2044,28 +1735,23 @@ TEST_FIELD(uint32_t, value.T); TEST_FIELD(uint32_t, value.C); TEST_FIELD(uint32_t, value.R2); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_86098")) { - #if (!defined(__xlc__)) - + struct Example_gh_86098 { uint8_t a :8; uint8_t b :8; uint32_t c :16; }; - struct Example_gh_86098 value = {0}; APPEND(PyUnicode_FromString("Example_gh_86098")); APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098))); @@ -2073,22 +1759,18 @@ TEST_FIELD(uint8_t, value.a); TEST_FIELD(uint8_t, value.b); TEST_FIELD(uint32_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_86098_pack")) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) - + #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Example_gh_86098_pack { uint8_t a :8; @@ -2096,7 +1778,6 @@ uint32_t c :16; }; #pragma pack(pop) - struct Example_gh_86098_pack value = {0}; APPEND(PyUnicode_FromString("Example_gh_86098_pack")); APPEND(PyLong_FromLong(sizeof(struct Example_gh_86098_pack))); @@ -2104,22 +1785,18 @@ TEST_FIELD(uint8_t, value.a); TEST_FIELD(uint8_t, value.b); TEST_FIELD(uint32_t, value.c); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - if (_PyUnicode_EqualToASCIIString(name, "AnonBitfields")) { - #if (!defined(__xlc__)) - + struct AnonBitfields { struct { signed char a :4; @@ -2127,7 +1804,6 @@ }; signed char y; }; - struct AnonBitfields value = {0}; APPEND(PyUnicode_FromString("AnonBitfields")); APPEND(PyLong_FromLong(sizeof(struct AnonBitfields))); @@ -2135,16 +1811,13 @@ TEST_FIELD(signed char, value.a); TEST_FIELD(unsigned char, value.b); TEST_FIELD(signed char, value.y); - #else APPEND(Py_NewRef(Py_None)); APPEND(PyUnicode_FromString("skipped on this compiler")); #endif - return result; } - Py_DECREF(result); PyErr_Format(PyExc_ValueError, "unknown testcase %R", name); @@ -2155,4 +1828,4 @@ #undef TEST_FIELD #undef SET_AND_APPEND #undef APPEND - + From 0cf10490aeb9d9470fc5a8c34c32bdf21b160426 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 2 May 2024 16:49:36 +0200 Subject: [PATCH 76/91] Use limited API again --- .../test_ctypes/test_generated_structs.py | 2 +- Modules/_ctypes/_ctypes_test.c | 6 +- Modules/_ctypes/_ctypes_test_generated.c.h | 138 +++++++++--------- 3 files changed, 74 insertions(+), 72 deletions(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index 4ab5680956db89..5467a74f47773c 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -620,7 +620,7 @@ def output(string): """) for name, cls in TESTCASES.items(): output(""" - if (_PyUnicode_EqualToASCIIString(name, %s)) { + if (PyUnicode_CompareWithASCIIString(name, %s) == 0) { """ % c_str_repr(name)) lines, requires = dump_ctype(cls, struct_or_union_tag=name, semi=';') if requires: diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 4d63feb7b7df3e..a37acaf57aa6ba 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,5 +1,8 @@ // Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED -// Need internal API for _PyUnicode_EqualToASCIIString +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED +# define Py_LIMITED_API 0x030c0000 +#endif // gh-85283: On Windows, Py_LIMITED_API requires Py_BUILD_CORE to not attempt // linking the extension to python3.lib (which fails). Py_BUILD_CORE_MODULE is @@ -9,7 +12,6 @@ #define Py_BUILD_CORE_MODULE #include -#include // _PyUnicode_EqualToASCIIString #include // printf() #include // qsort() diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index 6acb7cd436bd91..f25d623961a39c 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -50,7 +50,7 @@ return NULL; } - if (_PyUnicode_EqualToASCIIString(name, "SingleInt")) { + if (PyUnicode_CompareWithASCIIString(name, "SingleInt") == 0) { struct SingleInt { int a; @@ -63,7 +63,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "SingleInt_Union")) { + if (PyUnicode_CompareWithASCIIString(name, "SingleInt_Union") == 0) { union SingleInt_Union { int a; @@ -76,7 +76,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "SingleU32")) { + if (PyUnicode_CompareWithASCIIString(name, "SingleU32") == 0) { struct SingleU32 { uint32_t a; @@ -89,7 +89,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "SimpleStruct")) { + if (PyUnicode_CompareWithASCIIString(name, "SimpleStruct") == 0) { struct SimpleStruct { int32_t x; @@ -106,7 +106,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "SimpleUnion")) { + if (PyUnicode_CompareWithASCIIString(name, "SimpleUnion") == 0) { union SimpleUnion { int32_t x; @@ -123,7 +123,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "ManyTypes")) { + if (PyUnicode_CompareWithASCIIString(name, "ManyTypes") == 0) { struct ManyTypes { int8_t i8; @@ -150,7 +150,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "ManyTypesU")) { + if (PyUnicode_CompareWithASCIIString(name, "ManyTypesU") == 0) { union ManyTypesU { int8_t i8; @@ -177,7 +177,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Nested")) { + if (PyUnicode_CompareWithASCIIString(name, "Nested") == 0) { struct Nested { struct { @@ -212,7 +212,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Packed1")) { + if (PyUnicode_CompareWithASCIIString(name, "Packed1") == 0) { #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -236,7 +236,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Packed2")) { + if (PyUnicode_CompareWithASCIIString(name, "Packed2") == 0) { #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -260,7 +260,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Packed3")) { + if (PyUnicode_CompareWithASCIIString(name, "Packed3") == 0) { #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -284,7 +284,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Packed4")) { + if (PyUnicode_CompareWithASCIIString(name, "Packed4") == 0) { #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -308,7 +308,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "IntBits")) { + if (PyUnicode_CompareWithASCIIString(name, "IntBits") == 0) { struct IntBits { int A :1; @@ -337,7 +337,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Bits")) { + if (PyUnicode_CompareWithASCIIString(name, "Bits") == 0) { #if (!defined(__xlc__)) @@ -387,7 +387,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "IntBits_MSVC")) { + if (PyUnicode_CompareWithASCIIString(name, "IntBits_MSVC") == 0) { #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -423,7 +423,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Bits_MSVC")) { + if (PyUnicode_CompareWithASCIIString(name, "Bits_MSVC") == 0) { #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -473,7 +473,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "I64Bits")) { + if (PyUnicode_CompareWithASCIIString(name, "I64Bits") == 0) { #if (!defined(__xlc__)) @@ -497,7 +497,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "U64Bits")) { + if (PyUnicode_CompareWithASCIIString(name, "U64Bits") == 0) { #if (!defined(__xlc__)) @@ -521,7 +521,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_8")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct331_8") == 0) { #if (!defined(__xlc__)) @@ -545,7 +545,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_8")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_8") == 0) { #if (!defined(__xlc__)) @@ -569,7 +569,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_8")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_8") == 0) { #if (!defined(__xlc__)) @@ -595,7 +595,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_8")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_8") == 0) { #if (!defined(__xlc__)) @@ -619,7 +619,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_u8")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u8") == 0) { #if (!defined(__xlc__)) @@ -643,7 +643,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u8")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u8") == 0) { #if (!defined(__xlc__)) @@ -667,7 +667,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u8")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u8") == 0) { #if (!defined(__xlc__)) @@ -693,7 +693,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u8")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u8") == 0) { #if (!defined(__xlc__)) @@ -717,7 +717,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_16")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct331_16") == 0) { #if (!defined(__xlc__)) @@ -741,7 +741,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_16")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_16") == 0) { #if (!defined(__xlc__)) @@ -765,7 +765,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_16")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_16") == 0) { #if (!defined(__xlc__)) @@ -791,7 +791,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_16")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_16") == 0) { #if (!defined(__xlc__)) @@ -815,7 +815,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_u16")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u16") == 0) { #if (!defined(__xlc__)) @@ -839,7 +839,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u16")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u16") == 0) { #if (!defined(__xlc__)) @@ -863,7 +863,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u16")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u16") == 0) { #if (!defined(__xlc__)) @@ -889,7 +889,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u16")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u16") == 0) { #if (!defined(__xlc__)) @@ -913,7 +913,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_32")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct331_32") == 0) { #if (!defined(__xlc__)) @@ -937,7 +937,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_32")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_32") == 0) { #if (!defined(__xlc__)) @@ -961,7 +961,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_32")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_32") == 0) { #if (!defined(__xlc__)) @@ -987,7 +987,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_32")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_32") == 0) { #if (!defined(__xlc__)) @@ -1011,7 +1011,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_u32")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u32") == 0) { #if (!defined(__xlc__)) @@ -1035,7 +1035,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u32")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u32") == 0) { #if (!defined(__xlc__)) @@ -1059,7 +1059,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u32")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u32") == 0) { #if (!defined(__xlc__)) @@ -1085,7 +1085,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u32")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u32") == 0) { #if (!defined(__xlc__)) @@ -1109,7 +1109,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_64")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct331_64") == 0) { #if (!defined(__xlc__)) @@ -1133,7 +1133,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_64")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_64") == 0) { #if (!defined(__xlc__)) @@ -1157,7 +1157,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_64")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_64") == 0) { #if (!defined(__xlc__)) @@ -1183,7 +1183,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_64")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_64") == 0) { #if (!defined(__xlc__)) @@ -1207,7 +1207,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct331_u64")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct331_u64") == 0) { #if (!defined(__xlc__)) @@ -1231,7 +1231,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1x1_u64")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1x1_u64") == 0) { #if (!defined(__xlc__)) @@ -1255,7 +1255,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct1nx1_u64")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct1nx1_u64") == 0) { #if (!defined(__xlc__)) @@ -1281,7 +1281,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Struct3xx_u64")) { + if (PyUnicode_CompareWithASCIIString(name, "Struct3xx_u64") == 0) { #if (!defined(__xlc__)) @@ -1305,7 +1305,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed1")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed1") == 0) { #if (!defined(__xlc__)) @@ -1327,7 +1327,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed2")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed2") == 0) { #if (!defined(__xlc__)) @@ -1349,7 +1349,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed3")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed3") == 0) { #if (!defined(__xlc__)) @@ -1371,7 +1371,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed4")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed4") == 0) { #if (!defined(__xlc__)) @@ -1401,7 +1401,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed5")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed5") == 0) { #if (!defined(__xlc__)) @@ -1423,7 +1423,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed6")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed6") == 0) { #if (!defined(__xlc__)) @@ -1445,7 +1445,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed7")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed7") == 0) { #if (!defined(__xlc__)) @@ -1469,7 +1469,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed8_a")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed8_a") == 0) { #if (!defined(__xlc__)) @@ -1493,7 +1493,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed8_b")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed8_b") == 0) { #if (!defined(__xlc__)) @@ -1517,7 +1517,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed9")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed9") == 0) { #if (!defined(__xlc__)) @@ -1539,7 +1539,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Mixed10")) { + if (PyUnicode_CompareWithASCIIString(name, "Mixed10") == 0) { #if (!defined(__xlc__)) @@ -1561,7 +1561,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_95496")) { + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_95496") == 0) { #if (!defined(__xlc__)) @@ -1583,7 +1583,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_84039_bad")) { + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_bad") == 0) { #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -1623,7 +1623,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_84039_good_a")) { + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good_a") == 0) { #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -1659,7 +1659,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_84039_good")) { + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good") == 0) { #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -1703,7 +1703,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_73939")) { + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_73939") == 0) { #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -1743,7 +1743,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_86098")) { + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098") == 0) { #if (!defined(__xlc__)) @@ -1767,7 +1767,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "Example_gh_86098_pack")) { + if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098_pack") == 0) { #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) @@ -1793,7 +1793,7 @@ return result; } - if (_PyUnicode_EqualToASCIIString(name, "AnonBitfields")) { + if (PyUnicode_CompareWithASCIIString(name, "AnonBitfields") == 0) { #if (!defined(__xlc__)) From bc91549aa6b2d7220deb7a64f0b1cfeeda3a6807 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 3 May 2024 15:35:27 +0200 Subject: [PATCH 77/91] Appease the linter --- Lib/test/test_ctypes/test_generated_structs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index 5467a74f47773c..1099d820fdc47d 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -664,4 +664,3 @@ def output(string): #undef SET_AND_APPEND #undef APPEND """) - From 70bbc2670cd40ca7ef3353dd6f1622e057631b8a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 3 May 2024 15:36:34 +0200 Subject: [PATCH 78/91] Fix ReST typo --- Doc/whatsnew/3.13.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index f8d9e47806bf91..c760f60c27c431 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -432,7 +432,7 @@ ctypes match platform defaults (GCC/Clang or MSC). In particular, fields no longer overlap. (Contributed by Matthias Görgens in :gh:`97702`.) -* A :attr:`ctypes.Structure._layout_`` class attribute can be set +* A :attr:`ctypes.Structure._layout_` class attribute can be set to help match a non-default ABI. (Contributed by Petr Viktorin in :gh:`97702`.) From 6e23b3d7134f56fe687a5834f60591f86ad150cd Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 3 May 2024 16:18:27 +0200 Subject: [PATCH 79/91] Use nicer error output for memory dumps --- .../test_ctypes/test_generated_structs.py | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index 1099d820fdc47d..cf6669d9fc6a34 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -52,7 +52,7 @@ class c_bool(ctypes.c_bool): # Register structs and unions to test TESTCASES = {} -def register(name=None): +def register(name=None, set_name=False): def decorator(cls, name=name): if name is None: name = cls.__name__ @@ -60,6 +60,8 @@ def decorator(cls, name=name): assert name.isidentifier() # will be used as a C identifier assert name not in TESTCASES TESTCASES[name] = cls + if set_name: + cls.__name__ = name return cls return decorator @@ -230,26 +232,26 @@ class U64Bits(Structure): for signedness in '', 'u': ctype = globals()[f'c_{signedness}int{n}'] - @register(f'Struct331_{signedness}{n}') + @register(f'Struct331_{signedness}{n}', set_name=True) class _cls(Structure): _fields_ = [("a", ctype, 3), ("b", ctype, 3), ("c", ctype, 1)] - @register(f'Struct1x1_{signedness}{n}') + @register(f'Struct1x1_{signedness}{n}', set_name=True) class _cls(Structure): _fields_ = [("a", ctype, 1), ("b", ctype, n-2), ("c", ctype, 1)] - @register(f'Struct1nx1_{signedness}{n}') + @register(f'Struct1nx1_{signedness}{n}', set_name=True) class _cls(Structure): _fields_ = [("a", ctype, 1), ("full", ctype), ("b", ctype, n-2), ("c", ctype, 1)] - @register(f'Struct3xx_{signedness}{n}') + @register(f'Struct3xx_{signedness}{n}', set_name=True) class _cls(Structure): _fields_ = [("a", ctype, 3), ("b", ctype, n-2), @@ -427,8 +429,13 @@ def test_generated_data(self): for value in -1, 1, 0: with self.subTest(field=field.full_name, value=value): field.set_to(obj, value) - mem = string_at(ptr, sizeof(obj)) - self.assertEqual(mem, next(expected)) + py_mem = string_at(ptr, sizeof(obj)) + c_mem = next(expected) + if py_mem != c_mem: + # Generate a helpful failure message + lines, requires = dump_ctype(cls) + m = "\n".join([str(field), 'in:', *lines]) + self.assertEqual(py_mem.hex(), c_mem.hex(), m) # The rest of this file is generating C code from a ctypes type. @@ -547,6 +554,24 @@ def set_to(self, obj, new): obj = getattr(obj, attr_name) setattr(obj, self.attr_path[-1], new) + @cached_property + def root(self): + if self.parent is None: + return self + else: + return self.parent + + @cached_property + def descriptor(self): + return getattr(self.parent_type, self.name) + + def __repr__(self): + qname = f'{self.root.parent_type.__name__}.{self.full_name}' + try: + desc = self.descriptor + except AttributeError: + desc = '???' + return f'<{type(self).__name__} for {qname}: {desc}>' def iterfields(tp, parent=None): """Get *leaf* fields of a structure or union, as FieldInfo""" From 1b9084188db4af129a483bf25c3bac1a04cad94d Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 3 May 2024 17:16:37 +0200 Subject: [PATCH 80/91] Regen global strings --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 3 --- 4 files changed, 6 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 0dfc38a9e3e34b..b8e60ecc34f254 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -989,7 +989,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ident)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(identity_hint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ignore)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 414292512d1e1e..f879d49cd5bd68 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -478,7 +478,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) STRUCT_FOR_ID(hour) - STRUCT_FOR_ID(id) STRUCT_FOR_ID(ident) STRUCT_FOR_ID(identity_hint) STRUCT_FOR_ID(ignore) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 4bc836df2fc0c9..5a572a87fdb97a 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -987,7 +987,6 @@ extern "C" { INIT_ID(hi), \ INIT_ID(hook), \ INIT_ID(hour), \ - INIT_ID(id), \ INIT_ID(ident), \ INIT_ID(identity_hint), \ INIT_ID(ignore), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 03ed1ee278e31f..41f87b24976f5f 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1275,9 +1275,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(hour); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(id); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(ident); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); From 2c4874b7e8c56ab0e93699366996c3bfcd7a6405 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 3 May 2024 17:40:09 +0200 Subject: [PATCH 81/91] Allow alignment < size, as in int64_t on x86 (32-bit), GCC layout --- .../test_ctypes/test_generated_structs.py | 6 ++++ Modules/_ctypes/_ctypes_test_generated.c.h | 17 ++++++++++ Modules/_ctypes/cfield.c | 32 ++++++++++--------- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index cf6669d9fc6a34..4cde165aa08d3d 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -140,6 +140,12 @@ class Packed4(Structure): _fields_ = [('a', c_int8), ('b', c_int64)] _pack_ = 8 +@register() +class X86_32EdgeCase(Structure): + # On a Pentium, long long (int64) is 32-bit aligned, + # so these are packed tightly. + _fields_ = [('a', c_int32), ('b', c_int64), ('c', c_int32)] + @register() class IntBits(Structure): _fields_ = [("A", c_int, 1), diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index f25d623961a39c..a1c162b8fe3c27 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -308,6 +308,23 @@ return result; } + if (PyUnicode_CompareWithASCIIString(name, "X86_32EdgeCase") == 0) { + + struct X86_32EdgeCase { + int32_t a; + int64_t b; + int32_t c; + }; + struct X86_32EdgeCase value = {0}; + APPEND(PyUnicode_FromString("X86_32EdgeCase")); + APPEND(PyLong_FromLong(sizeof(struct X86_32EdgeCase))); + APPEND(PyLong_FromLong(_Alignof(struct X86_32EdgeCase))); + TEST_FIELD(int32_t, value.a); + TEST_FIELD(int64_t, value.b); + TEST_FIELD(int32_t, value.c); + return result; + } + if (PyUnicode_CompareWithASCIIString(name, "IntBits") == 0) { struct IntBits { diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 03a8b2f8e440ee..e003846158c842 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -64,8 +64,9 @@ The function expects to be called repeatedly for all fields in a struct or union. It uses helper functions PyCField_FromDesc_gcc and PyCField_FromDesc_msvc to simulate the corresponding compilers. -GCC mode places fields one after another, bit by bit. But when a field would -straddle an alignment boundary for its type, we insert a few bits of padding to +GCC mode places fields one after another, bit by bit. But "each bit field must +fit within a single object of its specified type" (GCC manual, section 15.8 +"Bit Field Packing"). When it doesn't, we insert a few bits of padding to avoid that. MSVC mode works similar except for bitfield packing. Adjacent bit-fields are @@ -99,27 +100,28 @@ PyCField_FromDesc_gcc(Py_ssize_t bitsize, Py_ssize_t *pbitofs, *palign = info->align; - if ((bitsize > 0) - && (round_down(*pbitofs, 8 * info->align) - < round_down(*pbitofs + bitsize - 1, 8 * info->align))) { - // We would be straddling alignment units. - *pbitofs = round_up(*pbitofs, 8*info->align); + if (bitsize > 0) { + // Determine whether the bit field, if placed at the next free bit, + // fits within a single object of its specified type. + // That is: determine a "slot", sized & aligned for the specified type, + // which contains the bitfield's beginning: + Py_ssize_t slot_start_bit = round_down(*pbitofs, 8 * info->align); + Py_ssize_t slot_end_bit = slot_start_bit + 8 * info->size; + // And see if it also contains the bitfield's last bit: + Py_ssize_t field_end_bit = *pbitofs + bitsize; + if (field_end_bit > slot_end_bit) { + // It doesn't: add padding (bump up to the next alignment boundary) + *pbitofs = round_up(*pbitofs, 8*info->align); + } } assert(*poffset == 0); + self->offset = round_down(*pbitofs, 8*info->align) / 8; if(is_bitfield) { - self->offset = round_down(*pbitofs, 8*info->size) / 8; Py_ssize_t effective_bitsof = *pbitofs - 8 * self->offset; self->size = BUILD_SIZE(bitsize, effective_bitsof); assert(effective_bitsof <= info->size * 8); - if (info->size != info->align) { - PyErr_SetString( - PyExc_TypeError, - "bitfield's base type size differs from alignment"); - return -1; - } } else { - self->offset = round_down(*pbitofs, 8*info->align) / 8; self->size = info->size; } From 738323f9f14967d6bd569bcc3b243a2e4ddb1834 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sat, 4 May 2024 14:33:06 +0200 Subject: [PATCH 82/91] Add examples from MS docs --- Lib/test/test_ctypes/test_generated_structs.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index 4cde165aa08d3d..fdc7382e6bd592 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -146,6 +146,23 @@ class X86_32EdgeCase(Structure): # so these are packed tightly. _fields_ = [('a', c_int32), ('b', c_int64), ('c', c_int32)] +@register() +class MSBitFieldExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('a', c_uint, 4), + ('b', c_uint, 5), + ('c', c_uint, 7)] + +@register() +class MSStraddlingExample(Structure): + # From https://learn.microsoft.com/en-us/cpp/c-language/c-bit-fields + _fields_ = [ + ('first', c_uint, 9), + ('second', c_uint, 7), + ('may_straddle', c_uint, 30), + ('last', c_uint, 18)] + @register() class IntBits(Structure): _fields_ = [("A", c_int, 1), From af7487c706ee3b36964b3de07f3e1dee4304b1bb Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sun, 5 May 2024 21:22:34 +0200 Subject: [PATCH 83/91] Use correct spelling for test skips --- Lib/test/test_ctypes/test_generated_structs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index fdc7382e6bd592..93b708f2eb794d 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -441,7 +441,7 @@ def test_generated_data(self): expected = iter(_ctypes_test.get_generated_test_data(name)) expected_name = next(expected) if expected_name is None: - unittest.skipTest(next(expected)) + self.skipTest(next(expected)) self.assertEqual(name, expected_name) self.assertEqual(sizeof(cls), next(expected)) with self.subTest('alignment'): From 53533529026fd93943787311b9862dba009049e5 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sun, 5 May 2024 21:23:40 +0200 Subject: [PATCH 84/91] Only use 'attribute((ms_struct))' on x86_64 & ppc; on GCC and clang --- Lib/test/test_ctypes/test_generated_structs.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index 93b708f2eb794d..f8f0c2b90ae66c 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -21,7 +21,6 @@ from ctypes import sizeof, alignment, pointer, string_at _ctypes_test = import_helper.import_module("_ctypes_test") -KNOWN_COMPILERS = 'defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)' # ctypes erases the difference between `c_int` and e.g.`c_int16`. # To keep it, we'll use custom subclasses with the C name stashed in `_c_name`: @@ -495,12 +494,16 @@ def dump_ctype(tp, struct_or_union_tag='', variable_name='', semi=''): pops = [] pack = getattr(tp, '_pack_', None) if pack is not None: - requires.add(KNOWN_COMPILERS) pushes.append(f'#pragma pack(push, {pack})') pops.append(f'#pragma pack(pop)') layout = getattr(tp, '_layout_', None) if layout == 'ms' or pack: - requires.add(KNOWN_COMPILERS) + # The 'ms_struct' attribute only works on x86 and PowerPC + requires.add( + 'defined(MS_WIN32) || (' + '(defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (' + 'defined(__GNUC__) || defined(__clang__)))' + ) attributes.append('ms_struct') if attributes: a = f' GCC_ATTR({", ".join(attributes)})' From fe76d459e7b6a3f20f5a00ce6c4a8375d924f953 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sun, 5 May 2024 21:49:54 +0200 Subject: [PATCH 85/91] Remove the `info->size == info->align` assertion for _layout_='ms' The MS compiler aligns data based on its size by default, but the 'ms' layout might be used with other ABIs. --- Modules/_ctypes/cfield.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index e003846158c842..98d05282a216eb 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -168,12 +168,6 @@ PyCField_FromDesc_msvc( self->offset = *poffset - (*pfield_size) / 8; if(is_bitfield) { - if (info->size != info->align) { - PyErr_SetString( - PyExc_TypeError, - "bitfield's base type size differs from alignment"); - return -1; - } assert(0 <= (*pfield_size + *pbitofs)); assert((*pfield_size + *pbitofs) < info->size * 8); self->size = BUILD_SIZE(bitsize, *pfield_size + *pbitofs); From c4714f1be1fee2f6db8735a7900e12322017da58 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sun, 5 May 2024 21:50:06 +0200 Subject: [PATCH 86/91] Fix silly mistake in the macro --- Lib/test/test_ctypes/test_generated_structs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_ctypes/test_generated_structs.py b/Lib/test/test_ctypes/test_generated_structs.py index f8f0c2b90ae66c..f93371c067e8bd 100644 --- a/Lib/test/test_ctypes/test_generated_structs.py +++ b/Lib/test/test_ctypes/test_generated_structs.py @@ -621,13 +621,14 @@ def output(string): // Append VALUE to the result. - #define APPEND(VAL) { \\ - if (!VAL) { \\ + #define APPEND(ITEM) { \\ + PyObject *item = ITEM; \\ + if (!item) { \\ Py_DECREF(result); \\ return NULL; \\ } \\ - int rv = PyList_Append(result, VAL); \\ - Py_DECREF(VAL); \\ + int rv = PyList_Append(result, item); \\ + Py_DECREF(item); \\ if (rv < 0) { \\ Py_DECREF(result); \\ return NULL; \\ From c79725df07e260fe1eb902c7ca5a93e06430a139 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sun, 5 May 2024 22:23:05 +0200 Subject: [PATCH 87/91] Regen --- Modules/_ctypes/_ctypes_test_generated.c.h | 67 +++++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/Modules/_ctypes/_ctypes_test_generated.c.h b/Modules/_ctypes/_ctypes_test_generated.c.h index a1c162b8fe3c27..46a3e4b01e2259 100644 --- a/Modules/_ctypes/_ctypes_test_generated.c.h +++ b/Modules/_ctypes/_ctypes_test_generated.c.h @@ -2,13 +2,14 @@ // Append VALUE to the result. - #define APPEND(VAL) { \ - if (!VAL) { \ + #define APPEND(ITEM) { \ + PyObject *item = ITEM; \ + if (!item) { \ Py_DECREF(result); \ return NULL; \ } \ - int rv = PyList_Append(result, VAL); \ - Py_DECREF(VAL); \ + int rv = PyList_Append(result, item); \ + Py_DECREF(item); \ if (rv < 0) { \ Py_DECREF(result); \ return NULL; \ @@ -214,7 +215,7 @@ if (PyUnicode_CompareWithASCIIString(name, "Packed1") == 0) { - #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Packed1 { @@ -238,7 +239,7 @@ if (PyUnicode_CompareWithASCIIString(name, "Packed2") == 0) { - #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) #pragma pack(push, 2) struct GCC_ATTR(ms_struct) Packed2 { @@ -262,7 +263,7 @@ if (PyUnicode_CompareWithASCIIString(name, "Packed3") == 0) { - #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) #pragma pack(push, 4) struct GCC_ATTR(ms_struct) Packed3 { @@ -286,7 +287,7 @@ if (PyUnicode_CompareWithASCIIString(name, "Packed4") == 0) { - #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) #pragma pack(push, 8) struct GCC_ATTR(ms_struct) Packed4 { @@ -325,6 +326,42 @@ return result; } + if (PyUnicode_CompareWithASCIIString(name, "MSBitFieldExample") == 0) { + + struct MSBitFieldExample { + unsigned int a :4; + unsigned int b :5; + unsigned int c :7; + }; + struct MSBitFieldExample value = {0}; + APPEND(PyUnicode_FromString("MSBitFieldExample")); + APPEND(PyLong_FromLong(sizeof(struct MSBitFieldExample))); + APPEND(PyLong_FromLong(_Alignof(struct MSBitFieldExample))); + TEST_FIELD(unsigned int, value.a); + TEST_FIELD(unsigned int, value.b); + TEST_FIELD(unsigned int, value.c); + return result; + } + + if (PyUnicode_CompareWithASCIIString(name, "MSStraddlingExample") == 0) { + + struct MSStraddlingExample { + unsigned int first :9; + unsigned int second :7; + unsigned int may_straddle :30; + unsigned int last :18; + }; + struct MSStraddlingExample value = {0}; + APPEND(PyUnicode_FromString("MSStraddlingExample")); + APPEND(PyLong_FromLong(sizeof(struct MSStraddlingExample))); + APPEND(PyLong_FromLong(_Alignof(struct MSStraddlingExample))); + TEST_FIELD(unsigned int, value.first); + TEST_FIELD(unsigned int, value.second); + TEST_FIELD(unsigned int, value.may_straddle); + TEST_FIELD(unsigned int, value.last); + return result; + } + if (PyUnicode_CompareWithASCIIString(name, "IntBits") == 0) { struct IntBits { @@ -406,7 +443,7 @@ if (PyUnicode_CompareWithASCIIString(name, "IntBits_MSVC") == 0) { - #if (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) struct GCC_ATTR(ms_struct) IntBits_MSVC { int A :1; @@ -442,7 +479,7 @@ if (PyUnicode_CompareWithASCIIString(name, "Bits_MSVC") == 0) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) struct GCC_ATTR(ms_struct) Bits_MSVC { int A :1; @@ -1602,7 +1639,7 @@ if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_bad") == 0) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Example_gh_84039_bad { @@ -1642,7 +1679,7 @@ if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good_a") == 0) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Example_gh_84039_good_a { @@ -1678,7 +1715,7 @@ if (PyUnicode_CompareWithASCIIString(name, "Example_gh_84039_good") == 0) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Example_gh_84039_good { @@ -1722,7 +1759,7 @@ if (PyUnicode_CompareWithASCIIString(name, "Example_gh_73939") == 0) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Example_gh_73939 { @@ -1786,7 +1823,7 @@ if (PyUnicode_CompareWithASCIIString(name, "Example_gh_86098_pack") == 0) { - #if (!defined(__xlc__)) && (defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__)) + #if (!defined(__xlc__)) && (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) #pragma pack(push, 1) struct GCC_ATTR(ms_struct) Example_gh_86098_pack { From cace0c9a9294de773b100bb4aeb93838b7e7a437 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sun, 5 May 2024 22:18:59 +0200 Subject: [PATCH 88/91] Include field name in error message --- Lib/test/test_ctypes/test_bitfields.py | 6 +++--- Modules/_ctypes/stgdict.c | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index eac30af70055ca..129b14df519360 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -212,10 +212,10 @@ def test_c_wchar(self): def test_single_bitfield_size(self): for c_typ in int_types: result = self.fail_fields(("a", c_typ, -1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) result = self.fail_fields(("a", c_typ, 0)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) class X(Structure): _fields_ = [("a", c_typ, 1)] @@ -226,7 +226,7 @@ class X(Structure): self.assertEqual(sizeof(X), sizeof(c_typ)) result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - self.assertEqual(result, (ValueError, 'number of bits invalid for bit field')) + self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) def test_multi_bitfields_size(self): class X(Structure): diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index a3f74e99eac794..a63a956dd326cc 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -497,8 +497,9 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct return -1; } if (bitsize <= 0 || bitsize > info->size * 8) { - PyErr_SetString(PyExc_ValueError, - "number of bits invalid for bit field"); + PyErr_Format(PyExc_ValueError, + "number of bits invalid for bit field %R", + name); Py_DECREF(pair); return -1; } From 61e92bfbddfcc4368abff1751b82e70e9ee8cd4e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sun, 5 May 2024 22:34:33 +0200 Subject: [PATCH 89/91] Conditionally skip tests that assume sizeof(int64) == alignment(int64) --- Lib/test/test_ctypes/test_bitfields.py | 90 +++++++++++++++----------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/Lib/test/test_ctypes/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py index 129b14df519360..e6509e6bf89e1d 100644 --- a/Lib/test/test_ctypes/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -149,35 +149,41 @@ class X(Structure): def test_signed(self): for c_typ in signed_int_types: - class X(Structure): - _fields_ = [("dummy", c_typ), - ("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)*2) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("dummy", c_typ), + ("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)*2) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, -1, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, -1, 0)) def test_unsigned(self): for c_typ in unsigned_int_types: - class X(Structure): - _fields_ = [("a", c_typ, 3), - ("b", c_typ, 3), - ("c", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) - - x = X() - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) - x.a = -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) - x.a, x.b = 0, -1 - self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + class X(Structure): + _fields_ = [("a", c_typ, 3), + ("b", c_typ, 3), + ("c", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)) + + x = X() + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 0, 0)) + x.a = -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 7, 0, 0)) + x.a, x.b = 0, -1 + self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) def fail_fields(self, *fields): return self.get_except(type(Structure), "X", (), @@ -211,22 +217,28 @@ def test_c_wchar(self): def test_single_bitfield_size(self): for c_typ in int_types: - result = self.fail_fields(("a", c_typ, -1)) - self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) + with self.subTest(c_typ): + if sizeof(c_typ) != alignment(c_typ): + self.skipTest('assumes size=alignment') + result = self.fail_fields(("a", c_typ, -1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) - result = self.fail_fields(("a", c_typ, 0)) - self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) + result = self.fail_fields(("a", c_typ, 0)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) - class X(Structure): - _fields_ = [("a", c_typ, 1)] - self.assertEqual(sizeof(X), sizeof(c_typ)) + class X(Structure): + _fields_ = [("a", c_typ, 1)] + self.assertEqual(sizeof(X), sizeof(c_typ)) - class X(Structure): - _fields_ = [("a", c_typ, sizeof(c_typ)*8)] - self.assertEqual(sizeof(X), sizeof(c_typ)) + class X(Structure): + _fields_ = [("a", c_typ, sizeof(c_typ)*8)] + self.assertEqual(sizeof(X), sizeof(c_typ)) - result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) - self.assertEqual(result, (ValueError, "number of bits invalid for bit field 'a'")) + result = self.fail_fields(("a", c_typ, sizeof(c_typ)*8 + 1)) + self.assertEqual(result, (ValueError, + "number of bits invalid for bit field 'a'")) def test_multi_bitfields_size(self): class X(Structure): @@ -318,6 +330,8 @@ class X(Structure): a.B = 1 self.assertEqual(1, a.B) + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') def test_mixed_7(self): class X(Structure): _fields_ = [ @@ -353,6 +367,8 @@ class X(Structure): else: self.assertEqual(4, sizeof(X)) + @unittest.skipIf(sizeof(c_uint64) != alignment(c_uint64), + 'assumes size=alignment') def test_mixed_10(self): class X(Structure): _fields_ = [ From ba610517ae17b0354ceaaae2bb9d5b8e6399daf8 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Sun, 5 May 2024 22:41:02 +0200 Subject: [PATCH 90/91] Use ms_struct only on Windows and x86/ppc GCC/clang --- Modules/_ctypes/_ctypes_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index a37acaf57aa6ba..2a8d2995052ff5 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -761,7 +761,7 @@ EXPORT(int) unpack_bitfields(struct BITS *bits, char name) return 999; } -#if defined(MS_WIN32) || defined(__GNUC__) || defined(__clang__) +#if (defined(MS_WIN32) || ((defined(__x86_64__) || defined(__i386__) || defined(__ppc64__)) && (defined(__GNUC__) || defined(__clang__)))) struct #ifndef MS_WIN32 __attribute__ ((ms_struct)) From 89914448877a37d453ac99bd9adb38b3eb79a499 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 6 May 2024 11:37:13 +0200 Subject: [PATCH 91/91] Use proper PyArg_ParseTuple code for Py_ssize_t --- Modules/_ctypes/stgdict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index a63a956dd326cc..52d8ec92380b30 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -443,7 +443,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct PyObject *prop; Py_ssize_t bitsize = 0; - if (!pair || !PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { + if (!pair || !PyArg_ParseTuple(pair, "UO|n", &name, &desc, &bitsize)) { PyErr_SetString(PyExc_TypeError, "'_fields_' must be a sequence of (name, C type) pairs"); Py_XDECREF(pair);