From f4fdbf229946aaeb816b31fed7923a7dcf2390e7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 2 Jul 2024 17:42:36 +0200 Subject: [PATCH 01/44] gh-102471: Add PyLong import and export API Co-authored-by: Sergey B Kirpichev --- Doc/c-api/long.rst | 153 ++++++++++++++++++ Doc/conf.py | 2 + Doc/using/configure.rst | 3 +- Doc/whatsnew/3.14.rst | 10 ++ Include/cpython/longintrepr.h | 52 +++++- Lib/test/test_capi/test_long.py | 66 ++++++++ ...-07-03-17-26-53.gh-issue-102471.XpmKYk.rst | 9 ++ Modules/_testcapi/long.c | 144 +++++++++++++++++ Objects/longobject.c | 75 +++++++++ Tools/c-analyzer/cpython/ignored.tsv | 1 + 10 files changed, 512 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 9f2c48d98b8344..c2014c7905196c 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -540,6 +540,9 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Exactly what values are considered compact is an implementation detail and is subject to change. + .. versionadded:: 3.12 + + .. c:function:: Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op) If *op* is compact, as determined by :c:func:`PyUnstable_Long_IsCompact`, @@ -547,3 +550,153 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Otherwise, the return value is undefined. + .. versionadded:: 3.12 + + +Export API +^^^^^^^^^^ + +.. versionadded:: 3.14 + +.. c:type:: Py_digit + + A single unsigned digit in the range [``0``; ``PyLong_BASE - 1``]. + + It is usually used in an *array of digits*, such as the + :c:member:`PyLong_DigitArray.digits` array. + + Its size depend on the :c:macro:`!PYLONG_BITS_IN_DIGIT` macro: + see the ``configure`` :option:`--enable-big-digits` option. + + See :c:member:`PyLong_LAYOUT.bits_per_digit` for the number of bits per + digit and :c:member:`PyLong_LAYOUT.digit_size` for the size of a digit (in + bytes). + + +.. c:struct:: PyLong_LAYOUT + + Layout of an array of digits, used by Python :class:`int` object. + + See also :attr:`sys.int_info` which exposes similar information to Python. + + .. c:member:: uint8_t bits_per_digit + + Bits per digit. + + .. c:member:: uint8_t digit_size + + Digit size in bytes. + + .. c:member:: int8_t word_endian + + Word endian: + + - ``1`` for most significant word first (big endian) + - ``-1`` for least significant first (little endian) + + .. c:member:: int8_t array_endian + + Array endian: + + - ``1`` for most significant byte first (big endian) + - ``-1`` for least significant first (little endian) + + +.. c:struct:: PyLong_DigitArray + + A Python :class:`int` object exported as an array of digits. + + See :c:struct:`PyLong_LAYOUT` for the :c:member:`digits` layout. + + .. c:member:: PyObject *obj + + Strong reference to the Python :class:`int` object. + + .. c:member:: int negative + + 1 if the number is negative, 0 otherwise. + + .. c:member:: Py_ssize_t ndigits + + Number of digits in :c:member:`digits` array. + + .. c:member:: const Py_digit *digits + + Read-only array of unsigned digits. + + +.. c:function:: int PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) + + Export a Python :class:`int` object as an array of digits. + + On success, set *\*array* and return 0. + On error, set an exception and return -1. + + This function always succeeds if *obj* is a Python :class:`int` object or a + subclass. + + :c:func:`PyLong_FreeDigitArray` must be called once done with using + *export*. + + +.. c:function:: void PyLong_FreeDigitArray(PyLong_DigitArray *array) + + Release the export *array* created by :c:func:`PyLong_AsDigitArray`. + + +PyLongWriter API +^^^^^^^^^^^^^^^^ + +The :c:type:`PyLongWriter` API can be used to import an integer. + +.. versionadded:: 3.14 + +.. c:struct:: PyLongWriter + + A Python :class:`int` writer instance. + + The instance must be destroyed by :c:func:`PyLongWriter_Finish`. + + +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, Py_digit **digits) + + Create a :c:type:`PyLongWriter`. + + On success, set *\*digits* and return a writer. + On error, set an exception and return ``NULL``. + + *negative* is ``1`` if the number is negative, or ``0`` otherwise. + + *ndigits* is the number of digits in the *digits* array. It must be + positive. + + The caller must initialize the array of digits *digits* and then call + :c:func:`PyLongWriter_Finish` to get a Python :class:`int`. Digits must be + in the range [``0``; ``PyLong_BASE - 1``]. Unused digits must be set to + ``0``. + + See :c:struct:`PyLong_LAYOUT` for the layout of an array of digits. + + +.. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) + + Finish a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + + On success, return a Python :class:`int` object. + On error, set an exception and return ``NULL``. + + +Example creating an integer from an array of digits:: + + PyObject * + long_import(int negative, Py_ssize_t ndigits, Py_digit *digits) + { + Py_digit *writer_digits; + PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, + &writer_digits); + if (writer == NULL) { + return NULL; + } + memcpy(writer_digits, digits, ndigits * sizeof(digit)); + return PyLongWriter_Finish(writer); + } diff --git a/Doc/conf.py b/Doc/conf.py index 3860d146a27e85..ef4477027b424c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -141,6 +141,8 @@ ('c:type', 'size_t'), ('c:type', 'ssize_t'), ('c:type', 'time_t'), + ('c:type', 'int8_t'), + ('c:type', 'uint8_t'), ('c:type', 'uint64_t'), ('c:type', 'uintmax_t'), ('c:type', 'uintptr_t'), diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 6a4a52bb6e8b12..d1265ff6da1dbb 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -129,7 +129,8 @@ General Options Define the ``PYLONG_BITS_IN_DIGIT`` to ``15`` or ``30``. - See :data:`sys.int_info.bits_per_digit `. + See :data:`sys.int_info.bits_per_digit ` and the + :c:type:`Py_digit` type. .. option:: --with-suffix=SUFFIX diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index aecc7cabd0d1f5..d857d6d5caea2d 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -405,6 +405,16 @@ New Features (Contributed by Victor Stinner in :gh:`119182`.) +* Add a new import and export API for Python :class:`int` objects: + + * :c:func:`PyLong_AsDigitArray`; + * :c:func:`PyLong_FreeDigitArray`; + * :c:func:`PyLongWriter_Create`; + * :c:func:`PyLongWriter_Finish`; + * :c:struct:`PyLong_LAYOUT`. + + (Contributed by Victor Stinner in :gh:`102471`.) + Porting to Python 3.14 ---------------------- diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index c60ccc463653f9..a36d823bbeb44f 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -58,8 +58,10 @@ typedef long stwodigits; /* signed variant of twodigits */ #else #error "PYLONG_BITS_IN_DIGIT should be 15 or 30" #endif -#define PyLong_BASE ((digit)1 << PyLong_SHIFT) -#define PyLong_MASK ((digit)(PyLong_BASE - 1)) +#define PyLong_BASE ((Py_digit)1 << PyLong_SHIFT) +#define PyLong_MASK ((Py_digit)(PyLong_BASE - 1)) + +typedef digit Py_digit; /* Long integer representation. @@ -139,6 +141,52 @@ _PyLong_CompactValue(const PyLongObject *op) #define PyUnstable_Long_CompactValue _PyLong_CompactValue +/* --- Import/Export API -------------------------------------------------- */ + +typedef struct PyLongLayout { + // Bits per digit + uint8_t bits_per_digit; + + // Digit size in bytes + uint8_t digit_size; + + // Word endian: + // * 1 for most significant word first (big endian) + // * -1 for least significant first (little endian) + int8_t word_endian; + + // Array endian: + // * 1 for most significant byte first (big endian) + // * -1 for least significant first (little endian) + int8_t array_endian; +} PyLongLayout; + +PyAPI_DATA(const PyLongLayout) PyLong_LAYOUT; + +typedef struct PyLong_DigitArray { + PyObject *obj; + int negative; + Py_ssize_t ndigits; + const Py_digit *digits; +} PyLong_DigitArray; + +PyAPI_FUNC(int) PyLong_AsDigitArray( + PyObject *obj, + PyLong_DigitArray *array); +PyAPI_FUNC(void) PyLong_FreeDigitArray( + PyLong_DigitArray *array); + + +/* --- PyLongWriter API --------------------------------------------------- */ + +typedef struct PyLongWriter PyLongWriter; + +PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( + int negative, + Py_ssize_t ndigits, + Py_digit **digits); +PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index bdbdd7bcfe0f2a..b5c713b0d16620 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -631,6 +631,72 @@ def test_long_getsign(self): # CRASHES getsign(NULL) + def test_long_layout(self): + # Test PyLong_LAYOUT + int_info = sys.int_info + layout = _testcapi.get_pylong_layout() + expected = { + 'array_endian': -1, + 'bits_per_digit': int_info.bits_per_digit, + 'digit_size': int_info.sizeof_digit, + 'word_endian': -1 if sys.byteorder == 'little' else 1, + } + self.assertEqual(layout, expected) + + def test_long_export(self): + # Test PyLong_Export() + layout = _testcapi.get_pylong_layout() + base = 2 ** layout['bits_per_digit'] + + pylong_export = _testcapi.pylong_export + self.assertEqual(pylong_export(0), (0, [0])) + self.assertEqual(pylong_export(123), (0, [123])) + self.assertEqual(pylong_export(-123), (1, [123])) + self.assertEqual(pylong_export(base**2 * 3 + base * 2 + 1), + (0, [1, 2, 3])) + + with self.assertRaises(TypeError): + pylong_export(1.0) + with self.assertRaises(TypeError): + pylong_export(0+1j) + with self.assertRaises(TypeError): + pylong_export("abc") + + def test_longwriter_create(self): + # Test PyLong_Import() + layout = _testcapi.get_pylong_layout() + base = 2 ** layout['bits_per_digit'] + + pylongwriter_create = _testcapi.pylongwriter_create + self.assertEqual(pylongwriter_create(0, []), 0) + self.assertEqual(pylongwriter_create(0, [0]), 0) + self.assertEqual(pylongwriter_create(0, [123]), 123) + self.assertEqual(pylongwriter_create(1, [123]), -123) + self.assertEqual(pylongwriter_create(1, [1, 2]), + -(base * 2 + 1)) + self.assertEqual(pylongwriter_create(0, [1, 2, 3]), + base**2 * 3 + base * 2 + 1) + max_digit = base - 1 + self.assertEqual(pylongwriter_create(0, [max_digit, max_digit, max_digit]), + base**2 * max_digit + base * max_digit + max_digit) + + # normalize + self.assertEqual(pylongwriter_create(0, [123, 0, 0]), 123) + + # test singletons + normalize + for num in (-2, 0, 1, 5, 42, 100): + self.assertIs(pylongwriter_create(bool(num < 0), [abs(num), 0]), + num) + + # round trip: Python int -> export -> Python int + pylong_export = _testcapi.pylong_export + numbers = [*range(0, 10), 12345, 0xdeadbeef, 2**100, 2**100-1] + numbers.extend(-num for num in list(numbers)) + for num in numbers: + with self.subTest(num=num): + export = pylong_export(num) + self.assertEqual(pylongwriter_create(*export), num, export) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst b/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst new file mode 100644 index 00000000000000..c29cb25833181c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst @@ -0,0 +1,9 @@ +Add a new import and export API for Python :class:`int` objects: + +* :c:func:`PyLong_AsDigitArray`; +* :c:func:`PyLong_FreeDigitArray`; +* :c:func:`PyLongWriter_Create`; +* :c:func:`PyLongWriter_Finish`; +* :c:struct:`PyLong_LAYOUT`. + +Patch by Victor Stinner. diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 2b5e85d5707522..ec08e8d2396dc8 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -117,6 +117,147 @@ pylong_aspid(PyObject *module, PyObject *arg) } +static PyObject * +pylong_export(PyObject *module, PyObject *obj) +{ + PyLong_DigitArray array; + if (PyLong_AsDigitArray(obj, &array) < 0) { + return NULL; + } + + PyObject *digits = PyList_New(0); + for (Py_ssize_t i=0; i < array.ndigits; i++) { + PyObject *digit = PyLong_FromUnsignedLong(array.digits[i]); + if (digit == NULL) { + Py_DECREF(digits); + goto error; + } + + if (PyList_Append(digits, digit) < 0) { + Py_DECREF(digits); + Py_DECREF(digit); + goto error; + } + Py_DECREF(digit); + } + + PyObject *res = Py_BuildValue("(iN)", array.negative, digits); + PyLong_FreeDigitArray(&array); + return res; + +error: + PyLong_FreeDigitArray(&array); + return NULL; +} + + +static PyObject * +pylongwriter_create(PyObject *module, PyObject *args) +{ + int negative; + PyObject *list; + if (!PyArg_ParseTuple(args, "iO!", &negative, &PyList_Type, &list)) { + return NULL; + } + Py_ssize_t ndigits = PyList_GET_SIZE(list); + + Py_digit *digits = PyMem_Malloc(ndigits * sizeof(Py_digit)); + if (digits == NULL) { + PyErr_NoMemory(); + return NULL; + } + + for (Py_ssize_t i=0; i < ndigits; i++) { + PyObject *item = PyList_GET_ITEM(list, i); + + long digit = PyLong_AsLong(item); + if (digit == -1 && PyErr_Occurred()) { + goto error; + } + + if (digit < 0 || digit >= PyLong_BASE) { + PyErr_SetString(PyExc_ValueError, "digit doesn't fit into Py_digit"); + goto error; + } + digits[i] = (Py_digit)digit; + } + + Py_digit *writer_digits; + PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, + &writer_digits); + if (writer == NULL) { + goto error; + } + memcpy(writer_digits, digits, ndigits * sizeof(digit)); + PyObject *res = PyLongWriter_Finish(writer); + PyMem_Free(digits); + + return res; + +error: + PyMem_Free(digits); + return NULL; +} + + +static PyObject * +get_pylong_layout(PyObject *module, PyObject *Py_UNUSED(args)) +{ + PyLongLayout layout = PyLong_LAYOUT; + + PyObject *dict = PyDict_New(); + if (dict == NULL) { + goto error; + } + + PyObject *value = PyLong_FromUnsignedLong(layout.bits_per_digit); + if (value == NULL) { + goto error; + } + int res = PyDict_SetItemString(dict, "bits_per_digit", value); + Py_DECREF(value); + if (res < 0) { + goto error; + } + + value = PyLong_FromUnsignedLong(layout.digit_size); + if (value == NULL) { + goto error; + } + res = PyDict_SetItemString(dict, "digit_size", value); + Py_DECREF(value); + if (res < 0) { + goto error; + } + + value = PyLong_FromLong(layout.word_endian); + if (value == NULL) { + goto error; + } + res = PyDict_SetItemString(dict, "word_endian", value); + Py_DECREF(value); + if (res < 0) { + goto error; + } + + value = PyLong_FromLong(layout.array_endian); + if (value == NULL) { + goto error; + } + res = PyDict_SetItemString(dict, "array_endian", value); + Py_DECREF(value); + if (res < 0) { + goto error; + } + + return dict; + +error: + Py_XDECREF(dict); + return NULL; +} + + static PyMethodDef test_methods[] = { _TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF {"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS}, @@ -124,6 +265,9 @@ static PyMethodDef test_methods[] = { {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, {"pylong_getsign", pylong_getsign, METH_O}, {"pylong_aspid", pylong_aspid, METH_O}, + {"pylong_export", pylong_export, METH_O}, + {"pylongwriter_create", pylongwriter_create, METH_VARARGS}, + {"get_pylong_layout", get_pylong_layout, METH_NOARGS}, {NULL}, }; diff --git a/Objects/longobject.c b/Objects/longobject.c index 050ce1a7303842..a7ffeffa90a4fc 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6727,3 +6727,78 @@ Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op) { return _PyLong_CompactValue((PyLongObject*)op); } + +const PyLongLayout PyLong_LAYOUT = { + .bits_per_digit = PyLong_SHIFT, + .word_endian = PY_LITTLE_ENDIAN ? -1 : 1, + .array_endian = -1, // least significant first + .digit_size = sizeof(digit), +}; + + +int +PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) +{ + if (!PyLong_Check(obj)) { + PyErr_Format(PyExc_TypeError, "expect int, got %T", obj); + return -1; + } + PyLongObject *self = (PyLongObject*)obj; + + array->obj = Py_NewRef(obj); + array->negative = _PyLong_IsNegative(self); + array->ndigits = _PyLong_DigitCount(self); + if (array->ndigits == 0) { + array->ndigits = 1; + } + array->digits = self->long_value.ob_digit; + return 0; +} + + +void +PyLong_FreeDigitArray(PyLong_DigitArray *array) +{ + Py_CLEAR(array->obj); + array->negative = 0; + array->ndigits = 0; + array->digits = NULL; +} + + +/* --- PyLongWriter API --------------------------------------------------- */ + +PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, Py_digit **digits) +{ + if (ndigits < 0) { + PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); + return NULL; + } + assert(digits != NULL); + + PyLongObject *obj = _PyLong_New(ndigits); + if (obj == NULL) { + return NULL; + } + if (ndigits == 0) { + assert(obj->long_value.ob_digit[0] == 0); + } + if (negative) { + _PyLong_FlipSign(obj); + } + + *digits = obj->long_value.ob_digit; + return (PyLongWriter*)obj; +} + + +PyObject* PyLongWriter_Finish(PyLongWriter *writer) +{ + PyLongObject *obj = (PyLongObject *)writer; + assert(Py_REFCNT(obj) == 1); + + // Normalize and get singleton if possible + obj = maybe_small_long(long_normalize(obj)); + + return (PyObject*)obj; +} diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 63b640e465ac6b..f17f8cfa0faa44 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -318,6 +318,7 @@ Objects/exceptions.c - static_exceptions - Objects/genobject.c - ASYNC_GEN_IGNORED_EXIT_MSG - Objects/genobject.c - NON_INIT_CORO_MSG - Objects/longobject.c - _PyLong_DigitValue - +Objects/longobject.c - PyLong_LAYOUT - Objects/object.c - _Py_SwappedOp - Objects/object.c - _Py_abstract_hack - Objects/object.c - last_final_reftotal - From c2e568e457efc082f73d8c0a8adab01d83e2c420 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 23 Jul 2024 17:27:46 +0200 Subject: [PATCH 02/44] Add layout --- Doc/c-api/long.rst | 27 ++- Doc/whatsnew/3.14.rst | 4 +- Include/cpython/longintrepr.h | 6 +- Lib/test/test_capi/test_long.py | 63 ++++-- ...-07-03-17-26-53.gh-issue-102471.XpmKYk.rst | 4 +- Modules/_testcapi/long.c | 205 +++++++++++++----- Objects/longobject.c | 17 +- 7 files changed, 231 insertions(+), 95 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index c2014c7905196c..1fa613a7286819 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -568,15 +568,18 @@ Export API Its size depend on the :c:macro:`!PYLONG_BITS_IN_DIGIT` macro: see the ``configure`` :option:`--enable-big-digits` option. - See :c:member:`PyLong_LAYOUT.bits_per_digit` for the number of bits per - digit and :c:member:`PyLong_LAYOUT.digit_size` for the size of a digit (in + See :c:member:`PyLongLayout.bits_per_digit` for the number of bits per + digit and :c:member:`PyLongLayout.digit_size` for the size of a digit (in bytes). -.. c:struct:: PyLong_LAYOUT +.. c:struct:: PyLongLayout Layout of an array of digits, used by Python :class:`int` object. + Use :c:func:`PyLong_GetNativeLayout` to get the native layout of Python + :class:`int` objects. + See also :attr:`sys.int_info` which exposes similar information to Python. .. c:member:: uint8_t bits_per_digit @@ -602,12 +605,17 @@ Export API - ``-1`` for least significant first (little endian) +.. c:function:: const PyLongLayout* PyLong_GetNativeLayout(void) + + Get the native layout of Python :class:`int` objects. + + See the :c:struct:`PyLongLayout` structure. + + .. c:struct:: PyLong_DigitArray A Python :class:`int` object exported as an array of digits. - See :c:struct:`PyLong_LAYOUT` for the :c:member:`digits` layout. - .. c:member:: PyObject *obj Strong reference to the Python :class:`int` object. @@ -624,6 +632,10 @@ Export API Read-only array of unsigned digits. + .. c:member:: const PyLongLayout *layout + + Layout of the :c:member:`digits`. + .. c:function:: int PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) @@ -658,7 +670,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. The instance must be destroyed by :c:func:`PyLongWriter_Finish`. -.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, Py_digit **digits) +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, Py_digit **digits, const PyLongLayout *layout) Create a :c:type:`PyLongWriter`. @@ -675,7 +687,8 @@ The :c:type:`PyLongWriter` API can be used to import an integer. in the range [``0``; ``PyLong_BASE - 1``]. Unused digits must be set to ``0``. - See :c:struct:`PyLong_LAYOUT` for the layout of an array of digits. + *layout* is the layout of *digits*: it must be equal to + :c:func:`PyLong_GetNativeLayout`. .. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index d857d6d5caea2d..4c7d25e3d872a8 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -407,11 +407,11 @@ New Features * Add a new import and export API for Python :class:`int` objects: + * :c:func:`PyLong_GetNativeLayout`; * :c:func:`PyLong_AsDigitArray`; * :c:func:`PyLong_FreeDigitArray`; * :c:func:`PyLongWriter_Create`; - * :c:func:`PyLongWriter_Finish`; - * :c:struct:`PyLong_LAYOUT`. + * :c:func:`PyLongWriter_Finish`. (Contributed by Victor Stinner in :gh:`102471`.) diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index a36d823bbeb44f..5ceb6c1b2827b3 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -161,13 +161,14 @@ typedef struct PyLongLayout { int8_t array_endian; } PyLongLayout; -PyAPI_DATA(const PyLongLayout) PyLong_LAYOUT; +PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); typedef struct PyLong_DigitArray { PyObject *obj; int negative; Py_ssize_t ndigits; const Py_digit *digits; + const PyLongLayout *layout; } PyLong_DigitArray; PyAPI_FUNC(int) PyLong_AsDigitArray( @@ -184,7 +185,8 @@ typedef struct PyLongWriter PyLongWriter; PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( int negative, Py_ssize_t ndigits, - Py_digit **digits); + Py_digit **digits, + const PyLongLayout *layout); PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); #ifdef __cplusplus diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index b5c713b0d16620..9b7e7a72d2cdfb 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -648,19 +648,19 @@ def test_long_export(self): layout = _testcapi.get_pylong_layout() base = 2 ** layout['bits_per_digit'] - pylong_export = _testcapi.pylong_export - self.assertEqual(pylong_export(0), (0, [0])) - self.assertEqual(pylong_export(123), (0, [123])) - self.assertEqual(pylong_export(-123), (1, [123])) - self.assertEqual(pylong_export(base**2 * 3 + base * 2 + 1), - (0, [1, 2, 3])) + pylong_asdigitarray = _testcapi.pylong_asdigitarray + self.assertEqual(pylong_asdigitarray(0), (0, [0], layout)) + self.assertEqual(pylong_asdigitarray(123), (0, [123], layout)) + self.assertEqual(pylong_asdigitarray(-123), (1, [123], layout)) + self.assertEqual(pylong_asdigitarray(base**2 * 3 + base * 2 + 1), + (0, [1, 2, 3], layout)) with self.assertRaises(TypeError): - pylong_export(1.0) + pylong_asdigitarray(1.0) with self.assertRaises(TypeError): - pylong_export(0+1j) + pylong_asdigitarray(0+1j) with self.assertRaises(TypeError): - pylong_export("abc") + pylong_asdigitarray("abc") def test_longwriter_create(self): # Test PyLong_Import() @@ -668,34 +668,55 @@ def test_longwriter_create(self): base = 2 ** layout['bits_per_digit'] pylongwriter_create = _testcapi.pylongwriter_create - self.assertEqual(pylongwriter_create(0, []), 0) - self.assertEqual(pylongwriter_create(0, [0]), 0) - self.assertEqual(pylongwriter_create(0, [123]), 123) - self.assertEqual(pylongwriter_create(1, [123]), -123) - self.assertEqual(pylongwriter_create(1, [1, 2]), + self.assertEqual(pylongwriter_create(0, [], layout), 0) + self.assertEqual(pylongwriter_create(0, [0], layout), 0) + self.assertEqual(pylongwriter_create(0, [123], layout), 123) + self.assertEqual(pylongwriter_create(1, [123], layout), -123) + self.assertEqual(pylongwriter_create(1, [1, 2], layout), -(base * 2 + 1)) - self.assertEqual(pylongwriter_create(0, [1, 2, 3]), + self.assertEqual(pylongwriter_create(0, [1, 2, 3], layout), base**2 * 3 + base * 2 + 1) max_digit = base - 1 - self.assertEqual(pylongwriter_create(0, [max_digit, max_digit, max_digit]), + self.assertEqual(pylongwriter_create(0, [max_digit, max_digit, max_digit], layout), base**2 * max_digit + base * max_digit + max_digit) # normalize - self.assertEqual(pylongwriter_create(0, [123, 0, 0]), 123) + self.assertEqual(pylongwriter_create(0, [123, 0, 0], layout), 123) # test singletons + normalize for num in (-2, 0, 1, 5, 42, 100): - self.assertIs(pylongwriter_create(bool(num < 0), [abs(num), 0]), + self.assertIs(pylongwriter_create(bool(num < 0), [abs(num), 0], layout), num) # round trip: Python int -> export -> Python int - pylong_export = _testcapi.pylong_export + pylong_asdigitarray = _testcapi.pylong_asdigitarray numbers = [*range(0, 10), 12345, 0xdeadbeef, 2**100, 2**100-1] numbers.extend(-num for num in list(numbers)) for num in numbers: with self.subTest(num=num): - export = pylong_export(num) - self.assertEqual(pylongwriter_create(*export), num, export) + negative, digits, layout = pylong_asdigitarray(num) + self.assertEqual(pylongwriter_create(negative, digits, layout), num, + (negative, digits, layout)) + + # invalid layout + with self.assertRaises(ValueError): + layout = _testcapi.get_pylong_layout() + layout['bits_per_digit'] += 1 + pylongwriter_create(0, [123], layout) + with self.assertRaises(ValueError): + layout = _testcapi.get_pylong_layout() + layout['digit_size'] *= 2 + pylongwriter_create(0, [123], layout) + def change_endian(endian): + return (-endian if endian else 1) + with self.assertRaises(ValueError): + layout = _testcapi.get_pylong_layout() + layout['word_endian'] = change_endian(layout['word_endian']) + pylongwriter_create(0, [123], layout) + with self.assertRaises(ValueError): + layout = _testcapi.get_pylong_layout() + layout['array_endian'] = change_endian(layout['array_endian']) + pylongwriter_create(0, [123], layout) if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst b/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst index c29cb25833181c..16b99959f2b203 100644 --- a/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst +++ b/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst @@ -1,9 +1,9 @@ Add a new import and export API for Python :class:`int` objects: +* :c:func:`PyLong_GetNativeLayout`; * :c:func:`PyLong_AsDigitArray`; * :c:func:`PyLong_FreeDigitArray`; * :c:func:`PyLongWriter_Create`; -* :c:func:`PyLongWriter_Finish`; -* :c:struct:`PyLong_LAYOUT`. +* :c:func:`PyLongWriter_Finish`. Patch by Victor Stinner. diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index ec08e8d2396dc8..10bbe8afe65934 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -118,7 +118,128 @@ pylong_aspid(PyObject *module, PyObject *arg) static PyObject * -pylong_export(PyObject *module, PyObject *obj) +layout_to_dict(const PyLongLayout *layout) +{ + PyObject *dict = PyDict_New(); + if (dict == NULL) { + goto error; + } + + PyObject *value = PyLong_FromUnsignedLong(layout->bits_per_digit); + if (value == NULL) { + goto error; + } + int res = PyDict_SetItemString(dict, "bits_per_digit", value); + Py_DECREF(value); + if (res < 0) { + goto error; + } + + value = PyLong_FromUnsignedLong(layout->digit_size); + if (value == NULL) { + goto error; + } + res = PyDict_SetItemString(dict, "digit_size", value); + Py_DECREF(value); + if (res < 0) { + goto error; + } + + value = PyLong_FromLong(layout->word_endian); + if (value == NULL) { + goto error; + } + res = PyDict_SetItemString(dict, "word_endian", value); + Py_DECREF(value); + if (res < 0) { + goto error; + } + + value = PyLong_FromLong(layout->array_endian); + if (value == NULL) { + goto error; + } + res = PyDict_SetItemString(dict, "array_endian", value); + Py_DECREF(value); + if (res < 0) { + goto error; + } + + return dict; + +error: + Py_XDECREF(dict); + return NULL; +} + + +static int +layout_from_dict_get(PyObject *dict, const char *key, int is_signed, long *pvalue) +{ + PyObject *item; + int res = PyDict_GetItemStringRef(dict, key, &item); + if (res <= 0) { + return -1; + } + + long value = PyLong_AsLong(item); + Py_DECREF(item); + if (value == -1 && PyErr_Occurred()) { + return -1; + } + + if (is_signed) { + if (value < INT8_MIN || value > INT8_MAX) { + return -1; + } + } + else { + if (value < 0 || value > (long)UINT8_MAX) { + return -1; + } + } + + *pvalue = value; + return 0; +} + + +static int +layout_from_dict(PyLongLayout *layout, PyObject *dict) +{ + long value; + if (layout_from_dict_get(dict, "bits_per_digit", 0, &value) < 0) { + goto error; + } + layout->bits_per_digit = (uint8_t)value; + + if (layout_from_dict_get(dict, "digit_size", 0, &value) < 0) { + goto error; + } + layout->digit_size = (uint8_t)value; + + if (layout_from_dict_get(dict, "word_endian", 1, &value) < 0) { + goto error; + } + layout->word_endian = (int8_t)value; + + if (layout_from_dict_get(dict, "array_endian", 1, &value) < 0) { + goto error; + } + layout->array_endian = (int8_t)value; + + return 0; + +error: + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, "invalid layout"); + } + return -1; +} + + +static PyObject * +pylong_asdigitarray(PyObject *module, PyObject *obj) { PyLong_DigitArray array; if (PyLong_AsDigitArray(obj, &array) < 0) { @@ -129,23 +250,27 @@ pylong_export(PyObject *module, PyObject *obj) for (Py_ssize_t i=0; i < array.ndigits; i++) { PyObject *digit = PyLong_FromUnsignedLong(array.digits[i]); if (digit == NULL) { - Py_DECREF(digits); goto error; } if (PyList_Append(digits, digit) < 0) { - Py_DECREF(digits); Py_DECREF(digit); goto error; } Py_DECREF(digit); } - PyObject *res = Py_BuildValue("(iN)", array.negative, digits); + PyObject *layout_dict = layout_to_dict(array.layout); + if (layout_dict == NULL) { + goto error; + } + + PyObject *res = Py_BuildValue("(iNN)", array.negative, digits, layout_dict); PyLong_FreeDigitArray(&array); return res; error: + Py_DECREF(digits); PyLong_FreeDigitArray(&array); return NULL; } @@ -155,12 +280,22 @@ static PyObject * pylongwriter_create(PyObject *module, PyObject *args) { int negative; - PyObject *list; - if (!PyArg_ParseTuple(args, "iO!", &negative, &PyList_Type, &list)) { + PyObject *list, *layout_dict; + if (!PyArg_ParseTuple(args, "iO!O!", + &negative, + &PyList_Type, &list, + &PyDict_Type, &layout_dict)) + { return NULL; } Py_ssize_t ndigits = PyList_GET_SIZE(list); + PyLongLayout layout; + memset(&layout, 0, sizeof(layout)); + if (layout_from_dict(&layout, layout_dict) < 0) { + return NULL; + } + Py_digit *digits = PyMem_Malloc(ndigits * sizeof(Py_digit)); if (digits == NULL) { PyErr_NoMemory(); @@ -184,7 +319,7 @@ pylongwriter_create(PyObject *module, PyObject *args) Py_digit *writer_digits; PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, - &writer_digits); + &writer_digits, &layout); if (writer == NULL) { goto error; } @@ -203,58 +338,8 @@ pylongwriter_create(PyObject *module, PyObject *args) static PyObject * get_pylong_layout(PyObject *module, PyObject *Py_UNUSED(args)) { - PyLongLayout layout = PyLong_LAYOUT; - - PyObject *dict = PyDict_New(); - if (dict == NULL) { - goto error; - } - - PyObject *value = PyLong_FromUnsignedLong(layout.bits_per_digit); - if (value == NULL) { - goto error; - } - int res = PyDict_SetItemString(dict, "bits_per_digit", value); - Py_DECREF(value); - if (res < 0) { - goto error; - } - - value = PyLong_FromUnsignedLong(layout.digit_size); - if (value == NULL) { - goto error; - } - res = PyDict_SetItemString(dict, "digit_size", value); - Py_DECREF(value); - if (res < 0) { - goto error; - } - - value = PyLong_FromLong(layout.word_endian); - if (value == NULL) { - goto error; - } - res = PyDict_SetItemString(dict, "word_endian", value); - Py_DECREF(value); - if (res < 0) { - goto error; - } - - value = PyLong_FromLong(layout.array_endian); - if (value == NULL) { - goto error; - } - res = PyDict_SetItemString(dict, "array_endian", value); - Py_DECREF(value); - if (res < 0) { - goto error; - } - - return dict; - -error: - Py_XDECREF(dict); - return NULL; + const PyLongLayout *layout = PyLong_GetNativeLayout(); + return layout_to_dict(layout); } @@ -265,7 +350,7 @@ static PyMethodDef test_methods[] = { {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, {"pylong_getsign", pylong_getsign, METH_O}, {"pylong_aspid", pylong_aspid, METH_O}, - {"pylong_export", pylong_export, METH_O}, + {"pylong_asdigitarray", pylong_asdigitarray, METH_O}, {"pylongwriter_create", pylongwriter_create, METH_VARARGS}, {"get_pylong_layout", get_pylong_layout, METH_NOARGS}, {NULL}, diff --git a/Objects/longobject.c b/Objects/longobject.c index a7ffeffa90a4fc..03b114adf0e445 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6736,6 +6736,12 @@ const PyLongLayout PyLong_LAYOUT = { }; +const PyLongLayout* PyLong_GetNativeLayout(void) +{ + return &PyLong_LAYOUT; +} + + int PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) { @@ -6752,6 +6758,7 @@ PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) array->ndigits = 1; } array->digits = self->long_value.ob_digit; + array->layout = &PyLong_LAYOUT; return 0; } @@ -6768,13 +6775,21 @@ PyLong_FreeDigitArray(PyLong_DigitArray *array) /* --- PyLongWriter API --------------------------------------------------- */ -PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, Py_digit **digits) +PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, Py_digit **digits, + const PyLongLayout *layout) { if (ndigits < 0) { PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); return NULL; } assert(digits != NULL); + // First, compare pointers (fast-path) since it's faster + if (layout != &PyLong_LAYOUT + && (memcmp(layout, &PyLong_LAYOUT, sizeof(*layout)) != 0)) + { + PyErr_SetString(PyExc_ValueError, "only the native layout is supported"); + return NULL; + } PyLongObject *obj = _PyLong_New(ndigits); if (obj == NULL) { From b19764f2d987f4b2314be5a2b39031002852f64b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 3 Sep 2024 18:57:16 +0200 Subject: [PATCH 03/44] Rename word_endian to digits_order Rename also array_endian to endian. --- Doc/c-api/long.rst | 4 ++-- Include/cpython/longintrepr.h | 4 ++-- Lib/test/test_capi/test_long.py | 8 ++++---- Modules/_testcapi/long.c | 16 ++++++++-------- Objects/longobject.c | 4 ++-- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index a5f1c813224c4f..184e9382da3bac 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -647,14 +647,14 @@ Export API Digit size in bytes. - .. c:member:: int8_t word_endian + .. c:member:: int8_t digits_order Word endian: - ``1`` for most significant word first (big endian) - ``-1`` for least significant first (little endian) - .. c:member:: int8_t array_endian + .. c:member:: int8_t endian Array endian: diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 5ceb6c1b2827b3..e3bd1c9b1a2854 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -153,12 +153,12 @@ typedef struct PyLongLayout { // Word endian: // * 1 for most significant word first (big endian) // * -1 for least significant first (little endian) - int8_t word_endian; + int8_t digits_order; // Array endian: // * 1 for most significant byte first (big endian) // * -1 for least significant first (little endian) - int8_t array_endian; + int8_t endian; } PyLongLayout; PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 5d38c577cd267b..210fbf9437e7d3 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -674,10 +674,10 @@ def test_long_layout(self): int_info = sys.int_info layout = _testcapi.get_pylong_layout() expected = { - 'array_endian': -1, + 'endian': -1, 'bits_per_digit': int_info.bits_per_digit, 'digit_size': int_info.sizeof_digit, - 'word_endian': -1 if sys.byteorder == 'little' else 1, + 'digits_order': -1 if sys.byteorder == 'little' else 1, } self.assertEqual(layout, expected) @@ -749,11 +749,11 @@ def change_endian(endian): return (-endian if endian else 1) with self.assertRaises(ValueError): layout = _testcapi.get_pylong_layout() - layout['word_endian'] = change_endian(layout['word_endian']) + layout['digits_order'] = change_endian(layout['digits_order']) pylongwriter_create(0, [123], layout) with self.assertRaises(ValueError): layout = _testcapi.get_pylong_layout() - layout['array_endian'] = change_endian(layout['array_endian']) + layout['endian'] = change_endian(layout['endian']) pylongwriter_create(0, [123], layout) diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 10bbe8afe65934..5e4a4db1e3e171 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -145,21 +145,21 @@ layout_to_dict(const PyLongLayout *layout) goto error; } - value = PyLong_FromLong(layout->word_endian); + value = PyLong_FromLong(layout->digits_order); if (value == NULL) { goto error; } - res = PyDict_SetItemString(dict, "word_endian", value); + res = PyDict_SetItemString(dict, "digits_order", value); Py_DECREF(value); if (res < 0) { goto error; } - value = PyLong_FromLong(layout->array_endian); + value = PyLong_FromLong(layout->endian); if (value == NULL) { goto error; } - res = PyDict_SetItemString(dict, "array_endian", value); + res = PyDict_SetItemString(dict, "endian", value); Py_DECREF(value); if (res < 0) { goto error; @@ -218,15 +218,15 @@ layout_from_dict(PyLongLayout *layout, PyObject *dict) } layout->digit_size = (uint8_t)value; - if (layout_from_dict_get(dict, "word_endian", 1, &value) < 0) { + if (layout_from_dict_get(dict, "digits_order", 1, &value) < 0) { goto error; } - layout->word_endian = (int8_t)value; + layout->digits_order = (int8_t)value; - if (layout_from_dict_get(dict, "array_endian", 1, &value) < 0) { + if (layout_from_dict_get(dict, "endian", 1, &value) < 0) { goto error; } - layout->array_endian = (int8_t)value; + layout->endian = (int8_t)value; return 0; diff --git a/Objects/longobject.c b/Objects/longobject.c index cce711b5d3833b..127a908c026747 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6778,8 +6778,8 @@ int PyLong_AsUInt64(PyObject *obj, uint64_t *value) const PyLongLayout PyLong_LAYOUT = { .bits_per_digit = PyLong_SHIFT, - .word_endian = PY_LITTLE_ENDIAN ? -1 : 1, - .array_endian = -1, // least significant first + .digits_order = PY_LITTLE_ENDIAN ? -1 : 1, + .endian = -1, // least significant first .digit_size = sizeof(digit), }; From 6f7fd11e360c55d71d423f919391ddfaa8f1141a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 3 Sep 2024 19:20:51 +0200 Subject: [PATCH 04/44] Replace Py_digit* type with void* --- Doc/c-api/long.rst | 10 ++++++---- Include/cpython/longintrepr.h | 4 ++-- Modules/_testcapi/long.c | 10 +++++++--- Objects/longobject.c | 2 +- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 184e9382da3bac..24564f93cdcaeb 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -685,7 +685,7 @@ Export API Number of digits in :c:member:`digits` array. - .. c:member:: const Py_digit *digits + .. c:member:: const void *digits Read-only array of unsigned digits. @@ -727,7 +727,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. The instance must be destroyed by :c:func:`PyLongWriter_Finish`. -.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, Py_digit **digits, const PyLongLayout *layout) +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits, const PyLongLayout *layout) Create a :c:type:`PyLongWriter`. @@ -761,12 +761,14 @@ Example creating an integer from an array of digits:: PyObject * long_import(int negative, Py_ssize_t ndigits, Py_digit *digits) { - Py_digit *writer_digits; + void *writer_digits; PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, &writer_digits); if (writer == NULL) { return NULL; } - memcpy(writer_digits, digits, ndigits * sizeof(digit)); + + assert(layout.digit_size == sizeof(Py_digit)); + memcpy(writer_digits, digits, ndigits * sizeof(Py_digit)); return PyLongWriter_Finish(writer); } diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index e3bd1c9b1a2854..a4d3e75e224252 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -167,7 +167,7 @@ typedef struct PyLong_DigitArray { PyObject *obj; int negative; Py_ssize_t ndigits; - const Py_digit *digits; + const void *digits; const PyLongLayout *layout; } PyLong_DigitArray; @@ -185,7 +185,7 @@ typedef struct PyLongWriter PyLongWriter; PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( int negative, Py_ssize_t ndigits, - Py_digit **digits, + void **digits, const PyLongLayout *layout); PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 5e4a4db1e3e171..e6e7db848d4945 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -246,9 +246,12 @@ pylong_asdigitarray(PyObject *module, PyObject *obj) return NULL; } + assert(array.layout->digit_size == sizeof(Py_digit)); + const Py_digit *array_digits = array.digits; + PyObject *digits = PyList_New(0); for (Py_ssize_t i=0; i < array.ndigits; i++) { - PyObject *digit = PyLong_FromUnsignedLong(array.digits[i]); + PyObject *digit = PyLong_FromUnsignedLong(array_digits[i]); if (digit == NULL) { goto error; } @@ -317,13 +320,14 @@ pylongwriter_create(PyObject *module, PyObject *args) digits[i] = (Py_digit)digit; } - Py_digit *writer_digits; + void *writer_digits; PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, &writer_digits, &layout); if (writer == NULL) { goto error; } - memcpy(writer_digits, digits, ndigits * sizeof(digit)); + assert(layout.digit_size == sizeof(Py_digit)); + memcpy(writer_digits, digits, ndigits * sizeof(Py_digit)); PyObject *res = PyLongWriter_Finish(writer); PyMem_Free(digits); diff --git a/Objects/longobject.c b/Objects/longobject.c index 127a908c026747..205cbfe69d6170 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6823,7 +6823,7 @@ PyLong_FreeDigitArray(PyLong_DigitArray *array) /* --- PyLongWriter API --------------------------------------------------- */ -PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, Py_digit **digits, +PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits, const PyLongLayout *layout) { if (ndigits < 0) { From 080e079a8be4936cc6339efdc4599b7cda0472ea Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Sep 2024 13:36:50 +0200 Subject: [PATCH 05/44] Add PyLongWriter_Discard() function --- Doc/c-api/long.rst | 5 +++++ Doc/whatsnew/3.14.rst | 3 ++- Include/cpython/longintrepr.h | 1 + .../C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst | 3 ++- Objects/longobject.c | 8 ++++++++ 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 24564f93cdcaeb..5f02192beb7f1a 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -756,6 +756,11 @@ The :c:type:`PyLongWriter` API can be used to import an integer. On error, set an exception and return ``NULL``. +.. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) + + Discard the internal object and destroy the writer instance. + + Example creating an integer from an array of digits:: PyObject * diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 93700404a41b30..976eaaeaa88955 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -534,7 +534,8 @@ New Features * :c:func:`PyLong_AsDigitArray`; * :c:func:`PyLong_FreeDigitArray`; * :c:func:`PyLongWriter_Create`; - * :c:func:`PyLongWriter_Finish`. + * :c:func:`PyLongWriter_Finish`; + * :c:func:`PyLongWriter_Discard`. (Contributed by Victor Stinner in :gh:`102471`.) diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index a4d3e75e224252..1e655678c54ba5 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -188,6 +188,7 @@ PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( void **digits, const PyLongLayout *layout); PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); +PyAPI_FUNC(PyObject*) PyLongWriter_Discard(PyLongWriter *writer); #ifdef __cplusplus } diff --git a/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst b/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst index 16b99959f2b203..58b7999435c64e 100644 --- a/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst +++ b/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst @@ -4,6 +4,7 @@ Add a new import and export API for Python :class:`int` objects: * :c:func:`PyLong_AsDigitArray`; * :c:func:`PyLong_FreeDigitArray`; * :c:func:`PyLongWriter_Create`; -* :c:func:`PyLongWriter_Finish`. +* :c:func:`PyLongWriter_Finish`; +* :c:func:`PyLongWriter_Discard`. Patch by Victor Stinner. diff --git a/Objects/longobject.c b/Objects/longobject.c index 205cbfe69d6170..c137d4b6c5c309 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6855,6 +6855,14 @@ PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digit } +void PyLongWriter_Discard(PyLongWriter *writer) +{ + PyLongObject *obj = (PyLongObject *)writer; + assert(Py_REFCNT(obj) == 1); + Py_DECREF(obj); +} + + PyObject* PyLongWriter_Finish(PyLongWriter *writer) { PyLongObject *obj = (PyLongObject *)writer; From 1a7902fb13ca55a56841b5af89d4afa1c0f92875 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Sep 2024 13:57:01 +0200 Subject: [PATCH 06/44] Fixes --- Include/cpython/longintrepr.h | 2 +- Lib/test/test_capi/test_long.py | 6 +++--- Objects/longobject.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 1e655678c54ba5..a7919508f1bf61 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -188,7 +188,7 @@ PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( void **digits, const PyLongLayout *layout); PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); -PyAPI_FUNC(PyObject*) PyLongWriter_Discard(PyLongWriter *writer); +PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); #ifdef __cplusplus } diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 210fbf9437e7d3..5c5b1a0ead7860 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -670,14 +670,14 @@ def test_long_asuint64(self): negative_value_error=ValueError) def test_long_layout(self): - # Test PyLong_LAYOUT + # Test PyLong_GetNativeLayout() int_info = sys.int_info layout = _testcapi.get_pylong_layout() expected = { - 'endian': -1, 'bits_per_digit': int_info.bits_per_digit, 'digit_size': int_info.sizeof_digit, - 'digits_order': -1 if sys.byteorder == 'little' else 1, + 'digits_order': -1, + 'endian': -1 if sys.byteorder == 'little' else 1, } self.assertEqual(layout, expected) diff --git a/Objects/longobject.c b/Objects/longobject.c index c137d4b6c5c309..5bce42d45d5ab1 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6778,8 +6778,8 @@ int PyLong_AsUInt64(PyObject *obj, uint64_t *value) const PyLongLayout PyLong_LAYOUT = { .bits_per_digit = PyLong_SHIFT, - .digits_order = PY_LITTLE_ENDIAN ? -1 : 1, - .endian = -1, // least significant first + .digits_order = -1, // least significant first + .endian = PY_LITTLE_ENDIAN ? -1 : 1, .digit_size = sizeof(digit), }; From b70a6ddd3e1fd4fea9527675f689d15481f5f8e1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Sep 2024 14:26:54 +0200 Subject: [PATCH 07/44] Use unsigned type for ndigits --- Doc/c-api/long.rst | 7 +++---- Include/cpython/longintrepr.h | 4 ++-- Modules/_testcapi/long.c | 5 +++-- Objects/longobject.c | 13 +++++++------ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 5f02192beb7f1a..4ea4ada76ffbc8 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -727,7 +727,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. The instance must be destroyed by :c:func:`PyLongWriter_Finish`. -.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits, const PyLongLayout *layout) +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, size_t ndigits, void **digits, const PyLongLayout *layout) Create a :c:type:`PyLongWriter`. @@ -736,8 +736,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. *negative* is ``1`` if the number is negative, or ``0`` otherwise. - *ndigits* is the number of digits in the *digits* array. It must be - positive. + *ndigits* is the number of digits in the *digits* array. The caller must initialize the array of digits *digits* and then call :c:func:`PyLongWriter_Finish` to get a Python :class:`int`. Digits must be @@ -764,7 +763,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. Example creating an integer from an array of digits:: PyObject * - long_import(int negative, Py_ssize_t ndigits, Py_digit *digits) + long_import(int negative, size_t ndigits, Py_digit *digits) { void *writer_digits; PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index a7919508f1bf61..e088fce02acdbb 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -166,7 +166,7 @@ PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); typedef struct PyLong_DigitArray { PyObject *obj; int negative; - Py_ssize_t ndigits; + size_t ndigits; const void *digits; const PyLongLayout *layout; } PyLong_DigitArray; @@ -184,7 +184,7 @@ typedef struct PyLongWriter PyLongWriter; PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( int negative, - Py_ssize_t ndigits, + size_t ndigits, void **digits, const PyLongLayout *layout); PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index e6e7db848d4945..a00287bf355993 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -250,7 +250,8 @@ pylong_asdigitarray(PyObject *module, PyObject *obj) const Py_digit *array_digits = array.digits; PyObject *digits = PyList_New(0); - for (Py_ssize_t i=0; i < array.ndigits; i++) { + assert(array.ndigits != SIZE_MAX); + for (size_t i=0; i < array.ndigits; i++) { PyObject *digit = PyLong_FromUnsignedLong(array_digits[i]); if (digit == NULL) { goto error; @@ -321,7 +322,7 @@ pylongwriter_create(PyObject *module, PyObject *args) } void *writer_digits; - PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, + PyLongWriter *writer = PyLongWriter_Create(negative, (size_t)ndigits, &writer_digits, &layout); if (writer == NULL) { goto error; diff --git a/Objects/longobject.c b/Objects/longobject.c index 5bce42d45d5ab1..1d55e18013f4f1 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6823,13 +6823,9 @@ PyLong_FreeDigitArray(PyLong_DigitArray *array) /* --- PyLongWriter API --------------------------------------------------- */ -PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits, +PyLongWriter* PyLongWriter_Create(int negative, size_t ndigits, void **digits, const PyLongLayout *layout) { - if (ndigits < 0) { - PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); - return NULL; - } assert(digits != NULL); // First, compare pointers (fast-path) since it's faster if (layout != &PyLong_LAYOUT @@ -6839,7 +6835,12 @@ PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digit return NULL; } - PyLongObject *obj = _PyLong_New(ndigits); + if (ndigits > (size_t)PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "too many digits in integer"); + return NULL; + } + PyLongObject *obj = _PyLong_New((Py_ssize_t)ndigits); if (obj == NULL) { return NULL; } From 07552a71ca2e209c9482a5d6263612a7c171cc19 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Sep 2024 14:33:13 +0200 Subject: [PATCH 08/44] Remove again layout * Remove layout parameter of PyLongWriter_Create() * Remove layout member of PyLong_DigitArray structure --- Doc/c-api/long.rst | 6 +-- Include/cpython/longintrepr.h | 4 +- Lib/test/test_capi/test_long.py | 52 ++++++------------- Modules/_testcapi/long.c | 91 +++------------------------------ Objects/longobject.c | 11 +--- 5 files changed, 26 insertions(+), 138 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 4ea4ada76ffbc8..ede7a415e9d2e3 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -689,10 +689,6 @@ Export API Read-only array of unsigned digits. - .. c:member:: const PyLongLayout *layout - - Layout of the :c:member:`digits`. - .. c:function:: int PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) @@ -727,7 +723,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. The instance must be destroyed by :c:func:`PyLongWriter_Finish`. -.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, size_t ndigits, void **digits, const PyLongLayout *layout) +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, size_t ndigits, void **digits) Create a :c:type:`PyLongWriter`. diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index e088fce02acdbb..640dcc6b336d2f 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -168,7 +168,6 @@ typedef struct PyLong_DigitArray { int negative; size_t ndigits; const void *digits; - const PyLongLayout *layout; } PyLong_DigitArray; PyAPI_FUNC(int) PyLong_AsDigitArray( @@ -185,8 +184,7 @@ typedef struct PyLongWriter PyLongWriter; PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( int negative, size_t ndigits, - void **digits, - const PyLongLayout *layout); + void **digits); PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 5c5b1a0ead7860..544764670e2036 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -687,11 +687,11 @@ def test_long_export(self): base = 2 ** layout['bits_per_digit'] pylong_asdigitarray = _testcapi.pylong_asdigitarray - self.assertEqual(pylong_asdigitarray(0), (0, [0], layout)) - self.assertEqual(pylong_asdigitarray(123), (0, [123], layout)) - self.assertEqual(pylong_asdigitarray(-123), (1, [123], layout)) + self.assertEqual(pylong_asdigitarray(0), (0, [0])) + self.assertEqual(pylong_asdigitarray(123), (0, [123])) + self.assertEqual(pylong_asdigitarray(-123), (1, [123])) self.assertEqual(pylong_asdigitarray(base**2 * 3 + base * 2 + 1), - (0, [1, 2, 3], layout)) + (0, [1, 2, 3])) with self.assertRaises(TypeError): pylong_asdigitarray(1.0) @@ -706,24 +706,24 @@ def test_longwriter_create(self): base = 2 ** layout['bits_per_digit'] pylongwriter_create = _testcapi.pylongwriter_create - self.assertEqual(pylongwriter_create(0, [], layout), 0) - self.assertEqual(pylongwriter_create(0, [0], layout), 0) - self.assertEqual(pylongwriter_create(0, [123], layout), 123) - self.assertEqual(pylongwriter_create(1, [123], layout), -123) - self.assertEqual(pylongwriter_create(1, [1, 2], layout), + self.assertEqual(pylongwriter_create(0, []), 0) + self.assertEqual(pylongwriter_create(0, [0]), 0) + self.assertEqual(pylongwriter_create(0, [123]), 123) + self.assertEqual(pylongwriter_create(1, [123]), -123) + self.assertEqual(pylongwriter_create(1, [1, 2]), -(base * 2 + 1)) - self.assertEqual(pylongwriter_create(0, [1, 2, 3], layout), + self.assertEqual(pylongwriter_create(0, [1, 2, 3]), base**2 * 3 + base * 2 + 1) max_digit = base - 1 - self.assertEqual(pylongwriter_create(0, [max_digit, max_digit, max_digit], layout), + self.assertEqual(pylongwriter_create(0, [max_digit, max_digit, max_digit]), base**2 * max_digit + base * max_digit + max_digit) # normalize - self.assertEqual(pylongwriter_create(0, [123, 0, 0], layout), 123) + self.assertEqual(pylongwriter_create(0, [123, 0, 0]), 123) # test singletons + normalize for num in (-2, 0, 1, 5, 42, 100): - self.assertIs(pylongwriter_create(bool(num < 0), [abs(num), 0], layout), + self.assertIs(pylongwriter_create(bool(num < 0), [abs(num), 0]), num) # round trip: Python int -> export -> Python int @@ -732,29 +732,9 @@ def test_longwriter_create(self): numbers.extend(-num for num in list(numbers)) for num in numbers: with self.subTest(num=num): - negative, digits, layout = pylong_asdigitarray(num) - self.assertEqual(pylongwriter_create(negative, digits, layout), num, - (negative, digits, layout)) - - # invalid layout - with self.assertRaises(ValueError): - layout = _testcapi.get_pylong_layout() - layout['bits_per_digit'] += 1 - pylongwriter_create(0, [123], layout) - with self.assertRaises(ValueError): - layout = _testcapi.get_pylong_layout() - layout['digit_size'] *= 2 - pylongwriter_create(0, [123], layout) - def change_endian(endian): - return (-endian if endian else 1) - with self.assertRaises(ValueError): - layout = _testcapi.get_pylong_layout() - layout['digits_order'] = change_endian(layout['digits_order']) - pylongwriter_create(0, [123], layout) - with self.assertRaises(ValueError): - layout = _testcapi.get_pylong_layout() - layout['endian'] = change_endian(layout['endian']) - pylongwriter_create(0, [123], layout) + negative, digits = pylong_asdigitarray(num) + self.assertEqual(pylongwriter_create(negative, digits), num, + (negative, digits)) if __name__ == "__main__": diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index a00287bf355993..607221523581ab 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -173,71 +173,6 @@ layout_to_dict(const PyLongLayout *layout) } -static int -layout_from_dict_get(PyObject *dict, const char *key, int is_signed, long *pvalue) -{ - PyObject *item; - int res = PyDict_GetItemStringRef(dict, key, &item); - if (res <= 0) { - return -1; - } - - long value = PyLong_AsLong(item); - Py_DECREF(item); - if (value == -1 && PyErr_Occurred()) { - return -1; - } - - if (is_signed) { - if (value < INT8_MIN || value > INT8_MAX) { - return -1; - } - } - else { - if (value < 0 || value > (long)UINT8_MAX) { - return -1; - } - } - - *pvalue = value; - return 0; -} - - -static int -layout_from_dict(PyLongLayout *layout, PyObject *dict) -{ - long value; - if (layout_from_dict_get(dict, "bits_per_digit", 0, &value) < 0) { - goto error; - } - layout->bits_per_digit = (uint8_t)value; - - if (layout_from_dict_get(dict, "digit_size", 0, &value) < 0) { - goto error; - } - layout->digit_size = (uint8_t)value; - - if (layout_from_dict_get(dict, "digits_order", 1, &value) < 0) { - goto error; - } - layout->digits_order = (int8_t)value; - - if (layout_from_dict_get(dict, "endian", 1, &value) < 0) { - goto error; - } - layout->endian = (int8_t)value; - - return 0; - -error: - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, "invalid layout"); - } - return -1; -} - - static PyObject * pylong_asdigitarray(PyObject *module, PyObject *obj) { @@ -246,7 +181,7 @@ pylong_asdigitarray(PyObject *module, PyObject *obj) return NULL; } - assert(array.layout->digit_size == sizeof(Py_digit)); + assert(PyLong_GetNativeLayout()->digit_size == sizeof(Py_digit)); const Py_digit *array_digits = array.digits; PyObject *digits = PyList_New(0); @@ -264,12 +199,7 @@ pylong_asdigitarray(PyObject *module, PyObject *obj) Py_DECREF(digit); } - PyObject *layout_dict = layout_to_dict(array.layout); - if (layout_dict == NULL) { - goto error; - } - - PyObject *res = Py_BuildValue("(iNN)", array.negative, digits, layout_dict); + PyObject *res = Py_BuildValue("(iN)", array.negative, digits); PyLong_FreeDigitArray(&array); return res; @@ -284,22 +214,15 @@ static PyObject * pylongwriter_create(PyObject *module, PyObject *args) { int negative; - PyObject *list, *layout_dict; - if (!PyArg_ParseTuple(args, "iO!O!", + PyObject *list; + if (!PyArg_ParseTuple(args, "iO!", &negative, - &PyList_Type, &list, - &PyDict_Type, &layout_dict)) + &PyList_Type, &list)) { return NULL; } Py_ssize_t ndigits = PyList_GET_SIZE(list); - PyLongLayout layout; - memset(&layout, 0, sizeof(layout)); - if (layout_from_dict(&layout, layout_dict) < 0) { - return NULL; - } - Py_digit *digits = PyMem_Malloc(ndigits * sizeof(Py_digit)); if (digits == NULL) { PyErr_NoMemory(); @@ -323,11 +246,11 @@ pylongwriter_create(PyObject *module, PyObject *args) void *writer_digits; PyLongWriter *writer = PyLongWriter_Create(negative, (size_t)ndigits, - &writer_digits, &layout); + &writer_digits); if (writer == NULL) { goto error; } - assert(layout.digit_size == sizeof(Py_digit)); + assert(PyLong_GetNativeLayout()->digit_size == sizeof(Py_digit)); memcpy(writer_digits, digits, ndigits * sizeof(Py_digit)); PyObject *res = PyLongWriter_Finish(writer); PyMem_Free(digits); diff --git a/Objects/longobject.c b/Objects/longobject.c index 1d55e18013f4f1..13ff01fd36b248 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6806,7 +6806,6 @@ PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) array->ndigits = 1; } array->digits = self->long_value.ob_digit; - array->layout = &PyLong_LAYOUT; return 0; } @@ -6823,17 +6822,9 @@ PyLong_FreeDigitArray(PyLong_DigitArray *array) /* --- PyLongWriter API --------------------------------------------------- */ -PyLongWriter* PyLongWriter_Create(int negative, size_t ndigits, void **digits, - const PyLongLayout *layout) +PyLongWriter* PyLongWriter_Create(int negative, size_t ndigits, void **digits) { assert(digits != NULL); - // First, compare pointers (fast-path) since it's faster - if (layout != &PyLong_LAYOUT - && (memcmp(layout, &PyLong_LAYOUT, sizeof(*layout)) != 0)) - { - PyErr_SetString(PyExc_ValueError, "only the native layout is supported"); - return NULL; - } if (ndigits > (size_t)PY_SSIZE_T_MAX) { PyErr_SetString(PyExc_OverflowError, From 0d0f942ab8ed9f2f2a06c8a41d4e8a09d0c789c0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Sep 2024 13:59:22 +0200 Subject: [PATCH 09/44] Revert "Use unsigned type for ndigits" This reverts commit b70a6ddd3e1fd4fea9527675f689d15481f5f8e1. --- Doc/c-api/long.rst | 7 ++++--- Include/cpython/longintrepr.h | 4 ++-- Modules/_testcapi/long.c | 5 ++--- Objects/longobject.c | 14 +++++++------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index ede7a415e9d2e3..8936abe65e4c1e 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -723,7 +723,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. The instance must be destroyed by :c:func:`PyLongWriter_Finish`. -.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, size_t ndigits, void **digits) +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) Create a :c:type:`PyLongWriter`. @@ -732,7 +732,8 @@ The :c:type:`PyLongWriter` API can be used to import an integer. *negative* is ``1`` if the number is negative, or ``0`` otherwise. - *ndigits* is the number of digits in the *digits* array. + *ndigits* is the number of digits in the *digits* array. It must be + positive. The caller must initialize the array of digits *digits* and then call :c:func:`PyLongWriter_Finish` to get a Python :class:`int`. Digits must be @@ -759,7 +760,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. Example creating an integer from an array of digits:: PyObject * - long_import(int negative, size_t ndigits, Py_digit *digits) + long_import(int negative, Py_ssize_t ndigits, Py_digit *digits) { void *writer_digits; PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 640dcc6b336d2f..fb5f6745e8d9de 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -166,7 +166,7 @@ PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); typedef struct PyLong_DigitArray { PyObject *obj; int negative; - size_t ndigits; + Py_ssize_t ndigits; const void *digits; } PyLong_DigitArray; @@ -183,7 +183,7 @@ typedef struct PyLongWriter PyLongWriter; PyAPI_FUNC(PyLongWriter*) PyLongWriter_Create( int negative, - size_t ndigits, + Py_ssize_t ndigits, void **digits); PyAPI_FUNC(PyObject*) PyLongWriter_Finish(PyLongWriter *writer); PyAPI_FUNC(void) PyLongWriter_Discard(PyLongWriter *writer); diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 607221523581ab..31b64f669cc125 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -185,8 +185,7 @@ pylong_asdigitarray(PyObject *module, PyObject *obj) const Py_digit *array_digits = array.digits; PyObject *digits = PyList_New(0); - assert(array.ndigits != SIZE_MAX); - for (size_t i=0; i < array.ndigits; i++) { + for (Py_ssize_t i=0; i < array.ndigits; i++) { PyObject *digit = PyLong_FromUnsignedLong(array_digits[i]); if (digit == NULL) { goto error; @@ -245,7 +244,7 @@ pylongwriter_create(PyObject *module, PyObject *args) } void *writer_digits; - PyLongWriter *writer = PyLongWriter_Create(negative, (size_t)ndigits, + PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, &writer_digits); if (writer == NULL) { goto error; diff --git a/Objects/longobject.c b/Objects/longobject.c index 13ff01fd36b248..69a8508ab79e27 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6822,16 +6822,16 @@ PyLong_FreeDigitArray(PyLong_DigitArray *array) /* --- PyLongWriter API --------------------------------------------------- */ -PyLongWriter* PyLongWriter_Create(int negative, size_t ndigits, void **digits) +PyLongWriter* +PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) { - assert(digits != NULL); - - if (ndigits > (size_t)PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "too many digits in integer"); + if (ndigits < 0) { + PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); return NULL; } - PyLongObject *obj = _PyLong_New((Py_ssize_t)ndigits); + assert(digits != NULL); + + PyLongObject *obj = _PyLong_New(ndigits); if (obj == NULL) { return NULL; } From 762c33aaf8c693c2005a7e8c7b89ffb2652d7c7e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 6 Sep 2024 14:08:35 +0200 Subject: [PATCH 10/44] doc: adjust ndigits documentation --- Doc/c-api/long.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 8936abe65e4c1e..dcf883ed150f01 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -733,7 +733,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. *negative* is ``1`` if the number is negative, or ``0`` otherwise. *ndigits* is the number of digits in the *digits* array. It must be - positive. + greater than or equal to 0. The caller must initialize the array of digits *digits* and then call :c:func:`PyLongWriter_Finish` to get a Python :class:`int`. Digits must be From 20be7a33809553cd26b3c61ba0e7bd17982c61e0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 13 Sep 2024 14:09:57 +0200 Subject: [PATCH 11/44] Update doc --- Doc/c-api/long.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index dcf883ed150f01..87d871d6bb9771 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -701,7 +701,7 @@ Export API subclass. :c:func:`PyLong_FreeDigitArray` must be called once done with using - *export*. + *array*. .. c:function:: void PyLong_FreeDigitArray(PyLong_DigitArray *array) @@ -740,9 +740,6 @@ The :c:type:`PyLongWriter` API can be used to import an integer. in the range [``0``; ``PyLong_BASE - 1``]. Unused digits must be set to ``0``. - *layout* is the layout of *digits*: it must be equal to - :c:func:`PyLong_GetNativeLayout`. - .. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) From d92bf1eb7c685dec68810f8a41b7f95e12bb39bb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 16 Sep 2024 10:56:11 +0200 Subject: [PATCH 12/44] Make PyLong_DigitArray.obj private --- Doc/c-api/long.rst | 8 ++++---- Include/cpython/longintrepr.h | 2 +- Modules/_testcapi/long.c | 6 ++++++ Objects/longobject.c | 9 ++++----- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 87d871d6bb9771..7a1ca7b188f1df 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -673,10 +673,6 @@ Export API A Python :class:`int` object exported as an array of digits. - .. c:member:: PyObject *obj - - Strong reference to the Python :class:`int` object. - .. c:member:: int negative 1 if the number is negative, 0 otherwise. @@ -689,6 +685,10 @@ Export API Read-only array of unsigned digits. + .. c:member:: Py_uintptr_t reserved + + Member used internally, must not be used for other purpose. + .. c:function:: int PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index fb5f6745e8d9de..bfe184761159d0 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -164,10 +164,10 @@ typedef struct PyLongLayout { PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); typedef struct PyLong_DigitArray { - PyObject *obj; int negative; Py_ssize_t ndigits; const void *digits; + Py_uintptr_t reserved; } PyLong_DigitArray; PyAPI_FUNC(int) PyLong_AsDigitArray( diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 31b64f669cc125..e2c377d40cf8cc 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -199,7 +199,13 @@ pylong_asdigitarray(PyObject *module, PyObject *obj) } PyObject *res = Py_BuildValue("(iN)", array.negative, digits); + PyLong_FreeDigitArray(&array); + assert(array.negative == 0); + assert(array.ndigits == 0); + assert(array.digits == NULL); + assert(array.reserved == 0); + return res; error: diff --git a/Objects/longobject.c b/Objects/longobject.c index 69a8508ab79e27..16ed9cf4a4e76c 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6799,13 +6799,13 @@ PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) } PyLongObject *self = (PyLongObject*)obj; - array->obj = Py_NewRef(obj); array->negative = _PyLong_IsNegative(self); array->ndigits = _PyLong_DigitCount(self); if (array->ndigits == 0) { array->ndigits = 1; } array->digits = self->long_value.ob_digit; + array->reserved = (Py_uintptr_t)Py_NewRef(obj); return 0; } @@ -6813,10 +6813,9 @@ PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) void PyLong_FreeDigitArray(PyLong_DigitArray *array) { - Py_CLEAR(array->obj); - array->negative = 0; - array->ndigits = 0; - array->digits = NULL; + PyObject *obj = (PyObject*)array->reserved; + memset(array, 0, sizeof(*array)); + Py_DECREF(obj); } From b3b02a27aee35075f241549174e612ad35d0d866 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 16 Sep 2024 13:22:36 +0200 Subject: [PATCH 13/44] Remove reserved documentation Rename reserved member to _reserved. --- Doc/c-api/long.rst | 4 ---- Include/cpython/longintrepr.h | 3 ++- Modules/_testcapi/long.c | 2 +- Objects/longobject.c | 4 ++-- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 7a1ca7b188f1df..1faaf6762cf644 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -685,10 +685,6 @@ Export API Read-only array of unsigned digits. - .. c:member:: Py_uintptr_t reserved - - Member used internally, must not be used for other purpose. - .. c:function:: int PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index bfe184761159d0..1261fba34ce3d4 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -167,7 +167,8 @@ typedef struct PyLong_DigitArray { int negative; Py_ssize_t ndigits; const void *digits; - Py_uintptr_t reserved; + // Member used internally, must not be used for other purpose. + Py_uintptr_t _reserved; } PyLong_DigitArray; PyAPI_FUNC(int) PyLong_AsDigitArray( diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index e2c377d40cf8cc..0df565d9e4aedd 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -204,7 +204,7 @@ pylong_asdigitarray(PyObject *module, PyObject *obj) assert(array.negative == 0); assert(array.ndigits == 0); assert(array.digits == NULL); - assert(array.reserved == 0); + assert(array._reserved == 0); return res; diff --git a/Objects/longobject.c b/Objects/longobject.c index 16ed9cf4a4e76c..797ed5a8276cb2 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6805,7 +6805,7 @@ PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) array->ndigits = 1; } array->digits = self->long_value.ob_digit; - array->reserved = (Py_uintptr_t)Py_NewRef(obj); + array->_reserved = (Py_uintptr_t)Py_NewRef(obj); return 0; } @@ -6813,7 +6813,7 @@ PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) void PyLong_FreeDigitArray(PyLong_DigitArray *array) { - PyObject *obj = (PyObject*)array->reserved; + PyObject *obj = (PyObject*)array->_reserved; memset(array, 0, sizeof(*array)); Py_DECREF(obj); } From caca2d7ce7ba606c0a6501caf54a976a6043364a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 16 Sep 2024 17:40:17 +0200 Subject: [PATCH 14/44] PyLong_FreeDigitArray() only clears _reserved --- Modules/_testcapi/long.c | 3 --- Objects/longobject.c | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 0df565d9e4aedd..d148cdb918bef9 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -201,9 +201,6 @@ pylong_asdigitarray(PyObject *module, PyObject *obj) PyObject *res = Py_BuildValue("(iN)", array.negative, digits); PyLong_FreeDigitArray(&array); - assert(array.negative == 0); - assert(array.ndigits == 0); - assert(array.digits == NULL); assert(array._reserved == 0); return res; diff --git a/Objects/longobject.c b/Objects/longobject.c index 797ed5a8276cb2..23556cd71811d2 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6814,7 +6814,7 @@ void PyLong_FreeDigitArray(PyLong_DigitArray *array) { PyObject *obj = (PyObject*)array->_reserved; - memset(array, 0, sizeof(*array)); + array->_reserved = 0; Py_DECREF(obj); } From 4221a4962b8aa553ada7050ba296bf19d9fe5493 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 16 Sep 2024 18:55:30 +0200 Subject: [PATCH 15/44] Make PyLong_LAYOUT static --- Objects/longobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 23556cd71811d2..65bf87c9914012 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6776,7 +6776,7 @@ int PyLong_AsUInt64(PyObject *obj, uint64_t *value) } -const PyLongLayout PyLong_LAYOUT = { +static const PyLongLayout PyLong_LAYOUT = { .bits_per_digit = PyLong_SHIFT, .digits_order = -1, // least significant first .endian = PY_LITTLE_ENDIAN ? -1 : 1, From 37b1d495b409f9ca045dd0d87c67e23bafbebf4e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 16 Sep 2024 18:33:07 +0200 Subject: [PATCH 16/44] Add PyLong_AsDigitArray.value * Rename functions and structure: * PyLong_AsDigitArray() => PyLong_Export() * PyLong_FreeDigitArray() => PyLong_FreeExport() * PyLong_DigitArray => PyLongExport * 'array' => 'export_long' --- Doc/c-api/long.rst | 8 +++--- Include/cpython/longintrepr.h | 15 ++++++----- Lib/test/test_capi/test_long.py | 48 +++++++++++++++++++++++---------- Modules/_testcapi/long.c | 27 +++++++++++-------- Objects/longobject.c | 36 ++++++++++++++++--------- 5 files changed, 86 insertions(+), 48 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 1faaf6762cf644..fa6c397ee718f4 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -686,7 +686,7 @@ Export API Read-only array of unsigned digits. -.. c:function:: int PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) +.. c:function:: int PyLong_Export(PyObject *obj, PyLong_DigitArray *array) Export a Python :class:`int` object as an array of digits. @@ -696,13 +696,13 @@ Export API This function always succeeds if *obj* is a Python :class:`int` object or a subclass. - :c:func:`PyLong_FreeDigitArray` must be called once done with using + :c:func:`PyLong_FreeExport` must be called once done with using *array*. -.. c:function:: void PyLong_FreeDigitArray(PyLong_DigitArray *array) +.. c:function:: void PyLong_FreeExport(PyLong_DigitArray *array) - Release the export *array* created by :c:func:`PyLong_AsDigitArray`. + Release the export *array* created by :c:func:`PyLong_Export`. PyLongWriter API diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 1261fba34ce3d4..d952ba13271877 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -163,19 +163,20 @@ typedef struct PyLongLayout { PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); -typedef struct PyLong_DigitArray { - int negative; +typedef struct PyLongExport { + int64_t value; + uint8_t negative; Py_ssize_t ndigits; const void *digits; // Member used internally, must not be used for other purpose. Py_uintptr_t _reserved; -} PyLong_DigitArray; +} PyLongExport; -PyAPI_FUNC(int) PyLong_AsDigitArray( +PyAPI_FUNC(int) PyLong_Export( PyObject *obj, - PyLong_DigitArray *array); -PyAPI_FUNC(void) PyLong_FreeDigitArray( - PyLong_DigitArray *array); + PyLongExport *export_long); +PyAPI_FUNC(void) PyLong_FreeExport( + PyLongExport *export_long); /* --- PyLongWriter API --------------------------------------------------- */ diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 544764670e2036..e26c646c2c5647 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -10,6 +10,7 @@ NULL = None + class IntSubclass(int): pass @@ -686,19 +687,23 @@ def test_long_export(self): layout = _testcapi.get_pylong_layout() base = 2 ** layout['bits_per_digit'] - pylong_asdigitarray = _testcapi.pylong_asdigitarray - self.assertEqual(pylong_asdigitarray(0), (0, [0])) - self.assertEqual(pylong_asdigitarray(123), (0, [123])) - self.assertEqual(pylong_asdigitarray(-123), (1, [123])) - self.assertEqual(pylong_asdigitarray(base**2 * 3 + base * 2 + 1), - (0, [1, 2, 3])) + pylong_export = _testcapi.pylong_export - with self.assertRaises(TypeError): - pylong_asdigitarray(1.0) - with self.assertRaises(TypeError): - pylong_asdigitarray(0+1j) - with self.assertRaises(TypeError): - pylong_asdigitarray("abc") + # value fits into int64_t + self.assertEqual(pylong_export(0), 0) + self.assertEqual(pylong_export(123), 123) + self.assertEqual(pylong_export(-123), -123) + + # use an array, doesn't fit into int64_t + self.assertEqual(pylong_export(base**10 * 2 + 1), + (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + self.assertEqual(pylong_export(-(base**10 * 2 + 1)), + (1, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + + for value in (1.0, 0+1j, "abc"): + with self.subTest(value=value): + with self.assertRaises(TypeError): + pylong_export(value) def test_longwriter_create(self): # Test PyLong_Import() @@ -726,13 +731,28 @@ def test_longwriter_create(self): self.assertIs(pylongwriter_create(bool(num < 0), [abs(num), 0]), num) + def to_digits(num): + digits = [] + while True: + num, digit = divmod(num, base) + digits.append(digit) + if not num: + break + return digits + # round trip: Python int -> export -> Python int - pylong_asdigitarray = _testcapi.pylong_asdigitarray + pylong_export = _testcapi.pylong_export numbers = [*range(0, 10), 12345, 0xdeadbeef, 2**100, 2**100-1] numbers.extend(-num for num in list(numbers)) for num in numbers: with self.subTest(num=num): - negative, digits = pylong_asdigitarray(num) + data = pylong_export(num) + if isinstance(data, tuple): + negative, digits = data + else: + value = data + negative = int(value < 0) + digits = to_digits(abs(value)) self.assertEqual(pylongwriter_create(negative, digits), num, (negative, digits)) diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index d148cdb918bef9..05eeb80dcb6449 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -174,19 +174,24 @@ layout_to_dict(const PyLongLayout *layout) static PyObject * -pylong_asdigitarray(PyObject *module, PyObject *obj) +pylong_export(PyObject *module, PyObject *obj) { - PyLong_DigitArray array; - if (PyLong_AsDigitArray(obj, &array) < 0) { + PyLongExport export_long; + if (PyLong_Export(obj, &export_long) < 0) { return NULL; } + if (export_long.digits == NULL) { + return PyLong_FromInt64(export_long.value); + // PyLong_FreeExport() is not needed in this case + } + assert(PyLong_GetNativeLayout()->digit_size == sizeof(Py_digit)); - const Py_digit *array_digits = array.digits; + const Py_digit *export_long_digits = export_long.digits; PyObject *digits = PyList_New(0); - for (Py_ssize_t i=0; i < array.ndigits; i++) { - PyObject *digit = PyLong_FromUnsignedLong(array_digits[i]); + for (Py_ssize_t i=0; i < export_long.ndigits; i++) { + PyObject *digit = PyLong_FromUnsignedLong(export_long_digits[i]); if (digit == NULL) { goto error; } @@ -198,16 +203,16 @@ pylong_asdigitarray(PyObject *module, PyObject *obj) Py_DECREF(digit); } - PyObject *res = Py_BuildValue("(iN)", array.negative, digits); + PyObject *res = Py_BuildValue("(iN)", export_long.negative, digits); - PyLong_FreeDigitArray(&array); - assert(array._reserved == 0); + PyLong_FreeExport(&export_long); + assert(export_long._reserved == 0); return res; error: Py_DECREF(digits); - PyLong_FreeDigitArray(&array); + PyLong_FreeExport(&export_long); return NULL; } @@ -280,7 +285,7 @@ static PyMethodDef test_methods[] = { {"pylong_fromnativebytes", pylong_fromnativebytes, METH_VARARGS}, {"pylong_getsign", pylong_getsign, METH_O}, {"pylong_aspid", pylong_aspid, METH_O}, - {"pylong_asdigitarray", pylong_asdigitarray, METH_O}, + {"pylong_export", pylong_export, METH_O}, {"pylongwriter_create", pylongwriter_create, METH_VARARGS}, {"get_pylong_layout", get_pylong_layout, METH_NOARGS}, {NULL}, diff --git a/Objects/longobject.c b/Objects/longobject.c index 65bf87c9914012..f78135b07478e1 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6791,31 +6791,43 @@ const PyLongLayout* PyLong_GetNativeLayout(void) int -PyLong_AsDigitArray(PyObject *obj, PyLong_DigitArray *array) +PyLong_Export(PyObject *obj, PyLongExport *export_long) { if (!PyLong_Check(obj)) { PyErr_Format(PyExc_TypeError, "expect int, got %T", obj); return -1; } - PyLongObject *self = (PyLongObject*)obj; + int64_t value; + if (PyLong_AsInt64(obj, &value) >= 0) { + export_long->value = value; + export_long->negative = 0; + export_long->ndigits = 0; + export_long->digits = 0; + export_long->_reserved = 0; + } + else { + PyErr_Clear(); - array->negative = _PyLong_IsNegative(self); - array->ndigits = _PyLong_DigitCount(self); - if (array->ndigits == 0) { - array->ndigits = 1; + PyLongObject *self = (PyLongObject*)obj; + export_long->value = 0; + export_long->negative = _PyLong_IsNegative(self); + export_long->ndigits = _PyLong_DigitCount(self); + if (export_long->ndigits == 0) { + export_long->ndigits = 1; + } + export_long->digits = self->long_value.ob_digit; + export_long->_reserved = (Py_uintptr_t)Py_NewRef(obj); } - array->digits = self->long_value.ob_digit; - array->_reserved = (Py_uintptr_t)Py_NewRef(obj); return 0; } void -PyLong_FreeDigitArray(PyLong_DigitArray *array) +PyLong_FreeExport(PyLongExport *export_long) { - PyObject *obj = (PyObject*)array->_reserved; - array->_reserved = 0; - Py_DECREF(obj); + PyObject *obj = (PyObject*)export_long->_reserved; + export_long->_reserved = 0; + Py_XDECREF(obj); } From d70a1214a675cab71cc87b8935cd2798c81c9f96 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 17 Sep 2024 10:55:42 +0200 Subject: [PATCH 17/44] Inline PyLong_AsInt64() to avoid the exception --- Objects/longobject.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index f78135b07478e1..81678382d19185 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6797,8 +6797,15 @@ PyLong_Export(PyObject *obj, PyLongExport *export_long) PyErr_Format(PyExc_TypeError, "expect int, got %T", obj); return -1; } + int64_t value; - if (PyLong_AsInt64(obj, &value) >= 0) { + int flags = Py_ASNATIVEBYTES_NATIVE_ENDIAN; + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, &value, sizeof(value), flags); + if (bytes < 0) { + return -1; + } + + if ((size_t)bytes <= sizeof(value)) { export_long->value = value; export_long->negative = 0; export_long->ndigits = 0; @@ -6806,8 +6813,6 @@ PyLong_Export(PyObject *obj, PyLongExport *export_long) export_long->_reserved = 0; } else { - PyErr_Clear(); - PyLongObject *self = (PyLongObject*)obj; export_long->value = 0; export_long->negative = _PyLong_IsNegative(self); From 4aa25f64eb01e00caa37bf4e9e445e47f1770993 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 17 Sep 2024 11:00:33 +0200 Subject: [PATCH 18/44] Remove Py_digit type; update the doc --- Doc/c-api/long.rst | 67 +++++++------------ Doc/whatsnew/3.14.rst | 4 +- Include/cpython/longintrepr.h | 6 +- ...-07-03-17-26-53.gh-issue-102471.XpmKYk.rst | 4 +- Modules/_testcapi/long.c | 30 ++++----- 5 files changed, 44 insertions(+), 67 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index fa6c397ee718f4..0df08f9bfe4305 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -615,21 +615,6 @@ Export API .. versionadded:: 3.14 -.. c:type:: Py_digit - - A single unsigned digit in the range [``0``; ``PyLong_BASE - 1``]. - - It is usually used in an *array of digits*, such as the - :c:member:`PyLong_DigitArray.digits` array. - - Its size depend on the :c:macro:`!PYLONG_BITS_IN_DIGIT` macro: - see the ``configure`` :option:`--enable-big-digits` option. - - See :c:member:`PyLongLayout.bits_per_digit` for the number of bits per - digit and :c:member:`PyLongLayout.digit_size` for the size of a digit (in - bytes). - - .. c:struct:: PyLongLayout Layout of an array of digits, used by Python :class:`int` object. @@ -669,11 +654,23 @@ Export API See the :c:struct:`PyLongLayout` structure. -.. c:struct:: PyLong_DigitArray +.. c:struct:: PyLongExport + + Export of a Python :class:`int` object. - A Python :class:`int` object exported as an array of digits. + There are two cases: - .. c:member:: int negative + * If :c:member:`digits` is ``NULL``, only use the :c:member:`value` member. + Calling :c:func:`PyLong_FreeExport` is optional in this case. + * If :c:member:`digits` is not ``NULL``, use :c:member:`negative`, + :c:member:`ndigits` and :c:member:`digits` members. + Calling :c:func:`PyLong_FreeExport` is mandatory in this case. + + .. c:member:: int64_t value + + Integer value if :c:member:`digits` is ``NULL``. + + .. c:member:: uint8_t negative 1 if the number is negative, 0 otherwise. @@ -683,26 +680,26 @@ Export API .. c:member:: const void *digits - Read-only array of unsigned digits. + Read-only array of unsigned digits. Can be ``NULL``. -.. c:function:: int PyLong_Export(PyObject *obj, PyLong_DigitArray *array) +.. c:function:: int PyLong_Export(PyObject *obj, PyLongExport *export_long) - Export a Python :class:`int` object as an array of digits. + Export a Python :class:`int` object. - On success, set *\*array* and return 0. + On success, set *\*export_long* and return 0. On error, set an exception and return -1. This function always succeeds if *obj* is a Python :class:`int` object or a subclass. - :c:func:`PyLong_FreeExport` must be called once done with using - *array*. + If *export_long.digits* is not ``NULL, :c:func:`PyLong_FreeExport` must be + called when the export is no longer needed. -.. c:function:: void PyLong_FreeExport(PyLong_DigitArray *array) +.. c:function:: void PyLong_FreeExport(PyLongExport *export_long) - Release the export *array* created by :c:func:`PyLong_Export`. + Release the export *export_long* created by :c:func:`PyLong_Export`. PyLongWriter API @@ -748,21 +745,3 @@ The :c:type:`PyLongWriter` API can be used to import an integer. .. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) Discard the internal object and destroy the writer instance. - - -Example creating an integer from an array of digits:: - - PyObject * - long_import(int negative, Py_ssize_t ndigits, Py_digit *digits) - { - void *writer_digits; - PyLongWriter *writer = PyLongWriter_Create(negative, ndigits, - &writer_digits); - if (writer == NULL) { - return NULL; - } - - assert(layout.digit_size == sizeof(Py_digit)); - memcpy(writer_digits, digits, ndigits * sizeof(Py_digit)); - return PyLongWriter_Finish(writer); - } diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 976eaaeaa88955..1fb96571d21dea 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -531,8 +531,8 @@ New Features * Add a new import and export API for Python :class:`int` objects: * :c:func:`PyLong_GetNativeLayout`; - * :c:func:`PyLong_AsDigitArray`; - * :c:func:`PyLong_FreeDigitArray`; + * :c:func:`PyLong_Export`; + * :c:func:`PyLong_FreeExport`; * :c:func:`PyLongWriter_Create`; * :c:func:`PyLongWriter_Finish`; * :c:func:`PyLongWriter_Discard`. diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index d952ba13271877..ac038dd23090e6 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -58,10 +58,8 @@ typedef long stwodigits; /* signed variant of twodigits */ #else #error "PYLONG_BITS_IN_DIGIT should be 15 or 30" #endif -#define PyLong_BASE ((Py_digit)1 << PyLong_SHIFT) -#define PyLong_MASK ((Py_digit)(PyLong_BASE - 1)) - -typedef digit Py_digit; +#define PyLong_BASE ((digit)1 << PyLong_SHIFT) +#define PyLong_MASK ((digit)(PyLong_BASE - 1)) /* Long integer representation. diff --git a/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst b/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst index 58b7999435c64e..852c0dbd3b6f26 100644 --- a/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst +++ b/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst @@ -1,8 +1,8 @@ Add a new import and export API for Python :class:`int` objects: * :c:func:`PyLong_GetNativeLayout`; -* :c:func:`PyLong_AsDigitArray`; -* :c:func:`PyLong_FreeDigitArray`; +* :c:func:`PyLong_Export`; +* :c:func:`PyLong_FreeExport`; * :c:func:`PyLongWriter_Create`; * :c:func:`PyLongWriter_Finish`; * :c:func:`PyLongWriter_Discard`. diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 05eeb80dcb6449..235cae80047b2f 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -186,21 +186,21 @@ pylong_export(PyObject *module, PyObject *obj) // PyLong_FreeExport() is not needed in this case } - assert(PyLong_GetNativeLayout()->digit_size == sizeof(Py_digit)); - const Py_digit *export_long_digits = export_long.digits; + assert(PyLong_GetNativeLayout()->digit_size == sizeof(digit)); + const digit *export_long_digits = export_long.digits; PyObject *digits = PyList_New(0); for (Py_ssize_t i=0; i < export_long.ndigits; i++) { - PyObject *digit = PyLong_FromUnsignedLong(export_long_digits[i]); - if (digit == NULL) { + PyObject *item = PyLong_FromUnsignedLong(export_long_digits[i]); + if (item == NULL) { goto error; } - if (PyList_Append(digits, digit) < 0) { - Py_DECREF(digit); + if (PyList_Append(digits, item) < 0) { + Py_DECREF(item); goto error; } - Py_DECREF(digit); + Py_DECREF(item); } PyObject *res = Py_BuildValue("(iN)", export_long.negative, digits); @@ -230,7 +230,7 @@ pylongwriter_create(PyObject *module, PyObject *args) } Py_ssize_t ndigits = PyList_GET_SIZE(list); - Py_digit *digits = PyMem_Malloc(ndigits * sizeof(Py_digit)); + digit *digits = PyMem_Malloc(ndigits * sizeof(digit)); if (digits == NULL) { PyErr_NoMemory(); return NULL; @@ -239,16 +239,16 @@ pylongwriter_create(PyObject *module, PyObject *args) for (Py_ssize_t i=0; i < ndigits; i++) { PyObject *item = PyList_GET_ITEM(list, i); - long digit = PyLong_AsLong(item); - if (digit == -1 && PyErr_Occurred()) { + long num = PyLong_AsLong(item); + if (num == -1 && PyErr_Occurred()) { goto error; } - if (digit < 0 || digit >= PyLong_BASE) { - PyErr_SetString(PyExc_ValueError, "digit doesn't fit into Py_digit"); + if (num < 0 || num >= PyLong_BASE) { + PyErr_SetString(PyExc_ValueError, "digit doesn't fit into digit"); goto error; } - digits[i] = (Py_digit)digit; + digits[i] = (digit)num; } void *writer_digits; @@ -257,8 +257,8 @@ pylongwriter_create(PyObject *module, PyObject *args) if (writer == NULL) { goto error; } - assert(PyLong_GetNativeLayout()->digit_size == sizeof(Py_digit)); - memcpy(writer_digits, digits, ndigits * sizeof(Py_digit)); + assert(PyLong_GetNativeLayout()->digit_size == sizeof(digit)); + memcpy(writer_digits, digits, ndigits * sizeof(digit)); PyObject *res = PyLongWriter_Finish(writer); PyMem_Free(digits); From 5d3e22496011f48cb0fbc297caeed9e4d12ac06c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 17 Sep 2024 13:10:30 +0200 Subject: [PATCH 19/44] Add long_asnativebytes() function --- Objects/longobject.c | 106 +++++++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 48 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 81678382d19185..d5ae168e9fd4e0 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1117,53 +1117,18 @@ _resolve_endianness(int *endianness) return 0; } -Py_ssize_t -PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) + +static Py_ssize_t +long_asnativebytes(PyLongObject *v, void* buffer, Py_ssize_t n, + int little_endian, int unsigned_buffer) { - PyLongObject *v; + Py_ssize_t res = 0; union { Py_ssize_t v; unsigned char b[sizeof(Py_ssize_t)]; } cv; - int do_decref = 0; - Py_ssize_t res = 0; - - if (vv == NULL || n < 0) { - PyErr_BadInternalCall(); - return -1; - } - - int little_endian = flags; - if (_resolve_endianness(&little_endian) < 0) { - return -1; - } - - if (PyLong_Check(vv)) { - v = (PyLongObject *)vv; - } - else if (flags != -1 && (flags & Py_ASNATIVEBYTES_ALLOW_INDEX)) { - v = (PyLongObject *)_PyNumber_Index(vv); - if (v == NULL) { - return -1; - } - do_decref = 1; - } - else { - PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); - return -1; - } - - if ((flags != -1 && (flags & Py_ASNATIVEBYTES_REJECT_NEGATIVE)) - && _PyLong_IsNegative(v)) { - PyErr_SetString(PyExc_ValueError, "Cannot convert negative int"); - if (do_decref) { - Py_DECREF(v); - } - return -1; - } if (_PyLong_IsCompact(v)) { - res = 0; cv.v = _PyLong_CompactValue(v); /* Most paths result in res = sizeof(compact value). Only the case * where 0 < n < sizeof(compact value) do we need to check and adjust @@ -1200,7 +1165,7 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) /* Positive values with the MSB set do not require an * additional bit when the caller's intent is to treat them * as unsigned. */ - if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { + if (unsigned_buffer) { res = n; } else { res = n + 1; @@ -1287,7 +1252,7 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) * as unsigned. */ unsigned char *b = (unsigned char *)buffer; if (b[little_endian ? n - 1 : 0] & 0x80) { - if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { + if (unsigned_buffer) { res = n; } else { res = n + 1; @@ -1296,6 +1261,54 @@ PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) } } } + return res; +} + + +Py_ssize_t +PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) +{ + PyLongObject *v; + int do_decref = 0; + Py_ssize_t res = 0; + + if (vv == NULL || n < 0) { + PyErr_BadInternalCall(); + return -1; + } + + int little_endian = flags; + if (_resolve_endianness(&little_endian) < 0) { + return -1; + } + + if (PyLong_Check(vv)) { + v = (PyLongObject *)vv; + } + else if (flags != -1 && (flags & Py_ASNATIVEBYTES_ALLOW_INDEX)) { + v = (PyLongObject *)_PyNumber_Index(vv); + if (v == NULL) { + return -1; + } + do_decref = 1; + } + else { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } + + if ((flags != -1 && (flags & Py_ASNATIVEBYTES_REJECT_NEGATIVE)) + && _PyLong_IsNegative(v)) { + PyErr_SetString(PyExc_ValueError, "Cannot convert negative int"); + if (do_decref) { + Py_DECREF(v); + } + return -1; + } + + int unsigned_buffer = (flags == -1 + || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)); + res = long_asnativebytes(v, buffer, n, little_endian, unsigned_buffer); if (do_decref) { Py_DECREF(v); @@ -6797,13 +6810,11 @@ PyLong_Export(PyObject *obj, PyLongExport *export_long) PyErr_Format(PyExc_TypeError, "expect int, got %T", obj); return -1; } + PyLongObject *self = (PyLongObject*)obj; int64_t value; - int flags = Py_ASNATIVEBYTES_NATIVE_ENDIAN; - Py_ssize_t bytes = PyLong_AsNativeBytes(obj, &value, sizeof(value), flags); - if (bytes < 0) { - return -1; - } + Py_ssize_t bytes = long_asnativebytes(self, &value, sizeof(value), + PY_LITTLE_ENDIAN, 0); if ((size_t)bytes <= sizeof(value)) { export_long->value = value; @@ -6813,7 +6824,6 @@ PyLong_Export(PyObject *obj, PyLongExport *export_long) export_long->_reserved = 0; } else { - PyLongObject *self = (PyLongObject*)obj; export_long->value = 0; export_long->negative = _PyLong_IsNegative(self); export_long->ndigits = _PyLong_DigitCount(self); From c7d7cb22567ecedfc89e4fc5e49b6b055b41fb3b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 17 Sep 2024 13:15:57 +0200 Subject: [PATCH 20/44] Remove reference to removed Py_digit type Fix also a typo in long.rst. --- Doc/c-api/long.rst | 2 +- Doc/using/configure.rst | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 0df08f9bfe4305..79c80642510579 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -693,7 +693,7 @@ Export API This function always succeeds if *obj* is a Python :class:`int` object or a subclass. - If *export_long.digits* is not ``NULL, :c:func:`PyLong_FreeExport` must be + If *export_long.digits* is not ``NULL``, :c:func:`PyLong_FreeExport` must be called when the export is no longer needed. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 7f9da4344e4438..10adf744c7ff52 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -129,8 +129,7 @@ General Options Define the ``PYLONG_BITS_IN_DIGIT`` to ``15`` or ``30``. - See :data:`sys.int_info.bits_per_digit ` and the - :c:type:`Py_digit` type. + See :data:`sys.int_info.bits_per_digit `. .. option:: --with-suffix=SUFFIX From a3d601a1bc0f2938ccbd5a5e8bb1fbef652ec583 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 17 Sep 2024 14:00:08 +0200 Subject: [PATCH 21/44] Address Antoine's review --- Doc/c-api/long.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 79c80642510579..9aa9fc77a29e1f 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -634,14 +634,14 @@ Export API .. c:member:: int8_t digits_order - Word endian: + Digits order: - - ``1`` for most significant word first (big endian) - - ``-1`` for least significant first (little endian) + - ``1`` for most significant digit first + - ``-1`` for least significant digit first .. c:member:: int8_t endian - Array endian: + Digit endianness: - ``1`` for most significant byte first (big endian) - ``-1`` for least significant first (little endian) @@ -668,15 +668,18 @@ Export API .. c:member:: int64_t value - Integer value if :c:member:`digits` is ``NULL``. + The native integer value of the exported :class:`int` object. + Only valid if :c:member:`digits` is ``NULL``. .. c:member:: uint8_t negative 1 if the number is negative, 0 otherwise. + Only valid if :c:member:`digits` is not ``NULL``. .. c:member:: Py_ssize_t ndigits Number of digits in :c:member:`digits` array. + Only valid if :c:member:`digits` is not ``NULL``. .. c:member:: const void *digits From 3e8d296c254f341f87027e30a7cb84fa8fd286ff Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 18 Sep 2024 10:36:06 +0300 Subject: [PATCH 22/44] Apply suggestions from code review I think this should fix warnings. --- Modules/_testcapi/long.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 235cae80047b2f..02c5c7ecaac6c6 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -230,7 +230,7 @@ pylongwriter_create(PyObject *module, PyObject *args) } Py_ssize_t ndigits = PyList_GET_SIZE(list); - digit *digits = PyMem_Malloc(ndigits * sizeof(digit)); + digit *digits = PyMem_Malloc((size_t)ndigits * sizeof(digit)); if (digits == NULL) { PyErr_NoMemory(); return NULL; @@ -258,7 +258,7 @@ pylongwriter_create(PyObject *module, PyObject *args) goto error; } assert(PyLong_GetNativeLayout()->digit_size == sizeof(digit)); - memcpy(writer_digits, digits, ndigits * sizeof(digit)); + memcpy(writer_digits, digits, (size_t)ndigits * sizeof(digit)); PyObject *res = PyLongWriter_Finish(writer); PyMem_Free(digits); From a8fd66990491ab6d190a0e9bf648b05d332ee0f4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Sep 2024 10:57:24 +0200 Subject: [PATCH 23/44] Revert "Add long_asnativebytes() function" This reverts commit 5d3e22496011f48cb0fbc297caeed9e4d12ac06c. --- Objects/longobject.c | 106 ++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 58 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index d5ae168e9fd4e0..81678382d19185 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1117,18 +1117,53 @@ _resolve_endianness(int *endianness) return 0; } - -static Py_ssize_t -long_asnativebytes(PyLongObject *v, void* buffer, Py_ssize_t n, - int little_endian, int unsigned_buffer) +Py_ssize_t +PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) { - Py_ssize_t res = 0; + PyLongObject *v; union { Py_ssize_t v; unsigned char b[sizeof(Py_ssize_t)]; } cv; + int do_decref = 0; + Py_ssize_t res = 0; + + if (vv == NULL || n < 0) { + PyErr_BadInternalCall(); + return -1; + } + + int little_endian = flags; + if (_resolve_endianness(&little_endian) < 0) { + return -1; + } + + if (PyLong_Check(vv)) { + v = (PyLongObject *)vv; + } + else if (flags != -1 && (flags & Py_ASNATIVEBYTES_ALLOW_INDEX)) { + v = (PyLongObject *)_PyNumber_Index(vv); + if (v == NULL) { + return -1; + } + do_decref = 1; + } + else { + PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); + return -1; + } + + if ((flags != -1 && (flags & Py_ASNATIVEBYTES_REJECT_NEGATIVE)) + && _PyLong_IsNegative(v)) { + PyErr_SetString(PyExc_ValueError, "Cannot convert negative int"); + if (do_decref) { + Py_DECREF(v); + } + return -1; + } if (_PyLong_IsCompact(v)) { + res = 0; cv.v = _PyLong_CompactValue(v); /* Most paths result in res = sizeof(compact value). Only the case * where 0 < n < sizeof(compact value) do we need to check and adjust @@ -1165,7 +1200,7 @@ long_asnativebytes(PyLongObject *v, void* buffer, Py_ssize_t n, /* Positive values with the MSB set do not require an * additional bit when the caller's intent is to treat them * as unsigned. */ - if (unsigned_buffer) { + if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { res = n; } else { res = n + 1; @@ -1252,7 +1287,7 @@ long_asnativebytes(PyLongObject *v, void* buffer, Py_ssize_t n, * as unsigned. */ unsigned char *b = (unsigned char *)buffer; if (b[little_endian ? n - 1 : 0] & 0x80) { - if (unsigned_buffer) { + if (flags == -1 || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)) { res = n; } else { res = n + 1; @@ -1261,54 +1296,6 @@ long_asnativebytes(PyLongObject *v, void* buffer, Py_ssize_t n, } } } - return res; -} - - -Py_ssize_t -PyLong_AsNativeBytes(PyObject* vv, void* buffer, Py_ssize_t n, int flags) -{ - PyLongObject *v; - int do_decref = 0; - Py_ssize_t res = 0; - - if (vv == NULL || n < 0) { - PyErr_BadInternalCall(); - return -1; - } - - int little_endian = flags; - if (_resolve_endianness(&little_endian) < 0) { - return -1; - } - - if (PyLong_Check(vv)) { - v = (PyLongObject *)vv; - } - else if (flags != -1 && (flags & Py_ASNATIVEBYTES_ALLOW_INDEX)) { - v = (PyLongObject *)_PyNumber_Index(vv); - if (v == NULL) { - return -1; - } - do_decref = 1; - } - else { - PyErr_Format(PyExc_TypeError, "expect int, got %T", vv); - return -1; - } - - if ((flags != -1 && (flags & Py_ASNATIVEBYTES_REJECT_NEGATIVE)) - && _PyLong_IsNegative(v)) { - PyErr_SetString(PyExc_ValueError, "Cannot convert negative int"); - if (do_decref) { - Py_DECREF(v); - } - return -1; - } - - int unsigned_buffer = (flags == -1 - || (flags & Py_ASNATIVEBYTES_UNSIGNED_BUFFER)); - res = long_asnativebytes(v, buffer, n, little_endian, unsigned_buffer); if (do_decref) { Py_DECREF(v); @@ -6810,11 +6797,13 @@ PyLong_Export(PyObject *obj, PyLongExport *export_long) PyErr_Format(PyExc_TypeError, "expect int, got %T", obj); return -1; } - PyLongObject *self = (PyLongObject*)obj; int64_t value; - Py_ssize_t bytes = long_asnativebytes(self, &value, sizeof(value), - PY_LITTLE_ENDIAN, 0); + int flags = Py_ASNATIVEBYTES_NATIVE_ENDIAN; + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, &value, sizeof(value), flags); + if (bytes < 0) { + return -1; + } if ((size_t)bytes <= sizeof(value)) { export_long->value = value; @@ -6824,6 +6813,7 @@ PyLong_Export(PyObject *obj, PyLongExport *export_long) export_long->_reserved = 0; } else { + PyLongObject *self = (PyLongObject*)obj; export_long->value = 0; export_long->negative = _PyLong_IsNegative(self); export_long->ndigits = _PyLong_DigitCount(self); From a04f9d0485c41ad23945cf8ef56e884e532d5554 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Sep 2024 10:53:59 +0200 Subject: [PATCH 24/44] Use PyLong_AsLongAndOverflow() --- Objects/longobject.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 81678382d19185..09b896ff2f3493 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6798,14 +6798,18 @@ PyLong_Export(PyObject *obj, PyLongExport *export_long) return -1; } - int64_t value; - int flags = Py_ASNATIVEBYTES_NATIVE_ENDIAN; - Py_ssize_t bytes = PyLong_AsNativeBytes(obj, &value, sizeof(value), flags); - if (bytes < 0) { - return -1; - } + int overflow; +#if SIZEOF_LONG == 8 + long value = PyLong_AsLongAndOverflow(obj, &overflow); +#elif SIZEOF_LONG_LONG == 8 + long long value = PyLong_AsLongLongAndOverflow(obj, &overflow); +#else +# error "unable to convert a long to int64_t" +#endif + // the function cannot fail since obj is a PyLongObject + assert(!(value == -1 && PyErr_Occurred())); - if ((size_t)bytes <= sizeof(value)) { + if (!overflow) { export_long->value = value; export_long->negative = 0; export_long->ndigits = 0; From b2be94a2b98d1b52ad90c915442930daafd05cb2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 18 Sep 2024 15:33:52 +0200 Subject: [PATCH 25/44] Try PyLong_AsLongLongAndOverflow() first --- Objects/longobject.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 09b896ff2f3493..8445c36d02f94f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6798,14 +6798,15 @@ PyLong_Export(PyObject *obj, PyLongExport *export_long) return -1; } + // Fast-path: try to convert to a int64_t int overflow; #if SIZEOF_LONG == 8 long value = PyLong_AsLongAndOverflow(obj, &overflow); -#elif SIZEOF_LONG_LONG == 8 - long long value = PyLong_AsLongLongAndOverflow(obj, &overflow); #else -# error "unable to convert a long to int64_t" + // Windows has 32-bit long, so use 64-bit long long instead + long long value = PyLong_AsLongLongAndOverflow(obj, &overflow); #endif + Py_BUILD_ASSERT(sizeof(value) == sizeof(int64_t)); // the function cannot fail since obj is a PyLongObject assert(!(value == -1 && PyErr_Occurred())); From 167d75e2508cc34680fc8127cb907236dacf8fa7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 19 Sep 2024 12:52:43 +0200 Subject: [PATCH 26/44] Update Doc/c-api/long.rst Co-authored-by: Sergey B Kirpichev --- Doc/c-api/long.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 7b0e9047a7c273..9a1160ccdba5e9 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -624,7 +624,7 @@ Export API Use :c:func:`PyLong_GetNativeLayout` to get the native layout of Python :class:`int` objects. - See also :attr:`sys.int_info` which exposes similar information to Python. + See also :data:`sys.int_info` which exposes similar information to Python. .. c:member:: uint8_t bits_per_digit From 5e53a5b7520efcfc7c0df37d204e0dae14ccbece Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 16 Oct 2024 16:05:59 +0300 Subject: [PATCH 27/44] Sync implementation with PEP (#8) --- Doc/c-api/long.rst | 28 +++++++++++++++++----------- Include/cpython/longintrepr.h | 10 +++++----- Lib/test/test_capi/test_long.py | 2 +- Modules/_testcapi/long.c | 4 ++-- Objects/longobject.c | 2 +- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 9a1160ccdba5e9..a35efe6d358e71 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -619,10 +619,12 @@ Export API .. c:struct:: PyLongLayout - Layout of an array of digits, used by Python :class:`int` object. + Layout of an array of "digits" ("limbs" in the GMP terminology), used to + represent absolute value for arbitrary precision integers. Use :c:func:`PyLong_GetNativeLayout` to get the native layout of Python - :class:`int` objects. + :class:`int` objects, used internally for integers with "big enough" + absolute value. See also :data:`sys.int_info` which exposes similar information to Python. @@ -655,6 +657,11 @@ Export API See the :c:struct:`PyLongLayout` structure. + The function must not be called before Python initialization nor after + Python finalization. The returned layout is valid until Python is + finalized. The layout is the same for all Python sub-interpreters and + so it can be cached. + .. c:struct:: PyLongExport @@ -695,9 +702,6 @@ Export API On success, set *\*export_long* and return 0. On error, set an exception and return -1. - This function always succeeds if *obj* is a Python :class:`int` object or a - subclass. - If *export_long.digits* is not ``NULL``, :c:func:`PyLong_FreeExport` must be called when the export is no longer needed. @@ -718,7 +722,8 @@ The :c:type:`PyLongWriter` API can be used to import an integer. A Python :class:`int` writer instance. - The instance must be destroyed by :c:func:`PyLongWriter_Finish`. + The instance must be destroyed by :c:func:`PyLongWriter_Finish` or + :c:func:`PyLongWriter_Discard`. .. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) @@ -733,10 +738,11 @@ The :c:type:`PyLongWriter` API can be used to import an integer. *ndigits* is the number of digits in the *digits* array. It must be greater than or equal to 0. - The caller must initialize the array of digits *digits* and then call - :c:func:`PyLongWriter_Finish` to get a Python :class:`int`. Digits must be - in the range [``0``; ``PyLong_BASE - 1``]. Unused digits must be set to - ``0``. + The caller can either initialize the array of digits *digits* and then call + :c:func:`PyLongWriter_Finish` to get a Python :class:`int`, or call + :c:func:`PyLongWriter_Discard` to destroy the writer instance. Digits must + be in the range [``0``; ``(1 << sys.int_info.bits_per_digit) - 1``]. Unused + digits must be set to ``0``. .. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) @@ -749,4 +755,4 @@ The :c:type:`PyLongWriter` API can be used to import an integer. .. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) - Discard the internal object and destroy the writer instance. + Discard a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index ac038dd23090e6..ff6d7740702b3b 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -148,15 +148,15 @@ typedef struct PyLongLayout { // Digit size in bytes uint8_t digit_size; - // Word endian: - // * 1 for most significant word first (big endian) - // * -1 for least significant first (little endian) + // Digits order: + // * 1 for most significant digit first + // * -1 for least significant first int8_t digits_order; - // Array endian: + // Digit endianness: // * 1 for most significant byte first (big endian) // * -1 for least significant first (little endian) - int8_t endian; + int8_t endianness; } PyLongLayout; PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index e26c646c2c5647..56cf78895ba168 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -678,7 +678,7 @@ def test_long_layout(self): 'bits_per_digit': int_info.bits_per_digit, 'digit_size': int_info.sizeof_digit, 'digits_order': -1, - 'endian': -1 if sys.byteorder == 'little' else 1, + 'endianness': -1 if sys.byteorder == 'little' else 1, } self.assertEqual(layout, expected) diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 02c5c7ecaac6c6..be42020855d625 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -155,11 +155,11 @@ layout_to_dict(const PyLongLayout *layout) goto error; } - value = PyLong_FromLong(layout->endian); + value = PyLong_FromLong(layout->endianness); if (value == NULL) { goto error; } - res = PyDict_SetItemString(dict, "endian", value); + res = PyDict_SetItemString(dict, "endianness", value); Py_DECREF(value); if (res < 0) { goto error; diff --git a/Objects/longobject.c b/Objects/longobject.c index 8445c36d02f94f..234ecef6f4ddf2 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6779,7 +6779,7 @@ int PyLong_AsUInt64(PyObject *obj, uint64_t *value) static const PyLongLayout PyLong_LAYOUT = { .bits_per_digit = PyLong_SHIFT, .digits_order = -1, // least significant first - .endian = PY_LITTLE_ENDIAN ? -1 : 1, + .endianness = PY_LITTLE_ENDIAN ? -1 : 1, .digit_size = sizeof(digit), }; From 0422f9de3e9b9778c762ecab6ea581a917fe1177 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 13 Nov 2024 07:08:15 +0300 Subject: [PATCH 28/44] fix NL in Doc/c-api/long.rst (sorry, damn web editor) --- Doc/c-api/long.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index a89bb28169b16e..6e0ff886fcc51d 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -797,4 +797,4 @@ The :c:type:`PyLongWriter` API can be used to import an integer. .. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) - Discard a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. \ No newline at end of file + Discard a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. From a529a48ebe149acef50999031e32a477becfbd77 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 13 Nov 2024 07:09:08 +0300 Subject: [PATCH 29/44] rename news --- .../2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Misc/NEWS.d/next/{C API => C_API}/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst (100%) diff --git a/Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst b/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst similarity index 100% rename from Misc/NEWS.d/next/C API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst rename to Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst From 3db44f30b368429cf92afada287bdca9bdea076b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 13 Nov 2024 12:10:19 +0100 Subject: [PATCH 30/44] Address Erlend's review --- Modules/_testcapi/long.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 46515949afb8bf..210578763e865e 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -246,9 +246,7 @@ pylongwriter_create(PyObject *module, PyObject *args) { int negative; PyObject *list; - if (!PyArg_ParseTuple(args, "iO!", - &negative, - &PyList_Type, &list)) + if (!PyArg_ParseTuple(args, "iO!", &negative, &PyList_Type, &list)) { return NULL; } @@ -256,8 +254,7 @@ pylongwriter_create(PyObject *module, PyObject *args) digit *digits = PyMem_Malloc((size_t)ndigits * sizeof(digit)); if (digits == NULL) { - PyErr_NoMemory(); - return NULL; + return PyErr_NoMemory(); } for (Py_ssize_t i=0; i < ndigits; i++) { From 1d2863ecad18e1fd6c75760d3412f92710ecea83 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 13 Nov 2024 12:12:21 +0100 Subject: [PATCH 31/44] Address Sergey's review --- Objects/longobject.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 2a69f7f2acfca5..034ef08b2db9a4 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6786,7 +6786,8 @@ static const PyLongLayout PyLong_LAYOUT = { }; -const PyLongLayout* PyLong_GetNativeLayout(void) +const PyLongLayout* +PyLong_GetNativeLayout(void) { return &PyLong_LAYOUT; } @@ -6838,8 +6839,10 @@ void PyLong_FreeExport(PyLongExport *export_long) { PyObject *obj = (PyObject*)export_long->_reserved; - export_long->_reserved = 0; - Py_XDECREF(obj); + if (obj) { + export_long->_reserved = 0; + Py_DECREF(obj); + } } From 033bd65f837e173398d8b45427be7182f33735f4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 28 Nov 2024 16:11:58 +0100 Subject: [PATCH 32/44] Update documentation from PEP 757 Remove doc from the header file, only keep doc in Doc/c-api/long.rst to avoid the documentation to be outdated in one place. --- Doc/c-api/long.rst | 13 ++++++++----- Include/cpython/longintrepr.h | 11 ----------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 4fa8d4f7ac5e5d..f09abea7429f14 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -724,7 +724,7 @@ Export API .. c:member:: uint8_t negative - 1 if the number is negative, 0 otherwise. + ``1`` if the number is negative, ``0`` otherwise. Only valid if :c:member:`digits` is not ``NULL``. .. c:member:: Py_ssize_t ndigits @@ -741,11 +741,11 @@ Export API Export a Python :class:`int` object. - On success, set *\*export_long* and return 0. - On error, set an exception and return -1. + On success, set *\*export_long* and return ``0``. + On error, set an exception and return ``-1``. - If *export_long.digits* is not ``NULL``, :c:func:`PyLong_FreeExport` must be - called when the export is no longer needed. + If *export_long->digits* is not ``NULL``, :c:func:`PyLong_FreeExport` must + be called when the export is no longer needed. .. c:function:: void PyLong_FreeExport(PyLongExport *export_long) @@ -794,6 +794,9 @@ The :c:type:`PyLongWriter` API can be used to import an integer. On success, return a Python :class:`int` object. On error, set an exception and return ``NULL``. + The function takes care of normalizing the digits and converts the object + to a compact integer if needed. + .. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index ff6d7740702b3b..9c6422395664e1 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -142,20 +142,9 @@ _PyLong_CompactValue(const PyLongObject *op) /* --- Import/Export API -------------------------------------------------- */ typedef struct PyLongLayout { - // Bits per digit uint8_t bits_per_digit; - - // Digit size in bytes uint8_t digit_size; - - // Digits order: - // * 1 for most significant digit first - // * -1 for least significant first int8_t digits_order; - - // Digit endianness: - // * 1 for most significant byte first (big endian) - // * -1 for least significant first (little endian) int8_t endianness; } PyLongLayout; From 36b87d4593cb0ffce7cf0d7cd6850a880009449e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 28 Nov 2024 17:17:27 +0100 Subject: [PATCH 33/44] Update Modules/_testcapi/long.c Co-authored-by: Sergey B Kirpichev --- Modules/_testcapi/long.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 210578763e865e..5b20b8c08c5e2a 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -145,6 +145,7 @@ static PyObject * layout_to_dict(const PyLongLayout *layout) { PyObject *dict = PyDict_New(); + if (dict == NULL) { goto error; } From 94d852e10b3276c22a655fa946f9242d7d30faa3 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Mon, 9 Dec 2024 13:06:18 +0300 Subject: [PATCH 34/44] Sync implementation with PEP (#9) --- Doc/c-api/long.rst | 19 ++++++++++++------- Include/cpython/longintrepr.h | 2 +- Lib/test/test_capi/test_long.py | 5 +++-- Modules/_testcapi/long.c | 8 ++++---- Objects/longobject.c | 7 ++----- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f09abea7429f14..7650d98fdea421 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -685,7 +685,7 @@ Export API - ``1`` for most significant digit first - ``-1`` for least significant digit first - .. c:member:: int8_t endian + .. c:member:: int8_t digit_endianness Digit endianness: @@ -772,19 +772,20 @@ The :c:type:`PyLongWriter` API can be used to import an integer. Create a :c:type:`PyLongWriter`. - On success, set *\*digits* and return a writer. + On success, allocate *\*digits* and return a writer. On error, set an exception and return ``NULL``. *negative* is ``1`` if the number is negative, or ``0`` otherwise. *ndigits* is the number of digits in the *digits* array. It must be - greater than or equal to 0. + greater than 0. - The caller can either initialize the array of digits *digits* and then call - :c:func:`PyLongWriter_Finish` to get a Python :class:`int`, or call + The caller can either initialize the array of digits *digits* and then + either call :c:func:`PyLongWriter_Finish` to get a Python :class:`int` or :c:func:`PyLongWriter_Discard` to destroy the writer instance. Digits must - be in the range [``0``; ``(1 << sys.int_info.bits_per_digit) - 1``]. Unused - digits must be set to ``0``. + be in the range [``0``; ``(1 << bits_per_digit) - 1``] (where the + :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits per digit). + The unused most-significant digits must be set to ``0``. .. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) @@ -797,7 +798,11 @@ The :c:type:`PyLongWriter` API can be used to import an integer. The function takes care of normalizing the digits and converts the object to a compact integer if needed. + The writer instance is invalid after the call. + .. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) Discard a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + + The writer instance is invalid after the call. diff --git a/Include/cpython/longintrepr.h b/Include/cpython/longintrepr.h index 9c6422395664e1..357477b60d9a5a 100644 --- a/Include/cpython/longintrepr.h +++ b/Include/cpython/longintrepr.h @@ -145,7 +145,7 @@ typedef struct PyLongLayout { uint8_t bits_per_digit; uint8_t digit_size; int8_t digits_order; - int8_t endianness; + int8_t digit_endianness; } PyLongLayout; PyAPI_FUNC(const PyLongLayout*) PyLong_GetNativeLayout(void); diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 894dea1d6bac25..4e6f86f3cf4b29 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -723,7 +723,7 @@ def test_long_layout(self): 'bits_per_digit': int_info.bits_per_digit, 'digit_size': int_info.sizeof_digit, 'digits_order': -1, - 'endianness': -1 if sys.byteorder == 'little' else 1, + 'digit_endianness': -1 if sys.byteorder == 'little' else 1, } self.assertEqual(layout, expected) @@ -756,7 +756,8 @@ def test_longwriter_create(self): base = 2 ** layout['bits_per_digit'] pylongwriter_create = _testcapi.pylongwriter_create - self.assertEqual(pylongwriter_create(0, []), 0) + self.assertRaises(ValueError, pylongwriter_create, 0, []) + self.assertRaises(ValueError, pylongwriter_create, -123, []) self.assertEqual(pylongwriter_create(0, [0]), 0) self.assertEqual(pylongwriter_create(0, [123]), 123) self.assertEqual(pylongwriter_create(1, [123]), -123) diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 5b20b8c08c5e2a..a5f118bd481080 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -180,11 +180,11 @@ layout_to_dict(const PyLongLayout *layout) goto error; } - value = PyLong_FromLong(layout->endianness); + value = PyLong_FromLong(layout->digit_endianness); if (value == NULL) { goto error; } - res = PyDict_SetItemString(dict, "endianness", value); + res = PyDict_SetItemString(dict, "digit_endianness", value); Py_DECREF(value); if (res < 0) { goto error; @@ -215,7 +215,7 @@ pylong_export(PyObject *module, PyObject *obj) const digit *export_long_digits = export_long.digits; PyObject *digits = PyList_New(0); - for (Py_ssize_t i=0; i < export_long.ndigits; i++) { + for (Py_ssize_t i = 0; i < export_long.ndigits; i++) { PyObject *item = PyLong_FromUnsignedLong(export_long_digits[i]); if (item == NULL) { goto error; @@ -258,7 +258,7 @@ pylongwriter_create(PyObject *module, PyObject *args) return PyErr_NoMemory(); } - for (Py_ssize_t i=0; i < ndigits; i++) { + for (Py_ssize_t i = 0; i < ndigits; i++) { PyObject *item = PyList_GET_ITEM(list, i); long num = PyLong_AsLong(item); diff --git a/Objects/longobject.c b/Objects/longobject.c index 034ef08b2db9a4..c61c6ce7f57d2a 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6781,7 +6781,7 @@ int PyLong_AsUInt64(PyObject *obj, uint64_t *value) static const PyLongLayout PyLong_LAYOUT = { .bits_per_digit = PyLong_SHIFT, .digits_order = -1, // least significant first - .endianness = PY_LITTLE_ENDIAN ? -1 : 1, + .digit_endianness = PY_LITTLE_ENDIAN ? -1 : 1, .digit_size = sizeof(digit), }; @@ -6851,7 +6851,7 @@ PyLong_FreeExport(PyLongExport *export_long) PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) { - if (ndigits < 0) { + if (ndigits <= 0) { PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); return NULL; } @@ -6861,9 +6861,6 @@ PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) if (obj == NULL) { return NULL; } - if (ndigits == 0) { - assert(obj->long_value.ob_digit[0] == 0); - } if (negative) { _PyLong_FlipSign(obj); } From 53d584bd7c251c0503e2eb5e5333d4dc913e003d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 9 Dec 2024 11:55:53 +0100 Subject: [PATCH 35/44] Cleanup --- Doc/whatsnew/3.14.rst | 2 +- Lib/test/test_capi/test_long.py | 2 +- ...-07-03-17-26-53.gh-issue-102471.XpmKYk.rst | 2 +- Modules/_testcapi/long.c | 58 ++++++------------- 4 files changed, 21 insertions(+), 43 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 88aa454b30f0c7..5ce398ab93d6b4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1018,7 +1018,7 @@ New features (Contributed by Victor Stinner in :gh:`107954`.) -* Add a new import and export API for Python :class:`int` objects: +* Add a new import and export API for Python :class:`int` objects (:pep:`757`): * :c:func:`PyLong_GetNativeLayout`; * :c:func:`PyLong_Export`; diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 4e6f86f3cf4b29..65daa728c903e1 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -751,7 +751,7 @@ def test_long_export(self): pylong_export(value) def test_longwriter_create(self): - # Test PyLong_Import() + # Test PyLongWriter_Create() layout = _testcapi.get_pylong_layout() base = 2 ** layout['bits_per_digit'] diff --git a/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst b/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst index 852c0dbd3b6f26..c18c159ac87d08 100644 --- a/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst +++ b/Misc/NEWS.d/next/C_API/2024-07-03-17-26-53.gh-issue-102471.XpmKYk.rst @@ -1,4 +1,4 @@ -Add a new import and export API for Python :class:`int` objects: +Add a new import and export API for Python :class:`int` objects (:pep:`757`): * :c:func:`PyLong_GetNativeLayout`; * :c:func:`PyLong_Export`; diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index a5f118bd481080..5c33120fc8d5c5 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -145,50 +145,28 @@ static PyObject * layout_to_dict(const PyLongLayout *layout) { PyObject *dict = PyDict_New(); - if (dict == NULL) { goto error; } - PyObject *value = PyLong_FromUnsignedLong(layout->bits_per_digit); - if (value == NULL) { - goto error; - } - int res = PyDict_SetItemString(dict, "bits_per_digit", value); - Py_DECREF(value); - if (res < 0) { - goto error; - } - - value = PyLong_FromUnsignedLong(layout->digit_size); - if (value == NULL) { - goto error; - } - res = PyDict_SetItemString(dict, "digit_size", value); - Py_DECREF(value); - if (res < 0) { - goto error; - } - - value = PyLong_FromLong(layout->digits_order); - if (value == NULL) { - goto error; - } - res = PyDict_SetItemString(dict, "digits_order", value); - Py_DECREF(value); - if (res < 0) { - goto error; - } - - value = PyLong_FromLong(layout->digit_endianness); - if (value == NULL) { - goto error; - } - res = PyDict_SetItemString(dict, "digit_endianness", value); - Py_DECREF(value); - if (res < 0) { - goto error; - } +#define SET_DICT(KEY, EXPR) \ + do { \ + PyObject *value = (EXPR); \ + if (value == NULL) { \ + goto error; \ + } \ + int res = PyDict_SetItemString(dict, KEY, value); \ + Py_DECREF(value); \ + if (res < 0) { \ + goto error; \ + } \ + } while (0) + + SET_DICT("bits_per_digit", PyLong_FromUnsignedLong(layout->bits_per_digit)); + SET_DICT("digit_size", PyLong_FromUnsignedLong(layout->digit_size)); + SET_DICT("digits_order", PyLong_FromLong(layout->digits_order)); + SET_DICT("digit_endianness", PyLong_FromLong(layout->digit_endianness)); +#undef SET_DICT return dict; From 577598a91754b6ea5b06d995189d893dead1dc6f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 9 Dec 2024 13:46:01 +0100 Subject: [PATCH 36/44] Update Doc/c-api/long.rst Co-authored-by: Steve Dower --- Doc/c-api/long.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 7650d98fdea421..f28f42e5546a21 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -668,7 +668,7 @@ Export API :class:`int` objects, used internally for integers with "big enough" absolute value. - See also :data:`sys.int_info` which exposes similar information to Python. + See also :data:`sys.int_info` which exposes similar information in Python. .. c:member:: uint8_t bits_per_digit From b08cd55cf191629e16e4681a94aae9b86ad2f368 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 9 Dec 2024 16:02:37 +0100 Subject: [PATCH 37/44] Address Steve's review --- Doc/c-api/long.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f28f42e5546a21..6b9dcc563729bd 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -672,11 +672,13 @@ Export API .. c:member:: uint8_t bits_per_digit - Bits per digit. + Bits per digit. For example, a 15 bit digit means that bits 0-14 contain + meaningful information. .. c:member:: uint8_t digit_size - Digit size in bytes. + Digit size in bytes. For example, a 15 bit digit will require at least 2 + bytes. .. c:member:: int8_t digits_order @@ -780,6 +782,8 @@ The :c:type:`PyLongWriter` API can be used to import an integer. *ndigits* is the number of digits in the *digits* array. It must be greater than 0. + *digits* must not be NULL. + The caller can either initialize the array of digits *digits* and then either call :c:func:`PyLongWriter_Finish` to get a Python :class:`int` or :c:func:`PyLongWriter_Discard` to destroy the writer instance. Digits must From eaebef39b0c727b9c9eb57791141618bad61c46b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 10 Dec 2024 09:59:14 +0100 Subject: [PATCH 38/44] =?UTF-8?q?Address=20B=C3=A9n=C3=A9dikt's=20review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Doc/c-api/long.rst | 12 ++++++++---- Lib/test/test_capi/test_long.py | 3 +++ Modules/_testcapi/long.c | 8 +++++--- Objects/longobject.c | 8 ++++++-- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 6b9dcc563729bd..3e527a6d1eace0 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -657,7 +657,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Export API ^^^^^^^^^^ -.. versionadded:: 3.14 +.. versionadded:: next .. c:struct:: PyLongLayout @@ -692,7 +692,7 @@ Export API Digit endianness: - ``1`` for most significant byte first (big endian) - - ``-1`` for least significant first (little endian) + - ``-1`` for least significant byte first (little endian) .. c:function:: const PyLongLayout* PyLong_GetNativeLayout(void) @@ -743,6 +743,8 @@ Export API Export a Python :class:`int` object. + *export_long* must not be ``NULL``. + On success, set *\*export_long* and return ``0``. On error, set an exception and return ``-1``. @@ -760,7 +762,7 @@ PyLongWriter API The :c:type:`PyLongWriter` API can be used to import an integer. -.. versionadded:: 3.14 +.. versionadded:: next .. c:struct:: PyLongWriter @@ -789,7 +791,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. :c:func:`PyLongWriter_Discard` to destroy the writer instance. Digits must be in the range [``0``; ``(1 << bits_per_digit) - 1``] (where the :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits per digit). - The unused most-significant digits must be set to ``0``. + The unused most significant digits must be set to ``0``. .. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) @@ -809,4 +811,6 @@ The :c:type:`PyLongWriter` API can be used to import an integer. Discard a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + *writer* must not be ``NULL``. + The writer instance is invalid after the call. diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 65daa728c903e1..18124bba2b9a85 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -738,12 +738,15 @@ def test_long_export(self): self.assertEqual(pylong_export(0), 0) self.assertEqual(pylong_export(123), 123) self.assertEqual(pylong_export(-123), -123) + self.assertEqual(pylong_export(IntSubclass(123)), 123) # use an array, doesn't fit into int64_t self.assertEqual(pylong_export(base**10 * 2 + 1), (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) self.assertEqual(pylong_export(-(base**10 * 2 + 1)), (1, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) + self.assertEqual(pylong_export(IntSubclass(base**10 * 2 + 1)), + (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) for value in (1.0, 0+1j, "abc"): with self.subTest(value=value): diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 5c33120fc8d5c5..999113716d9ecc 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -193,6 +193,9 @@ pylong_export(PyObject *module, PyObject *obj) const digit *export_long_digits = export_long.digits; PyObject *digits = PyList_New(0); + if (digits == NULL) { + goto error; + } for (Py_ssize_t i = 0; i < export_long.ndigits; i++) { PyObject *item = PyLong_FromUnsignedLong(export_long_digits[i]); if (item == NULL) { @@ -214,7 +217,7 @@ pylong_export(PyObject *module, PyObject *obj) return res; error: - Py_DECREF(digits); + Py_XDECREF(digits); PyLong_FreeExport(&export_long); return NULL; } @@ -225,8 +228,7 @@ pylongwriter_create(PyObject *module, PyObject *args) { int negative; PyObject *list; - if (!PyArg_ParseTuple(args, "iO!", &negative, &PyList_Type, &list)) - { + if (!PyArg_ParseTuple(args, "iO!", &negative, &PyList_Type, &list)) { return NULL; } Py_ssize_t ndigits = PyList_GET_SIZE(list); diff --git a/Objects/longobject.c b/Objects/longobject.c index c61c6ce7f57d2a..2262507c8dec41 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6853,13 +6853,13 @@ PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) { if (ndigits <= 0) { PyErr_SetString(PyExc_ValueError, "ndigits must be positive"); - return NULL; + goto error; } assert(digits != NULL); PyLongObject *obj = _PyLong_New(ndigits); if (obj == NULL) { - return NULL; + goto error; } if (negative) { _PyLong_FlipSign(obj); @@ -6867,6 +6867,10 @@ PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) *digits = obj->long_value.ob_digit; return (PyLongWriter*)obj; + +error: + *digits = NULL; + return NULL; } From 03248c705c3405f3b4b812f91f7946b3fbfab269 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 10 Dec 2024 14:21:14 +0100 Subject: [PATCH 39/44] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Objects/longobject.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Objects/longobject.c b/Objects/longobject.c index 2262507c8dec41..94ae7227f145ce 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6874,7 +6874,8 @@ PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) } -void PyLongWriter_Discard(PyLongWriter *writer) +void +PyLongWriter_Discard(PyLongWriter *writer) { PyLongObject *obj = (PyLongObject *)writer; assert(Py_REFCNT(obj) == 1); @@ -6882,7 +6883,8 @@ void PyLongWriter_Discard(PyLongWriter *writer) } -PyObject* PyLongWriter_Finish(PyLongWriter *writer) +PyObject* +PyLongWriter_Finish(PyLongWriter *writer) { PyLongObject *obj = (PyLongObject *)writer; assert(Py_REFCNT(obj) == 1); From 0a26f9782daf1f0187a71a1d132ab7e0aabde3ea Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 11 Dec 2024 11:57:57 +0100 Subject: [PATCH 40/44] Address Steve's review --- Doc/c-api/long.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 3e527a6d1eace0..5709f81cabfd77 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -749,7 +749,8 @@ Export API On error, set an exception and return ``-1``. If *export_long->digits* is not ``NULL``, :c:func:`PyLong_FreeExport` must - be called when the export is no longer needed. + be called when the export is no longer needed. Otherwise, calling + :c:func:`PyLong_FreeExport` is optional. .. c:function:: void PyLong_FreeExport(PyLongExport *export_long) From 88a62feff93ad401c4fb521013c0d8d43bcc3954 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 12 Dec 2024 17:09:31 +0100 Subject: [PATCH 41/44] Add PyLong_Export to Doc/data/refcounts.dat --- Doc/data/refcounts.dat | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 3f49c88c3cc028..70cc7864e299ff 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1299,6 +1299,10 @@ PyLong_GetSign:int::: PyLong_GetSign:PyObject*:v:0: PyLong_GetSign:int*:sign:: +PyLong_Export:int::: +PyLong_Export:PyObject*:obj:0: +PyLong_Export:PyLongExport*:export_long:: + PyMapping_Check:int::: PyMapping_Check:PyObject*:o:0: From 45517ab99a339763cc4ea8a6f997c68d023aeed3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 12 Dec 2024 17:13:23 +0100 Subject: [PATCH 42/44] Address Serhiy's review --- Lib/test/test_capi/test_long.py | 7 +++--- Modules/_testcapi/long.c | 39 +++++++++------------------------ Objects/longobject.c | 2 +- 3 files changed, 14 insertions(+), 34 deletions(-) diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index 18124bba2b9a85..d45ac75c822ea9 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -748,10 +748,9 @@ def test_long_export(self): self.assertEqual(pylong_export(IntSubclass(base**10 * 2 + 1)), (0, [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2])) - for value in (1.0, 0+1j, "abc"): - with self.subTest(value=value): - with self.assertRaises(TypeError): - pylong_export(value) + self.assertRaises(TypeError, pylong_export, 1.0) + self.assertRaises(TypeError, pylong_export, 0+1j) + self.assertRaises(TypeError, pylong_export, "abc") def test_longwriter_create(self): # Test PyLongWriter_Create() diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 999113716d9ecc..3e12eafec19109 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -144,35 +144,11 @@ pylong_aspid(PyObject *module, PyObject *arg) static PyObject * layout_to_dict(const PyLongLayout *layout) { - PyObject *dict = PyDict_New(); - if (dict == NULL) { - goto error; - } - -#define SET_DICT(KEY, EXPR) \ - do { \ - PyObject *value = (EXPR); \ - if (value == NULL) { \ - goto error; \ - } \ - int res = PyDict_SetItemString(dict, KEY, value); \ - Py_DECREF(value); \ - if (res < 0) { \ - goto error; \ - } \ - } while (0) - - SET_DICT("bits_per_digit", PyLong_FromUnsignedLong(layout->bits_per_digit)); - SET_DICT("digit_size", PyLong_FromUnsignedLong(layout->digit_size)); - SET_DICT("digits_order", PyLong_FromLong(layout->digits_order)); - SET_DICT("digit_endianness", PyLong_FromLong(layout->digit_endianness)); -#undef SET_DICT - - return dict; - -error: - Py_XDECREF(dict); - return NULL; + return Py_BuildValue("{sisisisi}", + "bits_per_digit", (int)layout->bits_per_digit, + "digit_size", (int)layout->digit_size, + "digits_order", (int)layout->digits_order, + "digit_endianness", (int)layout->digit_endianness); } @@ -185,6 +161,9 @@ pylong_export(PyObject *module, PyObject *obj) } if (export_long.digits == NULL) { + assert(export_long.negative == 0); + assert(export_long.ndigits == 0); + assert(export_long.digits == NULL); return PyLong_FromInt64(export_long.value); // PyLong_FreeExport() is not needed in this case } @@ -209,6 +188,7 @@ pylong_export(PyObject *module, PyObject *obj) Py_DECREF(item); } + assert(export_long.value == 0); PyObject *res = Py_BuildValue("(iN)", export_long.negative, digits); PyLong_FreeExport(&export_long); @@ -228,6 +208,7 @@ pylongwriter_create(PyObject *module, PyObject *args) { int negative; PyObject *list; + // TODO(vstinner): write test for negative ndigits and digits==NULL if (!PyArg_ParseTuple(args, "iO!", &negative, &PyList_Type, &list)) { return NULL; } diff --git a/Objects/longobject.c b/Objects/longobject.c index 94ae7227f145ce..548ce959b74b9f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6817,7 +6817,7 @@ PyLong_Export(PyObject *obj, PyLongExport *export_long) export_long->value = value; export_long->negative = 0; export_long->ndigits = 0; - export_long->digits = 0; + export_long->digits = NULL; export_long->_reserved = 0; } else { From 92007d1c2c0d58b4cefcda6c39a180a6c564c64e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 12 Dec 2024 17:38:37 +0100 Subject: [PATCH 43/44] Address Petr's review --- Doc/c-api/long.rst | 46 +++++++++++++++++++++++++--------------- Doc/conf.py | 1 - Modules/_testcapi/long.c | 5 +++-- Objects/longobject.c | 1 + 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 5709f81cabfd77..f48cd07a979f56 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -703,8 +703,8 @@ Export API The function must not be called before Python initialization nor after Python finalization. The returned layout is valid until Python is - finalized. The layout is the same for all Python sub-interpreters and - so it can be cached. + finalized. The layout is the same for all Python sub-interpreters + in a process, and so it can be cached. .. c:struct:: PyLongExport @@ -714,10 +714,8 @@ Export API There are two cases: * If :c:member:`digits` is ``NULL``, only use the :c:member:`value` member. - Calling :c:func:`PyLong_FreeExport` is optional in this case. * If :c:member:`digits` is not ``NULL``, use :c:member:`negative`, :c:member:`ndigits` and :c:member:`digits` members. - Calling :c:func:`PyLong_FreeExport` is mandatory in this case. .. c:member:: int64_t value @@ -743,20 +741,28 @@ Export API Export a Python :class:`int` object. - *export_long* must not be ``NULL``. + *export_long* must point to a :c:struct:`PyLongExport` structure allocated + by the caller. It must not be ``NULL``. - On success, set *\*export_long* and return ``0``. + On success, fill in *\*export_long* and return ``0``. On error, set an exception and return ``-1``. - If *export_long->digits* is not ``NULL``, :c:func:`PyLong_FreeExport` must - be called when the export is no longer needed. Otherwise, calling - :c:func:`PyLong_FreeExport` is optional. + :c:func:`PyLong_FreeExport` must be called when the export is no longer + needed. + + .. impl-detail:: + This function always succeeds if *obj* is a Python :class:`int` object + or a subclass. .. c:function:: void PyLong_FreeExport(PyLongExport *export_long) Release the export *export_long* created by :c:func:`PyLong_Export`. + .. impl-detail:: + Calling :c:func:`PyLong_FreeExport` is optional if *export_long->digits* + is ``NULL``. + PyLongWriter API ^^^^^^^^^^^^^^^^ @@ -787,12 +793,18 @@ The :c:type:`PyLongWriter` API can be used to import an integer. *digits* must not be NULL. - The caller can either initialize the array of digits *digits* and then - either call :c:func:`PyLongWriter_Finish` to get a Python :class:`int` or - :c:func:`PyLongWriter_Discard` to destroy the writer instance. Digits must - be in the range [``0``; ``(1 << bits_per_digit) - 1``] (where the - :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits per digit). - The unused most significant digits must be set to ``0``. + After a successful call to this function, the caller should fill in the + array of digits *digits* and then call :c:func:`PyLongWriter_Finish` to get + a Python :class:`int`. + The layout of *digits* is described by :c:func:`PyLong_GetNativeLayout`. + + Digits must be in the range [``0``; ``(1 << bits_per_digit) - 1``] + (where the :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits + per digit). + Any unused most significant digits must be set to ``0``. + + Alternately, call :c:func:`PyLongWriter_Discard` to destroy the writer + instance without creating an :class:`~int` object. .. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) @@ -805,7 +817,7 @@ The :c:type:`PyLongWriter` API can be used to import an integer. The function takes care of normalizing the digits and converts the object to a compact integer if needed. - The writer instance is invalid after the call. + The writer instance and the *digits* array are invalid after the call. .. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) @@ -814,4 +826,4 @@ The :c:type:`PyLongWriter` API can be used to import an integer. *writer* must not be ``NULL``. - The writer instance is invalid after the call. + The writer instance and the *digits* array are invalid after the call. diff --git a/Doc/conf.py b/Doc/conf.py index e7ac4af47cf738..9cde394cbaed69 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -158,7 +158,6 @@ ('c:type', 'size_t'), ('c:type', 'ssize_t'), ('c:type', 'time_t'), - ('c:type', 'int8_t'), ('c:type', 'uint8_t'), ('c:type', 'uint16_t'), ('c:type', 'uint32_t'), diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 3e12eafec19109..42243023a45768 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -164,8 +164,9 @@ pylong_export(PyObject *module, PyObject *obj) assert(export_long.negative == 0); assert(export_long.ndigits == 0); assert(export_long.digits == NULL); - return PyLong_FromInt64(export_long.value); - // PyLong_FreeExport() is not needed in this case + PyObject *res = PyLong_FromInt64(export_long.value); + PyLong_FreeExport(&export_long); + return res; } assert(PyLong_GetNativeLayout()->digit_size == sizeof(digit)); diff --git a/Objects/longobject.c b/Objects/longobject.c index 548ce959b74b9f..97cc37e488875f 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -6797,6 +6797,7 @@ int PyLong_Export(PyObject *obj, PyLongExport *export_long) { if (!PyLong_Check(obj)) { + memset(export_long, 0, sizeof(*export_long)); PyErr_Format(PyExc_TypeError, "expect int, got %T", obj); return -1; } From 6d3cb80a89a13de0ef15dbdc64a4b17f9ca2490b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 12 Dec 2024 17:47:11 +0100 Subject: [PATCH 44/44] Add PyLongWriter to Doc/data/refcounts.dat --- Doc/data/refcounts.dat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 70cc7864e299ff..7f2b5fb3ceee81 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -1303,6 +1303,9 @@ PyLong_Export:int::: PyLong_Export:PyObject*:obj:0: PyLong_Export:PyLongExport*:export_long:: +PyLongWriter_Finish:PyObject*::+1: +PyLongWriter_Finish:PyLongWriter*:writer:: + PyMapping_Check:int::: PyMapping_Check:PyObject*:o:0: