From 88455f22d423d21c96fb14aadd2104fc0525caff Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Sat, 4 Jul 2020 16:36:55 -0500 Subject: [PATCH 01/21] try to port pickle to multi-stage init --- ...2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst | 1 + Modules/_pickle.c | 259 +++++++++--------- 2 files changed, 123 insertions(+), 137 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst new file mode 100644 index 00000000000000..a44047351a307f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst @@ -0,0 +1 @@ +Port :mod:`_pickle` to multiphase initialization diff --git a/Modules/_pickle.c b/Modules/_pickle.c index a26732af8ba2a1..6c9707c625d612 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -20,6 +20,9 @@ PyDoc_STRVAR(pickle_module_doc, "Optimized C implementation for the Python pickle module."); +//TODO should we remove Pickler_Type from the clinic input here? +//similar entry was not removed in pull 20960 for _bz2 + /*[clinic input] module _pickle class _pickle.Pickler "PicklerObject *" "&Pickler_Type" @@ -192,6 +195,10 @@ typedef struct { /* functools.partial, used for implementing __newobj_ex__ with protocols 2 and 3 */ PyObject *partial; + + /* needed for multi-state initialization*/ + PyTypeObject *pickler_type; + PyTypeObject *unpickler_type; } PickleState; /* Forward declaration of the _pickle module definition. */ @@ -230,6 +237,8 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->codecs_encode); Py_CLEAR(st->getattr); Py_CLEAR(st->partial); + Py_CLEAR(st->pickler_type); + Py_CLEAR(st->unpickler_type); } /* Initialize the given pickle module state. */ @@ -712,8 +721,6 @@ typedef struct { /* Forward declarations */ static int save(PicklerObject *, PyObject *, int); static int save_reduce(PicklerObject *, PyObject *, PyObject *); -static PyTypeObject Pickler_Type; -static PyTypeObject Unpickler_Type; #include "clinic/_pickle.c.h" @@ -1105,11 +1112,12 @@ _Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t data_len) } static PicklerObject * -_Pickler_New(void) +_Pickler_New(PyObject *module) { PicklerObject *self; - self = PyObject_GC_New(PicklerObject, &Pickler_Type); + PickleState *st = _Pickle_GetState(module); + self = PyObject_GC_New(PicklerObject, st->pickler_type); if (self == NULL) return NULL; @@ -1586,11 +1594,12 @@ _Unpickler_MemoCleanup(UnpicklerObject *self) } static UnpicklerObject * -_Unpickler_New(void) +_Unpickler_New(PyObject *module) { UnpicklerObject *self; + PickleState *st = _Pickle_GetState(module); - self = PyObject_GC_New(UnpicklerObject, &Unpickler_Type); + self = PyObject_GC_New(UnpicklerObject, st->unpickler_type); if (self == NULL) return NULL; @@ -5049,47 +5058,28 @@ static PyGetSetDef Pickler_getsets[] = { {NULL} }; -static PyTypeObject Pickler_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.Pickler" , /*tp_name*/ - sizeof(PicklerObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)Pickler_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - _pickle_Pickler___init____doc__, /*tp_doc*/ - (traverseproc)Pickler_traverse, /*tp_traverse*/ - (inquiry)Pickler_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - Pickler_methods, /*tp_methods*/ - Pickler_members, /*tp_members*/ - Pickler_getsets, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - _pickle_Pickler___init__, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_GC_Del, /*tp_free*/ - 0, /*tp_is_gc*/ +static PyType_Slot pickle_pickler_type_slots[] = { + {Py_tp_dealloc, Pickler_dealloc}, + {Py_tp_methods, Pickler_methods}, + {Py_tp_members, Pickler_members}, + {Py_tp_getset, Pickler_getsets}, + {Py_tp_clear, Pickler_clear}, + {Py_tp_doc, (char*)_pickle_Pickler___init____doc__}, + {Py_tp_traverse, Pickler_traverse}, + {0, 0} +}; + +//TODO - how to handle this?? +// Calling PyType_GetModuleState() on a subclass is not safe. +// bz2_compressor_type_spec does not have Py_TPFLAGS_BASETYPE flag +// which prevents to create a subclass. +// So calling PyType_GetModuleState() in this file is always safe. + +static PyType_Spec pickle_pickler_type_spec = { + .name = "_pickle.Pickler", + .basicsize = sizeof(PicklerObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .slots = pickle_pickler_type_slots, }; /* Temporary helper for calling self.find_class(). @@ -7554,47 +7544,21 @@ static PyGetSetDef Unpickler_getsets[] = { {NULL} }; -static PyTypeObject Unpickler_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.Unpickler", /*tp_name*/ - sizeof(UnpicklerObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)Unpickler_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - _pickle_Unpickler___init____doc__, /*tp_doc*/ - (traverseproc)Unpickler_traverse, /*tp_traverse*/ - (inquiry)Unpickler_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - Unpickler_methods, /*tp_methods*/ - 0, /*tp_members*/ - Unpickler_getsets, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - _pickle_Unpickler___init__, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_GC_Del, /*tp_free*/ - 0, /*tp_is_gc*/ +static PyType_Slot pickle_unpickler_type_slots[] = { + {Py_tp_dealloc, Unpickler_dealloc}, + {Py_tp_doc, (char *)_pickle_Unpickler___init____doc__}, + {Py_tp_traverse, Unpickler_traverse}, + {Py_tp_clear, Unpickler_clear}, + {Py_tp_methods, Unpickler_methods}, + {Py_tp_getset, Unpickler_getsets}, + {0,0} +}; + +static PyType_Spec pickle_unpickler_type_spec = { + .name = "_pickle.Unpickler", + .basicsize = sizeof(UnpicklerObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .slots = pickle_unpickler_type_slots, }; /*[clinic input] @@ -7643,7 +7607,7 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, PyObject *buffer_callback) /*[clinic end generated code: output=706186dba996490c input=5ed6653da99cd97c]*/ { - PicklerObject *pickler = _Pickler_New(); + PicklerObject *pickler = _Pickler_New(module); if (pickler == NULL) return NULL; @@ -7708,7 +7672,7 @@ _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, /*[clinic end generated code: output=fbab0093a5580fdf input=e543272436c6f987]*/ { PyObject *result; - PicklerObject *pickler = _Pickler_New(); + PicklerObject *pickler = _Pickler_New(module); if (pickler == NULL) return NULL; @@ -7774,7 +7738,7 @@ _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports, /*[clinic end generated code: output=250452d141c23e76 input=46c7c31c92f4f371]*/ { PyObject *result; - UnpicklerObject *unpickler = _Unpickler_New(); + UnpicklerObject *unpickler = _Unpickler_New(module); if (unpickler == NULL) return NULL; @@ -7834,7 +7798,7 @@ _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports, /*[clinic end generated code: output=82ac1e6b588e6d02 input=b3615540d0535087]*/ { PyObject *result; - UnpicklerObject *unpickler = _Unpickler_New(); + UnpicklerObject *unpickler = _Unpickler_New(module); if (unpickler == NULL) return NULL; @@ -7898,81 +7862,102 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->codecs_encode); Py_VISIT(st->getattr); Py_VISIT(st->partial); + Py_VISIT(st->pickler_type); + Py_VISIT(st->unpickler_type); return 0; } -static struct PyModuleDef _picklemodule = { - PyModuleDef_HEAD_INIT, - "_pickle", /* m_name */ - pickle_module_doc, /* m_doc */ - sizeof(PickleState), /* m_size */ - pickle_methods, /* m_methods */ - NULL, /* m_reload */ - pickle_traverse, /* m_traverse */ - pickle_clear, /* m_clear */ - (freefunc)pickle_free /* m_free */ -}; - -PyMODINIT_FUNC -PyInit__pickle(void) +static int +_pickle_exec(PyObject *m) { - PyObject *m; PickleState *st; - m = PyState_FindModule(&_picklemodule); - if (m) { - return Py_NewRef(m); - } - if (PyType_Ready(&Pdata_Type) < 0) - return NULL; + return -1; if (PyType_Ready(&PicklerMemoProxyType) < 0) - return NULL; + return -1; if (PyType_Ready(&UnpicklerMemoProxyType) < 0) - return NULL; + return -1; - /* Create the module and add the functions. */ - m = PyModule_Create(&_picklemodule); - if (m == NULL) - return NULL; + st = _Pickle_GetState(m); /* Add types */ - if (PyModule_AddType(m, &Pickler_Type) < 0) { - return NULL; + if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) { + return -1; } - if (PyModule_AddType(m, &Unpickler_Type) < 0) { - return NULL; + + st->pickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickle_pickler_type_spec, NULL); + if (st->pickler_type == NULL) { + return -1; } - if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) { - return NULL; + + if (PyModule_AddType(m, st->pickler_type) < 0) { + return -1; } - st = _Pickle_GetState(m); + st->unpickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickle_unpickler_type_spec, NULL); + if (st->unpickler_type == NULL) { + return -1; + } + + if (PyModule_AddType(m, st->unpickler_type) < 0) { + return -1; + } /* Initialize the exceptions. */ st->PickleError = PyErr_NewException("_pickle.PickleError", NULL, NULL); if (st->PickleError == NULL) - return NULL; + return -1; st->PicklingError = \ PyErr_NewException("_pickle.PicklingError", st->PickleError, NULL); if (st->PicklingError == NULL) - return NULL; + return -1; st->UnpicklingError = \ PyErr_NewException("_pickle.UnpicklingError", st->PickleError, NULL); if (st->UnpicklingError == NULL) - return NULL; + return -1; - if (PyModule_AddObjectRef(m, "PickleError", st->PickleError) < 0) { - return NULL; + Py_INCREF(st->PickleError); + if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0) { + Py_DECREF((PyObject *)st->PickleError); + return -1; } - if (PyModule_AddObjectRef(m, "PicklingError", st->PicklingError) < 0) { - return NULL; + Py_INCREF(st->PicklingError); + if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0) { + Py_DECREF((PyObject *)st->PicklingError); + return -1; } - if (PyModule_AddObjectRef(m, "UnpicklingError", st->UnpicklingError) < 0) { - return NULL; + Py_INCREF(st->UnpicklingError); + if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0) { + Py_DECREF((PyObject *)st->UnpicklingError); + return -1; } + if (_Pickle_InitState(st) < 0) - return NULL; + return -1; - return m; + return 0; +} + +static PyModuleDef_Slot pickle_slots[] = { + {Py_mod_exec, _pickle_exec}, + {0, NULL} +}; + +static struct PyModuleDef _picklemodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_pickle", + .m_doc = pickle_module_doc, + .m_size = sizeof(PickleState), + .m_methods = pickle_methods, + .m_slots = pickle_slots, + .m_traverse = pickle_traverse, + .m_clear = pickle_clear, + .m_free = (freefunc)pickle_free +}; + +PyMODINIT_FUNC +PyInit__pickle(void) +{ + return PyModuleDef_Init(&_picklemodule); } From 7a6189a19ae63d451dbd853be73a481a14e99fa3 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Mon, 7 Sep 2020 15:09:00 -0500 Subject: [PATCH 02/21] remove global state access --- Modules/_pickle.c | 858 ++++++++++++++++++------------------- Modules/clinic/_pickle.c.h | 76 +++- 2 files changed, 482 insertions(+), 452 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 6c9707c625d612..ab16095769bca0 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -211,14 +211,6 @@ _Pickle_GetState(PyObject *module) return (PickleState *)_PyModule_GetState(module); } -/* Find the module instance imported in the currently running sub-interpreter - and get its state. */ -static PickleState * -_Pickle_GetGlobalState(void) -{ - return _Pickle_GetState(PyState_FindModule(&_picklemodule)); -} - /* Clear the given pickle module state. */ static void _Pickle_ClearState(PickleState *st) @@ -531,9 +523,8 @@ Pdata_grow(Pdata *self) } static int -Pdata_stack_underflow(Pdata *self) +Pdata_stack_underflow(PickleState *st, Pdata *self) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, self->mark_set ? "unexpected MARK found" : @@ -546,16 +537,16 @@ Pdata_stack_underflow(Pdata *self) * is raised and V is set to NULL. */ static PyObject * -Pdata_pop(Pdata *self) +Pdata_pop(PickleState *state, Pdata *self) { if (Py_SIZE(self) <= self->fence) { - Pdata_stack_underflow(self); + Pdata_stack_underflow(state, self); return NULL; } Py_SET_SIZE(self, Py_SIZE(self) - 1); return self->data[Py_SIZE(self)]; } -#define PDATA_POP(D, V) do { (V) = Pdata_pop((D)); } while (0) +#define PDATA_POP(S, D, V) do { (V) = Pdata_pop(S, (D)); } while (0) static int Pdata_push(Pdata *self, PyObject *obj) @@ -578,13 +569,13 @@ Pdata_push(Pdata *self, PyObject *obj) if (Pdata_push((D), (O)) < 0) return (ER); } while(0) static PyObject * -Pdata_poptuple(Pdata *self, Py_ssize_t start) +Pdata_poptuple(PickleState *state, Pdata *self, Py_ssize_t start) { PyObject *tuple; Py_ssize_t len, i, j; if (start < self->fence) { - Pdata_stack_underflow(self); + Pdata_stack_underflow(state, self); return NULL; } len = Py_SIZE(self) - start; @@ -719,8 +710,8 @@ typedef struct { } UnpicklerMemoProxyObject; /* Forward declarations */ -static int save(PicklerObject *, PyObject *, int); -static int save_reduce(PicklerObject *, PyObject *, PyObject *); +static int save(PickleState *state, PicklerObject *, PyObject *, int); +static int save_reduce(PickleState *, PicklerObject *, PyObject *, PyObject *); #include "clinic/_pickle.c.h" @@ -1228,10 +1219,9 @@ _Unpickler_SetStringInput(UnpicklerObject *self, PyObject *input) } static int -bad_readline(void) +bad_readline(PickleState *state) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, "pickle data was truncated"); + PyErr_SetString(state->UnpicklingError, "pickle data was truncated"); return -1; } @@ -1325,14 +1315,13 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) /* Don't call it directly: use _Unpickler_Read() */ static Py_ssize_t -_Unpickler_ReadImpl(UnpicklerObject *self, char **s, Py_ssize_t n) +_Unpickler_ReadImpl(UnpicklerObject *self, PickleState *state, char **s, Py_ssize_t n) { Py_ssize_t num_read; *s = NULL; if (self->next_read_idx > PY_SSIZE_T_MAX - n) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, + PyErr_SetString(state->UnpicklingError, "read would overflow (invalid bytecode)"); return -1; } @@ -1341,14 +1330,14 @@ _Unpickler_ReadImpl(UnpicklerObject *self, char **s, Py_ssize_t n) assert(self->next_read_idx + n > self->input_len); if (!self->read) - return bad_readline(); + return bad_readline(state); /* Extend the buffer to satisfy desired size */ num_read = _Unpickler_ReadFromFile(self, n); if (num_read < 0) return -1; if (num_read < n) - return bad_readline(); + return bad_readline(state); *s = self->input_buffer; self->next_read_idx = n; return n; @@ -1363,7 +1352,7 @@ _Unpickler_ReadImpl(UnpicklerObject *self, char **s, Py_ssize_t n) * _Unpickler_Read() is recommended in most cases. */ static Py_ssize_t -_Unpickler_ReadInto(UnpicklerObject *self, char *buf, Py_ssize_t n) +_Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, Py_ssize_t n) { assert(n != READ_WHOLE_LINE); @@ -1384,7 +1373,7 @@ _Unpickler_ReadInto(UnpicklerObject *self, char *buf, Py_ssize_t n) /* Read from file */ if (!self->read) { /* We're unpickling memory, this means the input is truncated */ - return bad_readline(); + return bad_readline(state); } if (_Unpickler_SkipConsumed(self) < 0) { return -1; @@ -1411,7 +1400,7 @@ _Unpickler_ReadInto(UnpicklerObject *self, char *buf, Py_ssize_t n) Py_ssize_t read_size = PyBytes_GET_SIZE(data); if (read_size < n) { Py_DECREF(data); - return bad_readline(); + return bad_readline(state); } memcpy(buf, PyBytes_AS_STRING(data), n); Py_DECREF(data); @@ -1438,7 +1427,7 @@ _Unpickler_ReadInto(UnpicklerObject *self, char *buf, Py_ssize_t n) return -1; } if (read_size < n) { - return bad_readline(); + return bad_readline(state); } return n; } @@ -1456,12 +1445,12 @@ _Unpickler_ReadInto(UnpicklerObject *self, char *buf, Py_ssize_t n) Returns -1 (with an exception set) on failure. On success, return the number of chars read. */ -#define _Unpickler_Read(self, s, n) \ +#define _Unpickler_Read(self, state, s, n) \ (((n) <= (self)->input_len - (self)->next_read_idx) \ ? (*(s) = (self)->input_buffer + (self)->next_read_idx, \ (self)->next_read_idx += (n), \ (n)) \ - : _Unpickler_ReadImpl(self, (s), (n))) + : _Unpickler_ReadImpl(self, state, (s), (n))) static Py_ssize_t _Unpickler_CopyLine(UnpicklerObject *self, char *line, Py_ssize_t len, @@ -1485,7 +1474,7 @@ _Unpickler_CopyLine(UnpicklerObject *self, char *line, Py_ssize_t len, Returns the number of chars read, or -1 on failure. */ static Py_ssize_t -_Unpickler_Readline(UnpicklerObject *self, char **result) +_Unpickler_Readline(PickleState *state, UnpicklerObject *self, char **result) { Py_ssize_t i, num_read; @@ -1498,13 +1487,13 @@ _Unpickler_Readline(UnpicklerObject *self, char **result) } } if (!self->read) - return bad_readline(); + return bad_readline(state); num_read = _Unpickler_ReadFromFile(self, READ_WHOLE_LINE); if (num_read < 0) return -1; if (num_read == 0 || self->input_buffer[num_read - 1] != '\n') - return bad_readline(); + return bad_readline(state); self->next_read_idx = num_read; return _Unpickler_CopyLine(self, self->input_buffer, num_read, result); } @@ -1704,7 +1693,7 @@ _Unpickler_SetBuffers(UnpicklerObject *self, PyObject *buffers) /* Generate a GET opcode for an object stored in the memo. */ static int -memo_get(PicklerObject *self, PyObject *key) +memo_get(PickleState *state, PicklerObject *self, PyObject *key) { Py_ssize_t *value; char pdata[30]; @@ -1737,8 +1726,7 @@ memo_get(PicklerObject *self, PyObject *key) len = 5; } else { /* unlikely */ - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->PicklingError, + PyErr_SetString(state->PicklingError, "memo id too large for LONG_BINGET"); return -1; } @@ -1753,7 +1741,7 @@ memo_get(PicklerObject *self, PyObject *key) /* Store an object in the memo, assign it a new unique ID based on the number of objects currently stored in the memo and generate a PUT opcode. */ static int -memo_put(PicklerObject *self, PyObject *obj) +memo_put(PickleState *state, PicklerObject *self, PyObject *obj) { char pdata[30]; Py_ssize_t len; @@ -1794,8 +1782,7 @@ memo_put(PicklerObject *self, PyObject *obj) len = 5; } else { /* unlikely */ - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->PicklingError, + PyErr_SetString(state->PicklingError, "memo id too large for LONG_BINPUT"); return -1; } @@ -2347,8 +2334,8 @@ _Pickler_write_bytes(PicklerObject *self, } static int -_save_bytes_data(PicklerObject *self, PyObject *obj, const char *data, - Py_ssize_t size) +_save_bytes_data(PickleState *state, PicklerObject *self, PyObject *obj, + const char *data, Py_ssize_t size) { assert(self->proto >= 3); @@ -2387,7 +2374,7 @@ _save_bytes_data(PicklerObject *self, PyObject *obj, const char *data, return -1; } - if (memo_put(self, obj) < 0) { + if (memo_put(state, self, obj) < 0) { return -1; } @@ -2395,7 +2382,7 @@ _save_bytes_data(PicklerObject *self, PyObject *obj, const char *data, } static int -save_bytes(PicklerObject *self, PyObject *obj) +save_bytes(PickleState *state, PicklerObject *self, PyObject *obj) { if (self->proto < 3) { /* Older pickle protocols do not have an opcode for pickling bytes @@ -2416,7 +2403,6 @@ save_bytes(PicklerObject *self, PyObject *obj) reduce_value = Py_BuildValue("(O())", (PyObject*)&PyBytes_Type); } else { - PickleState *st = _Pickle_GetGlobalState(); PyObject *unicode_str = PyUnicode_DecodeLatin1(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj), @@ -2425,7 +2411,7 @@ save_bytes(PicklerObject *self, PyObject *obj) if (unicode_str == NULL) return -1; reduce_value = Py_BuildValue("(O(OO))", - st->codecs_encode, unicode_str, + state->codecs_encode, unicode_str, &_Py_ID(latin1)); Py_DECREF(unicode_str); } @@ -2434,19 +2420,19 @@ save_bytes(PicklerObject *self, PyObject *obj) return -1; /* save_reduce() will memoize the object automatically. */ - status = save_reduce(self, reduce_value, obj); + status = save_reduce(state, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } else { - return _save_bytes_data(self, obj, PyBytes_AS_STRING(obj), + return _save_bytes_data(state, self, obj, PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); } } static int -_save_bytearray_data(PicklerObject *self, PyObject *obj, const char *data, - Py_ssize_t size) +_save_bytearray_data(PickleState *state, PicklerObject *self, PyObject *obj, + const char *data, Py_ssize_t size) { assert(self->proto >= 5); @@ -2464,7 +2450,7 @@ _save_bytearray_data(PicklerObject *self, PyObject *obj, const char *data, return -1; } - if (memo_put(self, obj) < 0) { + if (memo_put(state, self, obj) < 0) { return -1; } @@ -2472,7 +2458,7 @@ _save_bytearray_data(PicklerObject *self, PyObject *obj, const char *data, } static int -save_bytearray(PicklerObject *self, PyObject *obj) +save_bytearray(PickleState *state, PicklerObject *self, PyObject *obj) { if (self->proto < 5) { /* Older pickle protocols do not have an opcode for pickling @@ -2497,22 +2483,21 @@ save_bytearray(PicklerObject *self, PyObject *obj) return -1; /* save_reduce() will memoize the object automatically. */ - status = save_reduce(self, reduce_value, obj); + status = save_reduce(state, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } else { - return _save_bytearray_data(self, obj, PyByteArray_AS_STRING(obj), + return _save_bytearray_data(state, self, obj, PyByteArray_AS_STRING(obj), PyByteArray_GET_SIZE(obj)); } } static int -save_picklebuffer(PicklerObject *self, PyObject *obj) +save_picklebuffer(PickleState *state, PicklerObject *self, PyObject *obj) { if (self->proto < 5) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->PicklingError, + PyErr_SetString(state->PicklingError, "PickleBuffer can only pickled with protocol >= 5"); return -1; } @@ -2521,8 +2506,7 @@ save_picklebuffer(PicklerObject *self, PyObject *obj) return -1; } if (view->suboffsets != NULL || !PyBuffer_IsContiguous(view, 'A')) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->PicklingError, + PyErr_SetString(state->PicklingError, "PickleBuffer can not be pickled when " "pointing to a non-contiguous buffer"); return -1; @@ -2542,12 +2526,12 @@ save_picklebuffer(PicklerObject *self, PyObject *obj) if (in_band) { /* Write data in-band */ if (view->readonly) { - return _save_bytes_data(self, obj, (const char*) view->buf, + return _save_bytes_data(state, self, obj, (const char*) view->buf, view->len); } else { - return _save_bytearray_data(self, obj, (const char*) view->buf, - view->len); + return _save_bytearray_data(state, self, obj, + (const char*) view->buf, view->len); } } else { @@ -2701,7 +2685,7 @@ write_unicode_binary(PicklerObject *self, PyObject *obj) } static int -save_unicode(PicklerObject *self, PyObject *obj) +save_unicode(PickleState *state, PicklerObject *self, PyObject *obj) { if (self->bin) { if (write_unicode_binary(self, obj) < 0) @@ -2731,7 +2715,7 @@ save_unicode(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, "\n", 1) < 0) return -1; } - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) return -1; return 0; @@ -2739,7 +2723,7 @@ save_unicode(PicklerObject *self, PyObject *obj) /* A helper for save_tuple. Push the len elements in tuple t on the stack. */ static int -store_tuple_elements(PicklerObject *self, PyObject *t, Py_ssize_t len) +store_tuple_elements(PickleState *state, PicklerObject *self, PyObject *t, Py_ssize_t len) { Py_ssize_t i; @@ -2750,7 +2734,7 @@ store_tuple_elements(PicklerObject *self, PyObject *t, Py_ssize_t len) if (element == NULL) return -1; - if (save(self, element, 0) < 0) + if (save(state, self, element, 0) < 0) return -1; } @@ -2764,7 +2748,7 @@ store_tuple_elements(PicklerObject *self, PyObject *t, Py_ssize_t len) * magic so that it works in all cases. IOW, this is a long routine. */ static int -save_tuple(PicklerObject *self, PyObject *obj) +save_tuple(PickleState *state, PicklerObject *self, PyObject *obj) { Py_ssize_t len, i; @@ -2801,7 +2785,7 @@ save_tuple(PicklerObject *self, PyObject *obj) */ if (len <= 3 && self->proto >= 2) { /* Use TUPLE{1,2,3} opcodes. */ - if (store_tuple_elements(self, obj, len) < 0) + if (store_tuple_elements(state, self, obj, len) < 0) return -1; if (PyMemoTable_Get(self->memo, obj)) { @@ -2810,7 +2794,7 @@ save_tuple(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &pop_op, 1) < 0) return -1; /* fetch from memo */ - if (memo_get(self, obj) < 0) + if (memo_get(state, self, obj) < 0) return -1; return 0; @@ -2828,7 +2812,7 @@ save_tuple(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; - if (store_tuple_elements(self, obj, len) < 0) + if (store_tuple_elements(state, self, obj, len) < 0) return -1; if (PyMemoTable_Get(self->memo, obj)) { @@ -2846,7 +2830,7 @@ save_tuple(PicklerObject *self, PyObject *obj) return -1; } /* fetch from memo */ - if (memo_get(self, obj) < 0) + if (memo_get(state, self, obj) < 0) return -1; return 0; @@ -2857,7 +2841,7 @@ save_tuple(PicklerObject *self, PyObject *obj) } memoize: - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) return -1; return 0; @@ -2870,7 +2854,7 @@ save_tuple(PicklerObject *self, PyObject *obj) * Returns 0 on success, <0 on error. */ static int -batch_list(PicklerObject *self, PyObject *iter) +batch_list(PickleState *state, PicklerObject *self, PyObject *iter) { PyObject *obj = NULL; PyObject *firstitem = NULL; @@ -2896,7 +2880,7 @@ batch_list(PicklerObject *self, PyObject *iter) return -1; break; } - i = save(self, obj, 0); + i = save(state, self, obj, 0); Py_DECREF(obj); if (i < 0) return -1; @@ -2925,7 +2909,7 @@ batch_list(PicklerObject *self, PyObject *iter) goto error; /* Only one item to write */ - if (save(self, firstitem, 0) < 0) + if (save(state, self, firstitem, 0) < 0) goto error; if (_Pickler_Write(self, &append_op, 1) < 0) goto error; @@ -2939,14 +2923,14 @@ batch_list(PicklerObject *self, PyObject *iter) if (_Pickler_Write(self, &mark_op, 1) < 0) goto error; - if (save(self, firstitem, 0) < 0) + if (save(state, self, firstitem, 0) < 0) goto error; Py_CLEAR(firstitem); n = 1; /* Fetch and save up to BATCHSIZE items */ while (obj) { - if (save(self, obj, 0) < 0) + if (save(state, self, obj, 0) < 0) goto error; Py_CLEAR(obj); n += 1; @@ -2986,7 +2970,7 @@ batch_list(PicklerObject *self, PyObject *iter) * Note that this only works for protocols > 0. */ static int -batch_list_exact(PicklerObject *self, PyObject *obj) +batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *item = NULL; Py_ssize_t this_batch, total; @@ -3002,7 +2986,7 @@ batch_list_exact(PicklerObject *self, PyObject *obj) if (PyList_GET_SIZE(obj) == 1) { item = PyList_GET_ITEM(obj, 0); Py_INCREF(item); - int err = save(self, item, 0); + int err = save(state, self, item, 0); Py_DECREF(item); if (err < 0) return -1; @@ -3020,7 +3004,7 @@ batch_list_exact(PicklerObject *self, PyObject *obj) while (total < PyList_GET_SIZE(obj)) { item = PyList_GET_ITEM(obj, total); Py_INCREF(item); - int err = save(self, item, 0); + int err = save(state, self, item, 0); Py_DECREF(item); if (err < 0) return -1; @@ -3037,7 +3021,7 @@ batch_list_exact(PicklerObject *self, PyObject *obj) } static int -save_list(PicklerObject *self, PyObject *obj) +save_list(PickleState *state, PicklerObject *self, PyObject *obj) { char header[3]; Py_ssize_t len; @@ -3064,7 +3048,7 @@ save_list(PicklerObject *self, PyObject *obj) if ((len = PyList_Size(obj)) < 0) goto error; - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) goto error; if (len != 0) { @@ -3072,7 +3056,7 @@ save_list(PicklerObject *self, PyObject *obj) if (PyList_CheckExact(obj) && self->proto > 0) { if (_Py_EnterRecursiveCall(" while pickling an object")) goto error; - status = batch_list_exact(self, obj); + status = batch_list_exact(state, self, obj); _Py_LeaveRecursiveCall(); } else { PyObject *iter = PyObject_GetIter(obj); @@ -3083,7 +3067,7 @@ save_list(PicklerObject *self, PyObject *obj) Py_DECREF(iter); goto error; } - status = batch_list(self, iter); + status = batch_list(state, self, iter); _Py_LeaveRecursiveCall(); Py_DECREF(iter); } @@ -3111,7 +3095,7 @@ save_list(PicklerObject *self, PyObject *obj) * ugly to bear. */ static int -batch_dict(PicklerObject *self, PyObject *iter) +batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) { PyObject *obj = NULL; PyObject *firstitem = NULL; @@ -3137,9 +3121,9 @@ batch_dict(PicklerObject *self, PyObject *iter) "iterator must return 2-tuples"); return -1; } - i = save(self, PyTuple_GET_ITEM(obj, 0), 0); + i = save(state, self, PyTuple_GET_ITEM(obj, 0), 0); if (i >= 0) - i = save(self, PyTuple_GET_ITEM(obj, 1), 0); + i = save(state, self, PyTuple_GET_ITEM(obj, 1), 0); Py_DECREF(obj); if (i < 0) return -1; @@ -3173,9 +3157,9 @@ batch_dict(PicklerObject *self, PyObject *iter) goto error; /* Only one item to write */ - if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) goto error; - if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) goto error; if (_Pickler_Write(self, &setitem_op, 1) < 0) goto error; @@ -3189,9 +3173,9 @@ batch_dict(PicklerObject *self, PyObject *iter) if (_Pickler_Write(self, &mark_op, 1) < 0) goto error; - if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) goto error; - if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) goto error; Py_CLEAR(firstitem); n = 1; @@ -3203,8 +3187,8 @@ batch_dict(PicklerObject *self, PyObject *iter) "iterator must return 2-tuples"); goto error; } - if (save(self, PyTuple_GET_ITEM(obj, 0), 0) < 0 || - save(self, PyTuple_GET_ITEM(obj, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(obj, 0), 0) < 0 || + save(state, self, PyTuple_GET_ITEM(obj, 1), 0) < 0) goto error; Py_CLEAR(obj); n += 1; @@ -3242,7 +3226,7 @@ batch_dict(PicklerObject *self, PyObject *iter) * Note that this currently doesn't work for protocol 0. */ static int -batch_dict_exact(PicklerObject *self, PyObject *obj) +batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *key = NULL, *value = NULL; int i; @@ -3262,10 +3246,10 @@ batch_dict_exact(PicklerObject *self, PyObject *obj) PyDict_Next(obj, &ppos, &key, &value); Py_INCREF(key); Py_INCREF(value); - if (save(self, key, 0) < 0) { + if (save(state, self, key, 0) < 0) { goto error; } - if (save(self, value, 0) < 0) { + if (save(state, self, value, 0) < 0) { goto error; } Py_CLEAR(key); @@ -3283,10 +3267,10 @@ batch_dict_exact(PicklerObject *self, PyObject *obj) while (PyDict_Next(obj, &ppos, &key, &value)) { Py_INCREF(key); Py_INCREF(value); - if (save(self, key, 0) < 0) { + if (save(state, self, key, 0) < 0) { goto error; } - if (save(self, value, 0) < 0) { + if (save(state, self, value, 0) < 0) { goto error; } Py_CLEAR(key); @@ -3312,7 +3296,7 @@ batch_dict_exact(PicklerObject *self, PyObject *obj) } static int -save_dict(PicklerObject *self, PyObject *obj) +save_dict(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *items, *iter; char header[3]; @@ -3337,7 +3321,7 @@ save_dict(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, header, len) < 0) goto error; - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) goto error; if (PyDict_GET_SIZE(obj)) { @@ -3347,7 +3331,7 @@ save_dict(PicklerObject *self, PyObject *obj) not a dict subclass. */ if (_Py_EnterRecursiveCall(" while pickling an object")) goto error; - status = batch_dict_exact(self, obj); + status = batch_dict_exact(state, self, obj); _Py_LeaveRecursiveCall(); } else { items = PyObject_CallMethodNoArgs(obj, &_Py_ID(items)); @@ -3361,7 +3345,7 @@ save_dict(PicklerObject *self, PyObject *obj) Py_DECREF(iter); goto error; } - status = batch_dict(self, iter); + status = batch_dict(state, self, iter); _Py_LeaveRecursiveCall(); Py_DECREF(iter); } @@ -3379,7 +3363,7 @@ save_dict(PicklerObject *self, PyObject *obj) } static int -save_set(PicklerObject *self, PyObject *obj) +save_set(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *item; int i; @@ -3405,7 +3389,7 @@ save_set(PicklerObject *self, PyObject *obj) return -1; } /* save_reduce() will memoize the object automatically. */ - status = save_reduce(self, reduce_value, obj); + status = save_reduce(state, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } @@ -3413,7 +3397,7 @@ save_set(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &empty_set_op, 1) < 0) return -1; - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) return -1; set_size = PySet_GET_SIZE(obj); @@ -3427,7 +3411,7 @@ save_set(PicklerObject *self, PyObject *obj) return -1; while (_PySet_NextEntry(obj, &ppos, &item, &hash)) { Py_INCREF(item); - int err = save(self, item, 0); + int err = save(state, self, item, 0); Py_CLEAR(item); if (err < 0) return -1; @@ -3448,7 +3432,7 @@ save_set(PicklerObject *self, PyObject *obj) } static int -save_frozenset(PicklerObject *self, PyObject *obj) +save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *iter; @@ -3474,7 +3458,7 @@ save_frozenset(PicklerObject *self, PyObject *obj) return -1; } /* save_reduce() will memoize the object automatically. */ - status = save_reduce(self, reduce_value, obj); + status = save_reduce(state, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } @@ -3497,7 +3481,7 @@ save_frozenset(PicklerObject *self, PyObject *obj) } break; } - if (save(self, item, 0) < 0) { + if (save(state, self, item, 0) < 0) { Py_DECREF(item); Py_DECREF(iter); return -1; @@ -3514,30 +3498,29 @@ save_frozenset(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &pop_mark_op, 1) < 0) return -1; - if (memo_get(self, obj) < 0) + if (memo_get(state, self, obj) < 0) return -1; return 0; } if (_Pickler_Write(self, &frozenset_op, 1) < 0) return -1; - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) return -1; return 0; } static int -fix_imports(PyObject **module_name, PyObject **global_name) +fix_imports(PickleState *state, PyObject **module_name, PyObject **global_name) { PyObject *key; PyObject *item; - PickleState *st = _Pickle_GetGlobalState(); key = PyTuple_Pack(2, *module_name, *global_name); if (key == NULL) return -1; - item = PyDict_GetItemWithError(st->name_mapping_3to2, key); + item = PyDict_GetItemWithError(state->name_mapping_3to2, key); Py_DECREF(key); if (item) { PyObject *fixed_module_name; @@ -3572,7 +3555,7 @@ fix_imports(PyObject **module_name, PyObject **global_name) return -1; } - item = PyDict_GetItemWithError(st->import_mapping_3to2, *module_name); + item = PyDict_GetItemWithError(state->import_mapping_3to2, *module_name); if (item) { if (!PyUnicode_Check(item)) { PyErr_Format(PyExc_RuntimeError, @@ -3591,7 +3574,7 @@ fix_imports(PyObject **module_name, PyObject **global_name) } static int -save_global(PicklerObject *self, PyObject *obj, PyObject *name) +save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *name) { PyObject *global_name = NULL; PyObject *module_name = NULL; @@ -3600,7 +3583,6 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) PyObject *dotted_path = NULL; PyObject *lastname = NULL; PyObject *cls; - PickleState *st = _Pickle_GetGlobalState(); int status = 0; const char global_op = GLOBAL; @@ -3635,7 +3617,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) extra parameters of __import__ to fix that. */ module = PyImport_Import(module_name); if (module == NULL) { - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "Can't pickle %R: import of module %R failed", obj, module_name); goto error; @@ -3645,14 +3627,14 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) cls = get_deep_attribute(module, dotted_path, &parent); Py_CLEAR(dotted_path); if (cls == NULL) { - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "Can't pickle %R: attribute lookup %S on %S failed", obj, global_name, module_name); goto error; } if (cls != obj) { Py_DECREF(cls); - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "Can't pickle %R: it's not the same object as %S.%S", obj, module_name, global_name); goto error; @@ -3673,7 +3655,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) if (extension_key == NULL) { goto error; } - code_obj = PyDict_GetItemWithError(st->extension_registry, + code_obj = PyDict_GetItemWithError(state->extension_registry, extension_key); Py_DECREF(extension_key); /* The object is not registered in the extension registry. @@ -3691,7 +3673,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) /* Verify code_obj has the right type and value. */ if (!PyLong_Check(code_obj)) { - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "Can't pickle %R: extension code %R isn't an integer", obj, code_obj); goto error; @@ -3699,7 +3681,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) code = PyLong_AS_LONG(code_obj); if (code <= 0 || code > 0x7fffffffL) { if (!PyErr_Occurred()) - PyErr_Format(st->PicklingError, "Can't pickle %R: extension " + PyErr_Format(state->PicklingError, "Can't pickle %R: extension " "code %ld is out of range", obj, code); goto error; } @@ -3736,21 +3718,20 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) if (self->proto >= 4) { const char stack_global_op = STACK_GLOBAL; - if (save(self, module_name, 0) < 0) + if (save(state, self, module_name, 0) < 0) goto error; - if (save(self, global_name, 0) < 0) + if (save(state, self, global_name, 0) < 0) goto error; if (_Pickler_Write(self, &stack_global_op, 1) < 0) goto error; } else if (parent != module) { - PickleState *st = _Pickle_GetGlobalState(); PyObject *reduce_value = Py_BuildValue("(O(OO))", - st->getattr, parent, lastname); + state->getattr, parent, lastname); if (reduce_value == NULL) goto error; - status = save_reduce(self, reduce_value, NULL); + status = save_reduce(state, self, reduce_value, NULL); Py_DECREF(reduce_value); if (status < 0) goto error; @@ -3768,7 +3749,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) /* For protocol < 3 and if the user didn't request against doing so, we convert module names to the old 2.x module names. */ if (self->proto < 3 && self->fix_imports) { - if (fix_imports(&module_name, &global_name) < 0) { + if (fix_imports(state, &module_name, &global_name) < 0) { goto error; } } @@ -3787,7 +3768,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) encoded = unicode_encoder(module_name); if (encoded == NULL) { if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "can't pickle module identifier '%S' using " "pickle protocol %i", module_name, self->proto); @@ -3806,7 +3787,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) encoded = unicode_encoder(global_name); if (encoded == NULL) { if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "can't pickle global identifier '%S' using " "pickle protocol %i", global_name, self->proto); @@ -3822,7 +3803,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) goto error; } /* Memoize the object. */ - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) goto error; } @@ -3841,7 +3822,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) } static int -save_singleton_type(PicklerObject *self, PyObject *obj, PyObject *singleton) +save_singleton_type(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *singleton) { PyObject *reduce_value; int status; @@ -3850,28 +3831,28 @@ save_singleton_type(PicklerObject *self, PyObject *obj, PyObject *singleton) if (reduce_value == NULL) { return -1; } - status = save_reduce(self, reduce_value, obj); + status = save_reduce(state, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } static int -save_type(PicklerObject *self, PyObject *obj) +save_type(PickleState *state, PicklerObject *self, PyObject *obj) { if (obj == (PyObject *)&_PyNone_Type) { - return save_singleton_type(self, obj, Py_None); + return save_singleton_type(state, self, obj, Py_None); } else if (obj == (PyObject *)&PyEllipsis_Type) { - return save_singleton_type(self, obj, Py_Ellipsis); + return save_singleton_type(state, self, obj, Py_Ellipsis); } else if (obj == (PyObject *)&_PyNotImplemented_Type) { - return save_singleton_type(self, obj, Py_NotImplemented); + return save_singleton_type(state, self, obj, Py_NotImplemented); } - return save_global(self, obj, NULL); + return save_global(state, self, obj, NULL); } static int -save_pers(PicklerObject *self, PyObject *obj) +save_pers(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *pid = NULL; int status = 0; @@ -3885,7 +3866,7 @@ save_pers(PicklerObject *self, PyObject *obj) if (pid != Py_None) { if (self->bin) { - if (save(self, pid, 1) < 0 || + if (save(state, self, pid, 1) < 0 || _Pickler_Write(self, &binpersid_op, 1) < 0) goto error; } @@ -3899,7 +3880,7 @@ save_pers(PicklerObject *self, PyObject *obj) /* XXX: Should it check whether the pid contains embedded newlines? */ if (!PyUnicode_IS_ASCII(pid_str)) { - PyErr_SetString(_Pickle_GetGlobalState()->PicklingError, + PyErr_SetString(state->PicklingError, "persistent IDs in protocol 0 must be " "ASCII strings"); Py_DECREF(pid_str); @@ -3942,15 +3923,14 @@ get_class(PyObject *obj) * appropriate __reduce__ method for obj. */ static int -save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) +save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *obj) { PyObject *callable; PyObject *argtup; - PyObject *state = NULL; + PyObject *state2 = NULL; PyObject *listitems = Py_None; PyObject *dictitems = Py_None; PyObject *state_setter = Py_None; - PickleState *st = _Pickle_GetGlobalState(); Py_ssize_t size; int use_newobj = 0, use_newobj_ex = 0; @@ -3961,34 +3941,34 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) size = PyTuple_Size(args); if (size < 2 || size > 6) { - PyErr_SetString(st->PicklingError, "tuple returned by " + PyErr_SetString(state->PicklingError, "tuple returned by " "__reduce__ must contain 2 through 6 elements"); return -1; } if (!PyArg_UnpackTuple(args, "save_reduce", 2, 6, - &callable, &argtup, &state, &listitems, &dictitems, + &callable, &argtup, &state2, &listitems, &dictitems, &state_setter)) return -1; if (!PyCallable_Check(callable)) { - PyErr_SetString(st->PicklingError, "first item of the tuple " + PyErr_SetString(state->PicklingError, "first item of the tuple " "returned by __reduce__ must be callable"); return -1; } if (!PyTuple_Check(argtup)) { - PyErr_SetString(st->PicklingError, "second item of the tuple " + PyErr_SetString(state->PicklingError, "second item of the tuple " "returned by __reduce__ must be a tuple"); return -1; } - if (state == Py_None) - state = NULL; + if (state2 == Py_None) + state2 = NULL; if (listitems == Py_None) listitems = NULL; else if (!PyIter_Check(listitems)) { - PyErr_Format(st->PicklingError, "fourth element of the tuple " + PyErr_Format(state->PicklingError, "fourth element of the tuple " "returned by __reduce__ must be an iterator, not %s", Py_TYPE(listitems)->tp_name); return -1; @@ -3997,7 +3977,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) if (dictitems == Py_None) dictitems = NULL; else if (!PyIter_Check(dictitems)) { - PyErr_Format(st->PicklingError, "fifth element of the tuple " + PyErr_Format(state->PicklingError, "fifth element of the tuple " "returned by __reduce__ must be an iterator, not %s", Py_TYPE(dictitems)->tp_name); return -1; @@ -4006,7 +3986,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) if (state_setter == Py_None) state_setter = NULL; else if (!PyCallable_Check(state_setter)) { - PyErr_Format(st->PicklingError, "sixth element of the tuple " + PyErr_Format(state->PicklingError, "sixth element of the tuple " "returned by __reduce__ must be a function, not %s", Py_TYPE(state_setter)->tp_name); return -1; @@ -4033,7 +4013,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) PyObject *kwargs; if (PyTuple_GET_SIZE(argtup) != 3) { - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "length of the NEWOBJ_EX argument tuple must be " "exactly 3, not %zd", PyTuple_GET_SIZE(argtup)); return -1; @@ -4041,30 +4021,30 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) cls = PyTuple_GET_ITEM(argtup, 0); if (!PyType_Check(cls)) { - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "first item from NEWOBJ_EX argument tuple must " "be a class, not %.200s", Py_TYPE(cls)->tp_name); return -1; } args = PyTuple_GET_ITEM(argtup, 1); if (!PyTuple_Check(args)) { - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "second item from NEWOBJ_EX argument tuple must " "be a tuple, not %.200s", Py_TYPE(args)->tp_name); return -1; } kwargs = PyTuple_GET_ITEM(argtup, 2); if (!PyDict_Check(kwargs)) { - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "third item from NEWOBJ_EX argument tuple must " "be a dict, not %.200s", Py_TYPE(kwargs)->tp_name); return -1; } if (self->proto >= 4) { - if (save(self, cls, 0) < 0 || - save(self, args, 0) < 0 || - save(self, kwargs, 0) < 0 || + if (save(state, self, cls, 0) < 0 || + save(state, self, args, 0) < 0 || + save(state, self, kwargs, 0) < 0 || _Pickler_Write(self, &newobj_ex_op, 1) < 0) { return -1; } @@ -4090,7 +4070,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) PyTuple_SET_ITEM(newargs, i + 2, Py_NewRef(item)); } - callable = PyObject_Call(st->partial, newargs, kwargs); + callable = PyObject_Call(state->partial, newargs, kwargs); Py_DECREF(newargs); if (callable == NULL) return -1; @@ -4101,8 +4081,8 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) return -1; } - if (save(self, callable, 0) < 0 || - save(self, newargs, 0) < 0 || + if (save(state, self, callable, 0) < 0 || + save(state, self, newargs, 0) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0) { Py_DECREF(newargs); Py_DECREF(callable); @@ -4120,13 +4100,13 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) /* Sanity checks. */ if (PyTuple_GET_SIZE(argtup) < 1) { - PyErr_SetString(st->PicklingError, "__newobj__ arglist is empty"); + PyErr_SetString(state->PicklingError, "__newobj__ arglist is empty"); return -1; } cls = PyTuple_GET_ITEM(argtup, 0); if (!PyType_Check(cls)) { - PyErr_SetString(st->PicklingError, "args[0] from " + PyErr_SetString(state->PicklingError, "args[0] from " "__newobj__ args is not a type"); return -1; } @@ -4139,7 +4119,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) p = obj_class != cls; Py_DECREF(obj_class); if (p) { - PyErr_SetString(st->PicklingError, "args[0] from " + PyErr_SetString(state->PicklingError, "args[0] from " "__newobj__ args has the wrong class"); return -1; } @@ -4172,14 +4152,15 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) function. */ /* Save the class and its __new__ arguments. */ - if (save(self, cls, 0) < 0) + if (save(state, self, cls, 0) < 0) { return -1; + } newargtup = PyTuple_GetSlice(argtup, 1, PyTuple_GET_SIZE(argtup)); if (newargtup == NULL) return -1; - p = save(self, newargtup, 0); + p = save(state, self, newargtup, 0); Py_DECREF(newargtup); if (p < 0) return -1; @@ -4189,8 +4170,8 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) return -1; } else { /* Not using NEWOBJ. */ - if (save(self, callable, 0) < 0 || - save(self, argtup, 0) < 0 || + if (save(state, self, callable, 0) < 0 || + save(state, self, argtup, 0) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0) return -1; } @@ -4208,24 +4189,24 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) if (_Pickler_Write(self, &pop_op, 1) < 0) return -1; - if (memo_get(self, obj) < 0) + if (memo_get(state, self, obj) < 0) return -1; return 0; } - else if (memo_put(self, obj) < 0) + else if (memo_put(state, self, obj) < 0) return -1; } - if (listitems && batch_list(self, listitems) < 0) + if (listitems && batch_list(state, self, listitems) < 0) return -1; - if (dictitems && batch_dict(self, dictitems) < 0) + if (dictitems && batch_dict(state, self, dictitems) < 0) return -1; - if (state) { + if (state2) { if (state_setter == NULL) { - if (save(self, state, 0) < 0 || + if (save(state, self, state2, 0) < 0 || _Pickler_Write(self, &build_op, 1) < 0) return -1; } @@ -4242,8 +4223,8 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) const char tupletwo_op = TUPLE2; const char pop_op = POP; - if (save(self, state_setter, 0) < 0 || - save(self, obj, 0) < 0 || save(self, state, 0) < 0 || + if (save(state, self, state_setter, 0) < 0 || + save(state, self, obj, 0) < 0 || save(state, self, state2, 0) < 0 || _Pickler_Write(self, &tupletwo_op, 1) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0 || _Pickler_Write(self, &pop_op, 1) < 0) @@ -4254,7 +4235,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) } static int -save(PicklerObject *self, PyObject *obj, int pers_save) +save(PickleState *state, PicklerObject *self, PyObject *obj, int pers_save) { PyTypeObject *type; PyObject *reduce_func = NULL; @@ -4272,7 +4253,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) 0 if it did nothing successfully; 1 if a persistent id was saved. */ - if ((status = save_pers(self, obj)) != 0) + if ((status = save_pers(state, self, obj)) != 0) return status; } @@ -4302,14 +4283,14 @@ save(PicklerObject *self, PyObject *obj, int pers_save) a GET (or BINGET) opcode, instead of pickling the object once again. */ if (PyMemoTable_Get(self->memo, obj)) { - return memo_get(self, obj); + return memo_get(state, self, obj); } if (type == &PyBytes_Type) { - return save_bytes(self, obj); + return save_bytes(state, self, obj); } else if (type == &PyUnicode_Type) { - return save_unicode(self, obj); + return save_unicode(state, self, obj); } /* We're only calling _Py_EnterRecursiveCall here so that atomic @@ -4319,31 +4300,31 @@ save(PicklerObject *self, PyObject *obj, int pers_save) } if (type == &PyDict_Type) { - status = save_dict(self, obj); + status = save_dict(state, self, obj); goto done; } else if (type == &PySet_Type) { - status = save_set(self, obj); + status = save_set(state, self, obj); goto done; } else if (type == &PyFrozenSet_Type) { - status = save_frozenset(self, obj); + status = save_frozenset(state, self, obj); goto done; } else if (type == &PyList_Type) { - status = save_list(self, obj); + status = save_list(state, self, obj); goto done; } else if (type == &PyTuple_Type) { - status = save_tuple(self, obj); + status = save_tuple(state, self, obj); goto done; } else if (type == &PyByteArray_Type) { - status = save_bytearray(self, obj); + status = save_bytearray(state, self, obj); goto done; } else if (type == &PyPickleBuffer_Type) { - status = save_picklebuffer(self, obj); + status = save_picklebuffer(state, self, obj); goto done; } @@ -4363,11 +4344,11 @@ save(PicklerObject *self, PyObject *obj, int pers_save) } if (type == &PyType_Type) { - status = save_type(self, obj); + status = save_type(state, self, obj); goto done; } else if (type == &PyFunction_Type) { - status = save_global(self, obj, NULL); + status = save_global(state, self, obj, NULL); goto done; } @@ -4378,8 +4359,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) * __reduce_ex__ method, or the object's __reduce__ method. */ if (self->dispatch_table == NULL) { - PickleState *st = _Pickle_GetGlobalState(); - reduce_func = PyDict_GetItemWithError(st->dispatch_table, + reduce_func = PyDict_GetItemWithError(state->dispatch_table, (PyObject *)type); if (reduce_func == NULL) { if (PyErr_Occurred()) { @@ -4405,7 +4385,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) reduce_value = _Pickle_FastCall(reduce_func, Py_NewRef(obj)); } else if (PyType_IsSubtype(type, &PyType_Type)) { - status = save_global(self, obj, NULL); + status = save_global(state, self, obj, NULL); goto done; } else { @@ -4437,8 +4417,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) reduce_value = PyObject_CallNoArgs(reduce_func); } else { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "can't pickle '%.200s' object: %R", type->tp_name, obj); goto error; @@ -4451,18 +4430,17 @@ save(PicklerObject *self, PyObject *obj, int pers_save) reduce: if (PyUnicode_Check(reduce_value)) { - status = save_global(self, obj, reduce_value); + status = save_global(state, self, obj, reduce_value); goto done; } if (!PyTuple_Check(reduce_value)) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->PicklingError, + PyErr_SetString(state->PicklingError, "__reduce__ must return a string or tuple"); goto error; } - status = save_reduce(self, reduce_value, obj); + status = save_reduce(state, self, reduce_value, obj); if (0) { error: @@ -4478,7 +4456,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) } static int -dump(PicklerObject *self, PyObject *obj) +dump(PickleState *state, PicklerObject *self, PyObject *obj) { const char stop_op = STOP; int status = -1; @@ -4508,7 +4486,7 @@ dump(PicklerObject *self, PyObject *obj) self->framing = 1; } - if (save(self, obj, 0) < 0 || + if (save(state, self, obj, 0) < 0 || _Pickler_Write(self, &stop_op, 1) < 0 || _Pickler_CommitFrame(self) < 0) goto error; @@ -4555,6 +4533,7 @@ _pickle_Pickler_clear_memo_impl(PicklerObject *self) _pickle.Pickler.dump + cls: defining_class obj: object / @@ -4562,15 +4541,16 @@ Write a pickled representation of the given object to the open file. [clinic start generated code]*/ static PyObject * -_pickle_Pickler_dump(PicklerObject *self, PyObject *obj) -/*[clinic end generated code: output=87ecad1261e02ac7 input=552eb1c0f52260d9]*/ +_pickle_Pickler_dump_impl(PicklerObject *self, PyTypeObject *cls, + PyObject *obj) +/*[clinic end generated code: output=952cf7f68b1445bb input=f949d84151983594]*/ { + PickleState *state = PyType_GetModuleState(cls); /* Check whether the Pickler was initialized correctly (issue3664). Developers often forget to call __init__() in their subclasses, which would trigger a segfault without this check. */ if (self->write == NULL) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_Format(st->PicklingError, + PyErr_Format(state->PicklingError, "Pickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -4579,7 +4559,7 @@ _pickle_Pickler_dump(PicklerObject *self, PyObject *obj) if (_Pickler_ClearBuffer(self) < 0) return NULL; - if (dump(self, obj) < 0) + if (dump(state, self, obj) < 0) return NULL; if (_Pickler_FlushToFile(self) < 0) @@ -5097,17 +5077,14 @@ find_class(UnpicklerObject *self, PyObject *module_name, PyObject *global_name) } static Py_ssize_t -marker(UnpicklerObject *self) +marker(PickleState *state, UnpicklerObject *self) { - Py_ssize_t mark; - if (self->num_marks < 1) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, "could not find MARK"); + PyErr_SetString(state->UnpicklingError, "could not find MARK"); return -1; } - mark = self->marks[--self->num_marks]; + Py_ssize_t mark = self->marks[--self->num_marks]; self->stack->mark_set = self->num_marks != 0; self->stack->fence = self->num_marks ? self->marks[self->num_marks - 1] : 0; @@ -5115,24 +5092,24 @@ marker(UnpicklerObject *self) } static int -load_none(UnpicklerObject *self) +load_none(PickleState *state, UnpicklerObject *self) { PDATA_APPEND(self->stack, Py_None, -1); return 0; } static int -load_int(UnpicklerObject *self) +load_int(PickleState *state, UnpicklerObject *self) { PyObject *value; char *endptr, *s; Py_ssize_t len; long x; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); errno = 0; /* XXX: Should the base argument of strtol() be explicitly set to 10? @@ -5167,7 +5144,7 @@ load_int(UnpicklerObject *self) } static int -load_bool(UnpicklerObject *self, PyObject *boolean) +load_bool(PickleState *state, UnpicklerObject *self, PyObject *boolean) { assert(boolean == Py_True || boolean == Py_False); PDATA_APPEND(self->stack, boolean, -1); @@ -5247,49 +5224,49 @@ load_binintx(UnpicklerObject *self, char *s, int size) } static int -load_binint(UnpicklerObject *self) +load_binint(PickleState *state, UnpicklerObject *self) { char *s; - - if (_Unpickler_Read(self, &s, 4) < 0) + if (_Unpickler_Read(self, state, &s, 4) < 0) { return -1; + } return load_binintx(self, s, 4); } static int -load_binint1(UnpicklerObject *self) +load_binint1(PickleState *state, UnpicklerObject *self) { char *s; - - if (_Unpickler_Read(self, &s, 1) < 0) + if (_Unpickler_Read(self, state, &s, 1) < 0) { return -1; + } return load_binintx(self, s, 1); } static int -load_binint2(UnpicklerObject *self) +load_binint2(PickleState *state, UnpicklerObject *self) { char *s; - - if (_Unpickler_Read(self, &s, 2) < 0) + if (_Unpickler_Read(self, state, &s, 2) < 0) { return -1; + } return load_binintx(self, s, 2); } static int -load_long(UnpicklerObject *self) +load_long(PickleState *state, UnpicklerObject *self) { PyObject *value; char *s = NULL; Py_ssize_t len; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); /* s[len-2] will usually be 'L' (and s[len-1] is '\n'); we need to remove the 'L' before calling PyLong_FromString. In order to maintain @@ -5310,21 +5287,20 @@ load_long(UnpicklerObject *self) * data following. */ static int -load_counted_long(UnpicklerObject *self, int size) +load_counted_long(PickleState *state, UnpicklerObject *self, int size) { PyObject *value; char *nbytes; char *pdata; assert(size == 1 || size == 4); - if (_Unpickler_Read(self, &nbytes, size) < 0) + if (_Unpickler_Read(self, state, &nbytes, size) < 0) return -1; size = calc_binint(nbytes, size); if (size < 0) { - PickleState *st = _Pickle_GetGlobalState(); /* Corrupt or hostile pickle -- we never write one like this */ - PyErr_SetString(st->UnpicklingError, + PyErr_SetString(state->UnpicklingError, "LONG pickle has negative byte count"); return -1; } @@ -5333,7 +5309,7 @@ load_counted_long(UnpicklerObject *self, int size) value = PyLong_FromLong(0L); else { /* Read the raw little-endian bytes and convert. */ - if (_Unpickler_Read(self, &pdata, size) < 0) + if (_Unpickler_Read(self, state, &pdata, size) < 0) return -1; value = _PyLong_FromByteArray((unsigned char *)pdata, (size_t)size, 1 /* little endian */ , 1 /* signed */ ); @@ -5345,17 +5321,17 @@ load_counted_long(UnpicklerObject *self, int size) } static int -load_float(UnpicklerObject *self) +load_float(PickleState *state, UnpicklerObject *self) { PyObject *value; char *endptr, *s; Py_ssize_t len; double d; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); errno = 0; d = PyOS_string_to_double(s, &endptr, PyExc_OverflowError); @@ -5374,13 +5350,13 @@ load_float(UnpicklerObject *self) } static int -load_binfloat(UnpicklerObject *self) +load_binfloat(PickleState *state, UnpicklerObject *self) { PyObject *value; double x; char *s; - if (_Unpickler_Read(self, &s, 8) < 0) + if (_Unpickler_Read(self, state, &s, 8) < 0) return -1; x = PyFloat_Unpack8(s, 0); @@ -5395,14 +5371,14 @@ load_binfloat(UnpicklerObject *self) } static int -load_string(UnpicklerObject *self) +load_string(PickleState *state, UnpicklerObject *self) { PyObject *bytes; PyObject *obj; Py_ssize_t len; char *s, *p; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; /* Strip the newline */ len--; @@ -5412,8 +5388,7 @@ load_string(UnpicklerObject *self) len -= 2; } else { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, + PyErr_SetString(state->UnpicklingError, "the STRING opcode argument must be quoted"); return -1; } @@ -5443,25 +5418,24 @@ load_string(UnpicklerObject *self) } static int -load_counted_binstring(UnpicklerObject *self, int nbytes) +load_counted_binstring(PickleState *state, UnpicklerObject *self, int nbytes) { PyObject *obj; Py_ssize_t size; char *s; - if (_Unpickler_Read(self, &s, nbytes) < 0) + if (_Unpickler_Read(self, state, &s, nbytes) < 0) return -1; size = calc_binsize(s, nbytes); if (size < 0) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_Format(st->UnpicklingError, + PyErr_Format(state->UnpicklingError, "BINSTRING exceeds system's maximum size of %zd bytes", PY_SSIZE_T_MAX); return -1; } - if (_Unpickler_Read(self, &s, size) < 0) + if (_Unpickler_Read(self, state, &s, size) < 0) return -1; /* Convert Python 2.x strings to bytes if the *encoding* given to the @@ -5481,13 +5455,13 @@ load_counted_binstring(UnpicklerObject *self, int nbytes) } static int -load_counted_binbytes(UnpicklerObject *self, int nbytes) +load_counted_binbytes(PickleState *state, UnpicklerObject *self, int nbytes) { PyObject *bytes; Py_ssize_t size; char *s; - if (_Unpickler_Read(self, &s, nbytes) < 0) + if (_Unpickler_Read(self, state, &s, nbytes) < 0) return -1; size = calc_binsize(s, nbytes); @@ -5501,7 +5475,7 @@ load_counted_binbytes(UnpicklerObject *self, int nbytes) bytes = PyBytes_FromStringAndSize(NULL, size); if (bytes == NULL) return -1; - if (_Unpickler_ReadInto(self, PyBytes_AS_STRING(bytes), size) < 0) { + if (_Unpickler_ReadInto(state, self, PyBytes_AS_STRING(bytes), size) < 0) { Py_DECREF(bytes); return -1; } @@ -5511,13 +5485,13 @@ load_counted_binbytes(UnpicklerObject *self, int nbytes) } static int -load_counted_bytearray(UnpicklerObject *self) +load_counted_bytearray(PickleState *state, UnpicklerObject *self) { PyObject *bytearray; Py_ssize_t size; char *s; - if (_Unpickler_Read(self, &s, 8) < 0) { + if (_Unpickler_Read(self, state, &s, 8) < 0) { return -1; } @@ -5533,7 +5507,7 @@ load_counted_bytearray(UnpicklerObject *self) if (bytearray == NULL) { return -1; } - if (_Unpickler_ReadInto(self, PyByteArray_AS_STRING(bytearray), size) < 0) { + if (_Unpickler_ReadInto(state, self, PyByteArray_AS_STRING(bytearray), size) < 0) { Py_DECREF(bytearray); return -1; } @@ -5543,11 +5517,10 @@ load_counted_bytearray(UnpicklerObject *self) } static int -load_next_buffer(UnpicklerObject *self) +load_next_buffer(PickleState *state, UnpicklerObject *self) { if (self->buffers == NULL) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, + PyErr_SetString(state->UnpicklingError, "pickle stream refers to out-of-band data " "but no *buffers* argument was given"); return -1; @@ -5555,8 +5528,7 @@ load_next_buffer(UnpicklerObject *self) PyObject *buf = PyIter_Next(self->buffers); if (buf == NULL) { if (!PyErr_Occurred()) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, + PyErr_SetString(state->UnpicklingError, "not enough out-of-band buffers"); } return -1; @@ -5567,11 +5539,11 @@ load_next_buffer(UnpicklerObject *self) } static int -load_readonly_buffer(UnpicklerObject *self) +load_readonly_buffer(PickleState *state, UnpicklerObject *self) { Py_ssize_t len = Py_SIZE(self->stack); if (len <= self->stack->fence) { - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); } PyObject *obj = self->stack->data[len - 1]; @@ -5593,16 +5565,16 @@ load_readonly_buffer(UnpicklerObject *self) } static int -load_unicode(UnpicklerObject *self) +load_unicode(PickleState *state, UnpicklerObject *self) { PyObject *str; Py_ssize_t len; char *s = NULL; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 1) - return bad_readline(); + return bad_readline(state); str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL); if (str == NULL) @@ -5613,13 +5585,13 @@ load_unicode(UnpicklerObject *self) } static int -load_counted_binunicode(UnpicklerObject *self, int nbytes) +load_counted_binunicode(PickleState *state, UnpicklerObject *self, int nbytes) { PyObject *str; Py_ssize_t size; char *s; - if (_Unpickler_Read(self, &s, nbytes) < 0) + if (_Unpickler_Read(self, state, &s, nbytes) < 0) return -1; size = calc_binsize(s, nbytes); @@ -5630,7 +5602,7 @@ load_counted_binunicode(UnpicklerObject *self, int nbytes) return -1; } - if (_Unpickler_Read(self, &s, size) < 0) + if (_Unpickler_Read(self, state, &s, size) < 0) return -1; str = PyUnicode_DecodeUTF8(s, size, "surrogatepass"); @@ -5642,14 +5614,14 @@ load_counted_binunicode(UnpicklerObject *self, int nbytes) } static int -load_counted_tuple(UnpicklerObject *self, Py_ssize_t len) +load_counted_tuple(PickleState *state, UnpicklerObject *self, Py_ssize_t len) { PyObject *tuple; if (Py_SIZE(self->stack) < len) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); - tuple = Pdata_poptuple(self->stack, Py_SIZE(self->stack) - len); + tuple = Pdata_poptuple(state, self->stack, Py_SIZE(self->stack) - len); if (tuple == NULL) return -1; PDATA_PUSH(self->stack, tuple, -1); @@ -5657,56 +5629,56 @@ load_counted_tuple(UnpicklerObject *self, Py_ssize_t len) } static int -load_tuple(UnpicklerObject *self) +load_tuple(PickleState *state, UnpicklerObject *self) { Py_ssize_t i; - - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) { return -1; + } - return load_counted_tuple(self, Py_SIZE(self->stack) - i); + return load_counted_tuple(state, self, Py_SIZE(self->stack) - i); } static int -load_empty_list(UnpicklerObject *self) +load_empty_list(PickleState *state, UnpicklerObject *self) { PyObject *list; - - if ((list = PyList_New(0)) == NULL) + if ((list = PyList_New(0)) == NULL) { return -1; + } PDATA_PUSH(self->stack, list, -1); return 0; } static int -load_empty_dict(UnpicklerObject *self) +load_empty_dict(PickleState *state, UnpicklerObject *self) { PyObject *dict; - - if ((dict = PyDict_New()) == NULL) + if ((dict = PyDict_New()) == NULL) { return -1; + } PDATA_PUSH(self->stack, dict, -1); return 0; } static int -load_empty_set(UnpicklerObject *self) +load_empty_set(PickleState *state, UnpicklerObject *self) { PyObject *set; - - if ((set = PySet_New(NULL)) == NULL) + if ((set = PySet_New(NULL)) == NULL) { return -1; + } PDATA_PUSH(self->stack, set, -1); return 0; } static int -load_list(UnpicklerObject *self) +load_list(PickleState *state, UnpicklerObject *self) { PyObject *list; Py_ssize_t i; - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; list = Pdata_poplist(self->stack, i); @@ -5717,12 +5689,12 @@ load_list(UnpicklerObject *self) } static int -load_dict(UnpicklerObject *self) +load_dict(PickleState *state, UnpicklerObject *self) { PyObject *dict, *key, *value; Py_ssize_t i, j, k; - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; j = Py_SIZE(self->stack); @@ -5730,8 +5702,7 @@ load_dict(UnpicklerObject *self) return -1; if ((j - i) % 2 != 0) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, "odd number of items for DICT"); + PyErr_SetString(state->UnpicklingError, "odd number of items for DICT"); Py_DECREF(dict); return -1; } @@ -5750,16 +5721,16 @@ load_dict(UnpicklerObject *self) } static int -load_frozenset(UnpicklerObject *self) +load_frozenset(PickleState *state, UnpicklerObject *self) { PyObject *items; PyObject *frozenset; Py_ssize_t i; - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; - items = Pdata_poptuple(self->stack, i); + items = Pdata_poptuple(state, self->stack, i); if (items == NULL) return -1; @@ -5793,22 +5764,22 @@ instantiate(PyObject *cls, PyObject *args) } static int -load_obj(UnpicklerObject *self) +load_obj(PickleState *state, UnpicklerObject *self) { PyObject *cls, *args, *obj = NULL; Py_ssize_t i; - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; if (Py_SIZE(self->stack) - i < 1) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); - args = Pdata_poptuple(self->stack, i + 1); + args = Pdata_poptuple(state, self->stack, i + 1); if (args == NULL) return -1; - PDATA_POP(self->stack, cls); + PDATA_POP(state, self->stack, cls); if (cls) { obj = instantiate(cls, args); Py_DECREF(cls); @@ -5822,7 +5793,7 @@ load_obj(UnpicklerObject *self) } static int -load_inst(UnpicklerObject *self) +load_inst(PickleState *state, UnpicklerObject *self) { PyObject *cls = NULL; PyObject *args = NULL; @@ -5833,12 +5804,12 @@ load_inst(UnpicklerObject *self) Py_ssize_t i; char *s; - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); /* Here it is safe to use PyUnicode_DecodeASCII(), even though non-ASCII identifiers are permitted in Python 3.0, since the INST opcode is only @@ -5847,10 +5818,10 @@ load_inst(UnpicklerObject *self) if (module_name == NULL) return -1; - if ((len = _Unpickler_Readline(self, &s)) >= 0) { + if ((len = _Unpickler_Readline(state, self, &s)) >= 0) { if (len < 2) { Py_DECREF(module_name); - return bad_readline(); + return bad_readline(state); } class_name = PyUnicode_DecodeASCII(s, len - 1, "strict"); if (class_name != NULL) { @@ -5863,7 +5834,7 @@ load_inst(UnpicklerObject *self) if (cls == NULL) return -1; - if ((args = Pdata_poptuple(self->stack, i)) != NULL) { + if ((args = Pdata_poptuple(state, self->stack, i)) != NULL) { obj = instantiate(cls, args); Py_DECREF(args); } @@ -5877,16 +5848,15 @@ load_inst(UnpicklerObject *self) } static void -newobj_unpickling_error(const char * msg, int use_kwargs, PyObject *arg) +newobj_unpickling_error(PickleState *state, const char * msg, int use_kwargs, PyObject *arg) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_Format(st->UnpicklingError, msg, + PyErr_Format(state->UnpicklingError, msg, use_kwargs ? "NEWOBJ_EX" : "NEWOBJ", Py_TYPE(arg)->tp_name); } static int -load_newobj(UnpicklerObject *self, int use_kwargs) +load_newobj(PickleState *state, UnpicklerObject *self, int use_kwargs) { PyObject *cls, *args, *kwargs = NULL; PyObject *obj; @@ -5895,17 +5865,17 @@ load_newobj(UnpicklerObject *self, int use_kwargs) * cls.__new__(cls, *args, **kwargs). */ if (use_kwargs) { - PDATA_POP(self->stack, kwargs); + PDATA_POP(state, self->stack, kwargs); if (kwargs == NULL) { return -1; } } - PDATA_POP(self->stack, args); + PDATA_POP(state, self->stack, args); if (args == NULL) { Py_XDECREF(kwargs); return -1; } - PDATA_POP(self->stack, cls); + PDATA_POP(state, self->stack, cls); if (cls == NULL) { Py_XDECREF(kwargs); Py_DECREF(args); @@ -5913,22 +5883,22 @@ load_newobj(UnpicklerObject *self, int use_kwargs) } if (!PyType_Check(cls)) { - newobj_unpickling_error("%s class argument must be a type, not %.200s", + newobj_unpickling_error(state, "%s class argument must be a type, not %.200s", use_kwargs, cls); goto error; } if (((PyTypeObject *)cls)->tp_new == NULL) { - newobj_unpickling_error("%s class argument '%.200s' doesn't have __new__", + newobj_unpickling_error(state, "%s class argument '%.200s' doesn't have __new__", use_kwargs, cls); goto error; } if (!PyTuple_Check(args)) { - newobj_unpickling_error("%s args argument must be a tuple, not %.200s", + newobj_unpickling_error(state, "%s args argument must be a tuple, not %.200s", use_kwargs, args); goto error; } if (use_kwargs && !PyDict_Check(kwargs)) { - newobj_unpickling_error("%s kwargs argument must be a dict, not %.200s", + newobj_unpickling_error(state, "%s kwargs argument must be a dict, not %.200s", use_kwargs, kwargs); goto error; } @@ -5951,7 +5921,7 @@ load_newobj(UnpicklerObject *self, int use_kwargs) } static int -load_global(UnpicklerObject *self) +load_global(PickleState *state, UnpicklerObject *self) { PyObject *global = NULL; PyObject *module_name; @@ -5959,18 +5929,18 @@ load_global(UnpicklerObject *self) Py_ssize_t len; char *s; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); module_name = PyUnicode_DecodeUTF8(s, len - 1, "strict"); if (!module_name) return -1; - if ((len = _Unpickler_Readline(self, &s)) >= 0) { + if ((len = _Unpickler_Readline(state, self, &s)) >= 0) { if (len < 2) { Py_DECREF(module_name); - return bad_readline(); + return bad_readline(state); } global_name = PyUnicode_DecodeUTF8(s, len - 1, "strict"); if (global_name) { @@ -5987,18 +5957,17 @@ load_global(UnpicklerObject *self) } static int -load_stack_global(UnpicklerObject *self) +load_stack_global(PickleState *state, UnpicklerObject *self) { PyObject *global; PyObject *module_name; PyObject *global_name; - PDATA_POP(self->stack, global_name); - PDATA_POP(self->stack, module_name); + PDATA_POP(state, self->stack, global_name); + PDATA_POP(state, self->stack, module_name); if (module_name == NULL || !PyUnicode_CheckExact(module_name) || global_name == NULL || !PyUnicode_CheckExact(global_name)) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, "STACK_GLOBAL requires str"); + PyErr_SetString(state->UnpicklingError, "STACK_GLOBAL requires str"); Py_XDECREF(global_name); Py_XDECREF(module_name); return -1; @@ -6013,22 +5982,22 @@ load_stack_global(UnpicklerObject *self) } static int -load_persid(UnpicklerObject *self) +load_persid(PickleState *state, UnpicklerObject *self) { PyObject *pid, *obj; Py_ssize_t len; char *s; if (self->pers_func) { - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 1) - return bad_readline(); + return bad_readline(state); pid = PyUnicode_DecodeASCII(s, len - 1, "strict"); if (pid == NULL) { if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { - PyErr_SetString(_Pickle_GetGlobalState()->UnpicklingError, + PyErr_SetString(state->UnpicklingError, "persistent IDs in protocol 0 must be " "ASCII strings"); } @@ -6044,8 +6013,7 @@ load_persid(UnpicklerObject *self) return 0; } else { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, + PyErr_SetString(state->UnpicklingError, "A load persistent id instruction was encountered, " "but no persistent_load function was specified."); return -1; @@ -6053,12 +6021,12 @@ load_persid(UnpicklerObject *self) } static int -load_binpersid(UnpicklerObject *self) +load_binpersid(PickleState *state, UnpicklerObject *self) { PyObject *pid, *obj; if (self->pers_func) { - PDATA_POP(self->stack, pid); + PDATA_POP(state, self->stack, pid); if (pid == NULL) return -1; @@ -6071,8 +6039,7 @@ load_binpersid(UnpicklerObject *self) return 0; } else { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_SetString(st->UnpicklingError, + PyErr_SetString(state->UnpicklingError, "A load persistent id instruction was encountered, " "but no persistent_load function was specified."); return -1; @@ -6080,7 +6047,7 @@ load_binpersid(UnpicklerObject *self) } static int -load_pop(UnpicklerObject *self) +load_pop(PickleState *state, UnpicklerObject *self) { Py_ssize_t len = Py_SIZE(self->stack); @@ -6097,7 +6064,7 @@ load_pop(UnpicklerObject *self) self->stack->fence = self->num_marks ? self->marks[self->num_marks - 1] : 0; } else if (len <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); else { len--; Py_DECREF(self->stack->data[len]); @@ -6107,47 +6074,49 @@ load_pop(UnpicklerObject *self) } static int -load_pop_mark(UnpicklerObject *self) +load_pop_mark(PickleState *state, UnpicklerObject *self) { Py_ssize_t i; - - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) { return -1; + } Pdata_clear(self->stack, i); - return 0; } static int -load_dup(UnpicklerObject *self) +load_dup(PickleState *state, UnpicklerObject *self) { PyObject *last; Py_ssize_t len = Py_SIZE(self->stack); if (len <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); last = self->stack->data[len - 1]; PDATA_APPEND(self->stack, last, -1); return 0; } static int -load_get(UnpicklerObject *self) +load_get(PickleState *state, UnpicklerObject *self) { PyObject *key, *value; Py_ssize_t idx; Py_ssize_t len; char *s; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) { return -1; - if (len < 2) - return bad_readline(); + } + if (len < 2) { + return bad_readline(state); + } key = PyLong_FromString(s, NULL, 10); - if (key == NULL) + if (key == NULL) { return -1; + } idx = PyLong_AsSsize_t(key); if (idx == -1 && PyErr_Occurred()) { Py_DECREF(key); @@ -6157,8 +6126,7 @@ load_get(UnpicklerObject *self) value = _Unpickler_MemoGet(self, idx); if (value == NULL) { if (!PyErr_Occurred()) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); + PyErr_Format(state->UnpicklingError, "Memo value not found at index %ld", idx); } Py_DECREF(key); return -1; @@ -6170,13 +6138,13 @@ load_get(UnpicklerObject *self) } static int -load_binget(UnpicklerObject *self) +load_binget(PickleState *state, UnpicklerObject *self) { PyObject *value; Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, &s, 1) < 0) + if (_Unpickler_Read(self, state, &s, 1) < 0) return -1; idx = Py_CHARMASK(s[0]); @@ -6185,8 +6153,7 @@ load_binget(UnpicklerObject *self) if (value == NULL) { PyObject *key = PyLong_FromSsize_t(idx); if (key != NULL) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); + PyErr_Format(state->UnpicklingError, "Memo value not found at index %ld", idx); Py_DECREF(key); } return -1; @@ -6197,13 +6164,13 @@ load_binget(UnpicklerObject *self) } static int -load_long_binget(UnpicklerObject *self) +load_long_binget(PickleState *state, UnpicklerObject *self) { PyObject *value; Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, &s, 4) < 0) + if (_Unpickler_Read(self, state, &s, 4) < 0) return -1; idx = calc_binsize(s, 4); @@ -6212,8 +6179,7 @@ load_long_binget(UnpicklerObject *self) if (value == NULL) { PyObject *key = PyLong_FromSsize_t(idx); if (key != NULL) { - PickleState *st = _Pickle_GetGlobalState(); - PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); + PyErr_Format(state->UnpicklingError, "Memo value not found at index %ld", idx); Py_DECREF(key); } return -1; @@ -6227,7 +6193,7 @@ load_long_binget(UnpicklerObject *self) * the number of bytes following the opcode, holding the index (code) value. */ static int -load_extension(UnpicklerObject *self, int nbytes) +load_extension(PickleState *state, UnpicklerObject *self, int nbytes) { char *codebytes; /* the nbytes bytes after the opcode */ long code; /* calc_binint returns long */ @@ -6235,15 +6201,14 @@ load_extension(UnpicklerObject *self, int nbytes) PyObject *obj; /* the object to push */ PyObject *pair; /* (module_name, class_name) */ PyObject *module_name, *class_name; - PickleState *st = _Pickle_GetGlobalState(); assert(nbytes == 1 || nbytes == 2 || nbytes == 4); - if (_Unpickler_Read(self, &codebytes, nbytes) < 0) + if (_Unpickler_Read(self, state, &codebytes, nbytes) < 0) return -1; code = calc_binint(codebytes, nbytes); if (code <= 0) { /* note that 0 is forbidden */ /* Corrupt or hostile pickle. */ - PyErr_SetString(st->UnpicklingError, "EXT specifies code <= 0"); + PyErr_SetString(state->UnpicklingError, "EXT specifies code <= 0"); return -1; } @@ -6251,7 +6216,7 @@ load_extension(UnpicklerObject *self, int nbytes) py_code = PyLong_FromLong(code); if (py_code == NULL) return -1; - obj = PyDict_GetItemWithError(st->extension_cache, py_code); + obj = PyDict_GetItemWithError(state->extension_cache, py_code); if (obj != NULL) { /* Bingo. */ Py_DECREF(py_code); @@ -6264,7 +6229,7 @@ load_extension(UnpicklerObject *self, int nbytes) } /* Look up the (module_name, class_name) pair. */ - pair = PyDict_GetItemWithError(st->inverted_registry, py_code); + pair = PyDict_GetItemWithError(state->inverted_registry, py_code); if (pair == NULL) { Py_DECREF(py_code); if (!PyErr_Occurred()) { @@ -6297,7 +6262,7 @@ load_extension(UnpicklerObject *self, int nbytes) return -1; } /* Cache code -> obj. */ - code = PyDict_SetItem(st->extension_cache, py_code, obj); + code = PyDict_SetItem(state->extension_cache, py_code, obj); Py_DECREF(py_code); if (code < 0) { Py_DECREF(obj); @@ -6314,19 +6279,19 @@ load_extension(UnpicklerObject *self, int nbytes) } static int -load_put(UnpicklerObject *self) +load_put(PickleState *state, UnpicklerObject *self) { PyObject *key, *value; Py_ssize_t idx; Py_ssize_t len; char *s = NULL; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); if (Py_SIZE(self->stack) <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; key = PyLong_FromString(s, NULL, 10); @@ -6345,17 +6310,18 @@ load_put(UnpicklerObject *self) } static int -load_binput(UnpicklerObject *self) +load_binput(PickleState *state, UnpicklerObject *self) { PyObject *value; Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, &s, 1) < 0) + if (_Unpickler_Read(self, state, &s, 1) < 0) { return -1; + } if (Py_SIZE(self->stack) <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; idx = Py_CHARMASK(s[0]); @@ -6364,17 +6330,18 @@ load_binput(UnpicklerObject *self) } static int -load_long_binput(UnpicklerObject *self) +load_long_binput(PickleState *state, UnpicklerObject *self) { PyObject *value; Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, &s, 4) < 0) + if (_Unpickler_Read(self, state, &s, 4) < 0) { return -1; + } if (Py_SIZE(self->stack) <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; idx = calc_binsize(s, 4); @@ -6388,19 +6355,19 @@ load_long_binput(UnpicklerObject *self) } static int -load_memoize(UnpicklerObject *self) +load_memoize(PickleState *state, UnpicklerObject *self) { PyObject *value; if (Py_SIZE(self->stack) <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; return _Unpickler_MemoPut(self, self->memo_len, value); } static int -do_append(UnpicklerObject *self, Py_ssize_t x) +do_append(PickleState *state, UnpicklerObject *self, Py_ssize_t x) { PyObject *value; PyObject *slice; @@ -6410,7 +6377,7 @@ do_append(UnpicklerObject *self, Py_ssize_t x) len = Py_SIZE(self->stack); if (x > len || x <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); if (len == x) /* nothing to do */ return 0; @@ -6475,24 +6442,24 @@ do_append(UnpicklerObject *self, Py_ssize_t x) } static int -load_append(UnpicklerObject *self) +load_append(PickleState *state, UnpicklerObject *self) { if (Py_SIZE(self->stack) - 1 <= self->stack->fence) - return Pdata_stack_underflow(self->stack); - return do_append(self, Py_SIZE(self->stack) - 1); + return Pdata_stack_underflow(state, self->stack); + return do_append(state, self, Py_SIZE(self->stack) - 1); } static int -load_appends(UnpicklerObject *self) +load_appends(PickleState *state, UnpicklerObject *self) { - Py_ssize_t i = marker(self); + Py_ssize_t i = marker(state, self); if (i < 0) return -1; - return do_append(self, i); + return do_append(state, self, i); } static int -do_setitems(UnpicklerObject *self, Py_ssize_t x) +do_setitems(PickleState *state, UnpicklerObject *self, Py_ssize_t x) { PyObject *value, *key; PyObject *dict; @@ -6501,13 +6468,12 @@ do_setitems(UnpicklerObject *self, Py_ssize_t x) len = Py_SIZE(self->stack); if (x > len || x <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); if (len == x) /* nothing to do */ return 0; if ((len - x) % 2 != 0) { - PickleState *st = _Pickle_GetGlobalState(); /* Corrupt or hostile pickle -- we never write one like this. */ - PyErr_SetString(st->UnpicklingError, + PyErr_SetString(state->UnpicklingError, "odd number of items for SETITEMS"); return -1; } @@ -6530,32 +6496,33 @@ do_setitems(UnpicklerObject *self, Py_ssize_t x) } static int -load_setitem(UnpicklerObject *self) +load_setitem(PickleState *state, UnpicklerObject *self) { - return do_setitems(self, Py_SIZE(self->stack) - 2); + return do_setitems(state, self, Py_SIZE(self->stack) - 2); } static int -load_setitems(UnpicklerObject *self) +load_setitems(PickleState *state, UnpicklerObject *self) { - Py_ssize_t i = marker(self); - if (i < 0) + Py_ssize_t i = marker(state, self); + if (i < 0) { return -1; - return do_setitems(self, i); + } + return do_setitems(state, self, i); } static int -load_additems(UnpicklerObject *self) +load_additems(PickleState *state, UnpicklerObject *self) { PyObject *set; Py_ssize_t mark, len, i; - mark = marker(self); + mark = marker(state, self); if (mark < 0) return -1; len = Py_SIZE(self->stack); if (mark > len || mark <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); if (len == mark) /* nothing to do */ return 0; @@ -6565,7 +6532,7 @@ load_additems(UnpicklerObject *self) PyObject *items; int status; - items = Pdata_poptuple(self->stack, mark); + items = Pdata_poptuple(state, self->stack, mark); if (items == NULL) return -1; @@ -6599,9 +6566,9 @@ load_additems(UnpicklerObject *self) } static int -load_build(UnpicklerObject *self) +load_build(PickleState *st, UnpicklerObject *self) { - PyObject *state, *inst, *slotstate; + PyObject *inst, *slotstate; PyObject *setstate; int status = 0; @@ -6609,9 +6576,10 @@ load_build(UnpicklerObject *self) * the stack top, possibly mutated via instance.__setstate__(state). */ if (Py_SIZE(self->stack) - 2 < self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(st, self->stack); - PDATA_POP(self->stack, state); + PyObject *state; + PDATA_POP(st, self->stack, state); if (state == NULL) return -1; @@ -6655,7 +6623,6 @@ load_build(UnpicklerObject *self) Py_ssize_t i; if (!PyDict_Check(state)) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "state is not a dictionary"); goto error; } @@ -6685,7 +6652,6 @@ load_build(UnpicklerObject *self) Py_ssize_t i; if (!PyDict_Check(slotstate)) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "slot state is not a dictionary"); goto error; @@ -6708,7 +6674,7 @@ load_build(UnpicklerObject *self) } static int -load_mark(UnpicklerObject *self) +load_mark(PickleState *state, UnpicklerObject *self) { /* Note that we split the (pickle.py) stack into two stacks, an @@ -6735,16 +6701,16 @@ load_mark(UnpicklerObject *self) } static int -load_reduce(UnpicklerObject *self) +load_reduce(PickleState *state, UnpicklerObject *self) { PyObject *callable = NULL; PyObject *argtup = NULL; PyObject *obj = NULL; - PDATA_POP(self->stack, argtup); + PDATA_POP(state, self->stack, argtup); if (argtup == NULL) return -1; - PDATA_POP(self->stack, callable); + PDATA_POP(state, self->stack, callable); if (callable) { obj = PyObject_CallObject(callable, argtup); Py_DECREF(callable); @@ -6762,13 +6728,14 @@ load_reduce(UnpicklerObject *self) * is the first opcode for protocols >= 2. */ static int -load_proto(UnpicklerObject *self) +load_proto(PickleState *state, UnpicklerObject *self) { char *s; int i; - if (_Unpickler_Read(self, &s, 1) < 0) + if (_Unpickler_Read(self, state, &s, 1) < 0) { return -1; + } i = (unsigned char)s[0]; if (i <= HIGHEST_PROTOCOL) { @@ -6781,12 +6748,12 @@ load_proto(UnpicklerObject *self) } static int -load_frame(UnpicklerObject *self) +load_frame(PickleState *state, UnpicklerObject *self) { char *s; Py_ssize_t frame_len; - if (_Unpickler_Read(self, &s, 8) < 0) + if (_Unpickler_Read(self, state, &s, 8) < 0) return -1; frame_len = calc_binsize(s, 8); @@ -6797,7 +6764,7 @@ load_frame(UnpicklerObject *self) return -1; } - if (_Unpickler_Read(self, &s, frame_len) < 0) + if (_Unpickler_Read(self, state, &s, frame_len) < 0) return -1; /* Rewind to start of frame */ @@ -6806,7 +6773,7 @@ load_frame(UnpicklerObject *self) } static PyObject * -load(UnpicklerObject *self) +load(PickleState *state, UnpicklerObject *self) { PyObject *value = NULL; char *s = NULL; @@ -6820,15 +6787,14 @@ load(UnpicklerObject *self) /* Convenient macros for the dispatch while-switch loop just below. */ #define OP(opcode, load_func) \ - case opcode: if (load_func(self) < 0) break; continue; + case opcode: if (load_func(state, self) < 0) break; continue; #define OP_ARG(opcode, load_func, arg) \ - case opcode: if (load_func(self, (arg)) < 0) break; continue; + case opcode: if (load_func(state, self, (arg)) < 0) break; continue; while (1) { - if (_Unpickler_Read(self, &s, 1) < 0) { - PickleState *st = _Pickle_GetGlobalState(); - if (PyErr_ExceptionMatches(st->UnpicklingError)) { + if (_Unpickler_Read(self, state, &s, 1) < 0) { + if (PyErr_ExceptionMatches(state->UnpicklingError)) { PyErr_Format(PyExc_EOFError, "Ran out of input"); } return NULL; @@ -6908,14 +6874,13 @@ load(UnpicklerObject *self) default: { - PickleState *st = _Pickle_GetGlobalState(); unsigned char c = (unsigned char) *s; if (0x20 <= c && c <= 0x7e && c != '\'' && c != '\\') { - PyErr_Format(st->UnpicklingError, + PyErr_Format(state->UnpicklingError, "invalid load key, '%c'.", c); } else { - PyErr_Format(st->UnpicklingError, + PyErr_Format(state->UnpicklingError, "invalid load key, '\\x%02x'.", c); } return NULL; @@ -6932,7 +6897,7 @@ load(UnpicklerObject *self) if (_Unpickler_SkipConsumed(self) < 0) return NULL; - PDATA_POP(self->stack, value); + PDATA_POP(state, self->stack, value); return value; } @@ -6940,6 +6905,8 @@ load(UnpicklerObject *self) _pickle.Unpickler.load + cls: defining_class + Load a pickle. Read a pickled object representation from the open file object given @@ -6948,24 +6915,25 @@ specified therein. [clinic start generated code]*/ static PyObject * -_pickle_Unpickler_load_impl(UnpicklerObject *self) -/*[clinic end generated code: output=fdcc488aad675b14 input=acbb91a42fa9b7b9]*/ +_pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=cc88168f608e3007 input=f5d2f87e61d5f07f]*/ { UnpicklerObject *unpickler = (UnpicklerObject*)self; + PickleState *st = PyType_GetModuleState(cls); + /* Check whether the Unpickler was initialized correctly. This prevents segfaulting if a subclass overridden __init__ with a function that does not call Unpickler.__init__(). Here, we simply ensure that self->read is not NULL. */ if (unpickler->read == NULL) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_Format(st->UnpicklingError, "Unpickler.__init__() was not called by %s.__init__()", Py_TYPE(unpickler)->tp_name); return NULL; } - return load(unpickler); + return load(st, unpickler); } /* The name of find_class() is misleading. In newer pickle protocols, this @@ -6976,6 +6944,7 @@ _pickle_Unpickler_load_impl(UnpicklerObject *self) _pickle.Unpickler.find_class + cls: defining_class module_name: object global_name: object / @@ -6991,10 +6960,10 @@ needed. Both arguments passed are str objects. [clinic start generated code]*/ static PyObject * -_pickle_Unpickler_find_class_impl(UnpicklerObject *self, +_pickle_Unpickler_find_class_impl(UnpicklerObject *self, PyTypeObject *cls, PyObject *module_name, PyObject *global_name) -/*[clinic end generated code: output=becc08d7f9ed41e3 input=e2e6a865de093ef4]*/ +/*[clinic end generated code: output=99577948abb0be81 input=9577745719219fc7]*/ { PyObject *global; PyObject *module; @@ -7004,20 +6973,21 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, return NULL; } + PickleState *state = PyType_GetModuleState(cls); + /* Try to map the old names used in Python 2.x to the new ones used in Python 3.x. We do this only with old pickle protocols and when the user has not disabled the feature. */ if (self->proto < 3 && self->fix_imports) { PyObject *key; PyObject *item; - PickleState *st = _Pickle_GetGlobalState(); /* Check if the global (i.e., a function or a class) was renamed or moved to another module. */ key = PyTuple_Pack(2, module_name, global_name); if (key == NULL) return NULL; - item = PyDict_GetItemWithError(st->name_mapping_2to3, key); + item = PyDict_GetItemWithError(state->name_mapping_2to3, key); Py_DECREF(key); if (item) { if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) { @@ -7043,7 +7013,7 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, } else { /* Check if the module was renamed. */ - item = PyDict_GetItemWithError(st->import_mapping_2to3, module_name); + item = PyDict_GetItemWithError(state->import_mapping_2to3, module_name); if (item) { if (!PyUnicode_Check(item)) { PyErr_Format(PyExc_RuntimeError, @@ -7621,8 +7591,10 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, if (_Pickler_SetBufferCallback(pickler, buffer_callback) < 0) goto error; - if (dump(pickler, obj) < 0) + PickleState *state = PyModule_GetState(module); + if (dump(state, pickler, obj) < 0) { goto error; + } if (_Pickler_FlushToFile(pickler) < 0) goto error; @@ -7683,8 +7655,10 @@ _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, if (_Pickler_SetBufferCallback(pickler, buffer_callback) < 0) goto error; - if (dump(pickler, obj) < 0) + PickleState *state = PyModule_GetState(module); + if (dump(state, pickler, obj) < 0) { goto error; + } result = _Pickler_GetString(pickler); Py_DECREF(pickler); @@ -7740,6 +7714,8 @@ _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports, PyObject *result; UnpicklerObject *unpickler = _Unpickler_New(module); + PickleState *state = _Pickle_GetState(module); + if (unpickler == NULL) return NULL; @@ -7754,7 +7730,7 @@ _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports, unpickler->fix_imports = fix_imports; - result = load(unpickler); + result = load(state, unpickler); Py_DECREF(unpickler); return result; @@ -7797,7 +7773,6 @@ _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports, PyObject *buffers) /*[clinic end generated code: output=82ac1e6b588e6d02 input=b3615540d0535087]*/ { - PyObject *result; UnpicklerObject *unpickler = _Unpickler_New(module); if (unpickler == NULL) @@ -7814,7 +7789,8 @@ _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports, unpickler->fix_imports = fix_imports; - result = load(unpickler); + PickleState *state = _Pickle_GetState(module); + PyObject *result = load(state, unpickler); Py_DECREF(unpickler); return result; diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index adb3abc5eb2372..539acc34a05cc1 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -38,7 +38,42 @@ PyDoc_STRVAR(_pickle_Pickler_dump__doc__, "Write a pickled representation of the given object to the open file."); #define _PICKLE_PICKLER_DUMP_METHODDEF \ - {"dump", (PyCFunction)_pickle_Pickler_dump, METH_O, _pickle_Pickler_dump__doc__}, + {"dump", _PyCFunction_CAST(_pickle_Pickler_dump), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _pickle_Pickler_dump__doc__}, + +static PyObject * +_pickle_Pickler_dump_impl(PicklerObject *self, PyTypeObject *cls, + PyObject *obj); + +static PyObject * +_pickle_Pickler_dump(PicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dump", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *obj; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + obj = args[0]; + return_value = _pickle_Pickler_dump_impl(self, cls, obj); + +exit: + return return_value; +} PyDoc_STRVAR(_pickle_Pickler___sizeof____doc__, "__sizeof__($self, /)\n" @@ -242,15 +277,19 @@ PyDoc_STRVAR(_pickle_Unpickler_load__doc__, "specified therein."); #define _PICKLE_UNPICKLER_LOAD_METHODDEF \ - {"load", (PyCFunction)_pickle_Unpickler_load, METH_NOARGS, _pickle_Unpickler_load__doc__}, + {"load", _PyCFunction_CAST(_pickle_Unpickler_load), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _pickle_Unpickler_load__doc__}, static PyObject * -_pickle_Unpickler_load_impl(UnpicklerObject *self); +_pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls); static PyObject * -_pickle_Unpickler_load(UnpicklerObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_Unpickler_load(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _pickle_Unpickler_load_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "load() takes no arguments"); + return NULL; + } + return _pickle_Unpickler_load_impl(self, cls); } PyDoc_STRVAR(_pickle_Unpickler_find_class__doc__, @@ -267,26 +306,41 @@ PyDoc_STRVAR(_pickle_Unpickler_find_class__doc__, "needed. Both arguments passed are str objects."); #define _PICKLE_UNPICKLER_FIND_CLASS_METHODDEF \ - {"find_class", _PyCFunction_CAST(_pickle_Unpickler_find_class), METH_FASTCALL, _pickle_Unpickler_find_class__doc__}, + {"find_class", _PyCFunction_CAST(_pickle_Unpickler_find_class), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _pickle_Unpickler_find_class__doc__}, static PyObject * -_pickle_Unpickler_find_class_impl(UnpicklerObject *self, +_pickle_Unpickler_find_class_impl(UnpicklerObject *self, PyTypeObject *cls, PyObject *module_name, PyObject *global_name); static PyObject * -_pickle_Unpickler_find_class(UnpicklerObject *self, PyObject *const *args, Py_ssize_t nargs) +_pickle_Unpickler_find_class(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "find_class", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; PyObject *module_name; PyObject *global_name; - if (!_PyArg_CheckPositional("find_class", nargs, 2, 2)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { goto exit; } module_name = args[0]; global_name = args[1]; - return_value = _pickle_Unpickler_find_class_impl(self, module_name, global_name); + return_value = _pickle_Unpickler_find_class_impl(self, cls, module_name, global_name); exit: return return_value; @@ -980,4 +1034,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=730dc26938561313 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a0e04b85e7bae626 input=a9049054013a1b77]*/ From ce32a69ddbac17605c48aabe1e5a20bd3f1fb899 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Fri, 6 Nov 2020 19:40:40 -0600 Subject: [PATCH 03/21] finish port --- Modules/_pickle.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index ab16095769bca0..660a0d563c93d1 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -5046,15 +5046,13 @@ static PyType_Slot pickle_pickler_type_slots[] = { {Py_tp_clear, Pickler_clear}, {Py_tp_doc, (char*)_pickle_Pickler___init____doc__}, {Py_tp_traverse, Pickler_traverse}, + {Py_tp_init, _pickle_Pickler___init__}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_free, PyObject_GC_Del}, {0, 0} }; -//TODO - how to handle this?? -// Calling PyType_GetModuleState() on a subclass is not safe. -// bz2_compressor_type_spec does not have Py_TPFLAGS_BASETYPE flag -// which prevents to create a subclass. -// So calling PyType_GetModuleState() in this file is always safe. - static PyType_Spec pickle_pickler_type_spec = { .name = "_pickle.Pickler", .basicsize = sizeof(PicklerObject), @@ -7521,6 +7519,10 @@ static PyType_Slot pickle_unpickler_type_slots[] = { {Py_tp_clear, Unpickler_clear}, {Py_tp_methods, Unpickler_methods}, {Py_tp_getset, Unpickler_getsets}, + {Py_tp_init, _pickle_Unpickler___init__}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_free, PyObject_GC_Del}, {0,0} }; From bffb49b8914e0cb543497a7343bc095939481453 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 22 Mar 2023 23:41:45 +0100 Subject: [PATCH 04/21] Convert PData type to heap type and add it to state --- Modules/_pickle.c | 74 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 54 insertions(+), 20 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 660a0d563c93d1..d9689c1d16e2da 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -196,9 +196,10 @@ typedef struct { 2 and 3 */ PyObject *partial; - /* needed for multi-state initialization*/ + /* Types */ PyTypeObject *pickler_type; PyTypeObject *unpickler_type; + PyTypeObject *Pdata_type; } PickleState; /* Forward declaration of the _pickle module definition. */ @@ -211,6 +212,14 @@ _Pickle_GetState(PyObject *module) return (PickleState *)_PyModule_GetState(module); } +static PickleState * +_Pickle_FindStateByType(PyTypeObject *tp) +{ + PyObject *module = PyType_GetModuleByDef(tp, &_picklemodule); + assert(module != NULL); + return _Pickle_GetState(module); +} + /* Clear the given pickle module state. */ static void _Pickle_ClearState(PickleState *st) @@ -231,6 +240,7 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->partial); Py_CLEAR(st->pickler_type); Py_CLEAR(st->unpickler_type); + Py_CLEAR(st->Pdata_type); } /* Initialize the given pickle module state. */ @@ -440,41 +450,61 @@ typedef struct { Py_ssize_t allocated; /* number of slots in data allocated */ } Pdata; +static int +Pdata_traverse(Pdata *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + static void Pdata_dealloc(Pdata *self) { + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); Py_ssize_t i = Py_SIZE(self); while (--i >= 0) { Py_DECREF(self->data[i]); } PyMem_Free(self->data); - PyObject_Free(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } -static PyTypeObject Pdata_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.Pdata", /*tp_name*/ - sizeof(Pdata), /*tp_basicsize*/ - sizeof(PyObject *), /*tp_itemsize*/ - (destructor)Pdata_dealloc, /*tp_dealloc*/ +static PyType_Slot pdata_slots[] = { + {Py_tp_dealloc, Pdata_dealloc}, + {Py_tp_traverse, Pdata_traverse}, + {0, NULL}, +}; + +static PyType_Spec pdata_spec = { + .name = "_pickle.Pdata", + .basicsize = sizeof(Pdata), + .itemsize = sizeof(PyObject *), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pdata_slots, }; static PyObject * -Pdata_New(void) +Pdata_New(PickleState *state) { Pdata *self; - if (!(self = PyObject_New(Pdata, &Pdata_Type))) + if (!(self = PyObject_GC_New(Pdata, state->Pdata_type))) { return NULL; + } Py_SET_SIZE(self, 0); self->mark_set = 0; self->fence = 0; self->allocated = 8; self->data = PyMem_Malloc(self->allocated * sizeof(PyObject *)); - if (self->data) - return (PyObject *)self; - Py_DECREF(self); - return PyErr_NoMemory(); + if (self->data == NULL) { + Py_DECREF(self); + return PyErr_NoMemory(); + } + PyObject_GC_Track(self); + return (PyObject *)self; } @@ -1614,7 +1644,7 @@ _Unpickler_New(PyObject *module) self->memo_size = 32; self->memo_len = 0; self->memo = _Unpickler_NewMemo(self->memo_size); - self->stack = (Pdata *)Pdata_New(); + self->stack = (Pdata *)Pdata_New(st); if (self->memo == NULL || self->stack == NULL) { Py_DECREF(self); @@ -7198,7 +7228,9 @@ _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, return -1; } - self->stack = (Pdata *)Pdata_New(); + PyTypeObject *tp = Py_TYPE(self); + PickleState *state = _Pickle_FindStateByType(tp); + self->stack = (Pdata *)Pdata_New(state); if (self->stack == NULL) return -1; @@ -7842,23 +7874,25 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->partial); Py_VISIT(st->pickler_type); Py_VISIT(st->unpickler_type); + Py_VISIT(st->Pdata_type); return 0; } static int _pickle_exec(PyObject *m) { - PickleState *st; + PickleState *st = _Pickle_GetState(m); - if (PyType_Ready(&Pdata_Type) < 0) + st->Pdata_type = (PyTypeObject *)PyType_FromMetaclass(NULL, m, &pdata_spec, + NULL); + if (st->Pdata_type == NULL) { return -1; + } if (PyType_Ready(&PicklerMemoProxyType) < 0) return -1; if (PyType_Ready(&UnpicklerMemoProxyType) < 0) return -1; - st = _Pickle_GetState(m); - /* Add types */ if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) { return -1; From 5fcce82440228ec2e6bc1ace6c2379d20ef68eea Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Wed, 22 Mar 2023 23:51:19 +0100 Subject: [PATCH 05/21] Convert PicklerMemoProxyType to heap type and add it to state --- Modules/_pickle.c | 72 +++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d9689c1d16e2da..2225d942b2f10d 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -200,6 +200,7 @@ typedef struct { PyTypeObject *pickler_type; PyTypeObject *unpickler_type; PyTypeObject *Pdata_type; + PyTypeObject *PicklerMemoProxyType; } PickleState; /* Forward declaration of the _pickle module definition. */ @@ -212,6 +213,14 @@ _Pickle_GetState(PyObject *module) return (PickleState *)_PyModule_GetState(module); } +static inline PickleState * +_Pickle_GetStateByClass(PyTypeObject *cls) +{ + void *state = _PyType_GetModuleState(cls); + assert(state != NULL); + return (PickleState *)state; +} + static PickleState * _Pickle_FindStateByType(PyTypeObject *tp) { @@ -241,6 +250,7 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->pickler_type); Py_CLEAR(st->unpickler_type); Py_CLEAR(st->Pdata_type); + Py_CLEAR(st->PicklerMemoProxyType); } /* Initialize the given pickle module state. */ @@ -4889,15 +4899,18 @@ static PyMethodDef picklerproxy_methods[] = { static void PicklerMemoProxy_dealloc(PicklerMemoProxyObject *self) { + PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - Py_XDECREF(self->pickler); + Py_CLEAR(self->pickler); PyObject_GC_Del((PyObject *)self); + Py_DECREF(tp); } static int PicklerMemoProxy_traverse(PicklerMemoProxyObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->pickler); return 0; } @@ -4909,43 +4922,28 @@ PicklerMemoProxy_clear(PicklerMemoProxyObject *self) return 0; } -static PyTypeObject PicklerMemoProxyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.PicklerMemoProxy", /*tp_name*/ - sizeof(PicklerMemoProxyObject), /*tp_basicsize*/ - 0, - (destructor)PicklerMemoProxy_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - PyObject_HashNotImplemented, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - PyObject_GenericSetAttr, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - 0, /* tp_doc */ - (traverseproc)PicklerMemoProxy_traverse, /* tp_traverse */ - (inquiry)PicklerMemoProxy_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - picklerproxy_methods, /* tp_methods */ +static PyType_Slot memoproxy_slots[] = { + {Py_tp_dealloc, PicklerMemoProxy_dealloc}, + {Py_tp_traverse, PicklerMemoProxy_traverse}, + {Py_tp_clear, PicklerMemoProxy_clear}, + {Py_tp_methods, picklerproxy_methods}, + {0, NULL}, +}; + +static PyType_Spec memoproxy_spec = { + .name = "_pickle.PicklerMemoProxy", + .basicsize = sizeof(PicklerMemoProxyObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = memoproxy_slots, }; static PyObject * PicklerMemoProxy_New(PicklerObject *pickler) { PicklerMemoProxyObject *self; - - self = PyObject_GC_New(PicklerMemoProxyObject, &PicklerMemoProxyType); + PickleState *st = _Pickle_GetStateByClass(Py_TYPE(pickler)); + self = PyObject_GC_New(PicklerMemoProxyObject, st->PicklerMemoProxyType); if (self == NULL) return NULL; self->pickler = (PicklerObject*)Py_NewRef(pickler); @@ -4972,7 +4970,8 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored)) return -1; } - if (Py_IS_TYPE(obj, &PicklerMemoProxyType)) { + PickleState *st = _Pickle_GetStateByClass(Py_TYPE(self)); + if (Py_IS_TYPE(obj, st->PicklerMemoProxyType)) { PicklerObject *pickler = ((PicklerMemoProxyObject *)obj)->pickler; @@ -7888,8 +7887,13 @@ _pickle_exec(PyObject *m) if (st->Pdata_type == NULL) { return -1; } - if (PyType_Ready(&PicklerMemoProxyType) < 0) + + st->PicklerMemoProxyType = (PyTypeObject *)PyType_FromMetaclass( + NULL, m, &memoproxy_spec, NULL); + if (st->PicklerMemoProxyType == NULL) { return -1; + } + if (PyType_Ready(&UnpicklerMemoProxyType) < 0) return -1; From 3203188068b1b16864cd6530a198074251037ada Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 23 Mar 2023 12:18:55 +0100 Subject: [PATCH 06/21] WIP --- Modules/_pickle.c | 137 +++++++++++++++++++++------------------------- 1 file changed, 63 insertions(+), 74 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 2225d942b2f10d..861a92bd18375a 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -201,6 +201,7 @@ typedef struct { PyTypeObject *unpickler_type; PyTypeObject *Pdata_type; PyTypeObject *PicklerMemoProxyType; + PyTypeObject *UnpicklerMemoProxyType; } PickleState; /* Forward declaration of the _pickle module definition. */ @@ -251,6 +252,7 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->unpickler_type); Py_CLEAR(st->Pdata_type); Py_CLEAR(st->PicklerMemoProxyType); + Py_CLEAR(st->UnpicklerMemoProxyType); } /* Initialize the given pickle module state. */ @@ -1143,14 +1145,12 @@ _Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t data_len) } static PicklerObject * -_Pickler_New(PyObject *module) +_Pickler_New(PickleState *st) { - PicklerObject *self; - - PickleState *st = _Pickle_GetState(module); - self = PyObject_GC_New(PicklerObject, st->pickler_type); - if (self == NULL) + PicklerObject *self = PyObject_GC_New(PicklerObject, st->pickler_type); + if (self == NULL) { return NULL; + } self->pers_func = NULL; self->dispatch_table = NULL; @@ -4641,36 +4641,6 @@ static struct PyMethodDef Pickler_methods[] = { {NULL, NULL} /* sentinel */ }; -static void -Pickler_dealloc(PicklerObject *self) -{ - PyObject_GC_UnTrack(self); - - Py_XDECREF(self->output_buffer); - Py_XDECREF(self->write); - Py_XDECREF(self->pers_func); - Py_XDECREF(self->dispatch_table); - Py_XDECREF(self->fast_memo); - Py_XDECREF(self->reducer_override); - Py_XDECREF(self->buffer_callback); - - PyMemoTable_Del(self->memo); - - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static int -Pickler_traverse(PicklerObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->write); - Py_VISIT(self->pers_func); - Py_VISIT(self->dispatch_table); - Py_VISIT(self->fast_memo); - Py_VISIT(self->reducer_override); - Py_VISIT(self->buffer_callback); - return 0; -} - static int Pickler_clear(PicklerObject *self) { @@ -4690,6 +4660,29 @@ Pickler_clear(PicklerObject *self) return 0; } +static void +Pickler_dealloc(PicklerObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + (void)Pickler_clear(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); +} + +static int +Pickler_traverse(PicklerObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->write); + Py_VISIT(self->pers_func); + Py_VISIT(self->dispatch_table); + Py_VISIT(self->fast_memo); + Py_VISIT(self->reducer_override); + Py_VISIT(self->buffer_callback); + return 0; +} + /*[clinic input] @@ -7129,6 +7122,7 @@ Unpickler_dealloc(UnpicklerObject *self) static int Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->readline); Py_VISIT(self->readinto); Py_VISIT(self->read); @@ -7355,15 +7349,18 @@ static PyMethodDef unpicklerproxy_methods[] = { static void UnpicklerMemoProxy_dealloc(UnpicklerMemoProxyObject *self) { + PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - Py_XDECREF(self->unpickler); + Py_CLEAR(self->unpickler); PyObject_GC_Del((PyObject *)self); + Py_DECREF(tp); } static int UnpicklerMemoProxy_traverse(UnpicklerMemoProxyObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->unpickler); return 0; } @@ -7375,44 +7372,29 @@ UnpicklerMemoProxy_clear(UnpicklerMemoProxyObject *self) return 0; } -static PyTypeObject UnpicklerMemoProxyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.UnpicklerMemoProxy", /*tp_name*/ - sizeof(UnpicklerMemoProxyObject), /*tp_basicsize*/ - 0, - (destructor)UnpicklerMemoProxy_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - PyObject_HashNotImplemented, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - PyObject_GenericSetAttr, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - 0, /* tp_doc */ - (traverseproc)UnpicklerMemoProxy_traverse, /* tp_traverse */ - (inquiry)UnpicklerMemoProxy_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - unpicklerproxy_methods, /* tp_methods */ +static PyType_Slot unpickler_memoproxy_slots[] = { + {Py_tp_dealloc, UnpicklerMemoProxy_dealloc}, + {Py_tp_traverse, UnpicklerMemoProxy_traverse}, + {Py_tp_clear, UnpicklerMemoProxy_clear}, + {Py_tp_methods, unpicklerproxy_methods}, + {0, NULL}, +}; + +static PyType_Spec unpickler_memoproxy_spec = { + .name = "_pickle.UnpicklerMemoProxy", + .basicsize = sizeof(UnpicklerMemoProxyObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = unpickler_memoproxy_slots, }; static PyObject * UnpicklerMemoProxy_New(UnpicklerObject *unpickler) { UnpicklerMemoProxyObject *self; - + PickleState *state = _Pickle_GetStateByClass(Py_TYPE(unpickler)); self = PyObject_GC_New(UnpicklerMemoProxyObject, - &UnpicklerMemoProxyType); + state->UnpicklerMemoProxyType); if (self == NULL) return NULL; self->unpickler = (UnpicklerObject*)Py_NewRef(unpickler); @@ -7441,7 +7423,8 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored return -1; } - if (Py_IS_TYPE(obj, &UnpicklerMemoProxyType)) { + PickleState *state = _Pickle_GetStateByClass(Py_TYPE(self)); + if (Py_IS_TYPE(obj, state->UnpicklerMemoProxyType)) { UnpicklerObject *unpickler = ((UnpicklerMemoProxyObject *)obj)->unpickler; @@ -7610,7 +7593,8 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, PyObject *buffer_callback) /*[clinic end generated code: output=706186dba996490c input=5ed6653da99cd97c]*/ { - PicklerObject *pickler = _Pickler_New(module); + PickleState *state = _Pickle_GetState(module); + PicklerObject *pickler = _Pickler_New(state); if (pickler == NULL) return NULL; @@ -7624,7 +7608,6 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, if (_Pickler_SetBufferCallback(pickler, buffer_callback) < 0) goto error; - PickleState *state = PyModule_GetState(module); if (dump(state, pickler, obj) < 0) { goto error; } @@ -7677,7 +7660,8 @@ _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, /*[clinic end generated code: output=fbab0093a5580fdf input=e543272436c6f987]*/ { PyObject *result; - PicklerObject *pickler = _Pickler_New(module); + PickleState *state = _Pickle_GetState(module); + PicklerObject *pickler = _Pickler_New(state); if (pickler == NULL) return NULL; @@ -7688,7 +7672,6 @@ _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, if (_Pickler_SetBufferCallback(pickler, buffer_callback) < 0) goto error; - PickleState *state = PyModule_GetState(module); if (dump(state, pickler, obj) < 0) { goto error; } @@ -7874,6 +7857,9 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->pickler_type); Py_VISIT(st->unpickler_type); Py_VISIT(st->Pdata_type); + Py_VISIT(st->Pdata_type); + Py_VISIT(st->PicklerMemoProxyType); + Py_VISIT(st->UnpicklerMemoProxyType); return 0; } @@ -7894,8 +7880,11 @@ _pickle_exec(PyObject *m) return -1; } - if (PyType_Ready(&UnpicklerMemoProxyType) < 0) + st->UnpicklerMemoProxyType = (PyTypeObject *)PyType_FromMetaclass( + NULL, m, &unpickler_memoproxy_spec, NULL); + if (st->UnpicklerMemoProxyType == NULL) { return -1; + } /* Add types */ if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) { From f1ce8fbdec88e47b52c77df84ec02f9d49230cd6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 23 Mar 2023 23:31:03 +0100 Subject: [PATCH 07/21] Fixup dealloc and traverse functions --- Modules/_pickle.c | 68 ++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 42 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 861a92bd18375a..fe4f0c693fb3ed 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4895,7 +4895,7 @@ PicklerMemoProxy_dealloc(PicklerMemoProxyObject *self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); Py_CLEAR(self->pickler); - PyObject_GC_Del((PyObject *)self); + tp->tp_free((PyObject *)self); Py_DECREF(tp); } @@ -7094,45 +7094,6 @@ static struct PyMethodDef Unpickler_methods[] = { {NULL, NULL} /* sentinel */ }; -static void -Unpickler_dealloc(UnpicklerObject *self) -{ - PyObject_GC_UnTrack((PyObject *)self); - Py_XDECREF(self->readline); - Py_XDECREF(self->readinto); - Py_XDECREF(self->read); - Py_XDECREF(self->peek); - Py_XDECREF(self->stack); - Py_XDECREF(self->pers_func); - Py_XDECREF(self->buffers); - if (self->buffer.buf != NULL) { - PyBuffer_Release(&self->buffer); - self->buffer.buf = NULL; - } - - _Unpickler_MemoCleanup(self); - PyMem_Free(self->marks); - PyMem_Free(self->input_line); - PyMem_Free(self->encoding); - PyMem_Free(self->errors); - - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static int -Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg) -{ - Py_VISIT(Py_TYPE(self)); - Py_VISIT(self->readline); - Py_VISIT(self->readinto); - Py_VISIT(self->read); - Py_VISIT(self->peek); - Py_VISIT(self->stack); - Py_VISIT(self->pers_func); - Py_VISIT(self->buffers); - return 0; -} - static int Unpickler_clear(UnpicklerObject *self) { @@ -7161,6 +7122,30 @@ Unpickler_clear(UnpicklerObject *self) return 0; } +static void +Unpickler_dealloc(UnpicklerObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack((PyObject *)self); + (void)Unpickler_clear(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); +} + +static int +Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->readline); + Py_VISIT(self->readinto); + Py_VISIT(self->read); + Py_VISIT(self->peek); + Py_VISIT(self->stack); + Py_VISIT(self->pers_func); + Py_VISIT(self->buffers); + return 0; +} + /*[clinic input] _pickle.Unpickler.__init__ @@ -7352,7 +7337,7 @@ UnpicklerMemoProxy_dealloc(UnpicklerMemoProxyObject *self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); Py_CLEAR(self->unpickler); - PyObject_GC_Del((PyObject *)self); + tp->tp_free((PyObject *)self); Py_DECREF(tp); } @@ -7857,7 +7842,6 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->pickler_type); Py_VISIT(st->unpickler_type); Py_VISIT(st->Pdata_type); - Py_VISIT(st->Pdata_type); Py_VISIT(st->PicklerMemoProxyType); Py_VISIT(st->UnpicklerMemoProxyType); return 0; From a6679ff3755265ecba74c51c88fda0eb574d05ce Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 23 Mar 2023 23:35:16 +0100 Subject: [PATCH 08/21] State is unused in AC --- Modules/_pickle.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index fe4f0c693fb3ed..a2460f8ae29e16 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -20,17 +20,14 @@ PyDoc_STRVAR(pickle_module_doc, "Optimized C implementation for the Python pickle module."); -//TODO should we remove Pickler_Type from the clinic input here? -//similar entry was not removed in pull 20960 for _bz2 - /*[clinic input] module _pickle -class _pickle.Pickler "PicklerObject *" "&Pickler_Type" -class _pickle.PicklerMemoProxy "PicklerMemoProxyObject *" "&PicklerMemoProxyType" -class _pickle.Unpickler "UnpicklerObject *" "&Unpickler_Type" -class _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "&UnpicklerMemoProxyType" +class _pickle.Pickler "PicklerObject *" "" +class _pickle.PicklerMemoProxy "PicklerMemoProxyObject *" "" +class _pickle.Unpickler "UnpicklerObject *" "" +class _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4b3e113468a58e6c]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b6d7191ab6466cda]*/ /* Bump HIGHEST_PROTOCOL when new opcodes are added to the pickle protocol. Bump DEFAULT_PROTOCOL only when the oldest still supported version of Python From 35025d26692a4d92b7380fe5c9da0bf7d6d4c049 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 23 Mar 2023 23:36:32 +0100 Subject: [PATCH 09/21] Adjust NEWS --- .../2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst index a44047351a307f..4c5b6a849661c5 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst @@ -1 +1 @@ -Port :mod:`_pickle` to multiphase initialization +Adapt :mod:`_pickle` to :pep:`687`. Patch by Mohamed Koubaa and Erlend Aasland. From d8efd22db265702e769d9916b980890c96bac764 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 23 Mar 2023 23:37:14 +0100 Subject: [PATCH 10/21] Update globals-to-fix --- Tools/c-analyzer/cpython/globals-to-fix.tsv | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 57b8542fb46482..cfb6e09bdaa9d4 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -390,11 +390,6 @@ Modules/_decimal/_decimal.c - PyDecContextManager_Type - Modules/_decimal/_decimal.c - PyDecContext_Type - Modules/_decimal/_decimal.c - PyDecSignalDictMixin_Type - Modules/_decimal/_decimal.c - PyDec_Type - -Modules/_pickle.c - Pdata_Type - -Modules/_pickle.c - PicklerMemoProxyType - -Modules/_pickle.c - Pickler_Type - -Modules/_pickle.c - UnpicklerMemoProxyType - -Modules/_pickle.c - Unpickler_Type - Modules/ossaudiodev.c - OSSAudioType - Modules/ossaudiodev.c - OSSMixerType - Modules/socketmodule.c - sock_type - From 9fcc0d976415f1bd3104b410e4b3eb0feeac1e68 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 23 Mar 2023 23:38:32 +0100 Subject: [PATCH 11/21] Adjust get-state helpers --- Modules/_pickle.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index a2460f8ae29e16..8bb38b411e45df 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -205,10 +205,12 @@ typedef struct { static struct PyModuleDef _picklemodule; /* Given a module object, get its per-module state. */ -static PickleState * +static inline PickleState * _Pickle_GetState(PyObject *module) { - return (PickleState *)_PyModule_GetState(module); + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (PickleState *)state; } static inline PickleState * @@ -219,7 +221,7 @@ _Pickle_GetStateByClass(PyTypeObject *cls) return (PickleState *)state; } -static PickleState * +static inline PickleState * _Pickle_FindStateByType(PyTypeObject *tp) { PyObject *module = PyType_GetModuleByDef(tp, &_picklemodule); From add3ba5398eb82852d2bfb5452bfa1b4cc4322f2 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 24 Mar 2023 00:29:58 +0100 Subject: [PATCH 12/21] Reduce diff size; remove stylic changes --- Modules/_pickle.c | 470 +++++++++++++++++++++++----------------------- 1 file changed, 232 insertions(+), 238 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 8bb38b411e45df..45dc14c12fb4af 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -502,20 +502,19 @@ Pdata_New(PickleState *state) { Pdata *self; - if (!(self = PyObject_GC_New(Pdata, state->Pdata_type))) { + if (!(self = PyObject_GC_New(Pdata, state->Pdata_type))) return NULL; - } Py_SET_SIZE(self, 0); self->mark_set = 0; self->fence = 0; self->allocated = 8; self->data = PyMem_Malloc(self->allocated * sizeof(PyObject *)); - if (self->data == NULL) { - Py_DECREF(self); - return PyErr_NoMemory(); + if (self->data) { + PyObject_GC_Track(self); + return (PyObject *)self; } - PyObject_GC_Track(self); - return (PyObject *)self; + Py_DECREF(self); + return PyErr_NoMemory(); } @@ -1146,10 +1145,11 @@ _Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t data_len) static PicklerObject * _Pickler_New(PickleState *st) { - PicklerObject *self = PyObject_GC_New(PicklerObject, st->pickler_type); - if (self == NULL) { + PicklerObject *self; + + self = PyObject_GC_New(PicklerObject, st->pickler_type); + if (self == NULL) return NULL; - } self->pers_func = NULL; self->dispatch_table = NULL; @@ -1258,9 +1258,9 @@ _Unpickler_SetStringInput(UnpicklerObject *self, PyObject *input) } static int -bad_readline(PickleState *state) +bad_readline(PickleState *st) { - PyErr_SetString(state->UnpicklingError, "pickle data was truncated"); + PyErr_SetString(st->UnpicklingError, "pickle data was truncated"); return -1; } @@ -1354,13 +1354,13 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) /* Don't call it directly: use _Unpickler_Read() */ static Py_ssize_t -_Unpickler_ReadImpl(UnpicklerObject *self, PickleState *state, char **s, Py_ssize_t n) +_Unpickler_ReadImpl(UnpicklerObject *self, PickleState *st, char **s, Py_ssize_t n) { Py_ssize_t num_read; *s = NULL; if (self->next_read_idx > PY_SSIZE_T_MAX - n) { - PyErr_SetString(state->UnpicklingError, + PyErr_SetString(st->UnpicklingError, "read would overflow (invalid bytecode)"); return -1; } @@ -1369,14 +1369,14 @@ _Unpickler_ReadImpl(UnpicklerObject *self, PickleState *state, char **s, Py_ssiz assert(self->next_read_idx + n > self->input_len); if (!self->read) - return bad_readline(state); + return bad_readline(st); /* Extend the buffer to satisfy desired size */ num_read = _Unpickler_ReadFromFile(self, n); if (num_read < 0) return -1; if (num_read < n) - return bad_readline(state); + return bad_readline(st); *s = self->input_buffer; self->next_read_idx = n; return n; @@ -1391,7 +1391,8 @@ _Unpickler_ReadImpl(UnpicklerObject *self, PickleState *state, char **s, Py_ssiz * _Unpickler_Read() is recommended in most cases. */ static Py_ssize_t -_Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, Py_ssize_t n) +_Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, + Py_ssize_t n) { assert(n != READ_WHOLE_LINE); @@ -1732,7 +1733,7 @@ _Unpickler_SetBuffers(UnpicklerObject *self, PyObject *buffers) /* Generate a GET opcode for an object stored in the memo. */ static int -memo_get(PickleState *state, PicklerObject *self, PyObject *key) +memo_get(PickleState *st, PicklerObject *self, PyObject *key) { Py_ssize_t *value; char pdata[30]; @@ -1765,7 +1766,7 @@ memo_get(PickleState *state, PicklerObject *self, PyObject *key) len = 5; } else { /* unlikely */ - PyErr_SetString(state->PicklingError, + PyErr_SetString(st->PicklingError, "memo id too large for LONG_BINGET"); return -1; } @@ -1780,7 +1781,7 @@ memo_get(PickleState *state, PicklerObject *self, PyObject *key) /* Store an object in the memo, assign it a new unique ID based on the number of objects currently stored in the memo and generate a PUT opcode. */ static int -memo_put(PickleState *state, PicklerObject *self, PyObject *obj) +memo_put(PickleState *st, PicklerObject *self, PyObject *obj) { char pdata[30]; Py_ssize_t len; @@ -1821,7 +1822,7 @@ memo_put(PickleState *state, PicklerObject *self, PyObject *obj) len = 5; } else { /* unlikely */ - PyErr_SetString(state->PicklingError, + PyErr_SetString(st->PicklingError, "memo id too large for LONG_BINPUT"); return -1; } @@ -2373,7 +2374,7 @@ _Pickler_write_bytes(PicklerObject *self, } static int -_save_bytes_data(PickleState *state, PicklerObject *self, PyObject *obj, +_save_bytes_data(PickleState *st, PicklerObject *self, PyObject *obj, const char *data, Py_ssize_t size) { assert(self->proto >= 3); @@ -2413,7 +2414,7 @@ _save_bytes_data(PickleState *state, PicklerObject *self, PyObject *obj, return -1; } - if (memo_put(state, self, obj) < 0) { + if (memo_put(st, self, obj) < 0) { return -1; } @@ -2421,7 +2422,7 @@ _save_bytes_data(PickleState *state, PicklerObject *self, PyObject *obj, } static int -save_bytes(PickleState *state, PicklerObject *self, PyObject *obj) +save_bytes(PickleState *st, PicklerObject *self, PyObject *obj) { if (self->proto < 3) { /* Older pickle protocols do not have an opcode for pickling bytes @@ -2450,7 +2451,7 @@ save_bytes(PickleState *state, PicklerObject *self, PyObject *obj) if (unicode_str == NULL) return -1; reduce_value = Py_BuildValue("(O(OO))", - state->codecs_encode, unicode_str, + st->codecs_encode, unicode_str, &_Py_ID(latin1)); Py_DECREF(unicode_str); } @@ -2459,12 +2460,12 @@ save_bytes(PickleState *state, PicklerObject *self, PyObject *obj) return -1; /* save_reduce() will memoize the object automatically. */ - status = save_reduce(state, self, reduce_value, obj); + status = save_reduce(st, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } else { - return _save_bytes_data(state, self, obj, PyBytes_AS_STRING(obj), + return _save_bytes_data(st, self, obj, PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); } } @@ -2527,16 +2528,17 @@ save_bytearray(PickleState *state, PicklerObject *self, PyObject *obj) return status; } else { - return _save_bytearray_data(state, self, obj, PyByteArray_AS_STRING(obj), + return _save_bytearray_data(state, self, obj, + PyByteArray_AS_STRING(obj), PyByteArray_GET_SIZE(obj)); } } static int -save_picklebuffer(PickleState *state, PicklerObject *self, PyObject *obj) +save_picklebuffer(PickleState *st, PicklerObject *self, PyObject *obj) { if (self->proto < 5) { - PyErr_SetString(state->PicklingError, + PyErr_SetString(st->PicklingError, "PickleBuffer can only pickled with protocol >= 5"); return -1; } @@ -2545,7 +2547,7 @@ save_picklebuffer(PickleState *state, PicklerObject *self, PyObject *obj) return -1; } if (view->suboffsets != NULL || !PyBuffer_IsContiguous(view, 'A')) { - PyErr_SetString(state->PicklingError, + PyErr_SetString(st->PicklingError, "PickleBuffer can not be pickled when " "pointing to a non-contiguous buffer"); return -1; @@ -2565,12 +2567,12 @@ save_picklebuffer(PickleState *state, PicklerObject *self, PyObject *obj) if (in_band) { /* Write data in-band */ if (view->readonly) { - return _save_bytes_data(state, self, obj, (const char*) view->buf, + return _save_bytes_data(st, self, obj, (const char *)view->buf, view->len); } else { - return _save_bytearray_data(state, self, obj, - (const char*) view->buf, view->len); + return _save_bytearray_data(st, self, obj, (const char *)view->buf, + view->len); } } else { @@ -2762,7 +2764,8 @@ save_unicode(PickleState *state, PicklerObject *self, PyObject *obj) /* A helper for save_tuple. Push the len elements in tuple t on the stack. */ static int -store_tuple_elements(PickleState *state, PicklerObject *self, PyObject *t, Py_ssize_t len) +store_tuple_elements(PickleState *state, PicklerObject *self, PyObject *t, + Py_ssize_t len) { Py_ssize_t i; @@ -3551,7 +3554,7 @@ save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) } static int -fix_imports(PickleState *state, PyObject **module_name, PyObject **global_name) +fix_imports(PickleState *st, PyObject **module_name, PyObject **global_name) { PyObject *key; PyObject *item; @@ -3559,7 +3562,7 @@ fix_imports(PickleState *state, PyObject **module_name, PyObject **global_name) key = PyTuple_Pack(2, *module_name, *global_name); if (key == NULL) return -1; - item = PyDict_GetItemWithError(state->name_mapping_3to2, key); + item = PyDict_GetItemWithError(st->name_mapping_3to2, key); Py_DECREF(key); if (item) { PyObject *fixed_module_name; @@ -3594,7 +3597,7 @@ fix_imports(PickleState *state, PyObject **module_name, PyObject **global_name) return -1; } - item = PyDict_GetItemWithError(state->import_mapping_3to2, *module_name); + item = PyDict_GetItemWithError(st->import_mapping_3to2, *module_name); if (item) { if (!PyUnicode_Check(item)) { PyErr_Format(PyExc_RuntimeError, @@ -3613,7 +3616,8 @@ fix_imports(PickleState *state, PyObject **module_name, PyObject **global_name) } static int -save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *name) +save_global(PickleState *st, PicklerObject *self, PyObject *obj, + PyObject *name) { PyObject *global_name = NULL; PyObject *module_name = NULL; @@ -3656,7 +3660,7 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na extra parameters of __import__ to fix that. */ module = PyImport_Import(module_name); if (module == NULL) { - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "Can't pickle %R: import of module %R failed", obj, module_name); goto error; @@ -3666,14 +3670,14 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na cls = get_deep_attribute(module, dotted_path, &parent); Py_CLEAR(dotted_path); if (cls == NULL) { - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "Can't pickle %R: attribute lookup %S on %S failed", obj, global_name, module_name); goto error; } if (cls != obj) { Py_DECREF(cls); - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "Can't pickle %R: it's not the same object as %S.%S", obj, module_name, global_name); goto error; @@ -3694,7 +3698,7 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na if (extension_key == NULL) { goto error; } - code_obj = PyDict_GetItemWithError(state->extension_registry, + code_obj = PyDict_GetItemWithError(st->extension_registry, extension_key); Py_DECREF(extension_key); /* The object is not registered in the extension registry. @@ -3712,7 +3716,7 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na /* Verify code_obj has the right type and value. */ if (!PyLong_Check(code_obj)) { - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "Can't pickle %R: extension code %R isn't an integer", obj, code_obj); goto error; @@ -3720,7 +3724,7 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na code = PyLong_AS_LONG(code_obj); if (code <= 0 || code > 0x7fffffffL) { if (!PyErr_Occurred()) - PyErr_Format(state->PicklingError, "Can't pickle %R: extension " + PyErr_Format(st->PicklingError, "Can't pickle %R: extension " "code %ld is out of range", obj, code); goto error; } @@ -3757,9 +3761,9 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na if (self->proto >= 4) { const char stack_global_op = STACK_GLOBAL; - if (save(state, self, module_name, 0) < 0) + if (save(st, self, module_name, 0) < 0) goto error; - if (save(state, self, global_name, 0) < 0) + if (save(st, self, global_name, 0) < 0) goto error; if (_Pickler_Write(self, &stack_global_op, 1) < 0) @@ -3767,10 +3771,10 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na } else if (parent != module) { PyObject *reduce_value = Py_BuildValue("(O(OO))", - state->getattr, parent, lastname); + st->getattr, parent, lastname); if (reduce_value == NULL) goto error; - status = save_reduce(state, self, reduce_value, NULL); + status = save_reduce(st, self, reduce_value, NULL); Py_DECREF(reduce_value); if (status < 0) goto error; @@ -3788,7 +3792,7 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na /* For protocol < 3 and if the user didn't request against doing so, we convert module names to the old 2.x module names. */ if (self->proto < 3 && self->fix_imports) { - if (fix_imports(state, &module_name, &global_name) < 0) { + if (fix_imports(st, &module_name, &global_name) < 0) { goto error; } } @@ -3807,7 +3811,7 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na encoded = unicode_encoder(module_name); if (encoded == NULL) { if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "can't pickle module identifier '%S' using " "pickle protocol %i", module_name, self->proto); @@ -3826,7 +3830,7 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na encoded = unicode_encoder(global_name); if (encoded == NULL) { if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "can't pickle global identifier '%S' using " "pickle protocol %i", global_name, self->proto); @@ -3842,7 +3846,7 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na goto error; } /* Memoize the object. */ - if (memo_put(state, self, obj) < 0) + if (memo_put(st, self, obj) < 0) goto error; } @@ -3861,7 +3865,8 @@ save_global(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *na } static int -save_singleton_type(PickleState *state, PicklerObject *self, PyObject *obj, PyObject *singleton) +save_singleton_type(PickleState *state, PicklerObject *self, PyObject *obj, + PyObject *singleton) { PyObject *reduce_value; int status; @@ -3962,11 +3967,12 @@ get_class(PyObject *obj) * appropriate __reduce__ method for obj. */ static int -save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *obj) +save_reduce(PickleState *st, PicklerObject *self, PyObject *args, + PyObject *obj) { PyObject *callable; PyObject *argtup; - PyObject *state2 = NULL; + PyObject *state = NULL; PyObject *listitems = Py_None; PyObject *dictitems = Py_None; PyObject *state_setter = Py_None; @@ -3980,34 +3986,34 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o size = PyTuple_Size(args); if (size < 2 || size > 6) { - PyErr_SetString(state->PicklingError, "tuple returned by " + PyErr_SetString(st->PicklingError, "tuple returned by " "__reduce__ must contain 2 through 6 elements"); return -1; } if (!PyArg_UnpackTuple(args, "save_reduce", 2, 6, - &callable, &argtup, &state2, &listitems, &dictitems, + &callable, &argtup, &state, &listitems, &dictitems, &state_setter)) return -1; if (!PyCallable_Check(callable)) { - PyErr_SetString(state->PicklingError, "first item of the tuple " + PyErr_SetString(st->PicklingError, "first item of the tuple " "returned by __reduce__ must be callable"); return -1; } if (!PyTuple_Check(argtup)) { - PyErr_SetString(state->PicklingError, "second item of the tuple " + PyErr_SetString(st->PicklingError, "second item of the tuple " "returned by __reduce__ must be a tuple"); return -1; } - if (state2 == Py_None) - state2 = NULL; + if (state == Py_None) + state = NULL; if (listitems == Py_None) listitems = NULL; else if (!PyIter_Check(listitems)) { - PyErr_Format(state->PicklingError, "fourth element of the tuple " + PyErr_Format(st->PicklingError, "fourth element of the tuple " "returned by __reduce__ must be an iterator, not %s", Py_TYPE(listitems)->tp_name); return -1; @@ -4016,7 +4022,7 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o if (dictitems == Py_None) dictitems = NULL; else if (!PyIter_Check(dictitems)) { - PyErr_Format(state->PicklingError, "fifth element of the tuple " + PyErr_Format(st->PicklingError, "fifth element of the tuple " "returned by __reduce__ must be an iterator, not %s", Py_TYPE(dictitems)->tp_name); return -1; @@ -4025,7 +4031,7 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o if (state_setter == Py_None) state_setter = NULL; else if (!PyCallable_Check(state_setter)) { - PyErr_Format(state->PicklingError, "sixth element of the tuple " + PyErr_Format(st->PicklingError, "sixth element of the tuple " "returned by __reduce__ must be a function, not %s", Py_TYPE(state_setter)->tp_name); return -1; @@ -4052,7 +4058,7 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o PyObject *kwargs; if (PyTuple_GET_SIZE(argtup) != 3) { - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "length of the NEWOBJ_EX argument tuple must be " "exactly 3, not %zd", PyTuple_GET_SIZE(argtup)); return -1; @@ -4060,30 +4066,30 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o cls = PyTuple_GET_ITEM(argtup, 0); if (!PyType_Check(cls)) { - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "first item from NEWOBJ_EX argument tuple must " "be a class, not %.200s", Py_TYPE(cls)->tp_name); return -1; } args = PyTuple_GET_ITEM(argtup, 1); if (!PyTuple_Check(args)) { - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "second item from NEWOBJ_EX argument tuple must " "be a tuple, not %.200s", Py_TYPE(args)->tp_name); return -1; } kwargs = PyTuple_GET_ITEM(argtup, 2); if (!PyDict_Check(kwargs)) { - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "third item from NEWOBJ_EX argument tuple must " "be a dict, not %.200s", Py_TYPE(kwargs)->tp_name); return -1; } if (self->proto >= 4) { - if (save(state, self, cls, 0) < 0 || - save(state, self, args, 0) < 0 || - save(state, self, kwargs, 0) < 0 || + if (save(st, self, cls, 0) < 0 || + save(st, self, args, 0) < 0 || + save(st, self, kwargs, 0) < 0 || _Pickler_Write(self, &newobj_ex_op, 1) < 0) { return -1; } @@ -4109,7 +4115,7 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o PyTuple_SET_ITEM(newargs, i + 2, Py_NewRef(item)); } - callable = PyObject_Call(state->partial, newargs, kwargs); + callable = PyObject_Call(st->partial, newargs, kwargs); Py_DECREF(newargs); if (callable == NULL) return -1; @@ -4120,8 +4126,8 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o return -1; } - if (save(state, self, callable, 0) < 0 || - save(state, self, newargs, 0) < 0 || + if (save(st, self, callable, 0) < 0 || + save(st, self, newargs, 0) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0) { Py_DECREF(newargs); Py_DECREF(callable); @@ -4139,13 +4145,13 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o /* Sanity checks. */ if (PyTuple_GET_SIZE(argtup) < 1) { - PyErr_SetString(state->PicklingError, "__newobj__ arglist is empty"); + PyErr_SetString(st->PicklingError, "__newobj__ arglist is empty"); return -1; } cls = PyTuple_GET_ITEM(argtup, 0); if (!PyType_Check(cls)) { - PyErr_SetString(state->PicklingError, "args[0] from " + PyErr_SetString(st->PicklingError, "args[0] from " "__newobj__ args is not a type"); return -1; } @@ -4158,7 +4164,7 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o p = obj_class != cls; Py_DECREF(obj_class); if (p) { - PyErr_SetString(state->PicklingError, "args[0] from " + PyErr_SetString(st->PicklingError, "args[0] from " "__newobj__ args has the wrong class"); return -1; } @@ -4191,7 +4197,7 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o function. */ /* Save the class and its __new__ arguments. */ - if (save(state, self, cls, 0) < 0) { + if (save(st, self, cls, 0) < 0) { return -1; } @@ -4199,7 +4205,7 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o if (newargtup == NULL) return -1; - p = save(state, self, newargtup, 0); + p = save(st, self, newargtup, 0); Py_DECREF(newargtup); if (p < 0) return -1; @@ -4209,8 +4215,8 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o return -1; } else { /* Not using NEWOBJ. */ - if (save(state, self, callable, 0) < 0 || - save(state, self, argtup, 0) < 0 || + if (save(st, self, callable, 0) < 0 || + save(st, self, argtup, 0) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0) return -1; } @@ -4228,24 +4234,24 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o if (_Pickler_Write(self, &pop_op, 1) < 0) return -1; - if (memo_get(state, self, obj) < 0) + if (memo_get(st, self, obj) < 0) return -1; return 0; } - else if (memo_put(state, self, obj) < 0) + else if (memo_put(st, self, obj) < 0) return -1; } - if (listitems && batch_list(state, self, listitems) < 0) + if (listitems && batch_list(st, self, listitems) < 0) return -1; - if (dictitems && batch_dict(state, self, dictitems) < 0) + if (dictitems && batch_dict(st, self, dictitems) < 0) return -1; - if (state2) { + if (state) { if (state_setter == NULL) { - if (save(state, self, state2, 0) < 0 || + if (save(st, self, state, 0) < 0 || _Pickler_Write(self, &build_op, 1) < 0) return -1; } @@ -4262,8 +4268,8 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o const char tupletwo_op = TUPLE2; const char pop_op = POP; - if (save(state, self, state_setter, 0) < 0 || - save(state, self, obj, 0) < 0 || save(state, self, state2, 0) < 0 || + if (save(st, self, state_setter, 0) < 0 || + save(st, self, obj, 0) < 0 || save(st, self, state, 0) < 0 || _Pickler_Write(self, &tupletwo_op, 1) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0 || _Pickler_Write(self, &pop_op, 1) < 0) @@ -4274,7 +4280,7 @@ save_reduce(PickleState *state, PicklerObject *self, PyObject *args, PyObject *o } static int -save(PickleState *state, PicklerObject *self, PyObject *obj, int pers_save) +save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) { PyTypeObject *type; PyObject *reduce_func = NULL; @@ -4292,7 +4298,7 @@ save(PickleState *state, PicklerObject *self, PyObject *obj, int pers_save) 0 if it did nothing successfully; 1 if a persistent id was saved. */ - if ((status = save_pers(state, self, obj)) != 0) + if ((status = save_pers(st, self, obj)) != 0) return status; } @@ -4322,14 +4328,14 @@ save(PickleState *state, PicklerObject *self, PyObject *obj, int pers_save) a GET (or BINGET) opcode, instead of pickling the object once again. */ if (PyMemoTable_Get(self->memo, obj)) { - return memo_get(state, self, obj); + return memo_get(st, self, obj); } if (type == &PyBytes_Type) { - return save_bytes(state, self, obj); + return save_bytes(st, self, obj); } else if (type == &PyUnicode_Type) { - return save_unicode(state, self, obj); + return save_unicode(st, self, obj); } /* We're only calling _Py_EnterRecursiveCall here so that atomic @@ -4339,31 +4345,31 @@ save(PickleState *state, PicklerObject *self, PyObject *obj, int pers_save) } if (type == &PyDict_Type) { - status = save_dict(state, self, obj); + status = save_dict(st, self, obj); goto done; } else if (type == &PySet_Type) { - status = save_set(state, self, obj); + status = save_set(st, self, obj); goto done; } else if (type == &PyFrozenSet_Type) { - status = save_frozenset(state, self, obj); + status = save_frozenset(st, self, obj); goto done; } else if (type == &PyList_Type) { - status = save_list(state, self, obj); + status = save_list(st, self, obj); goto done; } else if (type == &PyTuple_Type) { - status = save_tuple(state, self, obj); + status = save_tuple(st, self, obj); goto done; } else if (type == &PyByteArray_Type) { - status = save_bytearray(state, self, obj); + status = save_bytearray(st, self, obj); goto done; } else if (type == &PyPickleBuffer_Type) { - status = save_picklebuffer(state, self, obj); + status = save_picklebuffer(st, self, obj); goto done; } @@ -4383,11 +4389,11 @@ save(PickleState *state, PicklerObject *self, PyObject *obj, int pers_save) } if (type == &PyType_Type) { - status = save_type(state, self, obj); + status = save_type(st, self, obj); goto done; } else if (type == &PyFunction_Type) { - status = save_global(state, self, obj, NULL); + status = save_global(st, self, obj, NULL); goto done; } @@ -4398,7 +4404,7 @@ save(PickleState *state, PicklerObject *self, PyObject *obj, int pers_save) * __reduce_ex__ method, or the object's __reduce__ method. */ if (self->dispatch_table == NULL) { - reduce_func = PyDict_GetItemWithError(state->dispatch_table, + reduce_func = PyDict_GetItemWithError(st->dispatch_table, (PyObject *)type); if (reduce_func == NULL) { if (PyErr_Occurred()) { @@ -4424,7 +4430,7 @@ save(PickleState *state, PicklerObject *self, PyObject *obj, int pers_save) reduce_value = _Pickle_FastCall(reduce_func, Py_NewRef(obj)); } else if (PyType_IsSubtype(type, &PyType_Type)) { - status = save_global(state, self, obj, NULL); + status = save_global(st, self, obj, NULL); goto done; } else { @@ -4456,7 +4462,7 @@ save(PickleState *state, PicklerObject *self, PyObject *obj, int pers_save) reduce_value = PyObject_CallNoArgs(reduce_func); } else { - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "can't pickle '%.200s' object: %R", type->tp_name, obj); goto error; @@ -4469,17 +4475,17 @@ save(PickleState *state, PicklerObject *self, PyObject *obj, int pers_save) reduce: if (PyUnicode_Check(reduce_value)) { - status = save_global(state, self, obj, reduce_value); + status = save_global(st, self, obj, reduce_value); goto done; } if (!PyTuple_Check(reduce_value)) { - PyErr_SetString(state->PicklingError, + PyErr_SetString(st->PicklingError, "__reduce__ must return a string or tuple"); goto error; } - status = save_reduce(state, self, reduce_value, obj); + status = save_reduce(st, self, reduce_value, obj); if (0) { error: @@ -4584,12 +4590,12 @@ _pickle_Pickler_dump_impl(PicklerObject *self, PyTypeObject *cls, PyObject *obj) /*[clinic end generated code: output=952cf7f68b1445bb input=f949d84151983594]*/ { - PickleState *state = PyType_GetModuleState(cls); + PickleState *st = _Pickle_GetStateByClass(cls); /* Check whether the Pickler was initialized correctly (issue3664). Developers often forget to call __init__() in their subclasses, which would trigger a segfault without this check. */ if (self->write == NULL) { - PyErr_Format(state->PicklingError, + PyErr_Format(st->PicklingError, "Pickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -4598,7 +4604,7 @@ _pickle_Pickler_dump_impl(PicklerObject *self, PyTypeObject *cls, if (_Pickler_ClearBuffer(self) < 0) return NULL; - if (dump(state, self, obj) < 0) + if (dump(st, self, obj) < 0) return NULL; if (_Pickler_FlushToFile(self) < 0) @@ -5059,7 +5065,7 @@ static PyGetSetDef Pickler_getsets[] = { {NULL} }; -static PyType_Slot pickle_pickler_type_slots[] = { +static PyType_Slot pickler_type_slots[] = { {Py_tp_dealloc, Pickler_dealloc}, {Py_tp_methods, Pickler_methods}, {Py_tp_members, Pickler_members}, @@ -5071,14 +5077,14 @@ static PyType_Slot pickle_pickler_type_slots[] = { {Py_tp_new, PyType_GenericNew}, {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_free, PyObject_GC_Del}, - {0, 0} + {0, NULL}, }; -static PyType_Spec pickle_pickler_type_spec = { +static PyType_Spec pickler_type_spec = { .name = "_pickle.Pickler", .basicsize = sizeof(PicklerObject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .slots = pickle_pickler_type_slots, + .slots = pickler_type_slots, }; /* Temporary helper for calling self.find_class(). @@ -5096,10 +5102,10 @@ find_class(UnpicklerObject *self, PyObject *module_name, PyObject *global_name) } static Py_ssize_t -marker(PickleState *state, UnpicklerObject *self) +marker(PickleState *st, UnpicklerObject *self) { if (self->num_marks < 1) { - PyErr_SetString(state->UnpicklingError, "could not find MARK"); + PyErr_SetString(st->UnpicklingError, "could not find MARK"); return -1; } @@ -5246,9 +5252,8 @@ static int load_binint(PickleState *state, UnpicklerObject *self) { char *s; - if (_Unpickler_Read(self, state, &s, 4) < 0) { + if (_Unpickler_Read(self, state, &s, 4) < 0) return -1; - } return load_binintx(self, s, 4); } @@ -5257,9 +5262,8 @@ static int load_binint1(PickleState *state, UnpicklerObject *self) { char *s; - if (_Unpickler_Read(self, state, &s, 1) < 0) { + if (_Unpickler_Read(self, state, &s, 1) < 0) return -1; - } return load_binintx(self, s, 1); } @@ -5268,9 +5272,8 @@ static int load_binint2(PickleState *state, UnpicklerObject *self) { char *s; - if (_Unpickler_Read(self, state, &s, 2) < 0) { + if (_Unpickler_Read(self, state, &s, 2) < 0) return -1; - } return load_binintx(self, s, 2); } @@ -5306,20 +5309,20 @@ load_long(PickleState *state, UnpicklerObject *self) * data following. */ static int -load_counted_long(PickleState *state, UnpicklerObject *self, int size) +load_counted_long(PickleState *st, UnpicklerObject *self, int size) { PyObject *value; char *nbytes; char *pdata; assert(size == 1 || size == 4); - if (_Unpickler_Read(self, state, &nbytes, size) < 0) + if (_Unpickler_Read(self, st, &nbytes, size) < 0) return -1; size = calc_binint(nbytes, size); if (size < 0) { /* Corrupt or hostile pickle -- we never write one like this */ - PyErr_SetString(state->UnpicklingError, + PyErr_SetString(st->UnpicklingError, "LONG pickle has negative byte count"); return -1; } @@ -5328,7 +5331,7 @@ load_counted_long(PickleState *state, UnpicklerObject *self, int size) value = PyLong_FromLong(0L); else { /* Read the raw little-endian bytes and convert. */ - if (_Unpickler_Read(self, state, &pdata, size) < 0) + if (_Unpickler_Read(self, st, &pdata, size) < 0) return -1; value = _PyLong_FromByteArray((unsigned char *)pdata, (size_t)size, 1 /* little endian */ , 1 /* signed */ ); @@ -5390,14 +5393,14 @@ load_binfloat(PickleState *state, UnpicklerObject *self) } static int -load_string(PickleState *state, UnpicklerObject *self) +load_string(PickleState *st, UnpicklerObject *self) { PyObject *bytes; PyObject *obj; Py_ssize_t len; char *s, *p; - if ((len = _Unpickler_Readline(state, self, &s)) < 0) + if ((len = _Unpickler_Readline(st, self, &s)) < 0) return -1; /* Strip the newline */ len--; @@ -5407,7 +5410,7 @@ load_string(PickleState *state, UnpicklerObject *self) len -= 2; } else { - PyErr_SetString(state->UnpicklingError, + PyErr_SetString(st->UnpicklingError, "the STRING opcode argument must be quoted"); return -1; } @@ -5437,24 +5440,24 @@ load_string(PickleState *state, UnpicklerObject *self) } static int -load_counted_binstring(PickleState *state, UnpicklerObject *self, int nbytes) +load_counted_binstring(PickleState *st, UnpicklerObject *self, int nbytes) { PyObject *obj; Py_ssize_t size; char *s; - if (_Unpickler_Read(self, state, &s, nbytes) < 0) + if (_Unpickler_Read(self, st, &s, nbytes) < 0) return -1; size = calc_binsize(s, nbytes); if (size < 0) { - PyErr_Format(state->UnpicklingError, + PyErr_Format(st->UnpicklingError, "BINSTRING exceeds system's maximum size of %zd bytes", PY_SSIZE_T_MAX); return -1; } - if (_Unpickler_Read(self, state, &s, size) < 0) + if (_Unpickler_Read(self, st, &s, size) < 0) return -1; /* Convert Python 2.x strings to bytes if the *encoding* given to the @@ -5526,7 +5529,8 @@ load_counted_bytearray(PickleState *state, UnpicklerObject *self) if (bytearray == NULL) { return -1; } - if (_Unpickler_ReadInto(state, self, PyByteArray_AS_STRING(bytearray), size) < 0) { + char *str = PyByteArray_AS_STRING(bytearray); + if (_Unpickler_ReadInto(state, self, str, size) < 0) { Py_DECREF(bytearray); return -1; } @@ -5536,10 +5540,10 @@ load_counted_bytearray(PickleState *state, UnpicklerObject *self) } static int -load_next_buffer(PickleState *state, UnpicklerObject *self) +load_next_buffer(PickleState *st, UnpicklerObject *self) { if (self->buffers == NULL) { - PyErr_SetString(state->UnpicklingError, + PyErr_SetString(st->UnpicklingError, "pickle stream refers to out-of-band data " "but no *buffers* argument was given"); return -1; @@ -5547,7 +5551,7 @@ load_next_buffer(PickleState *state, UnpicklerObject *self) PyObject *buf = PyIter_Next(self->buffers); if (buf == NULL) { if (!PyErr_Occurred()) { - PyErr_SetString(state->UnpicklingError, + PyErr_SetString(st->UnpicklingError, "not enough out-of-band buffers"); } return -1; @@ -5651,9 +5655,9 @@ static int load_tuple(PickleState *state, UnpicklerObject *self) { Py_ssize_t i; - if ((i = marker(state, self)) < 0) { + + if ((i = marker(state, self)) < 0) return -1; - } return load_counted_tuple(state, self, Py_SIZE(self->stack) - i); } @@ -5662,9 +5666,9 @@ static int load_empty_list(PickleState *state, UnpicklerObject *self) { PyObject *list; - if ((list = PyList_New(0)) == NULL) { + + if ((list = PyList_New(0)) == NULL) return -1; - } PDATA_PUSH(self->stack, list, -1); return 0; } @@ -5673,9 +5677,9 @@ static int load_empty_dict(PickleState *state, UnpicklerObject *self) { PyObject *dict; - if ((dict = PyDict_New()) == NULL) { + + if ((dict = PyDict_New()) == NULL) return -1; - } PDATA_PUSH(self->stack, dict, -1); return 0; } @@ -5684,9 +5688,9 @@ static int load_empty_set(PickleState *state, UnpicklerObject *self) { PyObject *set; - if ((set = PySet_New(NULL)) == NULL) { + + if ((set = PySet_New(NULL)) == NULL) return -1; - } PDATA_PUSH(self->stack, set, -1); return 0; } @@ -5708,12 +5712,12 @@ load_list(PickleState *state, UnpicklerObject *self) } static int -load_dict(PickleState *state, UnpicklerObject *self) +load_dict(PickleState *st, UnpicklerObject *self) { PyObject *dict, *key, *value; Py_ssize_t i, j, k; - if ((i = marker(state, self)) < 0) + if ((i = marker(st, self)) < 0) return -1; j = Py_SIZE(self->stack); @@ -5721,7 +5725,7 @@ load_dict(PickleState *state, UnpicklerObject *self) return -1; if ((j - i) % 2 != 0) { - PyErr_SetString(state->UnpicklingError, "odd number of items for DICT"); + PyErr_SetString(st->UnpicklingError, "odd number of items for DICT"); Py_DECREF(dict); return -1; } @@ -5867,9 +5871,10 @@ load_inst(PickleState *state, UnpicklerObject *self) } static void -newobj_unpickling_error(PickleState *state, const char * msg, int use_kwargs, PyObject *arg) +newobj_unpickling_error(PickleState *st, const char *msg, int use_kwargs, + PyObject *arg) { - PyErr_Format(state->UnpicklingError, msg, + PyErr_Format(st->UnpicklingError, msg, use_kwargs ? "NEWOBJ_EX" : "NEWOBJ", Py_TYPE(arg)->tp_name); } @@ -5902,22 +5907,26 @@ load_newobj(PickleState *state, UnpicklerObject *self, int use_kwargs) } if (!PyType_Check(cls)) { - newobj_unpickling_error(state, "%s class argument must be a type, not %.200s", + newobj_unpickling_error(state, + "%s class argument must be a type, not %.200s", use_kwargs, cls); goto error; } if (((PyTypeObject *)cls)->tp_new == NULL) { - newobj_unpickling_error(state, "%s class argument '%.200s' doesn't have __new__", + newobj_unpickling_error(state, + "%s class argument '%.200s' doesn't have __new__", use_kwargs, cls); goto error; } if (!PyTuple_Check(args)) { - newobj_unpickling_error(state, "%s args argument must be a tuple, not %.200s", + newobj_unpickling_error(state, + "%s args argument must be a tuple, not %.200s", use_kwargs, args); goto error; } if (use_kwargs && !PyDict_Check(kwargs)) { - newobj_unpickling_error(state, "%s kwargs argument must be a dict, not %.200s", + newobj_unpickling_error(state, + "%s kwargs argument must be a dict, not %.200s", use_kwargs, kwargs); goto error; } @@ -5976,17 +5985,17 @@ load_global(PickleState *state, UnpicklerObject *self) } static int -load_stack_global(PickleState *state, UnpicklerObject *self) +load_stack_global(PickleState *st, UnpicklerObject *self) { PyObject *global; PyObject *module_name; PyObject *global_name; - PDATA_POP(state, self->stack, global_name); - PDATA_POP(state, self->stack, module_name); + PDATA_POP(st, self->stack, global_name); + PDATA_POP(st, self->stack, module_name); if (module_name == NULL || !PyUnicode_CheckExact(module_name) || global_name == NULL || !PyUnicode_CheckExact(global_name)) { - PyErr_SetString(state->UnpicklingError, "STACK_GLOBAL requires str"); + PyErr_SetString(st->UnpicklingError, "STACK_GLOBAL requires str"); Py_XDECREF(global_name); Py_XDECREF(module_name); return -1; @@ -6001,22 +6010,22 @@ load_stack_global(PickleState *state, UnpicklerObject *self) } static int -load_persid(PickleState *state, UnpicklerObject *self) +load_persid(PickleState *st, UnpicklerObject *self) { PyObject *pid, *obj; Py_ssize_t len; char *s; if (self->pers_func) { - if ((len = _Unpickler_Readline(state, self, &s)) < 0) + if ((len = _Unpickler_Readline(st, self, &s)) < 0) return -1; if (len < 1) - return bad_readline(state); + return bad_readline(st); pid = PyUnicode_DecodeASCII(s, len - 1, "strict"); if (pid == NULL) { if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { - PyErr_SetString(state->UnpicklingError, + PyErr_SetString(st->UnpicklingError, "persistent IDs in protocol 0 must be " "ASCII strings"); } @@ -6032,7 +6041,7 @@ load_persid(PickleState *state, UnpicklerObject *self) return 0; } else { - PyErr_SetString(state->UnpicklingError, + PyErr_SetString(st->UnpicklingError, "A load persistent id instruction was encountered, " "but no persistent_load function was specified."); return -1; @@ -6040,12 +6049,12 @@ load_persid(PickleState *state, UnpicklerObject *self) } static int -load_binpersid(PickleState *state, UnpicklerObject *self) +load_binpersid(PickleState *st, UnpicklerObject *self) { PyObject *pid, *obj; if (self->pers_func) { - PDATA_POP(state, self->stack, pid); + PDATA_POP(st, self->stack, pid); if (pid == NULL) return -1; @@ -6058,7 +6067,7 @@ load_binpersid(PickleState *state, UnpicklerObject *self) return 0; } else { - PyErr_SetString(state->UnpicklingError, + PyErr_SetString(st->UnpicklingError, "A load persistent id instruction was encountered, " "but no persistent_load function was specified."); return -1; @@ -6096,11 +6105,11 @@ static int load_pop_mark(PickleState *state, UnpicklerObject *self) { Py_ssize_t i; - if ((i = marker(state, self)) < 0) { + if ((i = marker(state, self)) < 0) return -1; - } Pdata_clear(self->stack, i); + return 0; } @@ -6118,24 +6127,21 @@ load_dup(PickleState *state, UnpicklerObject *self) } static int -load_get(PickleState *state, UnpicklerObject *self) +load_get(PickleState *st, UnpicklerObject *self) { PyObject *key, *value; Py_ssize_t idx; Py_ssize_t len; char *s; - if ((len = _Unpickler_Readline(state, self, &s)) < 0) { + if ((len = _Unpickler_Readline(st, self, &s)) < 0) return -1; - } - if (len < 2) { - return bad_readline(state); - } + if (len < 2) + return bad_readline(st); key = PyLong_FromString(s, NULL, 10); - if (key == NULL) { + if (key == NULL) return -1; - } idx = PyLong_AsSsize_t(key); if (idx == -1 && PyErr_Occurred()) { Py_DECREF(key); @@ -6145,7 +6151,7 @@ load_get(PickleState *state, UnpicklerObject *self) value = _Unpickler_MemoGet(self, idx); if (value == NULL) { if (!PyErr_Occurred()) { - PyErr_Format(state->UnpicklingError, "Memo value not found at index %ld", idx); + PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); } Py_DECREF(key); return -1; @@ -6157,13 +6163,13 @@ load_get(PickleState *state, UnpicklerObject *self) } static int -load_binget(PickleState *state, UnpicklerObject *self) +load_binget(PickleState *st, UnpicklerObject *self) { PyObject *value; Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, state, &s, 1) < 0) + if (_Unpickler_Read(self, st, &s, 1) < 0) return -1; idx = Py_CHARMASK(s[0]); @@ -6172,7 +6178,7 @@ load_binget(PickleState *state, UnpicklerObject *self) if (value == NULL) { PyObject *key = PyLong_FromSsize_t(idx); if (key != NULL) { - PyErr_Format(state->UnpicklingError, "Memo value not found at index %ld", idx); + PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); Py_DECREF(key); } return -1; @@ -6183,13 +6189,13 @@ load_binget(PickleState *state, UnpicklerObject *self) } static int -load_long_binget(PickleState *state, UnpicklerObject *self) +load_long_binget(PickleState *st, UnpicklerObject *self) { PyObject *value; Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, state, &s, 4) < 0) + if (_Unpickler_Read(self, st, &s, 4) < 0) return -1; idx = calc_binsize(s, 4); @@ -6198,7 +6204,7 @@ load_long_binget(PickleState *state, UnpicklerObject *self) if (value == NULL) { PyObject *key = PyLong_FromSsize_t(idx); if (key != NULL) { - PyErr_Format(state->UnpicklingError, "Memo value not found at index %ld", idx); + PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); Py_DECREF(key); } return -1; @@ -6212,7 +6218,7 @@ load_long_binget(PickleState *state, UnpicklerObject *self) * the number of bytes following the opcode, holding the index (code) value. */ static int -load_extension(PickleState *state, UnpicklerObject *self, int nbytes) +load_extension(PickleState *st, UnpicklerObject *self, int nbytes) { char *codebytes; /* the nbytes bytes after the opcode */ long code; /* calc_binint returns long */ @@ -6222,12 +6228,12 @@ load_extension(PickleState *state, UnpicklerObject *self, int nbytes) PyObject *module_name, *class_name; assert(nbytes == 1 || nbytes == 2 || nbytes == 4); - if (_Unpickler_Read(self, state, &codebytes, nbytes) < 0) + if (_Unpickler_Read(self, st, &codebytes, nbytes) < 0) return -1; code = calc_binint(codebytes, nbytes); if (code <= 0) { /* note that 0 is forbidden */ /* Corrupt or hostile pickle. */ - PyErr_SetString(state->UnpicklingError, "EXT specifies code <= 0"); + PyErr_SetString(st->UnpicklingError, "EXT specifies code <= 0"); return -1; } @@ -6235,7 +6241,7 @@ load_extension(PickleState *state, UnpicklerObject *self, int nbytes) py_code = PyLong_FromLong(code); if (py_code == NULL) return -1; - obj = PyDict_GetItemWithError(state->extension_cache, py_code); + obj = PyDict_GetItemWithError(st->extension_cache, py_code); if (obj != NULL) { /* Bingo. */ Py_DECREF(py_code); @@ -6248,7 +6254,7 @@ load_extension(PickleState *state, UnpicklerObject *self, int nbytes) } /* Look up the (module_name, class_name) pair. */ - pair = PyDict_GetItemWithError(state->inverted_registry, py_code); + pair = PyDict_GetItemWithError(st->inverted_registry, py_code); if (pair == NULL) { Py_DECREF(py_code); if (!PyErr_Occurred()) { @@ -6281,7 +6287,7 @@ load_extension(PickleState *state, UnpicklerObject *self, int nbytes) return -1; } /* Cache code -> obj. */ - code = PyDict_SetItem(state->extension_cache, py_code, obj); + code = PyDict_SetItem(st->extension_cache, py_code, obj); Py_DECREF(py_code); if (code < 0) { Py_DECREF(obj); @@ -6335,9 +6341,8 @@ load_binput(PickleState *state, UnpicklerObject *self) Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, state, &s, 1) < 0) { + if (_Unpickler_Read(self, state, &s, 1) < 0) return -1; - } if (Py_SIZE(self->stack) <= self->stack->fence) return Pdata_stack_underflow(state, self->stack); @@ -6355,9 +6360,8 @@ load_long_binput(PickleState *state, UnpicklerObject *self) Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, state, &s, 4) < 0) { + if (_Unpickler_Read(self, state, &s, 4) < 0) return -1; - } if (Py_SIZE(self->stack) <= self->stack->fence) return Pdata_stack_underflow(state, self->stack); @@ -6478,7 +6482,7 @@ load_appends(PickleState *state, UnpicklerObject *self) } static int -do_setitems(PickleState *state, UnpicklerObject *self, Py_ssize_t x) +do_setitems(PickleState *st, UnpicklerObject *self, Py_ssize_t x) { PyObject *value, *key; PyObject *dict; @@ -6487,12 +6491,12 @@ do_setitems(PickleState *state, UnpicklerObject *self, Py_ssize_t x) len = Py_SIZE(self->stack); if (x > len || x <= self->stack->fence) - return Pdata_stack_underflow(state, self->stack); + return Pdata_stack_underflow(st, self->stack); if (len == x) /* nothing to do */ return 0; if ((len - x) % 2 != 0) { /* Corrupt or hostile pickle -- we never write one like this. */ - PyErr_SetString(state->UnpicklingError, + PyErr_SetString(st->UnpicklingError, "odd number of items for SETITEMS"); return -1; } @@ -6524,9 +6528,8 @@ static int load_setitems(PickleState *state, UnpicklerObject *self) { Py_ssize_t i = marker(state, self); - if (i < 0) { + if (i < 0) return -1; - } return do_setitems(state, self, i); } @@ -6752,9 +6755,8 @@ load_proto(PickleState *state, UnpicklerObject *self) char *s; int i; - if (_Unpickler_Read(self, state, &s, 1) < 0) { + if (_Unpickler_Read(self, state, &s, 1) < 0) return -1; - } i = (unsigned char)s[0]; if (i <= HIGHEST_PROTOCOL) { @@ -6792,7 +6794,7 @@ load_frame(PickleState *state, UnpicklerObject *self) } static PyObject * -load(PickleState *state, UnpicklerObject *self) +load(PickleState *st, UnpicklerObject *self) { PyObject *value = NULL; char *s = NULL; @@ -6806,14 +6808,14 @@ load(PickleState *state, UnpicklerObject *self) /* Convenient macros for the dispatch while-switch loop just below. */ #define OP(opcode, load_func) \ - case opcode: if (load_func(state, self) < 0) break; continue; + case opcode: if (load_func(st, self) < 0) break; continue; #define OP_ARG(opcode, load_func, arg) \ - case opcode: if (load_func(state, self, (arg)) < 0) break; continue; + case opcode: if (load_func(st, self, (arg)) < 0) break; continue; while (1) { - if (_Unpickler_Read(self, state, &s, 1) < 0) { - if (PyErr_ExceptionMatches(state->UnpicklingError)) { + if (_Unpickler_Read(self, st, &s, 1) < 0) { + if (PyErr_ExceptionMatches(st->UnpicklingError)) { PyErr_Format(PyExc_EOFError, "Ran out of input"); } return NULL; @@ -6895,11 +6897,11 @@ load(PickleState *state, UnpicklerObject *self) { unsigned char c = (unsigned char) *s; if (0x20 <= c && c <= 0x7e && c != '\'' && c != '\\') { - PyErr_Format(state->UnpicklingError, + PyErr_Format(st->UnpicklingError, "invalid load key, '%c'.", c); } else { - PyErr_Format(state->UnpicklingError, + PyErr_Format(st->UnpicklingError, "invalid load key, '\\x%02x'.", c); } return NULL; @@ -6916,7 +6918,7 @@ load(PickleState *state, UnpicklerObject *self) if (_Unpickler_SkipConsumed(self) < 0) return NULL; - PDATA_POP(state, self->stack, value); + PDATA_POP(st, self->stack, value); return value; } @@ -6939,7 +6941,7 @@ _pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls) { UnpicklerObject *unpickler = (UnpicklerObject*)self; - PickleState *st = PyType_GetModuleState(cls); + PickleState *st = _Pickle_GetStateByClass(cls); /* Check whether the Unpickler was initialized correctly. This prevents segfaulting if a subclass overridden __init__ with a function that does @@ -6992,21 +6994,20 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, PyTypeObject *cls, return NULL; } - PickleState *state = PyType_GetModuleState(cls); - /* Try to map the old names used in Python 2.x to the new ones used in Python 3.x. We do this only with old pickle protocols and when the user has not disabled the feature. */ if (self->proto < 3 && self->fix_imports) { PyObject *key; PyObject *item; + PickleState *st = _Pickle_GetStateByClass(cls); /* Check if the global (i.e., a function or a class) was renamed or moved to another module. */ key = PyTuple_Pack(2, module_name, global_name); if (key == NULL) return NULL; - item = PyDict_GetItemWithError(state->name_mapping_2to3, key); + item = PyDict_GetItemWithError(st->name_mapping_2to3, key); Py_DECREF(key); if (item) { if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) { @@ -7032,7 +7033,7 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, PyTypeObject *cls, } else { /* Check if the module was renamed. */ - item = PyDict_GetItemWithError(state->import_mapping_2to3, module_name); + item = PyDict_GetItemWithError(st->import_mapping_2to3, module_name); if (item) { if (!PyUnicode_Check(item)) { PyErr_Format(PyExc_RuntimeError, @@ -7376,6 +7377,7 @@ static PyObject * UnpicklerMemoProxy_New(UnpicklerObject *unpickler) { UnpicklerMemoProxyObject *self; + PickleState *state = _Pickle_GetStateByClass(Py_TYPE(unpickler)); self = PyObject_GC_New(UnpicklerMemoProxyObject, state->UnpicklerMemoProxyType); @@ -7510,7 +7512,7 @@ static PyGetSetDef Unpickler_getsets[] = { {NULL} }; -static PyType_Slot pickle_unpickler_type_slots[] = { +static PyType_Slot unpickler_type_slots[] = { {Py_tp_dealloc, Unpickler_dealloc}, {Py_tp_doc, (char *)_pickle_Unpickler___init____doc__}, {Py_tp_traverse, Unpickler_traverse}, @@ -7521,14 +7523,14 @@ static PyType_Slot pickle_unpickler_type_slots[] = { {Py_tp_alloc, PyType_GenericAlloc}, {Py_tp_new, PyType_GenericNew}, {Py_tp_free, PyObject_GC_Del}, - {0,0} + {0, NULL}, }; -static PyType_Spec pickle_unpickler_type_spec = { +static PyType_Spec unpickler_type_spec = { .name = "_pickle.Unpickler", .basicsize = sizeof(UnpicklerObject), .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .slots = pickle_unpickler_type_slots, + .slots = unpickler_type_slots, }; /*[clinic input] @@ -7592,9 +7594,8 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, if (_Pickler_SetBufferCallback(pickler, buffer_callback) < 0) goto error; - if (dump(state, pickler, obj) < 0) { + if (dump(state, pickler, obj) < 0) goto error; - } if (_Pickler_FlushToFile(pickler) < 0) goto error; @@ -7656,9 +7657,8 @@ _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, if (_Pickler_SetBufferCallback(pickler, buffer_callback) < 0) goto error; - if (dump(state, pickler, obj) < 0) { + if (dump(state, pickler, obj) < 0) goto error; - } result = _Pickler_GetString(pickler); Py_DECREF(pickler); @@ -7714,8 +7714,6 @@ _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports, PyObject *result; UnpicklerObject *unpickler = _Unpickler_New(module); - PickleState *state = _Pickle_GetState(module); - if (unpickler == NULL) return NULL; @@ -7730,6 +7728,7 @@ _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports, unpickler->fix_imports = fix_imports; + PickleState *state = _Pickle_GetState(module); result = load(state, unpickler); Py_DECREF(unpickler); return result; @@ -7773,6 +7772,7 @@ _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports, PyObject *buffers) /*[clinic end generated code: output=82ac1e6b588e6d02 input=b3615540d0535087]*/ { + PyObject *result; UnpicklerObject *unpickler = _Unpickler_New(module); if (unpickler == NULL) @@ -7790,7 +7790,7 @@ _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports, unpickler->fix_imports = fix_imports; PickleState *state = _Pickle_GetState(module); - PyObject *result = load(state, unpickler); + result = load(state, unpickler); Py_DECREF(unpickler); return result; @@ -7874,7 +7874,7 @@ _pickle_exec(PyObject *m) return -1; } - st->pickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickle_pickler_type_spec, NULL); + st->pickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickler_type_spec, NULL); if (st->pickler_type == NULL) { return -1; } @@ -7883,7 +7883,7 @@ _pickle_exec(PyObject *m) return -1; } - st->unpickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickle_unpickler_type_spec, NULL); + st->unpickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &unpickler_type_spec, NULL); if (st->unpickler_type == NULL) { return -1; } @@ -7905,19 +7905,13 @@ _pickle_exec(PyObject *m) if (st->UnpicklingError == NULL) return -1; - Py_INCREF(st->PickleError); - if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0) { - Py_DECREF((PyObject *)st->PickleError); + if (PyModule_AddObjectRef(m, "PickleError", st->PickleError) < 0) { return -1; } - Py_INCREF(st->PicklingError); - if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0) { - Py_DECREF((PyObject *)st->PicklingError); + if (PyModule_AddObjectRef(m, "PicklingError", st->PicklingError) < 0) { return -1; } - Py_INCREF(st->UnpicklingError); - if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0) { - Py_DECREF((PyObject *)st->UnpicklingError); + if (PyModule_AddObjectRef(m, "UnpicklingError", st->UnpicklingError) < 0) { return -1; } From b915f06a32bb96ab7de8414f62e1c187999ff985 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 24 Mar 2023 00:35:08 +0100 Subject: [PATCH 13/21] Nit --- Modules/_pickle.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 45dc14c12fb4af..0eb80cb3cbf8bc 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -7923,7 +7923,7 @@ _pickle_exec(PyObject *m) static PyModuleDef_Slot pickle_slots[] = { {Py_mod_exec, _pickle_exec}, - {0, NULL} + {0, NULL}, }; static struct PyModuleDef _picklemodule = { @@ -7935,7 +7935,7 @@ static struct PyModuleDef _picklemodule = { .m_slots = pickle_slots, .m_traverse = pickle_traverse, .m_clear = pickle_clear, - .m_free = (freefunc)pickle_free + .m_free = (freefunc)pickle_free, }; PyMODINIT_FUNC From 223e143392338f496b5e5cdcc18c0ed69d6f0fc5 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 24 Mar 2023 00:37:26 +0100 Subject: [PATCH 14/21] Revert type name changes --- Modules/_pickle.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 0eb80cb3cbf8bc..95954424fc1f3e 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -194,8 +194,8 @@ typedef struct { PyObject *partial; /* Types */ - PyTypeObject *pickler_type; - PyTypeObject *unpickler_type; + PyTypeObject *Pickler_Type; + PyTypeObject *Unpickler_Type; PyTypeObject *Pdata_type; PyTypeObject *PicklerMemoProxyType; PyTypeObject *UnpicklerMemoProxyType; @@ -247,8 +247,8 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->codecs_encode); Py_CLEAR(st->getattr); Py_CLEAR(st->partial); - Py_CLEAR(st->pickler_type); - Py_CLEAR(st->unpickler_type); + Py_CLEAR(st->Pickler_Type); + Py_CLEAR(st->Unpickler_Type); Py_CLEAR(st->Pdata_type); Py_CLEAR(st->PicklerMemoProxyType); Py_CLEAR(st->UnpicklerMemoProxyType); @@ -1147,7 +1147,7 @@ _Pickler_New(PickleState *st) { PicklerObject *self; - self = PyObject_GC_New(PicklerObject, st->pickler_type); + self = PyObject_GC_New(PicklerObject, st->Pickler_Type); if (self == NULL) return NULL; @@ -1628,7 +1628,7 @@ _Unpickler_New(PyObject *module) UnpicklerObject *self; PickleState *st = _Pickle_GetState(module); - self = PyObject_GC_New(UnpicklerObject, st->unpickler_type); + self = PyObject_GC_New(UnpicklerObject, st->Unpickler_Type); if (self == NULL) return NULL; @@ -7838,8 +7838,8 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->codecs_encode); Py_VISIT(st->getattr); Py_VISIT(st->partial); - Py_VISIT(st->pickler_type); - Py_VISIT(st->unpickler_type); + Py_VISIT(st->Pickler_Type); + Py_VISIT(st->Unpickler_Type); Py_VISIT(st->Pdata_type); Py_VISIT(st->PicklerMemoProxyType); Py_VISIT(st->UnpicklerMemoProxyType); @@ -7874,21 +7874,21 @@ _pickle_exec(PyObject *m) return -1; } - st->pickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickler_type_spec, NULL); - if (st->pickler_type == NULL) { + st->Pickler_Type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickler_type_spec, NULL); + if (st->Pickler_Type == NULL) { return -1; } - if (PyModule_AddType(m, st->pickler_type) < 0) { + if (PyModule_AddType(m, st->Pickler_Type) < 0) { return -1; } - st->unpickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &unpickler_type_spec, NULL); - if (st->unpickler_type == NULL) { + st->Unpickler_Type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &unpickler_type_spec, NULL); + if (st->Unpickler_Type == NULL) { return -1; } - if (PyModule_AddType(m, st->unpickler_type) < 0) { + if (PyModule_AddType(m, st->Unpickler_Type) < 0) { return -1; } From 288a76431ac67dc91ea99b34a00459a464120ee3 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 24 Mar 2023 00:39:58 +0100 Subject: [PATCH 15/21] fixup! Revert type name changes --- Modules/_pickle.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 95954424fc1f3e..013a40baa4fd75 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -196,7 +196,7 @@ typedef struct { /* Types */ PyTypeObject *Pickler_Type; PyTypeObject *Unpickler_Type; - PyTypeObject *Pdata_type; + PyTypeObject *Pdata_Type; PyTypeObject *PicklerMemoProxyType; PyTypeObject *UnpicklerMemoProxyType; } PickleState; @@ -249,7 +249,7 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->partial); Py_CLEAR(st->Pickler_Type); Py_CLEAR(st->Unpickler_Type); - Py_CLEAR(st->Pdata_type); + Py_CLEAR(st->Pdata_Type); Py_CLEAR(st->PicklerMemoProxyType); Py_CLEAR(st->UnpicklerMemoProxyType); } @@ -502,7 +502,7 @@ Pdata_New(PickleState *state) { Pdata *self; - if (!(self = PyObject_GC_New(Pdata, state->Pdata_type))) + if (!(self = PyObject_GC_New(Pdata, state->Pdata_Type))) return NULL; Py_SET_SIZE(self, 0); self->mark_set = 0; @@ -7840,7 +7840,7 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->partial); Py_VISIT(st->Pickler_Type); Py_VISIT(st->Unpickler_Type); - Py_VISIT(st->Pdata_type); + Py_VISIT(st->Pdata_Type); Py_VISIT(st->PicklerMemoProxyType); Py_VISIT(st->UnpicklerMemoProxyType); return 0; @@ -7851,9 +7851,9 @@ _pickle_exec(PyObject *m) { PickleState *st = _Pickle_GetState(m); - st->Pdata_type = (PyTypeObject *)PyType_FromMetaclass(NULL, m, &pdata_spec, + st->Pdata_Type = (PyTypeObject *)PyType_FromMetaclass(NULL, m, &pdata_spec, NULL); - if (st->Pdata_type == NULL) { + if (st->Pdata_Type == NULL) { return -1; } From 0551d52dbd30f756a353f62db7132145afac8281 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 24 Mar 2023 00:43:20 +0100 Subject: [PATCH 16/21] Add CREATE_TYPE helper --- Modules/_pickle.c | 44 +++++++++++++++----------------------------- 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 013a40baa4fd75..0185ca51461a8f 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -7851,43 +7851,29 @@ _pickle_exec(PyObject *m) { PickleState *st = _Pickle_GetState(m); - st->Pdata_Type = (PyTypeObject *)PyType_FromMetaclass(NULL, m, &pdata_spec, - NULL); - if (st->Pdata_Type == NULL) { - return -1; - } - - st->PicklerMemoProxyType = (PyTypeObject *)PyType_FromMetaclass( - NULL, m, &memoproxy_spec, NULL); - if (st->PicklerMemoProxyType == NULL) { - return -1; - } - - st->UnpicklerMemoProxyType = (PyTypeObject *)PyType_FromMetaclass( - NULL, m, &unpickler_memoproxy_spec, NULL); - if (st->UnpicklerMemoProxyType == NULL) { - return -1; - } +#define CREATE_TYPE(mod, type, spec) \ + do { \ + type = (PyTypeObject *)PyType_FromMetaclass(NULL, mod, spec, NULL); \ + if (type == NULL) { \ + return -1; \ + } \ + } while (0) + + CREATE_TYPE(m, st->Pdata_Type, &pdata_spec); + CREATE_TYPE(m, st->PicklerMemoProxyType, &memoproxy_spec); + CREATE_TYPE(m, st->UnpicklerMemoProxyType, &unpickler_memoproxy_spec); + CREATE_TYPE(m, st->Pickler_Type, &pickler_type_spec); + CREATE_TYPE(m, st->Unpickler_Type, &unpickler_type_spec); + +#undef CREATE_TYPE /* Add types */ if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) { return -1; } - - st->Pickler_Type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickler_type_spec, NULL); - if (st->Pickler_Type == NULL) { - return -1; - } - if (PyModule_AddType(m, st->Pickler_Type) < 0) { return -1; } - - st->Unpickler_Type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &unpickler_type_spec, NULL); - if (st->Unpickler_Type == NULL) { - return -1; - } - if (PyModule_AddType(m, st->Unpickler_Type) < 0) { return -1; } From ae0f6af8cd3b6c255c301104c4bb1be5e1b9c63f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sat, 1 Apr 2023 23:58:21 +0530 Subject: [PATCH 17/21] Update Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst --- .../2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst index 4c5b6a849661c5..9f9c8121bea5ae 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst @@ -1 +1 @@ -Adapt :mod:`_pickle` to :pep:`687`. Patch by Mohamed Koubaa and Erlend Aasland. +Adapt :mod:`!_pickle` to :pep:`687`. Patch by Mohamed Koubaa and Erlend Aasland. From 6c1621eaa49b6feab201b859959fa91d838b3b8d Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Apr 2023 15:12:59 +0200 Subject: [PATCH 18/21] Address review: in memo proxy types set tp_hash to PyObject_HashNotImplemented --- Modules/_pickle.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 0185ca51461a8f..2ca7fc97f7de4f 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4925,6 +4925,7 @@ static PyType_Slot memoproxy_slots[] = { {Py_tp_traverse, PicklerMemoProxy_traverse}, {Py_tp_clear, PicklerMemoProxy_clear}, {Py_tp_methods, picklerproxy_methods}, + {Py_tp_hash, PyObject_HashNotImplemented}, {0, NULL}, }; @@ -7362,6 +7363,7 @@ static PyType_Slot unpickler_memoproxy_slots[] = { {Py_tp_traverse, UnpicklerMemoProxy_traverse}, {Py_tp_clear, UnpicklerMemoProxy_clear}, {Py_tp_methods, unpicklerproxy_methods}, + {Py_tp_hash, PyObject_HashNotImplemented}, {0, NULL}, }; From ffc25adc0424312589d25672aaa8b2b37cb4e2ee Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Apr 2023 15:23:47 +0200 Subject: [PATCH 19/21] Address review: find state by module def in get/set methods --- Modules/_pickle.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 2ca7fc97f7de4f..106701259fae6f 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -4941,7 +4941,7 @@ static PyObject * PicklerMemoProxy_New(PicklerObject *pickler) { PicklerMemoProxyObject *self; - PickleState *st = _Pickle_GetStateByClass(Py_TYPE(pickler)); + PickleState *st = _Pickle_FindStateByType(Py_TYPE(pickler)); self = PyObject_GC_New(PicklerMemoProxyObject, st->PicklerMemoProxyType); if (self == NULL) return NULL; @@ -4969,7 +4969,7 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored)) return -1; } - PickleState *st = _Pickle_GetStateByClass(Py_TYPE(self)); + PickleState *st = _Pickle_FindStateByType(Py_TYPE(self)); if (Py_IS_TYPE(obj, st->PicklerMemoProxyType)) { PicklerObject *pickler = ((PicklerMemoProxyObject *)obj)->pickler; @@ -7380,7 +7380,7 @@ UnpicklerMemoProxy_New(UnpicklerObject *unpickler) { UnpicklerMemoProxyObject *self; - PickleState *state = _Pickle_GetStateByClass(Py_TYPE(unpickler)); + PickleState *state = _Pickle_FindStateByType(Py_TYPE(self)); self = PyObject_GC_New(UnpicklerMemoProxyObject, state->UnpicklerMemoProxyType); if (self == NULL) @@ -7411,7 +7411,7 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored return -1; } - PickleState *state = _Pickle_GetStateByClass(Py_TYPE(self)); + PickleState *state = _Pickle_FindStateByType(Py_TYPE(self)); if (Py_IS_TYPE(obj, state->UnpicklerMemoProxyType)) { UnpicklerObject *unpickler = ((UnpicklerMemoProxyObject *)obj)->unpickler; From e363fed46612bcebd9f9ee5a54c2f3e631e596b6 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Apr 2023 17:32:18 +0200 Subject: [PATCH 20/21] Address review --- Modules/_pickle.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 106701259fae6f..6b10ee8693d8db 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -7378,9 +7378,8 @@ static PyType_Spec unpickler_memoproxy_spec = { static PyObject * UnpicklerMemoProxy_New(UnpicklerObject *unpickler) { + PickleState *state = _Pickle_FindStateByType(Py_TYPE(unpickler)); UnpicklerMemoProxyObject *self; - - PickleState *state = _Pickle_FindStateByType(Py_TYPE(self)); self = PyObject_GC_New(UnpicklerMemoProxyObject, state->UnpicklerMemoProxyType); if (self == NULL) From bb397fe238dd27e9f974f9a5b454edeb6ecfde72 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 3 Apr 2023 22:45:30 +0200 Subject: [PATCH 21/21] Address review: make memo types immutable and add tests --- Lib/test/test_pickle.py | 28 ++++++++++++++++++++++++++++ Modules/_pickle.c | 6 ++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 80e7a4d23a4ba8..1a55da39bdc58d 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -293,6 +293,34 @@ class CustomCPicklerClass(_pickle.Pickler, AbstractCustomPicklerClass): pass pickler_class = CustomCPicklerClass + @support.cpython_only + class HeapTypesTests(unittest.TestCase): + def setUp(self): + pickler = _pickle.Pickler(io.BytesIO()) + unpickler = _pickle.Unpickler(io.BytesIO()) + + self._types = ( + _pickle.Pickler, + _pickle.Unpickler, + type(pickler.memo), + type(unpickler.memo), + + # We cannot test the _pickle.Pdata; + # there's no way to get to it. + ) + + def test_have_gc(self): + import gc + for tp in self._types: + with self.subTest(tp=tp): + self.assertTrue(gc.is_tracked(tp)) + + def test_immutable(self): + for tp in self._types: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foo = "bar" + @support.cpython_only class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 6b10ee8693d8db..360c7910f67187 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -5084,7 +5084,8 @@ static PyType_Slot pickler_type_slots[] = { static PyType_Spec pickler_type_spec = { .name = "_pickle.Pickler", .basicsize = sizeof(PicklerObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), .slots = pickler_type_slots, }; @@ -7530,7 +7531,8 @@ static PyType_Slot unpickler_type_slots[] = { static PyType_Spec unpickler_type_spec = { .name = "_pickle.Unpickler", .basicsize = sizeof(UnpicklerObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), .slots = unpickler_type_slots, };