#include "Python.h"

#include "structmember.h"

PyDoc_STRVAR(pickle_module_doc,

"Optimized C implementation for the Python pickle module.");

/* Bump this when new opcodes are added to the pickle protocol. */

enum {

HIGHEST_PROTOCOL = 3,

DEFAULT_PROTOCOL = 3

};

/* Pickle opcodes. These must be kept updated with pickle.py.

Extensive docs are in pickletools.py. */

enum opcode {

MARK = '(',

STOP = '.',

POP = '0',

POP_MARK = '1',

DUP = '2',

FLOAT = 'F',

INT = 'I',

BININT = 'J',

BININT1 = 'K',

LONG = 'L',

BININT2 = 'M',

NONE = 'N',

PERSID = 'P',

BINPERSID = 'Q',

REDUCE = 'R',

STRING = 'S',

BINSTRING = 'T',

SHORT_BINSTRING = 'U',

UNICODE = 'V',

BINUNICODE = 'X',

APPEND = 'a',

BUILD = 'b',

GLOBAL = 'c',

DICT = 'd',

EMPTY_DICT = '}',

APPENDS = 'e',

GET = 'g',

BINGET = 'h',

INST = 'i',

LONG_BINGET = 'j',

LIST = 'l',

EMPTY_LIST = ']',

OBJ = 'o',

PUT = 'p',

BINPUT = 'q',

LONG_BINPUT = 'r',

SETITEM = 's',

TUPLE = 't',

EMPTY_TUPLE = ')',

SETITEMS = 'u',

BINFLOAT = 'G',

/* Protocol 2. */

PROTO = '\x80',

NEWOBJ = '\x81',

EXT1 = '\x82',

EXT2 = '\x83',

EXT4 = '\x84',

TUPLE1 = '\x85',

TUPLE2 = '\x86',

TUPLE3 = '\x87',

NEWTRUE = '\x88',

NEWFALSE = '\x89',

LONG1 = '\x8a',

LONG4 = '\x8b',

/* Protocol 3 (Python 3.x) */

BINBYTES = 'B',

SHORT_BINBYTES = 'C',

};

/* These aren't opcodes -- they're ways to pickle bools before protocol 2

* so that unpicklers written before bools were introduced unpickle them

* as ints, but unpicklers after can recognize that bools were intended.

* Note that protocol 2 added direct ways to pickle bools.

*/

#undef TRUE

#define TRUE "I01\n"

#undef FALSE

#define FALSE "I00\n"

enum {

/* Keep in synch with pickle.Pickler._BATCHSIZE. This is how many elements

batch_list/dict() pumps out before doing APPENDS/SETITEMS. Nothing will

break if this gets out of synch with pickle.py, but it's unclear that would

help anything either. */

BATCHSIZE = 1000,

/* Nesting limit until Pickler, when running in "fast mode", starts

checking for self-referential data-structures. */

FAST_NESTING_LIMIT = 50,

/* Size of the write buffer of Pickler. Higher values will reduce the

number of calls to the write() method of the output stream. */

WRITE_BUF_SIZE = 256,

};

/* Exception classes for pickle. These should override the ones defined in

pickle.py, when the C-optimized Pickler and Unpickler are used. */

static PyObject *PickleError;

static PyObject *PicklingError;

static PyObject *UnpicklingError;

/* copyreg.dispatch_table, {type_object: pickling_function} */

static PyObject *dispatch_table;

/* For EXT[124] opcodes. */

/* copyreg._extension_registry, {(module_name, function_name): code} */

static PyObject *extension_registry;

/* copyreg._inverted_registry, {code: (module_name, function_name)} */

static PyObject *inverted_registry;

/* copyreg._extension_cache, {code: object} */

static PyObject *extension_cache;

/* XXX: Are these really nescessary? */

/* As the name says, an empty tuple. */

static PyObject *empty_tuple;

/* For looking up name pairs in copyreg._extension_registry. */

static PyObject *two_tuple;

static int

stack_underflow(void)

{

PyErr_SetString(UnpicklingError, "unpickling stack underflow");

return -1;

}

/* Internal data type used as the unpickling stack. */

typedef struct {

PyObject_HEAD

int length; /* number of initial slots in data currently used */

int size; /* number of slots in data allocated */

PyObject **data;

} Pdata;

static void

Pdata_dealloc(Pdata *self)

{

int i;

PyObject **p;

for (i = self->length, p = self->data; --i >= 0; p++) {

Py_DECREF(*p);

}

if (self->data)

PyMem_Free(self->data);

PyObject_Del(self);

}

static PyTypeObject Pdata_Type = {

PyVarObject_HEAD_INIT(NULL, 0)

"_pickle.Pdata", /*tp_name*/

sizeof(Pdata), /*tp_basicsize*/

0, /*tp_itemsize*/

(destructor)Pdata_dealloc, /*tp_dealloc*/

};

static PyObject *

Pdata_New(void)

{

Pdata *self;

if (!(self = PyObject_New(Pdata, &Pdata_Type)))

return NULL;

self->size = 8;

self->length = 0;

self->data = PyMem_Malloc(self->size * sizeof(PyObject *));

if (self->data)

return (PyObject *)self;

Py_DECREF(self);

return PyErr_NoMemory();

}

/* Retain only the initial clearto items. If clearto >= the current

* number of items, this is a (non-erroneous) NOP.

*/

static int

Pdata_clear(Pdata *self, int clearto)

{

int i;

PyObject **p;

if (clearto < 0)

return stack_underflow();

if (clearto >= self->length)

return 0;

for (i = self->length, p = self->data + clearto; --i >= clearto; p++) {

Py_CLEAR(*p);

}

self->length = clearto;

return 0;

}

static int

Pdata_grow(Pdata *self)

{

int bigger;

size_t nbytes;

PyObject **tmp;

bigger = (self->size << 1) + 1;

if (bigger <= 0) /* was 0, or new value overflows */

goto nomemory;

if ((int)(size_t)bigger != bigger)

goto nomemory;

nbytes = (size_t)bigger * sizeof(PyObject *);

if (nbytes / sizeof(PyObject *) != (size_t)bigger)

goto nomemory;

tmp = PyMem_Realloc(self->data, nbytes);

if (tmp == NULL)

goto nomemory;

self->data = tmp;

self->size = bigger;

return 0;

nomemory:

PyErr_NoMemory();

return -1;

}

/* D is a Pdata*. Pop the topmost element and store it into V, which

* must be an lvalue holding PyObject*. On stack underflow, UnpicklingError

* is raised and V is set to NULL.

*/

static PyObject *

Pdata_pop(Pdata *self)

{

if (self->length == 0) {

PyErr_SetString(UnpicklingError, "bad pickle data");

return NULL;

}

return self->data[--(self->length)];

}

#define PDATA_POP(D, V) do { (V) = Pdata_pop((D)); } while (0)

static int

Pdata_push(Pdata *self, PyObject *obj)

{

if (self->length == self->size && Pdata_grow(self) < 0) {

return -1;

}

self->data[self->length++] = obj;

return 0;

}

/* Push an object on stack, transferring its ownership to the stack. */

#define PDATA_PUSH(D, O, ER) do { \

if (Pdata_push((D), (O)) < 0) return (ER); } while(0)

/* Push an object on stack, adding a new reference to the object. */

#define PDATA_APPEND(D, O, ER) do { \

Py_INCREF((O)); \

if (Pdata_push((D), (O)) < 0) return (ER); } while(0)

static PyObject *

Pdata_poptuple(Pdata *self, Py_ssize_t start)

{

PyObject *tuple;

Py_ssize_t len, i, j;

len = self->length - start;

tuple = PyTuple_New(len);

if (tuple == NULL)

return NULL;

for (i = start, j = 0; j < len; i++, j++)

PyTuple_SET_ITEM(tuple, j, self->data[i]);

self->length = start;

return tuple;

}

static PyObject *

Pdata_poplist(Pdata *self, Py_ssize_t start)

{

PyObject *list;

Py_ssize_t len, i, j;

len = self->length - start;

list = PyList_New(len);

if (list == NULL)

return NULL;

for (i = start, j = 0; j < len; i++, j++)

PyList_SET_ITEM(list, j, self->data[i]);

self->length = start;

return list;

}

typedef struct PicklerObject {

PyObject_HEAD

PyObject *write; /* write() method of the output stream */

PyObject *memo; /* Memo dictionary, keep track of the seen

objects to support self-referential objects

pickling. */

PyObject *pers_func; /* persistent_id() method, can be NULL */

PyObject *arg;

int proto; /* Pickle protocol number, >= 0 */

int bin; /* Boolean, true if proto > 0 */

int buf_size; /* Size of the current buffered pickle data */

char *write_buf; /* Write buffer, this is to avoid calling the

write() method of the output stream too

often. */

int fast; /* Enable fast mode if set to a true value.

The fast mode disable the usage of memo,

therefore speeding the pickling process by

not generating superfluous PUT opcodes. It

should not be used if with self-referential

objects. */

int fast_nesting;

PyObject *fast_memo;

} PicklerObject;

typedef struct UnpicklerObject {

PyObject_HEAD

Pdata *stack; /* Pickle data stack, store unpickled objects. */

PyObject *readline; /* readline() method of the output stream */

PyObject *read; /* read() method of the output stream */

PyObject *memo; /* Memo dictionary, provide the objects stored

using the PUT opcodes. */

PyObject *arg;

PyObject *pers_func; /* persistent_load() method, can be NULL. */

PyObject *last_string; /* Reference to the last string read by the

readline() method. */

char *buffer; /* Reading buffer. */

char *encoding; /* Name of the encoding to be used for

decoding strings pickled using Python

2.x. The default value is "ASCII" */

char *errors; /* Name of errors handling scheme to used when

decoding strings. The default value is

"strict". */

int *marks; /* Mark stack, used for unpickling container

objects. */

Py_ssize_t num_marks; /* Number of marks in the mark stack. */

Py_ssize_t marks_size; /* Current allocated size of the mark stack. */

} UnpicklerObject;

/* Forward declarations */

static int save(PicklerObject *, PyObject *, int);

static int save_reduce(PicklerObject *, PyObject *, PyObject *);

static PyTypeObject Pickler_Type;

static PyTypeObject Unpickler_Type;

/* Helpers for creating the argument tuple passed to functions. This has the

performance advantage of calling PyTuple_New() only once. */

#define ARG_TUP(self, obj) do { \

if ((self)->arg || ((self)->arg=PyTuple_New(1))) { \

Py_XDECREF(PyTuple_GET_ITEM((self)->arg, 0)); \

PyTuple_SET_ITEM((self)->arg, 0, (obj)); \

} \

else { \

Py_DECREF((obj)); \

} \

} while (0)

#define FREE_ARG_TUP(self) do { \

if ((self)->arg->ob_refcnt > 1) \

Py_CLEAR((self)->arg); \

} while (0)

/* A temporary cleaner API for fast single argument function call.

XXX: Does caching the argument tuple provides any real performance benefits?

A quick benchmark, on a 2.0GHz Athlon64 3200+ running Linux 2.6.24 with

glibc 2.7, tells me that it takes roughly 20,000,000 PyTuple_New(1) calls

when the tuple is retrieved from the freelist (i.e, call PyTuple_New() then

immediately DECREF it) and 1,200,000 calls when allocating brand new tuples

(i.e, call PyTuple_New() and store the returned value in an array), to save

one second (wall clock time). Either ways, the loading time a pickle stream

large enough to generate this number of calls would be massively

overwhelmed by other factors, like I/O throughput, the GC traversal and

object allocation overhead. So, I really doubt these functions provide any

real benefits.

On the other hand, oprofile reports that pickle spends a lot of time in

these functions. But, that is probably more related to the function call

overhead, than the argument tuple allocation.

XXX: And, what is the reference behavior of these? Steal, borrow? At first

glance, it seems to steal the reference of 'arg' and borrow the reference

of 'func'.

*/

static PyObject *

pickler_call(PicklerObject *self, PyObject *func, PyObject *arg)

{

PyObject *result = NULL;

ARG_TUP(self, arg);

if (self->arg) {

result = PyObject_Call(func, self->arg, NULL);

FREE_ARG_TUP(self);

}

return result;

}

static PyObject *

unpickler_call(UnpicklerObject *self, PyObject *func, PyObject *arg)

{

PyObject *result = NULL;

ARG_TUP(self, arg);

if (self->arg) {

result = PyObject_Call(func, self->arg, NULL);

FREE_ARG_TUP(self);

}

return result;

}

static Py_ssize_t

pickler_write(PicklerObject *self, const char *s, Py_ssize_t n)

{

PyObject *data, *result;

if (s == NULL) {

if (!(self->buf_size))

return 0;

data = PyBytes_FromStringAndSize(self->write_buf, self->buf_size);

if (data == NULL)

return -1;

}

else {

if (self->buf_size && (n + self->buf_size) > WRITE_BUF_SIZE) {

if (pickler_write(self, NULL, 0) < 0)

return -1;

}

if (n > WRITE_BUF_SIZE) {

if (!(data = PyBytes_FromStringAndSize(s, n)))

return -1;

}

else {

memcpy(self->write_buf + self->buf_size, s, n);

self->buf_size += n;

return n;

}

}

/* object with write method */

result = pickler_call(self, self->write, data);

if (result == NULL)

return -1;

Py_DECREF(result);

self->buf_size = 0;

return n;

}

/* XXX: These read/readline functions ought to be optimized. Buffered I/O

might help a lot, especially with the new (but much slower) io library.

On the other hand, the added complexity might not worth it.

*/

/* Read at least n characters from the input stream and set s to the current

reading position. */

static Py_ssize_t

unpickler_read(UnpicklerObject *self, char **s, Py_ssize_t n)

{

PyObject *len;

PyObject *data;

len = PyLong_FromSsize_t(n);

if (len == NULL)

return -1;

data = unpickler_call(self, self->read, len);

if (data == NULL)

return -1;

/* XXX: Should bytearray be supported too? */

if (!PyBytes_Check(data)) {

PyErr_SetString(PyExc_ValueError,

"read() from the underlying stream did not"

"return bytes");

return -1;

}

Py_XDECREF(self->last_string);

self->last_string = data;

if (!(*s = PyBytes_AS_STRING(data)))

return -1;

return n;

}

static Py_ssize_t

unpickler_readline(UnpicklerObject *self, char **s)

{

PyObject *data;

data = PyObject_CallObject(self->readline, empty_tuple);

if (data == NULL)

return -1;

/* XXX: Should bytearray be supported too? */

if (!PyBytes_Check(data)) {

PyErr_SetString(PyExc_ValueError,

"readline() from the underlying stream did not"

"return bytes");

return -1;

}

Py_XDECREF(self->last_string);

self->last_string = data;

if (!(*s = PyBytes_AS_STRING(data)))

return -1;

return PyBytes_GET_SIZE(data);

}

/* Generate a GET opcode for an object stored in the memo. The 'key' argument

should be the address of the object as returned by PyLong_FromVoidPtr(). */

static int

memo_get(PicklerObject *self, PyObject *key)

{

PyObject *value;

PyObject *memo_id;

long x;

char pdata[30];

int len;

value = PyDict_GetItemWithError(self->memo, key);

if (value == NULL) {

if (!PyErr_Occurred())

PyErr_SetObject(PyExc_KeyError, key);

return -1;

}

memo_id = PyTuple_GetItem(value, 0);

if (memo_id == NULL)

return -1;

if (!PyLong_Check(memo_id)) {

PyErr_SetString(PicklingError, "memo id must be an integer");

return -1;

}

x = PyLong_AsLong(memo_id);

if (x == -1 && PyErr_Occurred())

return -1;

if (!self->bin) {

pdata[0] = GET;

PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ld\n", x);

len = (int)strlen(pdata);

}

else {

if (x < 256) {

pdata[0] = BINGET;

pdata[1] = (unsigned char)(x & 0xff);

len = 2;

}

else if (x <= 0xffffffffL) {

pdata[0] = LONG_BINGET;

pdata[1] = (unsigned char)(x & 0xff);

pdata[2] = (unsigned char)((x >> 8) & 0xff);

pdata[3] = (unsigned char)((x >> 16) & 0xff);

pdata[4] = (unsigned char)((x >> 24) & 0xff);

len = 5;

}

else { /* unlikely */

PyErr_SetString(PicklingError,

"memo id too large for LONG_BINGET");

return -1;

}

}

if (pickler_write(self, pdata, len) < 0)

return -1;

return 0;

}

/* Store an object in the memo, assign it a new unique ID based on the number

of objects currently stored in the memo and generate a PUT opcode. */

static int

memo_put(PicklerObject *self, PyObject *obj)

{

PyObject *key = NULL;

PyObject *memo_id = NULL;

PyObject *tuple = NULL;

long x;

char pdata[30];

int len;

int status = 0;

if (self->fast)

return 0;

key = PyLong_FromVoidPtr(obj);

if (key == NULL)

goto error;

if ((x = PyDict_Size(self->memo)) < 0)

goto error;

memo_id = PyLong_FromLong(x);

if (memo_id == NULL)

goto error;

tuple = PyTuple_New(2);

if (tuple == NULL)

goto error;

Py_INCREF(memo_id);

PyTuple_SET_ITEM(tuple, 0, memo_id);

Py_INCREF(obj);

PyTuple_SET_ITEM(tuple, 1, obj);

if (PyDict_SetItem(self->memo, key, tuple) < 0)

goto error;

if (!self->bin) {

pdata[0] = PUT;

PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ld\n", x);

len = strlen(pdata);

}

else {

if (x < 256) {

pdata[0] = BINPUT;

pdata[1] = (unsigned char)x;

len = 2;

}

else if (x <= 0xffffffffL) {

pdata[0] = LONG_BINPUT;

pdata[1] = (unsigned char)(x & 0xff);

pdata[2] = (unsigned char)((x >> 8) & 0xff);

pdata[3] = (unsigned char)((x >> 16) & 0xff);

pdata[4] = (unsigned char)((x >> 24) & 0xff);

len = 5;

}

else { /* unlikely */

PyErr_SetString(PicklingError,

"memo id too large for LONG_BINPUT");

return -1;

}

}

if (pickler_write(self, pdata, len) < 0)

goto error;

if (0) {

error:

status = -1;

}

Py_XDECREF(key);

Py_XDECREF(memo_id);

Py_XDECREF(tuple);

return status;

}

static PyObject *

whichmodule(PyObject *global, PyObject *global_name)

{

Py_ssize_t i, j;

static PyObject *module_str = NULL;

static PyObject *main_str = NULL;

PyObject *module_name;

PyObject *modules_dict;

PyObject *module;

PyObject *obj;

if (module_str == NULL) {

module_str = PyUnicode_InternFromString("__module__");

if (module_str == NULL)

return NULL;

main_str = PyUnicode_InternFromString("__main__");

if (main_str == NULL)

return NULL;

}

module_name = PyObject_GetAttr(global, module_str);

/* In some rare cases (e.g., random.getrandbits), __module__ can be

None. If it is so, then search sys.modules for the module of

global. */

if (module_name == Py_None) {

Py_DECREF(module_name);

goto search;

}

if (module_name) {

return module_name;

}

if (PyErr_ExceptionMatches(PyExc_AttributeError))

PyErr_Clear();

else

return NULL;

search:

modules_dict = PySys_GetObject("modules");

if (modules_dict == NULL)

return NULL;

i = 0;

module_name = NULL;

while ((j = PyDict_Next(modules_dict, &i, &module_name, &module))) {

if (PyObject_Compare(module_name, main_str) == 0)

continue;

obj = PyObject_GetAttr(module, global_name);

if (obj == NULL) {

if (PyErr_ExceptionMatches(PyExc_AttributeError))

PyErr_Clear();

else

return NULL;

continue;

}

if (obj != global) {

Py_DECREF(obj);

continue;

}

Py_DECREF(obj);

break;

}

/* If no module is found, use __main__. */

if (!j) {

module_name = main_str;

}

Py_INCREF(module_name);

return module_name;

}

/* fast_save_enter() and fast_save_leave() are guards against recursive

objects when Pickler is used with the "fast mode" (i.e., with object

memoization disabled). If the nesting of a list or dict object exceed

FAST_NESTING_LIMIT, these guards will start keeping an internal

reference to the seen list or dict objects and check whether these objects

are recursive. These are not strictly necessary, since save() has a

hard-coded recursion limit, but they give a nicer error message than the

typical RuntimeError. */

static int

fast_save_enter(PicklerObject *self, PyObject *obj)

{

/* if fast_nesting < 0, we're doing an error exit. */

if (++self->fast_nesting >= FAST_NESTING_LIMIT) {

PyObject *key = NULL;

if (self->fast_memo == NULL) {

self->fast_memo = PyDict_New();

if (self->fast_memo == NULL) {

self->fast_nesting = -1;

return 0;

}

}

key = PyLong_FromVoidPtr(obj);

if (key == NULL)

return 0;

if (PyDict_GetItem(self->fast_memo, key)) {

Py_DECREF(key);

PyErr_Format(PyExc_ValueError,

"fast mode: can't pickle cyclic objects "

"including object type %.200s at %p",

obj->ob_type->tp_name, obj);

self->fast_nesting = -1;

return 0;

}

if (PyDict_SetItem(self->fast_memo, key, Py_None) < 0) {

Py_DECREF(key);

self->fast_nesting = -1;

return 0;

}

Py_DECREF(key);

}

return 1;

}

static int

fast_save_leave(PicklerObject *self, PyObject *obj)

{

if (self->fast_nesting-- >= FAST_NESTING_LIMIT) {

PyObject *key = PyLong_FromVoidPtr(obj);

if (key == NULL)

return 0;

if (PyDict_DelItem(self->fast_memo, key) < 0) {

Py_DECREF(key);

return 0;

}

Py_DECREF(key);

}

return 1;

}

static int

save_none(PicklerObject *self, PyObject *obj)

{

const char none_op = NONE;

if (pickler_write(self, &none_op, 1) < 0)

return -1;

return 0;

}

static int

save_bool(PicklerObject *self, PyObject *obj)

{

static const char *buf[2] = { FALSE, TRUE };

const char len[2] = {sizeof(FALSE) - 1, sizeof(TRUE) - 1};

int p = (obj == Py_True);

if (self->proto >= 2) {

const char bool_op = p ? NEWTRUE : NEWFALSE;

if (pickler_write(self, &bool_op, 1) < 0)

return -1;

}

else if (pickler_write(self, buf[p], len[p]) < 0)

return -1;

return 0;

}

static int

save_int(PicklerObject *self, long x)

{

char pdata[32];

int len = 0;

if (!self->bin

#if SIZEOF_LONG > 4

|| x > 0x7fffffffL || x < -0x80000000L

#endif

) {

/* Text-mode pickle, or long too big to fit in the 4-byte

* signed BININT format: store as a string.

*/

pdata[0] = LONG; /* use LONG for consistence with pickle.py */

PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ld\n", x);

if (pickler_write(self, pdata, strlen(pdata)) < 0)

return -1;

}

else {

/* Binary pickle and x fits in a signed 4-byte int. */

pdata[1] = (unsigned char)(x & 0xff);

pdata[2] = (unsigned char)((x >> 8) & 0xff);

pdata[3] = (unsigned char)((x >> 16) & 0xff);

pdata[4] = (unsigned char)((x >> 24) & 0xff);

if ((pdata[4] == 0) && (pdata[3] == 0)) {

if (pdata[2] == 0) {

pdata[0] = BININT1;

len = 2;

}

else {

pdata[0] = BININT2;

len = 3;

}

}

else {

pdata[0] = BININT;

len = 5;

}

if (pickler_write(self, pdata, len) < 0)

return -1;

}

return 0;

}

static int

save_long(PicklerObject *self, PyObject *obj)

{

PyObject *repr = NULL;

Py_ssize_t size;

long val = PyLong_AsLong(obj);

int status = 0;

const char long_op = LONG;

if (val == -1 && PyErr_Occurred()) {

/* out of range for int pickling */

PyErr_Clear();

}

else

return save_int(self, val);

if (self->proto >= 2) {

/* Linear-time pickling. */

size_t nbits;

size_t nbytes;

unsigned char *pdata;

char header[5];

int i;

int sign = _PyLong_Sign(obj);

if (sign == 0) {

header[0] = LONG1;

header[1] = 0; /* It's 0 -- an empty bytestring. */

if (pickler_write(self, header, 2) < 0)

goto error;

return 0;

}

nbits = _PyLong_NumBits(obj);

if (nbits == (size_t)-1 && PyErr_Occurred())

goto error;

/* How many bytes do we need? There are nbits >> 3 full

* bytes of data, and nbits & 7 leftover bits. If there

* are any leftover bits, then we clearly need another

* byte. Wnat's not so obvious is that we *probably*

* need another byte even if there aren't any leftovers:

* the most-significant bit of the most-significant byte

* acts like a sign bit, and it's usually got a sense

* opposite of the one we need. The exception is longs

* of the form -(2**(8*j-1)) for j > 0. Such a long is

* its own 256's-complement, so has the right sign bit

* even without the extra byte. That's a pain to check

* for in advance, though, so we always grab an extra

* byte at the start, and cut it back later if possible.

*/

nbytes = (nbits >> 3) + 1;

if (nbytes > INT_MAX) {

PyErr_SetString(PyExc_OverflowError,

"long too large to pickle");

goto error;

}

repr = PyUnicode_FromStringAndSize(NULL, (int)nbytes);

if (repr == NULL)

goto error;

pdata = (unsigned char *)_PyUnicode_AsString(repr);

i = _PyLong_AsByteArray((PyLongObject *)obj,

pdata, nbytes,

1 /* little endian */ , 1 /* signed */ );

if (i < 0)

goto error;

/* If the long is negative, this may be a byte more than

* needed. This is so iff the MSB is all redundant sign

* bits.

*/

if (sign < 0 &&

nbytes > 1 &&

pdata[nbytes - 1] == 0xff &&

(pdata[nbytes - 2] & 0x80) != 0) {

nbytes--;

}

if (nbytes < 256) {

header[0] = LONG1;

header[1] = (unsigned char)nbytes;

size = 2;

}

else {

header[0] = LONG4;

size = (int)nbytes;

for (i = 1; i < 5; i++) {

header[i] = (unsigned char)(size & 0xff);

size >>= 8;

}

size = 5;

}

if (pickler_write(self, header, size) < 0 ||

pickler_write(self, (char *)pdata, (int)nbytes) < 0)

goto error;

}

else {

char *string;

/* proto < 2: write the repr and newline. This is quadratic-time

(in the number of digits), in both directions. */

repr = PyObject_Repr(obj);

if (repr == NULL)

goto error;

string = _PyUnicode_AsStringAndSize(repr, &size);

if (string == NULL)

goto error;

if (pickler_write(self, &long_op, 1) < 0 ||

pickler_write(self, string, size) < 0 ||

pickler_write(self, "\n", 1) < 0)

goto error;

}

if (0) {

error:

status = -1;

}

Py_XDECREF(repr);

return status;

}

static int

save_float(PicklerObject *self, PyObject *obj)

{

double x = PyFloat_AS_DOUBLE((PyFloatObject *)obj);

if (self->bin) {

char pdata[9];