#include "Python.h"

#include "cStringIO.h"

#include "structmember.h"

PyDoc_STRVAR(cPickle_module_documentation,

"C implementation and optimization of the Python pickle module\n"

"\n"

"cPickle.c,v 1.71 1999/07/11 13:30:34 jim Exp\n");

#ifndef Py_eval_input

#include <graminit.h>

#define Py_eval_input eval_input

#endif /* Py_eval_input */

#include <errno.h>

#define DEL_LIST_SLICE(list, from, to) (PyList_SetSlice(list, from, to, NULL))

#define WRITE_BUF_SIZE 256

/* --------------------------------------------------------------------------

NOTES on format codes.

XXX much more is needed here

Integer types

BININT1 8-bit unsigned integer; followed by 1 byte.

BININT2 16-bit unsigned integer; followed by 2 bytes, little-endian.

BININT 32-bit signed integer; followed by 4 bytes, little-endian.

INT Integer; natural decimal string conversion, then newline.

CAUTION: INT-reading code can't assume that what follows

fits in a Python int, because the size of Python ints varies

across platforms.

LONG Long (unbounded) integer; repr(i), then newline.

-------------------------------------------------------------------------- */

#define MARK '('

#define STOP '.'

#define POP '0'

#define POP_MARK '1'

#define DUP '2'

#define FLOAT 'F'

#define BINFLOAT 'G'

#define INT 'I'

#define BININT 'J'

#define BININT1 'K'

#define LONG 'L'

#define BININT2 'M'

#define NONE 'N'

#define PERSID 'P'

#define BINPERSID 'Q'

#define REDUCE 'R'

#define STRING 'S'

#define BINSTRING 'T'

#define SHORT_BINSTRING 'U'

#define UNICODE 'V'

#define BINUNICODE 'X'

#define APPEND 'a'

#define BUILD 'b'

#define GLOBAL 'c'

#define DICT 'd'

#define EMPTY_DICT '}'

#define APPENDS 'e'

#define GET 'g'

#define BINGET 'h'

#define INST 'i'

#define LONG_BINGET 'j'

#define LIST 'l'

#define EMPTY_LIST ']'

#define OBJ 'o'

#define PUT 'p'

#define BINPUT 'q'

#define LONG_BINPUT 'r'

#define SETITEM 's'

#define TUPLE 't'

#define EMPTY_TUPLE ')'

#define SETITEMS 'u'

#define TRUE "I01\n"

#define FALSE "I00\n"

static char MARKv = MARK;

static PyObject *PickleError;

static PyObject *PicklingError;

static PyObject *UnpickleableError;

static PyObject *UnpicklingError;

static PyObject *BadPickleGet;

static PyObject *dispatch_table;

static PyObject *safe_constructors;

static PyObject *empty_tuple;

static PyObject *__class___str, *__getinitargs___str, *__dict___str,

*__getstate___str, *__setstate___str, *__name___str, *__reduce___str,

*write_str, *__safe_for_unpickling___str, *append_str,

*read_str, *readline_str, *__main___str, *__basicnew___str,

*copy_reg_str, *dispatch_table_str, *safe_constructors_str;

/*************************************************************************

Internal Data type for pickle data. */

typedef struct {

PyObject_HEAD

int length, size;

PyObject **data;

} Pdata;

static void

Pdata_dealloc(Pdata *self)

{

int i;

PyObject **p;

for (i=self->length, p=self->data; --i >= 0; p++) Py_DECREF(*p);

if (self->data) free(self->data);

PyObject_Del(self);

}

static PyTypeObject PdataType = {

PyObject_HEAD_INIT(NULL) 0, "cPickle.Pdata", sizeof(Pdata), 0,

(destructor)Pdata_dealloc,

0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0L,0L,0L,0L, ""

};

#define Pdata_Check(O) ((O)->ob_type == &PdataType)

static PyObject *

Pdata_New(void)

{

Pdata *self;

if (!( self = PyObject_New(Pdata, &PdataType))) return NULL;

self->size=8;

self->length=0;

self->data=malloc(self->size * sizeof(PyObject*));

if (self->data) return (PyObject*)self;

Py_DECREF(self);

return PyErr_NoMemory();

}

static int

stackUnderflow(void)

{

PyErr_SetString(UnpicklingError, "unpickling stack underflow");

return -1;

}

static int

Pdata_clear(Pdata *self, int clearto)

{

int i;

PyObject **p;

if (clearto < 0) return stackUnderflow();

if (clearto >= self->length) return 0;

for (i=self->length, p=self->data+clearto; --i >= clearto; p++)

Py_DECREF(*p);

self->length=clearto;

return 0;

}

static int

Pdata_grow(Pdata *self)

{

if (! self->size) {

PyErr_NoMemory();

return -1;

}

self->size *= 2;

self->data = realloc(self->data, self->size*sizeof(PyObject*));

if (! self->data) {

self->size = 0;

PyErr_NoMemory();

return -1;

}

return 0;

}

#define PDATA_POP(D,V) { \

if ((D)->length) V=D->data[--((D)->length)]; \

else { \

PyErr_SetString(UnpicklingError, "bad pickle data"); \

V=NULL; \

} \

}

static PyObject *

Pdata_popTuple(Pdata *self, int start)

{

PyObject *r;

int i, j, l;

l=self->length-start;

if (!( r=PyTuple_New(l))) return NULL;

for (i=start, j=0 ; j < l; i++, j++)

PyTuple_SET_ITEM(r, j, self->data[i]);

self->length=start;

return r;

}

static PyObject *

Pdata_popList(Pdata *self, int start)

{

PyObject *r;

int i, j, l;

l=self->length-start;

if (!( r=PyList_New(l))) return NULL;

for (i=start, j=0 ; j < l; i++, j++)

PyList_SET_ITEM(r, j, self->data[i]);

self->length=start;

return r;

}

#define PDATA_APPEND_(D,O,ER) { \

if (Pdata_Append(((Pdata*)(D)), O) < 0) return ER; \

}

#define PDATA_APPEND(D,O,ER) { \

if (((Pdata*)(D))->length == ((Pdata*)(D))->size && \

Pdata_grow((Pdata*)(D)) < 0) \

return ER; \

Py_INCREF(O); \

((Pdata*)(D))->data[((Pdata*)(D))->length++]=O; \

}

#define PDATA_PUSH(D,O,ER) { \

if (((Pdata*)(D))->length == ((Pdata*)(D))->size && \

Pdata_grow((Pdata*)(D)) < 0) { \

Py_DECREF(O); \

return ER; \

} \

((Pdata*)(D))->data[((Pdata*)(D))->length++]=O; \

}

/*************************************************************************/

#define ARG_TUP(self, o) { \

if (self->arg || (self->arg=PyTuple_New(1))) { \

Py_XDECREF(PyTuple_GET_ITEM(self->arg,0)); \

PyTuple_SET_ITEM(self->arg,0,o); \

} \

else { \

Py_DECREF(o); \

} \

}

#define FREE_ARG_TUP(self) { \

if (self->arg->ob_refcnt > 1) { \

Py_DECREF(self->arg); \

self->arg=NULL; \

} \

}

typedef struct Picklerobject {

PyObject_HEAD

FILE *fp;

PyObject *write;

PyObject *file;

PyObject *memo;

PyObject *arg;

PyObject *pers_func;

PyObject *inst_pers_func;

int bin;

int fast; /* Fast mode doesn't save in memo, don't use if circ ref */

int (*write_func)(struct Picklerobject *, char *, int);

char *write_buf;

int buf_size;

PyObject *dispatch_table;

int fast_container; /* count nested container dumps */

PyObject *fast_memo;

} Picklerobject;

#ifndef PY_CPICKLE_FAST_LIMIT

#define PY_CPICKLE_FAST_LIMIT 50

#endif

staticforward PyTypeObject Picklertype;

typedef struct Unpicklerobject {

PyObject_HEAD

FILE *fp;

PyObject *file;

PyObject *readline;

PyObject *read;

PyObject *memo;

PyObject *arg;

Pdata *stack;

PyObject *mark;

PyObject *pers_func;

PyObject *last_string;

int *marks;

int num_marks;

int marks_size;

int (*read_func)(struct Unpicklerobject *, char **, int);

int (*readline_func)(struct Unpicklerobject *, char **);

int buf_size;

char *buf;

PyObject *safe_constructors;

PyObject *find_class;

} Unpicklerobject;

staticforward PyTypeObject Unpicklertype;

/* Forward decls that need the above structs */

static int save(Picklerobject *, PyObject *, int);

static int put2(Picklerobject *, PyObject *);

int

cPickle_PyMapping_HasKey(PyObject *o, PyObject *key)

{

PyObject *v;

if ((v = PyObject_GetItem(o,key))) {

Py_DECREF(v);

return 1;

}

PyErr_Clear();

return 0;

}

static

PyObject *

cPickle_ErrFormat(PyObject *ErrType, char *stringformat, char *format, ...)

{

va_list va;

PyObject *args=0, *retval=0;

va_start(va, format);

if (format) args = Py_VaBuildValue(format, va);

va_end(va);

if (format && ! args) return NULL;

if (stringformat && !(retval=PyString_FromString(stringformat)))

return NULL;

if (retval) {

if (args) {

PyObject *v;

v=PyString_Format(retval, args);

Py_DECREF(retval);

Py_DECREF(args);

if (! v) return NULL;

retval=v;

}

}

else

if (args) retval=args;

else {

PyErr_SetObject(ErrType,Py_None);

return NULL;

}

PyErr_SetObject(ErrType,retval);

Py_DECREF(retval);

return NULL;

}

static int

write_file(Picklerobject *self, char *s, int n)

{

size_t nbyteswritten;

if (s == NULL) {

return 0;

}

Py_BEGIN_ALLOW_THREADS

nbyteswritten = fwrite(s, sizeof(char), n, self->fp);

Py_END_ALLOW_THREADS

if (nbyteswritten != (size_t)n) {

PyErr_SetFromErrno(PyExc_IOError);

return -1;

}

return n;

}

static int

write_cStringIO(Picklerobject *self, char *s, int n)

{

if (s == NULL) {

return 0;

}

if (PycStringIO->cwrite((PyObject *)self->file, s, n) != n) {

return -1;

}

return n;

}

static int

write_none(Picklerobject *self, char *s, int n)

{

if (s == NULL) return 0;

return n;

}

static int

write_other(Picklerobject *self, char *s, int n)

{

PyObject *py_str = 0, *junk = 0;

if (s == NULL) {

if (!( self->buf_size )) return 0;

py_str = PyString_FromStringAndSize(self->write_buf,

self->buf_size);

if (!py_str)

return -1;

}

else {

if (self->buf_size && (n + self->buf_size) > WRITE_BUF_SIZE) {

if (write_other(self, NULL, 0) < 0)

return -1;

}

if (n > WRITE_BUF_SIZE) {

if (!( py_str =

PyString_FromStringAndSize(s, n)))

return -1;

}

else {

memcpy(self->write_buf + self->buf_size, s, n);

self->buf_size += n;

return n;

}

}

if (self->write) {

/* object with write method */

ARG_TUP(self, py_str);

if (self->arg) {

junk = PyObject_Call(self->write, self->arg, NULL);

FREE_ARG_TUP(self);

}

if (junk) Py_DECREF(junk);

else return -1;

}

else

PDATA_PUSH(self->file, py_str, -1);

self->buf_size = 0;

return n;

}

static int

read_file(Unpicklerobject *self, char **s, int n)

{

size_t nbytesread;

if (self->buf_size == 0) {

int size;

size = ((n < 32) ? 32 : n);

if (!( self->buf = (char *)malloc(size * sizeof(char)))) {

PyErr_NoMemory();

return -1;

}

self->buf_size = size;

}

else if (n > self->buf_size) {

self->buf = (char *)realloc(self->buf, n * sizeof(char));

if (!self->buf) {

PyErr_NoMemory();

return -1;

}

self->buf_size = n;

}

Py_BEGIN_ALLOW_THREADS

nbytesread = fread(self->buf, sizeof(char), n, self->fp);

Py_END_ALLOW_THREADS

if (nbytesread != (size_t)n) {

if (feof(self->fp)) {

PyErr_SetNone(PyExc_EOFError);

return -1;

}

PyErr_SetFromErrno(PyExc_IOError);

return -1;

}

*s = self->buf;

return n;

}

static int

readline_file(Unpicklerobject *self, char **s)

{

int i;

if (self->buf_size == 0) {

if (!( self->buf = (char *)malloc(40 * sizeof(char)))) {

PyErr_NoMemory();

return -1;

}

self->buf_size = 40;

}

i = 0;

while (1) {

for (; i < (self->buf_size - 1); i++) {

if (feof(self->fp) ||

(self->buf[i] = getc(self->fp)) == '\n') {

self->buf[i + 1] = '\0';

*s = self->buf;

return i + 1;

}

}

self->buf = (char *)realloc(self->buf,

(self->buf_size * 2) * sizeof(char));

if (!self->buf) {

PyErr_NoMemory();

return -1;

}

self->buf_size *= 2;

}

}

static int

read_cStringIO(Unpicklerobject *self, char **s, int n)

{

char *ptr;

if (PycStringIO->cread((PyObject *)self->file, &ptr, n) != n) {

PyErr_SetNone(PyExc_EOFError);

return -1;

}

*s = ptr;

return n;

}

static int

readline_cStringIO(Unpicklerobject *self, char **s)

{

int n;

char *ptr;

if ((n = PycStringIO->creadline((PyObject *)self->file, &ptr)) < 0) {

return -1;

}

*s = ptr;

return n;

}

static int

read_other(Unpicklerobject *self, char **s, int n)

{

PyObject *bytes, *str=0;

if (!( bytes = PyInt_FromLong(n))) return -1;

ARG_TUP(self, bytes);

if (self->arg) {

str = PyObject_Call(self->read, self->arg, NULL);

FREE_ARG_TUP(self);

}

if (! str) return -1;

Py_XDECREF(self->last_string);

self->last_string = str;

if (! (*s = PyString_AsString(str))) return -1;

return n;

}

static int

readline_other(Unpicklerobject *self, char **s)

{

PyObject *str;

int str_size;

if (!( str = PyObject_CallObject(self->readline, empty_tuple))) {

return -1;

}

if ((str_size = PyString_Size(str)) < 0)

return -1;

Py_XDECREF(self->last_string);

self->last_string = str;

if (! (*s = PyString_AsString(str)))

return -1;

return str_size;

}

static char *

pystrndup(char *s, int l)

{

char *r;

if (!( r=malloc((l+1)*sizeof(char)))) return (char*)PyErr_NoMemory();

memcpy(r,s,l);

r[l]=0;

return r;

}

static int

get(Picklerobject *self, PyObject *id)

{

PyObject *value, *mv;

long c_value;

char s[30];

size_t len;

if (!( mv = PyDict_GetItem(self->memo, id))) {

PyErr_SetObject(PyExc_KeyError, id);

return -1;

}

if (!( value = PyTuple_GetItem(mv, 0)))

return -1;

if (!( PyInt_Check(value))) {

PyErr_SetString(PicklingError, "no int where int expected in memo");

return -1;

}

c_value = PyInt_AS_LONG((PyIntObject*)value);

if (!self->bin) {

s[0] = GET;

PyOS_snprintf(s + 1, sizeof(s) - 1, "%ld\n", c_value);

len = strlen(s);

}

else if (Pdata_Check(self->file)) {

if (write_other(self, NULL, 0) < 0) return -1;

PDATA_APPEND(self->file, mv, -1);

return 0;

}

else {

if (c_value < 256) {

s[0] = BINGET;

s[1] = (int)(c_value & 0xff);

len = 2;

}

else {

s[0] = LONG_BINGET;

s[1] = (int)(c_value & 0xff);

s[2] = (int)((c_value >> 8) & 0xff);

s[3] = (int)((c_value >> 16) & 0xff);

s[4] = (int)((c_value >> 24) & 0xff);

len = 5;

}

}

if ((*self->write_func)(self, s, len) < 0)

return -1;

return 0;

}

static int

put(Picklerobject *self, PyObject *ob)

{

if (ob->ob_refcnt < 2 || self->fast)

return 0;

return put2(self, ob);

}

static int

put2(Picklerobject *self, PyObject *ob)

{

char c_str[30];

int p;

size_t len;

int res = -1;

PyObject *py_ob_id = 0, *memo_len = 0, *t = 0;

if (self->fast)

return 0;

if ((p = PyDict_Size(self->memo)) < 0)

goto finally;

/* Make sure memo keys are positive! */

p++;

if (!( py_ob_id = PyLong_FromVoidPtr(ob)))

goto finally;

if (!( memo_len = PyInt_FromLong(p)))

goto finally;

if (!( t = PyTuple_New(2)))

goto finally;

PyTuple_SET_ITEM(t, 0, memo_len);

Py_INCREF(memo_len);

PyTuple_SET_ITEM(t, 1, ob);

Py_INCREF(ob);

if (PyDict_SetItem(self->memo, py_ob_id, t) < 0)

goto finally;

if (!self->bin) {

c_str[0] = PUT;

PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, "%d\n", p);

len = strlen(c_str);

}

else if (Pdata_Check(self->file)) {

if (write_other(self, NULL, 0) < 0) return -1;

PDATA_APPEND(self->file, memo_len, -1);

res=0; /* Job well done ;) */

goto finally;

}

else {

if (p >= 256) {

c_str[0] = LONG_BINPUT;

c_str[1] = (int)(p & 0xff);

c_str[2] = (int)((p >> 8) & 0xff);

c_str[3] = (int)((p >> 16) & 0xff);

c_str[4] = (int)((p >> 24) & 0xff);

len = 5;

}

else {

c_str[0] = BINPUT;

c_str[1] = p;

len = 2;

}

}

if ((*self->write_func)(self, c_str, len) < 0)

goto finally;

res = 0;

finally:

Py_XDECREF(py_ob_id);

Py_XDECREF(memo_len);

Py_XDECREF(t);

return res;

}

#define PyImport_Import cPickle_Import

static PyObject *

PyImport_Import(PyObject *module_name)

{

static PyObject *silly_list=0, *__builtins___str=0, *__import___str;

static PyObject *standard_builtins=0;

PyObject *globals=0, *__import__=0, *__builtins__=0, *r=0;

if (!( silly_list )) {

if (!( __import___str=PyString_FromString("__import__")))

return NULL;

if (!( __builtins___str=PyString_FromString("__builtins__")))

return NULL;

if (!( silly_list=Py_BuildValue("[s]","__doc__")))

return NULL;

}

if ((globals=PyEval_GetGlobals())) {

Py_INCREF(globals);

__builtins__=PyObject_GetItem(globals,__builtins___str);

if (!__builtins__)

goto err;

}

else {

PyErr_Clear();

if (!(standard_builtins ||

(standard_builtins=PyImport_ImportModule("__builtin__"))))

return NULL;

__builtins__=standard_builtins;

Py_INCREF(__builtins__);

globals = Py_BuildValue("{sO}", "__builtins__", __builtins__);

if (!globals)

goto err;

}

if (PyDict_Check(__builtins__)) {

__import__=PyObject_GetItem(__builtins__,__import___str);

if (!__import__) goto err;

}

else {

__import__=PyObject_GetAttr(__builtins__,__import___str);

if (!__import__) goto err;

}

r=PyObject_CallFunction(__import__,"OOOO",

module_name, globals, globals, silly_list);

if (!r)

goto err;

Py_DECREF(globals);

Py_DECREF(__builtins__);

Py_DECREF(__import__);

return r;

err:

Py_XDECREF(globals);

Py_XDECREF(__builtins__);

Py_XDECREF(__import__);

return NULL;

}

static PyObject *

whichmodule(PyObject *global, PyObject *global_name)

{

int i, j;

PyObject *module = 0, *modules_dict = 0,

*global_name_attr = 0, *name = 0;

module = PyObject_GetAttrString(global, "__module__");

if (module) return module;

PyErr_Clear();

if (!( modules_dict = PySys_GetObject("modules")))

return NULL;

i = 0;

while ((j = PyDict_Next(modules_dict, &i, &name, &module))) {

if (PyObject_Compare(name, __main___str)==0) continue;

global_name_attr = PyObject_GetAttr(module, global_name);

if (!global_name_attr) {

PyErr_Clear();

continue;

}

if (global_name_attr != global) {

Py_DECREF(global_name_attr);

continue;

}

Py_DECREF(global_name_attr);

break;

}

/* The following implements the rule in pickle.py added in 1.5

that used __main__ if no module is found. I don't actually

like this rule. jlf

*/

if (!j) {

j=1;

name=__main___str;

}

Py_INCREF(name);

return name;

}

static int

fast_save_enter(Picklerobject *self, PyObject *obj)

{

/* if fast_container < 0, we're doing an error exit. */

if (++self->fast_container >= PY_CPICKLE_FAST_LIMIT) {

PyObject *key = NULL;

if (self->fast_memo == NULL) {

self->fast_memo = PyDict_New();

if (self->fast_memo == NULL) {

self->fast_container = -1;

return 0;

}

}

key = PyLong_FromVoidPtr(obj);

if (key == NULL)

return 0;

if (PyDict_GetItem(self->fast_memo, key)) {

PyErr_Format(PyExc_ValueError,

"fast mode: can't pickle cyclic objects including object type %s at %p",

obj->ob_type->tp_name, obj);

self->fast_container = -1;

return 0;

}

if (PyDict_SetItem(self->fast_memo, key, Py_None) < 0) {

self->fast_container = -1;

return 0;

}

}

return 1;

}

int

fast_save_leave(Picklerobject *self, PyObject *obj)

{

if (self->fast_container-- >= PY_CPICKLE_FAST_LIMIT) {

PyObject *key = PyLong_FromVoidPtr(obj);

if (key == NULL)

return 0;

if (PyDict_DelItem(self->fast_memo, key) < 0) {

return 0;

}

}

return 1;

}

static int

save_none(Picklerobject *self, PyObject *args)

{

static char none = NONE;

if ((*self->write_func)(self, &none, 1) < 0)

return -1;

return 0;

}

static int

save_bool(Picklerobject *self, PyObject *args)

{

static char *buf[2] = {FALSE, TRUE};

static char len[2] = {sizeof(FALSE)-1, sizeof(TRUE)-1};

long l = PyInt_AS_LONG((PyIntObject *)args);

if ((*self->write_func)(self, buf[l], len[l]) < 0)

return -1;

return 0;

}

static int

save_int(Picklerobject *self, PyObject *args)

{

char c_str[32];

long l = PyInt_AS_LONG((PyIntObject *)args);

int len = 0;

if (!self->bin

#if SIZEOF_LONG > 4

|| l > 0x7fffffffL

|| l < -0x80000000L

#endif

) {

/* Text-mode pickle, or long too big to fit in the 4-byte

* signed BININT format: store as a string.

*/

c_str[0] = INT;

PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, "%ld\n", l);

if ((*self->write_func)(self, c_str, strlen(c_str)) < 0)

return -1;

}

else {

/* Binary pickle and l fits in a signed 4-byte int. */

c_str[1] = (int)( l & 0xff);

c_str[2] = (int)((l >> 8) & 0xff);

c_str[3] = (int)((l >> 16) & 0xff);

c_str[4] = (int)((l >> 24) & 0xff);

if ((c_str[4] == 0) && (c_str[3] == 0)) {

if (c_str[2] == 0) {

c_str[0] = BININT1;

len = 2;

}

else {

c_str[0] = BININT2;

len = 3;

}

}

else {

c_str[0] = BININT;

len = 5;

}

if ((*self->write_func)(self, c_str, len) < 0)

return -1;

}

return 0;

}

static int

save_long(Picklerobject *self, PyObject *args)