/* Write Python objects to files and read them back.

This is intended for writing and reading compiled Python code only;

a true persistent storage facility would be much harder, since

it would have to take circular links and sharing into account. */

#include "Python.h"

#include "longintrepr.h"

#include "compile.h"

#include "marshal.h"

/* High water mark to determine when the marshalled object is dangerously deep

* and risks coring the interpreter. When the object stack gets this deep,

* raise an exception instead of continuing.

*/

#define MAX_MARSHAL_STACK_DEPTH 5000

#define TYPE_NULL '0'

#define TYPE_NONE 'N'

#define TYPE_FALSE 'F'

#define TYPE_TRUE 'T'

#define TYPE_STOPITER 'S'

#define TYPE_ELLIPSIS '.'

#define TYPE_INT 'i'

#define TYPE_INT64 'I'

#define TYPE_FLOAT 'f'

#define TYPE_COMPLEX 'x'

#define TYPE_LONG 'l'

#define TYPE_STRING 's'

#define TYPE_TUPLE '('

#define TYPE_LIST '['

#define TYPE_DICT '{'

#define TYPE_CODE 'c'

#define TYPE_UNICODE 'u'

#define TYPE_UNKNOWN '?'

typedef struct {

FILE *fp;

int error;

int depth;

/* If fp == NULL, the following are valid: */

PyObject *str;

char *ptr;

char *end;

} WFILE;

#define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \

else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \

else w_more(c, p)

static void

w_more(int c, WFILE *p)

{

int size, newsize;

if (p->str == NULL)

return; /* An error already occurred */

size = PyString_Size(p->str);

newsize = size + 1024;

if (_PyString_Resize(&p->str, newsize) != 0) {

p->ptr = p->end = NULL;

}

else {

p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;

p->end =

PyString_AS_STRING((PyStringObject *)p->str) + newsize;

*p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);

}

}

static void

w_string(char *s, int n, WFILE *p)

{

if (p->fp != NULL) {

fwrite(s, 1, n, p->fp);

}

else {

while (--n >= 0) {

w_byte(*s, p);

s++;

}

}

}

static void

w_short(int x, WFILE *p)

{

w_byte((char)( x & 0xff), p);

w_byte((char)((x>> 8) & 0xff), p);

}

static void

w_long(long x, WFILE *p)

{

w_byte((char)( x & 0xff), p);

w_byte((char)((x>> 8) & 0xff), p);

w_byte((char)((x>>16) & 0xff), p);

w_byte((char)((x>>24) & 0xff), p);

}

#if SIZEOF_LONG > 4

static void

w_long64(long x, WFILE *p)

{

w_long(x, p);

w_long(x>>32, p);

}

#endif

static void

w_object(PyObject *v, WFILE *p)

{

int i, n;

p->depth++;

if (p->depth > MAX_MARSHAL_STACK_DEPTH) {

p->error = 2;

}

else if (v == NULL) {

w_byte(TYPE_NULL, p);

}

else if (v == Py_None) {

w_byte(TYPE_NONE, p);

}

else if (v == PyExc_StopIteration) {

w_byte(TYPE_STOPITER, p);

}

else if (v == Py_Ellipsis) {

w_byte(TYPE_ELLIPSIS, p);

}

else if (v == Py_False) {

w_byte(TYPE_FALSE, p);

}

else if (v == Py_True) {

w_byte(TYPE_TRUE, p);

}

else if (PyInt_Check(v)) {

long x = PyInt_AS_LONG((PyIntObject *)v);

#if SIZEOF_LONG > 4

long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);

if (y && y != -1) {

w_byte(TYPE_INT64, p);

w_long64(x, p);

}

else

#endif

{

w_byte(TYPE_INT, p);

w_long(x, p);

}

}

else if (PyLong_Check(v)) {

PyLongObject *ob = (PyLongObject *)v;

w_byte(TYPE_LONG, p);

n = ob->ob_size;

w_long((long)n, p);

if (n < 0)

n = -n;

for (i = 0; i < n; i++)

w_short(ob->ob_digit[i], p);

}

else if (PyFloat_Check(v)) {

char buf[256]; /* Plenty to format any double */

PyFloat_AsReprString(buf, (PyFloatObject *)v);

n = strlen(buf);

w_byte(TYPE_FLOAT, p);

w_byte(n, p);

w_string(buf, n, p);

}

#ifndef WITHOUT_COMPLEX

else if (PyComplex_Check(v)) {

char buf[256]; /* Plenty to format any double */

PyFloatObject *temp;

w_byte(TYPE_COMPLEX, p);

temp = (PyFloatObject*)PyFloat_FromDouble(

PyComplex_RealAsDouble(v));

PyFloat_AsReprString(buf, temp);

Py_DECREF(temp);

n = strlen(buf);

w_byte(n, p);

w_string(buf, n, p);

temp = (PyFloatObject*)PyFloat_FromDouble(

PyComplex_ImagAsDouble(v));

PyFloat_AsReprString(buf, temp);

Py_DECREF(temp);

n = strlen(buf);

w_byte(n, p);

w_string(buf, n, p);

}

#endif

else if (PyString_Check(v)) {

w_byte(TYPE_STRING, p);

n = PyString_GET_SIZE(v);

w_long((long)n, p);

w_string(PyString_AS_STRING(v), n, p);

}

#ifdef Py_USING_UNICODE

else if (PyUnicode_Check(v)) {

PyObject *utf8;

utf8 = PyUnicode_AsUTF8String(v);

if (utf8 == NULL) {

p->depth--;

p->error = 1;

return;

}

w_byte(TYPE_UNICODE, p);

n = PyString_GET_SIZE(utf8);

w_long((long)n, p);

w_string(PyString_AS_STRING(utf8), n, p);

Py_DECREF(utf8);

}

#endif

else if (PyTuple_Check(v)) {

w_byte(TYPE_TUPLE, p);

n = PyTuple_Size(v);

w_long((long)n, p);

for (i = 0; i < n; i++) {

w_object(PyTuple_GET_ITEM(v, i), p);

}

}

else if (PyList_Check(v)) {

w_byte(TYPE_LIST, p);

n = PyList_GET_SIZE(v);

w_long((long)n, p);

for (i = 0; i < n; i++) {

w_object(PyList_GET_ITEM(v, i), p);

}

}

else if (PyDict_Check(v)) {

int pos;

PyObject *key, *value;

w_byte(TYPE_DICT, p);

/* This one is NULL object terminated! */

pos = 0;

while (PyDict_Next(v, &pos, &key, &value)) {

w_object(key, p);

w_object(value, p);

}

w_object((PyObject *)NULL, p);

}

else if (PyCode_Check(v)) {

PyCodeObject *co = (PyCodeObject *)v;

w_byte(TYPE_CODE, p);

w_long(co->co_argcount, p);

w_long(co->co_nlocals, p);

w_long(co->co_stacksize, p);

w_long(co->co_flags, p);

w_object(co->co_code, p);

w_object(co->co_consts, p);

w_object(co->co_names, p);

w_object(co->co_varnames, p);

w_object(co->co_freevars, p);

w_object(co->co_cellvars, p);

w_object(co->co_filename, p);

w_object(co->co_name, p);

w_long(co->co_firstlineno, p);

w_object(co->co_lnotab, p);

}

else if (PyObject_CheckReadBuffer(v)) {

/* Write unknown buffer-style objects as a string */

char *s;

PyBufferProcs *pb = v->ob_type->tp_as_buffer;

w_byte(TYPE_STRING, p);

n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);

w_long((long)n, p);

w_string(s, n, p);

}

else {

w_byte(TYPE_UNKNOWN, p);

p->error = 1;

}

p->depth--;

}

void

PyMarshal_WriteLongToFile(long x, FILE *fp)

{

WFILE wf;

wf.fp = fp;

wf.error = 0;

wf.depth = 0;

w_long(x, &wf);

}

void

PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp)

{

WFILE wf;

wf.fp = fp;

wf.error = 0;

wf.depth = 0;

w_object(x, &wf);

}

typedef WFILE RFILE; /* Same struct with different invariants */

#define rs_byte(p) (((p)->ptr != (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)

#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))

static int

r_string(char *s, int n, RFILE *p)

{

if (p->fp != NULL)

return fread(s, 1, n, p->fp);

if (p->end - p->ptr < n)

n = p->end - p->ptr;

memcpy(s, p->ptr, n);

p->ptr += n;

return n;

}

static int

r_short(RFILE *p)

{

register short x;

x = r_byte(p);

x |= r_byte(p) << 8;

/* Sign-extension, in case short greater than 16 bits */

x |= -(x & 0x8000);

return x;

}

static long

r_long(RFILE *p)

{

register long x;

register FILE *fp = p->fp;

if (fp) {

x = getc(fp);

x |= (long)getc(fp) << 8;

x |= (long)getc(fp) << 16;

x |= (long)getc(fp) << 24;

}

else {

x = rs_byte(p);

x |= (long)rs_byte(p) << 8;

x |= (long)rs_byte(p) << 16;

x |= (long)rs_byte(p) << 24;

}

#if SIZEOF_LONG > 4

/* Sign extension for 64-bit machines */

x |= -(x & 0x80000000L);

#endif

return x;

}

/* r_long64 deals with the TYPE_INT64 code. On a machine with

sizeof(long) > 4, it returns a Python int object, else a Python long

object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,

so there's no inefficiency here in returning a PyLong on 32-bit boxes

for everything written via TYPE_INT64 (i.e., if an int is written via

TYPE_INT64, it *needs* more than 32 bits).

*/

static PyObject *

r_long64(RFILE *p)

{

long lo4 = r_long(p);

long hi4 = r_long(p);

#if SIZEOF_LONG > 4

long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);

return PyInt_FromLong(x);

#else

unsigned char buf[8];

int one = 1;

int is_little_endian = (int)*(char*)&one;

if (is_little_endian) {

memcpy(buf, &lo4, 4);

memcpy(buf+4, &hi4, 4);

}

else {

memcpy(buf, &hi4, 4);

memcpy(buf+4, &lo4, 4);

}

return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);

#endif

}

static PyObject *

r_object(RFILE *p)

{

PyObject *v, *v2;

long i, n;

int type = r_byte(p);

switch (type) {

case EOF:

PyErr_SetString(PyExc_EOFError,

"EOF read where object expected");

return NULL;

case TYPE_NULL:

return NULL;

case TYPE_NONE:

Py_INCREF(Py_None);

return Py_None;

case TYPE_STOPITER:

Py_INCREF(PyExc_StopIteration);

return PyExc_StopIteration;

case TYPE_ELLIPSIS:

Py_INCREF(Py_Ellipsis);

return Py_Ellipsis;

case TYPE_FALSE:

Py_INCREF(Py_False);

return Py_False;

case TYPE_TRUE:

Py_INCREF(Py_True);

return Py_True;

case TYPE_INT:

return PyInt_FromLong(r_long(p));

case TYPE_INT64:

return r_long64(p);

case TYPE_LONG:

{

int size;

PyLongObject *ob;

n = r_long(p);

size = n<0 ? -n : n;

ob = _PyLong_New(size);

if (ob == NULL)

return NULL;

ob->ob_size = n;

for (i = 0; i < size; i++)

ob->ob_digit[i] = r_short(p);

return (PyObject *)ob;

}

case TYPE_FLOAT:

{

char buf[256];

double dx;

n = r_byte(p);

if (r_string(buf, (int)n, p) != n) {

PyErr_SetString(PyExc_EOFError,

"EOF read where object expected");

return NULL;

}

buf[n] = '\0';

PyFPE_START_PROTECT("atof", return 0)

dx = atof(buf);

PyFPE_END_PROTECT(dx)

return PyFloat_FromDouble(dx);

}

#ifndef WITHOUT_COMPLEX

case TYPE_COMPLEX:

{

char buf[256];

Py_complex c;

n = r_byte(p);

if (r_string(buf, (int)n, p) != n) {

PyErr_SetString(PyExc_EOFError,

"EOF read where object expected");

return NULL;

}

buf[n] = '\0';

PyFPE_START_PROTECT("atof", return 0)

c.real = atof(buf);

PyFPE_END_PROTECT(c)

n = r_byte(p);

if (r_string(buf, (int)n, p) != n) {

PyErr_SetString(PyExc_EOFError,

"EOF read where object expected");

return NULL;

}

buf[n] = '\0';

PyFPE_START_PROTECT("atof", return 0)

c.imag = atof(buf);

PyFPE_END_PROTECT(c)

return PyComplex_FromCComplex(c);

}

#endif

case TYPE_STRING:

n = r_long(p);

if (n < 0) {

PyErr_SetString(PyExc_ValueError, "bad marshal data");

return NULL;

}

v = PyString_FromStringAndSize((char *)NULL, n);

if (v != NULL) {

if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {

Py_DECREF(v);

v = NULL;

PyErr_SetString(PyExc_EOFError,

"EOF read where object expected");

}

}

return v;

#ifdef Py_USING_UNICODE

case TYPE_UNICODE:

{

char *buffer;

n = r_long(p);

if (n < 0) {

PyErr_SetString(PyExc_ValueError, "bad marshal data");

return NULL;

}

buffer = PyMem_NEW(char, n);

if (buffer == NULL)

return PyErr_NoMemory();

if (r_string(buffer, (int)n, p) != n) {

PyMem_DEL(buffer);

PyErr_SetString(PyExc_EOFError,

"EOF read where object expected");

return NULL;

}

v = PyUnicode_DecodeUTF8(buffer, n, NULL);

PyMem_DEL(buffer);

return v;

}

#endif

case TYPE_TUPLE:

n = r_long(p);

if (n < 0) {

PyErr_SetString(PyExc_ValueError, "bad marshal data");

return NULL;

}

v = PyTuple_New((int)n);

if (v == NULL)

return v;

for (i = 0; i < n; i++) {

v2 = r_object(p);

if ( v2 == NULL ) {

Py_DECREF(v);

v = NULL;

break;

}

PyTuple_SET_ITEM(v, (int)i, v2);

}

return v;

case TYPE_LIST:

n = r_long(p);

if (n < 0) {

PyErr_SetString(PyExc_ValueError, "bad marshal data");

return NULL;

}

v = PyList_New((int)n);

if (v == NULL)

return v;

for (i = 0; i < n; i++) {

v2 = r_object(p);

if ( v2 == NULL ) {

Py_DECREF(v);

v = NULL;

break;

}

PyList_SetItem(v, (int)i, v2);

}

return v;

case TYPE_DICT:

v = PyDict_New();

if (v == NULL)

return NULL;

for (;;) {

PyObject *key, *val;

key = r_object(p);

if (key == NULL)

break; /* XXX Assume TYPE_NULL, not an error */

val = r_object(p);

if (val != NULL)

PyDict_SetItem(v, key, val);

Py_DECREF(key);

Py_XDECREF(val);

}

return v;

case TYPE_CODE:

if (PyEval_GetRestricted()) {

PyErr_SetString(PyExc_RuntimeError,

"cannot unmarshal code objects in "

"restricted execution mode");

return NULL;

}

else {

int argcount = r_long(p);

int nlocals = r_long(p);

int stacksize = r_long(p);

int flags = r_long(p);

PyObject *code = NULL;

PyObject *consts = NULL;

PyObject *names = NULL;

PyObject *varnames = NULL;

PyObject *freevars = NULL;

PyObject *cellvars = NULL;

PyObject *filename = NULL;

PyObject *name = NULL;

int firstlineno = 0;

PyObject *lnotab = NULL;

code = r_object(p);

if (code) consts = r_object(p);

if (consts) names = r_object(p);

if (names) varnames = r_object(p);

if (varnames) freevars = r_object(p);

if (freevars) cellvars = r_object(p);

if (cellvars) filename = r_object(p);

if (filename) name = r_object(p);

if (name) {

firstlineno = r_long(p);

lnotab = r_object(p);

}

if (!PyErr_Occurred()) {

v = (PyObject *) PyCode_New(

argcount, nlocals, stacksize, flags,

code, consts, names, varnames,

freevars, cellvars, filename, name,

firstlineno, lnotab);

}

else

v = NULL;

Py_XDECREF(code);

Py_XDECREF(consts);

Py_XDECREF(names);

Py_XDECREF(varnames);

Py_XDECREF(freevars);

Py_XDECREF(cellvars);

Py_XDECREF(filename);

Py_XDECREF(name);

Py_XDECREF(lnotab);

}

return v;

default:

/* Bogus data got written, which isn't ideal.

This will let you keep working and recover. */

PyErr_SetString(PyExc_ValueError, "bad marshal data");

return NULL;

}

}

int

PyMarshal_ReadShortFromFile(FILE *fp)

{

RFILE rf;

rf.fp = fp;

return r_short(&rf);

}

long

PyMarshal_ReadLongFromFile(FILE *fp)

{

RFILE rf;

rf.fp = fp;

return r_long(&rf);

}

#ifdef HAVE_FSTAT

/* Return size of file in bytes; < 0 if unknown. */

static off_t

getfilesize(FILE *fp)

{

struct stat st;

if (fstat(fileno(fp), &st) != 0)

return -1;

else

return st.st_size;

}

#endif

/* If we can get the size of the file up-front, and it's reasonably small,

* read it in one gulp and delegate to ...FromString() instead. Much quicker

* than reading a byte at a time from file; speeds .pyc imports.

* CAUTION: since this may read the entire remainder of the file, don't

* call it unless you know you're done with the file.

*/

PyObject *

PyMarshal_ReadLastObjectFromFile(FILE *fp)

{

/* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.

* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.

*/

#define SMALL_FILE_LIMIT (1L << 14)

#define REASONABLE_FILE_LIMIT (1L << 18)

#ifdef HAVE_FSTAT

off_t filesize;

#endif

if (PyErr_Occurred()) {

fprintf(stderr, "XXX rd_object called with exception set\n");

return NULL;

}

#ifdef HAVE_FSTAT

filesize = getfilesize(fp);

if (filesize > 0) {

char buf[SMALL_FILE_LIMIT];

char* pBuf = NULL;

if (filesize <= SMALL_FILE_LIMIT)

pBuf = buf;

else if (filesize <= REASONABLE_FILE_LIMIT)

pBuf = (char *)PyMem_MALLOC(filesize);

if (pBuf != NULL) {

PyObject* v;

size_t n = fread(pBuf, 1, filesize, fp);

v = PyMarshal_ReadObjectFromString(pBuf, n);

if (pBuf != buf)

PyMem_FREE(pBuf);

return v;

}

}

#endif

/* We don't have fstat, or we do but the file is larger than

* REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.

*/

return PyMarshal_ReadObjectFromFile(fp);

#undef SMALL_FILE_LIMIT

#undef REASONABLE_FILE_LIMIT

}

PyObject *

PyMarshal_ReadObjectFromFile(FILE *fp)

{

RFILE rf;

if (PyErr_Occurred()) {

fprintf(stderr, "XXX rd_object called with exception set\n");

return NULL;

}

rf.fp = fp;

return r_object(&rf);

}

PyObject *

PyMarshal_ReadObjectFromString(char *str, int len)

{

RFILE rf;

if (PyErr_Occurred()) {

fprintf(stderr, "XXX rds_object called with exception set\n");

return NULL;

}

rf.fp = NULL;

rf.str = NULL;

rf.ptr = str;

rf.end = str + len;

return r_object(&rf);

}

PyObject *

PyMarshal_WriteObjectToString(PyObject *x) /* wrs_object() */

{

WFILE wf;

wf.fp = NULL;

wf.str = PyString_FromStringAndSize((char *)NULL, 50);

if (wf.str == NULL)

return NULL;

wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);

wf.end = wf.ptr + PyString_Size(wf.str);

wf.error = 0;

wf.depth = 0;

w_object(x, &wf);

if (wf.str != NULL)

_PyString_Resize(&wf.str,

(int) (wf.ptr -

PyString_AS_STRING((PyStringObject *)wf.str)));

if (wf.error) {

Py_XDECREF(wf.str);

PyErr_SetString(PyExc_ValueError,

(wf.error==1)?"unmarshallable object"

:"object too deeply nested to marshal");

return NULL;

}

return wf.str;

}

/* And an interface for Python programs... */

static PyObject *

marshal_dump(PyObject *self, PyObject *args)

{

WFILE wf;

PyObject *x;

PyObject *f;

if (!PyArg_ParseTuple(args, "OO:dump", &x, &f))

return NULL;

if (!PyFile_Check(f)) {

PyErr_SetString(PyExc_TypeError,

"marshal.dump() 2nd arg must be file");

return NULL;

}

wf.fp = PyFile_AsFile(f);

wf.str = NULL;

wf.ptr = wf.end = NULL;

wf.error = 0;

wf.depth = 0;

w_object(x, &wf);

if (wf.error) {

PyErr_SetString(PyExc_ValueError,

(wf.error==1)?"unmarshallable object"

:"object too deeply nested to marshal");

return NULL;

}

Py_INCREF(Py_None);

return Py_None;

}

static PyObject *

marshal_load(PyObject *self, PyObject *args)

{

RFILE rf;

PyObject *f;

PyObject *v;

if (!PyArg_ParseTuple(args, "O:load", &f))

return NULL;

if (!PyFile_Check(f)) {

PyErr_SetString(PyExc_TypeError,

"marshal.load() arg must be file");

return NULL;

}

rf.fp = PyFile_AsFile(f);

rf.str = NULL;

rf.ptr = rf.end = NULL;

PyErr_Clear();

v = r_object(&rf);

if (PyErr_Occurred()) {

Py_XDECREF(v);

v = NULL;

}

return v;

}

static PyObject *

marshal_dumps(PyObject *self, PyObject *args)

{

PyObject *x;

if (!PyArg_ParseTuple(args, "O:dumps", &x))

return NULL;

return PyMarshal_WriteObjectToString(x);

}

static PyObject *

marshal_loads(PyObject *self, PyObject *args)

{

RFILE rf;

PyObject *v;

char *s;

int n;

if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))

return NULL;

rf.fp = NULL;

rf.str = args;

rf.ptr = s;

rf.end = s + n;

PyErr_Clear();

v = r_object(&rf);

if (PyErr_Occurred()) {

Py_XDECREF(v);

v = NULL;

}

return v;

}

static PyMethodDef marshal_methods[] = {

{"dump", marshal_dump, METH_VARARGS},

{"load", marshal_load, METH_VARARGS},

{"dumps", marshal_dumps, METH_VARARGS},

{"loads", marshal_loads, METH_VARARGS},

{NULL, NULL} /* sentinel */

};

PyMODINIT_FUNC

PyMarshal_Init(void)

{

(void) Py_InitModule("marshal", marshal_methods);

}