It's unclear whether PyMarshal_XXX() are part of the public or private API.

tim-one · tim-one · commit d9b9ac855c6b · 2001-01-28T00:27:39.000Z
They're named as if public, so I did a Bad Thing by changing
PyMarshal_ReadObjectFromFile() to suck up the remainder of the file in one
gulp:  anyone who counted on that leaving the file pointer merely at the
end of the next object would be screwed.  So restored
PyMarshal_ReadObjectFromFile() to its earlier state, renamed the new greedy
code to PyMarshal_ReadLastObjectFromFile(), and changed Python internals to
call the latter instead.
diff --git a/Include/marshal.h b/Include/marshal.h
@@ -15,6 +15,7 @@ DL_IMPORT(PyObject *) PyMarshal_WriteObjectToString(PyObject *);
 DL_IMPORT(long) PyMarshal_ReadLongFromFile(FILE *);
 DL_IMPORT(int) PyMarshal_ReadShortFromFile(FILE *);
 DL_IMPORT(PyObject *) PyMarshal_ReadObjectFromFile(FILE *);
+DL_IMPORT(PyObject *) PyMarshal_ReadLastObjectFromFile(FILE *);
 DL_IMPORT(PyObject *) PyMarshal_ReadObjectFromString(char *, int);
 
 #ifdef __cplusplus
diff --git a/Python/import.c b/Python/import.c
@@ -555,7 +555,7 @@ read_compiled_module(char *cpathname, FILE *fp)
 {
 	PyObject *co;
 
-	co = PyMarshal_ReadObjectFromFile(fp);
+	co = PyMarshal_ReadLastObjectFromFile(fp);
 	/* Ugly: rd_object() may return NULL with or without error */
 	if (co == NULL || !PyCode_Check(co)) {
 		if (!PyErr_Occurred())
diff --git a/Python/marshal.c b/Python/marshal.c
@@ -110,10 +110,10 @@ w_object(PyObject *v, WFILE *p)
 	PyBufferProcs *pb;
 
 	p->depth++;
-	
+
 	if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
 		p->error = 2;
-	} 
+	}
 	else if (v == NULL) {
 		w_byte(TYPE_NULL, p);
 	}
@@ -363,31 +363,31 @@ r_object(RFILE *p)
 	PyObject *v, *v2;
 	long i, n;
 	int type = r_byte(p);
-	
+
 	switch (type) {
-	
+
 	case EOF:
 		PyErr_SetString(PyExc_EOFError,
 				"EOF read where object expected");
 		return NULL;
-	
+
 	case TYPE_NULL:
 		return NULL;
-	
+
 	case TYPE_NONE:
 		Py_INCREF(Py_None);
 		return Py_None;
-	
+
 	case TYPE_ELLIPSIS:
 		Py_INCREF(Py_Ellipsis);
 		return Py_Ellipsis;
-	
+
 	case TYPE_INT:
 		return PyInt_FromLong(r_long(p));
-	
+
 	case TYPE_INT64:
 		return PyInt_FromLong(r_long64(p));
-	
+
 	case TYPE_LONG:
 		{
 			int size;
@@ -402,7 +402,7 @@ r_object(RFILE *p)
 				ob->ob_digit[i] = r_short(p);
 			return (PyObject *)ob;
 		}
-	
+
 	case TYPE_FLOAT:
 		{
 			char buf[256];
@@ -419,7 +419,7 @@ r_object(RFILE *p)
 			PyFPE_END_PROTECT(dx)
 			return PyFloat_FromDouble(dx);
 		}
-	
+
 #ifndef WITHOUT_COMPLEX
 	case TYPE_COMPLEX:
 		{
@@ -448,7 +448,7 @@ r_object(RFILE *p)
 			return PyComplex_FromCComplex(c);
 		}
 #endif
-	
+
 	case TYPE_STRING:
 		n = r_long(p);
 		if (n < 0) {
@@ -465,7 +465,7 @@ r_object(RFILE *p)
 			}
 		}
 		return v;
-	
+
 	case TYPE_UNICODE:
 	    {
 		char *buffer;
@@ -488,7 +488,7 @@ r_object(RFILE *p)
 		PyMem_DEL(buffer);
 		return v;
 	    }
-	    
+
 	case TYPE_TUPLE:
 		n = r_long(p);
 		if (n < 0) {
@@ -508,7 +508,7 @@ r_object(RFILE *p)
 			PyTuple_SET_ITEM(v, (int)i, v2);
 		}
 		return v;
-	
+
 	case TYPE_LIST:
 		n = r_long(p);
 		if (n < 0) {
@@ -528,7 +528,7 @@ r_object(RFILE *p)
 			PyList_SetItem(v, (int)i, v2);
 		}
 		return v;
-	
+
 	case TYPE_DICT:
 		v = PyDict_New();
 		if (v == NULL)
@@ -545,7 +545,7 @@ r_object(RFILE *p)
 			Py_XDECREF(val);
 		}
 		return v;
-	
+
 	case TYPE_CODE:
 		{
 			int argcount = r_short(p);
@@ -562,7 +562,7 @@ r_object(RFILE *p)
 			PyObject *name = NULL;
 			int firstlineno = 0;
 			PyObject *lnotab = NULL;
-			
+
 			code = r_object(p);
 			if (code) consts = r_object(p);
 			if (consts) names = r_object(p);
@@ -575,13 +575,13 @@ r_object(RFILE *p)
 				firstlineno = r_short(p);
 				lnotab = r_object(p);
 			}
-			
+
 			if (!PyErr_Occurred()) {
 				v = (PyObject *) PyCode_New(
-					argcount, nlocals, stacksize, flags,  
+					argcount, nlocals, stacksize, flags,
 					code, consts, names, varnames,
-					freevars, cellvars, filename, name, 
-					firstlineno, lnotab);  
+					freevars, cellvars, filename, name,
+					firstlineno, lnotab);
 			}
 			else
 				v = NULL;
@@ -597,13 +597,13 @@ r_object(RFILE *p)
 
 		}
 		return v;
-	
+
 	default:
 		/* Bogus data got written, which isn't ideal.
 		   This will let you keep working and recover. */
 		PyErr_SetString(PyExc_ValueError, "bad marshal data");
 		return NULL;
-	
+
 	}
 }
 
@@ -627,20 +627,21 @@ getfilesize(FILE *fp)
 		return st.st_size;
 }
 #endif
- 
+
 /* If we can get the size of the file up-front, and it's reasonably small,
  * read it in one gulp and delegate to ...FromString() instead.  Much quicker
  * than reading a byte at a time from file; speeds .pyc imports.
+ * CAUTION:  since this may read the entire remainder of the file, don't
+ * call it unless you know you're done with the file.
  */
 PyObject *
-PyMarshal_ReadObjectFromFile(FILE *fp)
+PyMarshal_ReadLastObjectFromFile(FILE *fp)
 {
 /* 75% of 2.1's .pyc files can exploit SMALL_FILE_LIMIT.
  * REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc.
  */
 #define SMALL_FILE_LIMIT (1L << 14)
 #define REASONABLE_FILE_LIMIT (1L << 18)
-	RFILE rf;
 #ifdef HAVE_FSTAT
 	off_t filesize;
 #endif
@@ -665,15 +666,30 @@ PyMarshal_ReadObjectFromFile(FILE *fp)
 				PyMem_FREE(pBuf);
 			return v;
 		}
-		
+
 	}
 #endif
-	rf.fp = fp;
-	return r_object(&rf);
+	/* We don't have fstat, or we do but the file is larger than
+	 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
+	 */
+	return PyMarshal_ReadObjectFromFile(fp);
+
 #undef SMALL_FILE_LIMIT
 #undef REASONABLE_FILE_LIMIT
 }
 
+PyObject *
+PyMarshal_ReadObjectFromFile(FILE *fp)
+{
+	RFILE rf;
+	if (PyErr_Occurred()) {
+		fprintf(stderr, "XXX rd_object called with exception set\n");
+		return NULL;
+	}
+	rf.fp = fp;
+	return r_object(&rf);
+}
+
 PyObject *
 PyMarshal_ReadObjectFromString(char *str, int len)
 {
@@ -708,7 +724,7 @@ PyMarshal_WriteObjectToString(PyObject *x) /* wrs_object() */
 			   PyString_AS_STRING((PyStringObject *)wf.str)));
 	if (wf.error) {
 		Py_XDECREF(wf.str);
-		PyErr_SetString(PyExc_ValueError, 
+		PyErr_SetString(PyExc_ValueError,
 				(wf.error==1)?"unmarshallable object"
 				:"object too deeply nested to marshal");
 		return NULL;
@@ -738,7 +754,7 @@ marshal_dump(PyObject *self, PyObject *args)
 	wf.depth = 0;
 	w_object(x, &wf);
 	if (wf.error) {
-		PyErr_SetString(PyExc_ValueError, 
+		PyErr_SetString(PyExc_ValueError,
 				(wf.error==1)?"unmarshallable object"
 				:"object too deeply nested to marshal");
 		return NULL;
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
@@ -936,7 +936,7 @@ run_pyc_file(FILE *fp, char *filename, PyObject *globals, PyObject *locals)
 		return NULL;
 	}
 	(void) PyMarshal_ReadLongFromFile(fp);
-	v = PyMarshal_ReadObjectFromFile(fp);
+	v = PyMarshal_ReadLastObjectFromFile(fp);
 	fclose(fp);
 	if (v == NULL || !PyCode_Check(v)) {
 		Py_XDECREF(v);

Original file line number	Diff line number	Diff line change
`@@ -555,7 +555,7 @@ read_compiled_module(char cpathname, FILE fp)`
`555`	`555`	`{`
`556`	`556`	`PyObject *co;`
`557`	`557`
`558`		`- co = PyMarshal_ReadObjectFromFile(fp);`
	`558`	`+ co = PyMarshal_ReadLastObjectFromFile(fp);`
`559`	`559`	`/* Ugly: rd_object() may return NULL with or without error */`
`560`	`560`	`if (co == NULL \|\| !PyCode_Check(co)) {`
`561`	`561`	`if (!PyErr_Occurred())`
Original file line number	Diff line number	Diff line change
`@@ -936,7 +936,7 @@ run_pyc_file(FILE fp, char filename, PyObject globals, PyObject locals)`
`936`	`936`	`return NULL;`
`937`	`937`	`}`
`938`	`938`	`(void) PyMarshal_ReadLongFromFile(fp);`
`939`		`- v = PyMarshal_ReadObjectFromFile(fp);`
	`939`	`+ v = PyMarshal_ReadLastObjectFromFile(fp);`
`940`	`940`	`fclose(fp);`
`941`	`941`	`if (v == NULL \|\| !PyCode_Check(v)) {`
`942`	`942`	`Py_XDECREF(v);`