Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5b22213

Browse files
loewisloewis
committed
Make identifiers str (not str8) objects throughout.
This affects the parser, various object implementations, and all places that put identifiers into C string literals. In testing, a number of crashes occurred as code would fail when the recursion limit was reached (such as the Unicode interning dictionary having key/value pairs where key is not value). To solve these, I added an overflowed flag, which allows for 50 more recursions after the limit was reached and the exception was raised, and a recursion_critical flag, which indicates that recursion absolutely must be allowed, i.e. that a certain call must not cause a stack overflow exception. There are still some places where both str and str8 are accepted as identifiers; these should eventually be removed.
1 parent 38e43c2 commit 5b22213

40 files changed

Lines changed: 462 additions & 289 deletions

‎Include/ceval.h‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,10 @@ PyAPI_FUNC(int) Py_GetRecursionLimit(void);
5050
(_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) && \
5151
_Py_CheckRecursiveCall(where))
5252
#define Py_LeaveRecursiveCall() \
53-
(--PyThreadState_GET()->recursion_depth)
53+
do{ if((--PyThreadState_GET()->recursion_depth) < \
54+
_Py_CheckRecursionLimit - 50); \
55+
PyThreadState_GET()->overflowed = 0; \
56+
} while(0)
5457
PyAPI_FUNC(int) _Py_CheckRecursiveCall(char *where);
5558
PyAPI_DATA(int) _Py_CheckRecursionLimit;
5659
#ifdef USE_STACKCHECK
@@ -59,6 +62,14 @@ PyAPI_DATA(int) _Py_CheckRecursionLimit;
5962
# define _Py_MakeRecCheck(x) (++(x) > _Py_CheckRecursionLimit)
6063
#endif
6164

65+
#define Py_ALLOW_RECURSION \
66+
do { unsigned char _old = PyThreadState_GET()->recursion_critical;\
67+
PyThreadState_GET()->recursion_critical = 1;
68+
69+
#define Py_END_ALLOW_RECURSION \
70+
PyThreadState_GET()->recursion_critical = _old; \
71+
} while(0);
72+
6273
PyAPI_FUNC(const char *) PyEval_GetFuncName(PyObject *);
6374
PyAPI_FUNC(const char *) PyEval_GetFuncDesc(PyObject *);
6475

‎Include/pystate.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,10 @@ typedef struct _ts {
6161

6262
struct _frame *frame;
6363
int recursion_depth;
64+
char overflowed; /* The stack has overflowed. Allow 50 more calls
65+
to handle the runtime error. */
66+
char recursion_critical; /* The current calls must not cause
67+
a stack overflow. */
6468
/* 'tracing' keeps track of the execution depth when tracing/profiling.
6569
This is to prevent the actual trace/profile code from being recorded in
6670
the trace/profile. */

‎Include/stringobject.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,8 @@ PyAPI_FUNC(void) _Py_ReleaseInternedStrings(void);
8484
#define PyString_CHECK_INTERNED(op) (((PyStringObject *)(op))->ob_sstate)
8585

8686
/* Macro, trading safety for speed */
87-
#define PyString_AS_STRING(op) (((PyStringObject *)(op))->ob_sval)
88-
#define PyString_GET_SIZE(op) (((PyStringObject *)(op))->ob_size)
87+
#define PyString_AS_STRING(op) (assert(PyString_Check(op)),(((PyStringObject *)(op))->ob_sval))
88+
#define PyString_GET_SIZE(op) (assert(PyString_Check(op)),(((PyStringObject *)(op))->ob_size))
8989

9090
/* _PyString_Join(sep, x) is like sep.join(x). sep must be PyStringObject*,
9191
x must be an iterable object. */

‎Include/unicodeobject.h‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -410,13 +410,13 @@ PyAPI_DATA(PyTypeObject) PyUnicode_Type;
410410

411411
/* Fast access macros */
412412
#define PyUnicode_GET_SIZE(op) \
413-
(((PyUnicodeObject *)(op))->length)
413+
(assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->length))
414414
#define PyUnicode_GET_DATA_SIZE(op) \
415-
(((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE))
415+
(assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->length * sizeof(Py_UNICODE)))
416416
#define PyUnicode_AS_UNICODE(op) \
417-
(((PyUnicodeObject *)(op))->str)
417+
(assert(PyUnicode_Check(op)),(((PyUnicodeObject *)(op))->str))
418418
#define PyUnicode_AS_DATA(op) \
419-
((const char *)((PyUnicodeObject *)(op))->str)
419+
(assert(PyUnicode_Check(op)),((const char *)((PyUnicodeObject *)(op))->str))
420420

421421
/* --- Constants ---------------------------------------------------------- */
422422

@@ -627,6 +627,13 @@ PyAPI_FUNC(PyObject*) PyUnicode_FromOrdinal(int ordinal);
627627
PyAPI_FUNC(PyObject *) _PyUnicode_AsDefaultEncodedString(
628628
PyObject *, const char *);
629629

630+
/* Return a char* holding the default encoded value of the
631+
Unicode object.
632+
*/
633+
634+
PyAPI_FUNC(char *) PyUnicode_AsString(PyObject*);
635+
636+
630637
/* Returns the currently active default encoding.
631638
632639
The default encoding is currently implemented as run-time settable
@@ -1193,6 +1200,11 @@ PyAPI_FUNC(int) PyUnicode_Compare(
11931200
PyObject *right /* Right string */
11941201
);
11951202

1203+
PyAPI_FUNC(int) PyUnicode_CompareWithASCIIString(
1204+
PyObject *left,
1205+
const char *right
1206+
);
1207+
11961208
/* Rich compare two strings and return one of the following:
11971209
11981210
- NULL in case an exception was raised
@@ -1310,6 +1322,22 @@ PyAPI_FUNC(int) _PyUnicode_IsAlpha(
13101322
Py_UNICODE ch /* Unicode character */
13111323
);
13121324

1325+
PyAPI_FUNC(size_t) Py_UNICODE_strlen(const Py_UNICODE *u);
1326+
1327+
PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strcpy(
1328+
Py_UNICODE *s1, const Py_UNICODE *s2);
1329+
1330+
PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strncpy(
1331+
Py_UNICODE *s1, const Py_UNICODE *s2, size_t n);
1332+
1333+
PyAPI_FUNC(int) Py_UNICODE_strcmp(
1334+
const Py_UNICODE *s1, const Py_UNICODE *s2);
1335+
1336+
PyAPI_FUNC(Py_UNICODE*) Py_UNICODE_strchr(
1337+
const Py_UNICODE *s, Py_UNICODE c
1338+
);
1339+
1340+
13131341
#ifdef __cplusplus
13141342
}
13151343
#endif

‎Lib/test/test_frozen.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
from test.test_support import TestFailed
1111
import sys, os
1212

13+
raise TestFailed, "test currently causes assertion in debug mode"
14+
1315
try:
1416
import __hello__
1517
except ImportError as x:

‎Lib/test/test_new.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ def f(a): pass
143143
firstlineno, lnotab)
144144

145145
# new.code used to be a way to mutate a tuple...
146-
class S(str8):
146+
class S(str):
147147
pass
148148
t = (S("ab"),)
149149
d = new.code(argcount, kwonlyargcount, nlocals, stacksize,

‎Lib/test/test_sys.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -321,12 +321,6 @@ def __hash__(self):
321321

322322
self.assertRaises(TypeError, sys.intern, S("abc"))
323323

324-
# It's still safe to pass these strings to routines that
325-
# call intern internally, e.g. PyObject_SetAttr().
326-
s = S("abc")
327-
setattr(s, s, s)
328-
self.assertEqual(getattr(s, s), s)
329-
330324
s = "never interned as unicode before"
331325
self.assert_(sys.intern(s) is s)
332326
s2 = s.swapcase().swapcase()
@@ -338,6 +332,12 @@ def __hash__(self):
338332

339333
self.assertRaises(TypeError, sys.intern, U("abc"))
340334

335+
# It's still safe to pass these strings to routines that
336+
# call intern internally, e.g. PyObject_SetAttr().
337+
s = U("abc")
338+
setattr(s, s, s)
339+
self.assertEqual(getattr(s, s), s)
340+
341341

342342
def test_main():
343343
test.test_support.run_unittest(SysModuleTest)

‎Modules/_codecsmodule.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ escape_encode(PyObject *self,
172172
&PyString_Type, &str, &errors))
173173
return NULL;
174174

175-
size = PyUnicode_GET_SIZE(str);
175+
size = PyString_GET_SIZE(str);
176176
newsize = 4*size;
177177
if (newsize > PY_SSIZE_T_MAX || newsize / 4 != size) {
178178
PyErr_SetString(PyExc_OverflowError,

‎Modules/_hotshot.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -810,7 +810,7 @@ get_fileno(ProfilerObject *self, PyCodeObject *fcode)
810810
PyObject *name = PyDict_GetItem(dict, obj);
811811
if (name == NULL) {
812812
if (pack_define_func(self, fileno, fcode->co_firstlineno,
813-
PyString_AS_STRING(fcode->co_name)) < 0) {
813+
PyUnicode_AsString(fcode->co_name)) < 0) {
814814
Py_DECREF(obj);
815815
return -1;
816816
}

‎Modules/cPickle.c‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1829,8 +1829,8 @@ save_global(Picklerobject *self, PyObject *args, PyObject *name)
18291829
(name_size = PyString_Size(global_name)) < 0)
18301830
goto finally;
18311831

1832-
module_str = PyString_AS_STRING((PyStringObject *)module);
1833-
name_str = PyString_AS_STRING((PyStringObject *)global_name);
1832+
module_str = PyUnicode_AsString(module);
1833+
name_str = PyUnicode_AsString(global_name);
18341834

18351835
/* XXX This can be doing a relative import. Clearly it shouldn't,
18361836
but I don't know how to stop it. :-( */
@@ -1842,7 +1842,7 @@ save_global(Picklerobject *self, PyObject *args, PyObject *name)
18421842
"OS", args, module);
18431843
goto finally;
18441844
}
1845-
klass = PyObject_GetAttrString(mod, name_str);
1845+
klass = PyObject_GetAttr(mod, global_name);
18461846
if (klass == NULL) {
18471847
cPickle_ErrFormat(PicklingError,
18481848
"Can't pickle %s: attribute lookup %s.%s "
@@ -2223,7 +2223,7 @@ save(Picklerobject *self, PyObject *args, int pers_save)
22232223
res = save_string(self, args, 0);
22242224
goto finally;
22252225
}
2226-
if ((type == &PyUnicode_Type) && (PyString_GET_SIZE(args) < 2)) {
2226+
if ((type == &PyUnicode_Type) && (PyUnicode_GET_SIZE(args) < 2)) {
22272227
res = save_unicode(self, args, 0);
22282228
goto finally;
22292229
}
@@ -3584,7 +3584,7 @@ load_global(Unpicklerobject *self)
35843584
Py_DECREF(module_name);
35853585
return bad_readline();
35863586
}
3587-
if ((class_name = PyString_FromStringAndSize(s, len - 1))) {
3587+
if ((class_name = PyUnicode_FromStringAndSize(s, len - 1))) {
35883588
class = find_class(module_name, class_name,
35893589
self->find_class);
35903590
Py_DECREF(class_name);
@@ -5379,7 +5379,7 @@ init_stuff(PyObject *module_dict)
53795379
{
53805380
PyObject *copy_reg, *t, *r;
53815381

5382-
#define INIT_STR(S) if (!( S ## _str=PyString_InternFromString(#S))) return -1;
5382+
#define INIT_STR(S) if (!( S ## _str=PyUnicode_InternFromString(#S))) return -1;
53835383

53845384
if (PyType_Ready(&Unpicklertype) < 0)
53855385
return -1;

0 commit comments

Comments
 (0)