Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 4d6ec85

Browse files
tirantiran
committed
Merged revisions 61952-61953 via svnmerge from
svn+ssh://[email protected]/python/trunk ........ r61952 | mark.dickinson | 2008-03-26 22:41:36 +0100 (Wed, 26 Mar 2008) | 2 lines Typo: "objects reference count" -> "object's reference count" ........ r61953 | christian.heimes | 2008-03-26 23:01:37 +0100 (Wed, 26 Mar 2008) | 4 lines Patch #2477: Added from __future__ import unicode_literals The new PyParser_*Ex() functions are based on Neal's suggestion and initial patch. The new __future__ feature makes all '' and r'' unicode strings. b'' and br'' stay (byte) strings. ........
1 parent 9edef04 commit 4d6ec85

11 files changed

Lines changed: 92 additions & 30 deletions

File tree

‎Doc/c-api/structures.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ definition of all other Python objects.
2020

2121
All object types are extensions of this type. This is a type which contains the
2222
information Python needs to treat a pointer to an object as an object. In a
23-
normal "release" build, it contains only the objects reference count and a
23+
normal "release" build, it contains only the object's reference count and a
2424
pointer to the corresponding type object. It corresponds to the fields defined
2525
by the expansion of the ``PyObject_HEAD`` macro.
2626

‎Include/code.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ typedef struct {
4949
#define CO_FUTURE_ABSOLUTE_IMPORT 0x4000 /* do absolute imports by default */
5050
#define CO_FUTURE_WITH_STATEMENT 0x8000
5151
#define CO_FUTURE_PRINT_FUNCTION 0x10000
52+
#define CO_FUTURE_UNICODE_LITERALS 0x20000
5253
#endif
5354

5455
/* This should be defined if a future statement modifies the syntax.

‎Include/compile.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ typedef struct {
2525
#define FUTURE_ABSOLUTE_IMPORT "absolute_import"
2626
#define FUTURE_WITH_STATEMENT "with_statement"
2727
#define FUTURE_PRINT_FUNCTION "print_function"
28+
#define FUTURE_UNICODE_LITERALS "unicode_literals"
2829

2930
struct _mod; /* Declare the existence of this type */
3031
PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,

‎Include/parsetok.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ typedef struct {
2525

2626
#if 0
2727
#define PyPARSE_WITH_IS_KEYWORD 0x0003
28+
#define PyPARSE_PRINT_IS_FUNCTION 0x0004
29+
#define PyPARSE_UNICODE_LITERALS 0x0008
2830
#endif
2931

3032
PyAPI_FUNC(node *) PyParser_ParseString(const char *, grammar *, int,
@@ -38,11 +40,19 @@ PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
3840
const char*, grammar *,
3941
int, char *, char *,
4042
perrdetail *, int);
43+
PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(FILE *, const char *,
44+
const char*, grammar *,
45+
int, char *, char *,
46+
perrdetail *, int *);
4147

4248
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
4349
const char *,
4450
grammar *, int,
4551
perrdetail *, int);
52+
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(const char *,
53+
const char *,
54+
grammar *, int,
55+
perrdetail *, int *);
4656

4757
/* Note that he following function is defined in pythonrun.c not parsetok.c. */
4858
PyAPI_FUNC(void) PyParser_SetError(perrdetail *);

‎Lib/__future__.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
"absolute_import",
5555
"with_statement",
5656
"print_function",
57+
"unicode_literals",
5758
]
5859

5960
__all__ = ["all_feature_names"] + all_feature_names
@@ -68,6 +69,7 @@
6869
CO_FUTURE_ABSOLUTE_IMPORT = 0x4000 # perform absolute imports by default
6970
CO_FUTURE_WITH_STATEMENT = 0x8000 # with statement
7071
CO_FUTURE_PRINT_FUNCTION = 0x10000 # print function
72+
CO_FUTURE_UNICODE_LITERALS = 0x20000 # unicode string literals
7173

7274
class _Feature:
7375
def __init__(self, optionalRelease, mandatoryRelease, compiler_flag):
@@ -120,3 +122,7 @@ def __repr__(self):
120122
print_function = _Feature((2, 6, 0, "alpha", 2),
121123
(3, 0, 0, "alpha", 0),
122124
CO_FUTURE_PRINT_FUNCTION)
125+
126+
unicode_literals = _Feature((2, 6, 0, "alpha", 2),
127+
(3, 0, 0, "alpha", 0),
128+
CO_FUTURE_UNICODE_LITERALS)

‎Parser/parser.c‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -206,10 +206,18 @@ future_hack(parser_state *ps)
206206

207207
for (i = 0; i < NCH(ch); i += 2) {
208208
cch = CHILD(ch, i);
209-
if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME &&
210-
strcmp(STR(CHILD(cch, 0)), "with_statement") == 0) {
211-
ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
212-
break;
209+
if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) {
210+
char *str_ch = STR(CHILD(cch, 0));
211+
if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) {
212+
ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
213+
break;
214+
} else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) {
215+
ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
216+
break;
217+
} else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) {
218+
ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
219+
break;
220+
}
213221
}
214222
}
215223
}

‎Parser/parsetok.c‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ int Py_TabcheckFlag;
1414

1515

1616
/* Forward */
17-
static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
17+
static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
1818
static void initerr(perrdetail *err_ret, const char* filename);
1919

2020
/* Parse input coming from a string. Return error code, print some errors. */
@@ -36,6 +36,16 @@ node *
3636
PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
3737
grammar *g, int start,
3838
perrdetail *err_ret, int flags)
39+
{
40+
int iflags = flags;
41+
return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
42+
err_ret, &iflags);
43+
}
44+
45+
node *
46+
PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
47+
grammar *g, int start,
48+
perrdetail *err_ret, int *flags)
3949
{
4050
struct tok_state *tok;
4151

@@ -64,9 +74,19 @@ PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
6474
}
6575

6676
node *
67-
PyParser_ParseFileFlags(FILE *fp, const char *filename, const char* enc,
77+
PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc,
6878
grammar *g, int start,
6979
char *ps1, char *ps2, perrdetail *err_ret, int flags)
80+
{
81+
int iflags = flags;
82+
return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1,
83+
ps2, err_ret, &iflags);
84+
}
85+
86+
node *
87+
PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
88+
const char *enc, grammar *g, int start,
89+
char *ps1, char *ps2, perrdetail *err_ret, int *flags)
7090
{
7191
struct tok_state *tok;
7292

@@ -104,7 +124,7 @@ warn(const char *msg, const char *filename, int lineno)
104124

105125
static node *
106126
parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
107-
int flags)
127+
int *flags)
108128
{
109129
parser_state *ps;
110130
node *n;
@@ -117,7 +137,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
117137
return NULL;
118138
}
119139
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
120-
if (flags & PyPARSE_WITH_IS_KEYWORD)
140+
if (*flags & PyPARSE_WITH_IS_KEYWORD)
121141
ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
122142
#endif
123143

@@ -141,7 +161,7 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
141161
except if a certain flag is given --
142162
codeop.py uses this. */
143163
if (tok->indent &&
144-
!(flags & PyPARSE_DONT_IMPLY_DEDENT))
164+
!(*flags & PyPARSE_DONT_IMPLY_DEDENT))
145165
{
146166
tok->pendin = -tok->indent;
147167
tok->indent = 0;
@@ -205,7 +225,9 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
205225
}
206226
else
207227
n = NULL;
208-
228+
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
229+
*flags = ps->p_flags;
230+
#endif
209231
PyParser_Delete(ps);
210232

211233
if (n == NULL) {

‎Python/ast.c‎

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ static stmt_ty ast_for_classdef(struct compiling *, const node *, asdl_seq *);
3535
static expr_ty ast_for_call(struct compiling *, const node *, expr_ty);
3636

3737
static PyObject *parsenumber(const char *);
38-
static PyObject *parsestr(const node *n, const char *encoding, int *bytesmode);
38+
static PyObject *parsestr(struct compiling *, const node *n, int *bytesmode);
3939
static PyObject *parsestrplus(struct compiling *, const node *n,
4040
int *bytesmode);
4141

@@ -3191,14 +3191,13 @@ decode_unicode(const char *s, size_t len, int rawmode, const char *encoding)
31913191
* parsestr parses it, and returns the decoded Python string object.
31923192
*/
31933193
static PyObject *
3194-
parsestr(const node *n, const char *encoding, int *bytesmode)
3194+
parsestr(struct compiling *c, const node *n, int *bytesmode)
31953195
{
31963196
size_t len;
31973197
const char *s = STR(n);
31983198
int quote = Py_CHARMASK(*s);
31993199
int rawmode = 0;
32003200
int need_encoding;
3201-
32023201
if (isalpha(quote)) {
32033202
if (quote == 'b' || quote == 'B') {
32043203
quote = *++s;
@@ -3233,7 +3232,7 @@ parsestr(const node *n, const char *encoding, int *bytesmode)
32333232
}
32343233
}
32353234
if (!*bytesmode && !rawmode) {
3236-
return decode_unicode(s, len, rawmode, encoding);
3235+
return decode_unicode(s, len, rawmode, c->c_encoding);
32373236
}
32383237
if (*bytesmode) {
32393238
/* Disallow non-ascii characters (but not escapes) */
@@ -3246,28 +3245,27 @@ parsestr(const node *n, const char *encoding, int *bytesmode)
32463245
}
32473246
}
32483247
}
3249-
need_encoding = (!*bytesmode && encoding != NULL &&
3250-
strcmp(encoding, "utf-8") != 0 &&
3251-
strcmp(encoding, "iso-8859-1") != 0);
3248+
need_encoding = (!*bytesmode && c->c_encoding != NULL &&
3249+
strcmp(c->c_encoding, "utf-8") != 0 &&
3250+
strcmp(c->c_encoding, "iso-8859-1") != 0);
32523251
if (rawmode || strchr(s, '\\') == NULL) {
32533252
if (need_encoding) {
32543253
PyObject *v, *u = PyUnicode_DecodeUTF8(s, len, NULL);
32553254
if (u == NULL || !*bytesmode)
32563255
return u;
3257-
v = PyUnicode_AsEncodedString(u, encoding, NULL);
3256+
v = PyUnicode_AsEncodedString(u, c->c_encoding, NULL);
32583257
Py_DECREF(u);
32593258
return v;
32603259
} else if (*bytesmode) {
32613260
return PyString_FromStringAndSize(s, len);
3262-
} else if (strcmp(encoding, "utf-8") == 0) {
3261+
} else if (strcmp(c->c_encoding, "utf-8") == 0) {
32633262
return PyUnicode_FromStringAndSize(s, len);
32643263
} else {
32653264
return PyUnicode_DecodeLatin1(s, len, NULL);
32663265
}
32673266
}
3268-
32693267
return PyString_DecodeEscape(s, len, NULL, 1,
3270-
need_encoding ? encoding : NULL);
3268+
need_encoding ? c->c_encoding : NULL);
32713269
}
32723270

32733271
/* Build a Python string object out of a STRING+ atom. This takes care of
@@ -3280,13 +3278,13 @@ parsestrplus(struct compiling *c, const node *n, int *bytesmode)
32803278
PyObject *v;
32813279
int i;
32823280
REQ(CHILD(n, 0), STRING);
3283-
v = parsestr(CHILD(n, 0), c->c_encoding, bytesmode);
3281+
v = parsestr(c, CHILD(n, 0), bytesmode);
32843282
if (v != NULL) {
32853283
/* String literal concatenation */
32863284
for (i = 1; i < NCH(n); i++) {
32873285
PyObject *s;
32883286
int subbm = 0;
3289-
s = parsestr(CHILD(n, i), c->c_encoding, &subbm);
3287+
s = parsestr(c, CHILD(n, i), &subbm);
32903288
if (s == NULL)
32913289
goto onError;
32923290
if (*bytesmode != subbm) {

‎Python/future.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ future_check_features(PyFutureFeatures *ff, stmt_ty s, const char *filename)
3535
continue;
3636
} else if (strcmp(feature, FUTURE_PRINT_FUNCTION) == 0) {
3737
continue;
38+
} else if (strcmp(feature, FUTURE_UNICODE_LITERALS) == 0) {
39+
continue;
3840
} else if (strcmp(feature, "braces") == 0) {
3941
PyErr_SetString(PyExc_SyntaxError,
4042
"not a chance");

‎Python/import.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -821,12 +821,13 @@ parse_source_module(const char *pathname, FILE *fp)
821821
{
822822
PyCodeObject *co = NULL;
823823
mod_ty mod;
824+
PyCompilerFlags flags;
824825
PyArena *arena = PyArena_New();
825826
if (arena == NULL)
826827
return NULL;
827828

828829
mod = PyParser_ASTFromFile(fp, pathname, NULL,
829-
Py_file_input, 0, 0, 0,
830+
Py_file_input, 0, 0, &flags,
830831
NULL, arena);
831832
if (mod) {
832833
co = PyAST_Compile(mod, pathname, NULL, arena);

0 commit comments

Comments
 (0)