Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 85bcc66

Browse files
loewisloewis
committed
Convert code from sys.stdin.encoding to UTF-8 in
interactive mode. Fixes #1100.
1 parent 53de190 commit 85bcc66

7 files changed

Lines changed: 70 additions & 16 deletions

File tree

‎Include/parsetok.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ PyAPI_FUNC(node *) PyParser_ParseFile (FILE *, const char *, grammar *, int,
3434

3535
PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
3636
perrdetail *, int);
37-
PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, grammar *,
37+
PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
38+
const char*, grammar *,
3839
int, char *, char *,
3940
perrdetail *, int);
4041

‎Include/pythonrun.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags
4040
PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *,
4141
int, PyCompilerFlags *flags,
4242
PyArena *);
43-
PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int,
43+
PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *,
44+
const char*, int,
4445
char *, char *,
4546
PyCompilerFlags *, int *,
4647
PyArena *);

‎Parser/parsetok.c‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,19 +59,20 @@ node *
5959
PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
6060
char *ps1, char *ps2, perrdetail *err_ret)
6161
{
62-
return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
63-
err_ret, 0);
62+
return PyParser_ParseFileFlags(fp, filename, NULL,
63+
g, start, ps1, ps2, err_ret, 0);
6464
}
6565

6666
node *
67-
PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
67+
PyParser_ParseFileFlags(FILE *fp, const char *filename, const char* enc,
68+
grammar *g, int start,
6869
char *ps1, char *ps2, perrdetail *err_ret, int flags)
6970
{
7071
struct tok_state *tok;
7172

7273
initerr(err_ret, filename);
7374

74-
if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
75+
if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
7576
err_ret->error = E_NOMEM;
7677
return NULL;
7778
}

‎Parser/tokenizer.c‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,7 @@ PyTokenizer_FromString(const char *str)
677677
/* Set up tokenizer for file */
678678

679679
struct tok_state *
680-
PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
680+
PyTokenizer_FromFile(FILE *fp, char* enc, char *ps1, char *ps2)
681681
{
682682
struct tok_state *tok = tok_new();
683683
if (tok == NULL)
@@ -691,6 +691,17 @@ PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
691691
tok->fp = fp;
692692
tok->prompt = ps1;
693693
tok->nextprompt = ps2;
694+
if (enc != NULL) {
695+
/* Must copy encoding declaration since it
696+
gets copied into the parse tree. */
697+
tok->encoding = PyMem_MALLOC(strlen(enc)+1);
698+
if (!tok->encoding) {
699+
PyTokenizer_Free(tok);
700+
return NULL;
701+
}
702+
strcpy(tok->encoding, enc);
703+
tok->decoding_state = -1;
704+
}
694705
return tok;
695706
}
696707

@@ -742,6 +753,29 @@ tok_nextc(register struct tok_state *tok)
742753
}
743754
if (tok->prompt != NULL) {
744755
char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
756+
#ifndef PGEN
757+
if (tok->encoding && newtok && *newtok) {
758+
/* Recode to UTF-8 */
759+
Py_ssize_t buflen;
760+
const char* buf;
761+
PyObject *u = translate_into_utf8(newtok, tok->encoding);
762+
PyMem_FREE(newtok);
763+
if (!u) {
764+
tok->done = E_DECODE;
765+
return EOF;
766+
}
767+
buflen = PyBytes_Size(u);
768+
buf = PyBytes_AsString(u);
769+
if (!buf) {
770+
Py_DECREF(u);
771+
tok->done = E_DECODE;
772+
return EOF;
773+
}
774+
newtok = PyMem_MALLOC(buflen+1);
775+
strcpy(newtok, buf);
776+
Py_DECREF(u);
777+
}
778+
#endif
745779
if (tok->nextprompt != NULL)
746780
tok->prompt = tok->nextprompt;
747781
if (newtok == NULL)

‎Parser/tokenizer.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ struct tok_state {
5555
};
5656

5757
extern struct tok_state *PyTokenizer_FromString(const char *);
58-
extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
58+
extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
59+
char *, char *);
5960
extern void PyTokenizer_Free(struct tok_state *);
6061
extern int PyTokenizer_Get(struct tok_state *, char **, char **);
6162

‎Python/import.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -809,7 +809,8 @@ parse_source_module(const char *pathname, FILE *fp)
809809
if (arena == NULL)
810810
return NULL;
811811

812-
mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, 0,
812+
mod = PyParser_ASTFromFile(fp, pathname, NULL,
813+
Py_file_input, 0, 0, 0,
813814
NULL, arena);
814815
if (mod) {
815816
co = PyAST_Compile(mod, pathname, NULL, arena);

‎Python/pythonrun.c‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -744,12 +744,22 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag
744744
int
745745
PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
746746
{
747-
PyObject *m, *d, *v, *w;
747+
PyObject *m, *d, *v, *w, *oenc = NULL;
748748
mod_ty mod;
749749
PyArena *arena;
750-
char *ps1 = "", *ps2 = "";
750+
char *ps1 = "", *ps2 = "", *enc = NULL;
751751
int errcode = 0;
752752

753+
if (fp == stdin) {
754+
/* Fetch encoding from sys.stdin */
755+
v = PySys_GetObject("stdin");
756+
if (!v)
757+
return -1;
758+
oenc = PyObject_GetAttrString(v, "encoding");
759+
if (!oenc)
760+
return -1;
761+
enc = PyUnicode_AsString(oenc);
762+
}
753763
v = PySys_GetObject("ps1");
754764
if (v != NULL) {
755765
v = PyObject_Str(v);
@@ -770,13 +780,15 @@ PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags
770780
if (arena == NULL) {
771781
Py_XDECREF(v);
772782
Py_XDECREF(w);
783+
Py_XDECREF(oenc);
773784
return -1;
774785
}
775-
mod = PyParser_ASTFromFile(fp, filename,
786+
mod = PyParser_ASTFromFile(fp, filename, enc,
776787
Py_single_input, ps1, ps2,
777788
flags, &errcode, arena);
778789
Py_XDECREF(v);
779790
Py_XDECREF(w);
791+
Py_XDECREF(oenc);
780792
if (mod == NULL) {
781793
PyArena_Free(arena);
782794
if (errcode == E_EOF) {
@@ -1254,7 +1266,7 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals,
12541266
if (arena == NULL)
12551267
return NULL;
12561268

1257-
mod = PyParser_ASTFromFile(fp, filename, start, 0, 0,
1269+
mod = PyParser_ASTFromFile(fp, filename, NULL, start, 0, 0,
12581270
flags, NULL, arena);
12591271
if (closeit)
12601272
fclose(fp);
@@ -1379,13 +1391,15 @@ PyParser_ASTFromString(const char *s, const char *filename, int start,
13791391
}
13801392

13811393
mod_ty
1382-
PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1,
1394+
PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc,
1395+
int start, char *ps1,
13831396
char *ps2, PyCompilerFlags *flags, int *errcode,
13841397
PyArena *arena)
13851398
{
13861399
mod_ty mod;
13871400
perrdetail err;
1388-
node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
1401+
node *n = PyParser_ParseFileFlags(fp, filename, enc,
1402+
&_PyParser_Grammar,
13891403
start, ps1, ps2, &err, PARSER_FLAGS(flags));
13901404
if (n) {
13911405
mod = PyAST_FromNode(n, flags, filename, arena);
@@ -1406,7 +1420,8 @@ node *
14061420
PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags)
14071421
{
14081422
perrdetail err;
1409-
node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
1423+
node *n = PyParser_ParseFileFlags(fp, filename, NULL,
1424+
&_PyParser_Grammar,
14101425
start, NULL, NULL, &err, flags);
14111426
if (n == NULL)
14121427
err_input(&err);

0 commit comments

Comments
 (0)