From 56f08a1417bd03f5094f431d13e2db5f3320f72c Mon Sep 17 00:00:00 2001 From: Moreal Date: Sat, 13 Aug 2022 02:22:09 +0900 Subject: [PATCH 01/10] Make eval able to receive bytes also --- vm/src/stdlib/builtins.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 8f3d99e473..4d11faf277 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -248,11 +248,23 @@ mod builtins { #[cfg(feature = "rustpython-compiler")] #[pyfunction] fn eval( - source: Either>, + source: Either, PyRef>, scope: ScopeArgs, vm: &VirtualMachine, ) -> PyResult { - run_code(vm, source, scope, compile::Mode::Eval, "eval") + // source as string + let code = match source { + Either::A(either) => { + let source: &[u8] = match either { + Either::A(ref a) => a.as_str().as_bytes(), + Either::B(ref b) => b.as_bytes(), + }; + + Ok(Either::A(vm.ctx.new_str(std::str::from_utf8(source).unwrap()))) + }, + Either::B(code) => Ok(Either::B(code)), + }?; + run_code(vm, code, scope, compile::Mode::Eval, "eval") } /// Implements `exec` From bb8526ac28bc87223f76ac8ec341f19c00208328 Mon Sep 17 00:00:00 2001 From: Moreal Date: Sat, 13 Aug 2022 02:25:36 +0900 Subject: [PATCH 02/10] Raise ValueError if null character exists for eval argument --- vm/src/stdlib/builtins.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 4d11faf277..4c5579695f 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -259,6 +259,11 @@ mod builtins { Either::A(ref a) => a.as_str().as_bytes(), Either::B(ref b) => b.as_bytes(), }; + for x in source { + if *x == 0 { + return Err(vm.new_value_error("source code string cannot contain null bytes".to_owned())); + } + } Ok(Either::A(vm.ctx.new_str(std::str::from_utf8(source).unwrap()))) }, From c7689c97cbf168e361edb40c803fb60bc3306a77 Mon Sep 17 00:00:00 2001 From: Moreal Date: Sat, 13 Aug 2022 02:26:02 +0900 Subject: [PATCH 03/10] Remove todo comment of resolved issue --- extra_tests/snippets/syntax_non_utf8.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extra_tests/snippets/syntax_non_utf8.py b/extra_tests/snippets/syntax_non_utf8.py index 731b36889c..660532ad9d 100644 --- a/extra_tests/snippets/syntax_non_utf8.py +++ b/extra_tests/snippets/syntax_non_utf8.py @@ -5,8 +5,7 @@ dir_path = os.path.dirname(os.path.realpath(__file__)) -# TODO: RUSTPYTHON, RustPython raises a SyntaxError here, but cpython raise a ValueError -error = SyntaxError if platform.python_implementation() == 'RustPython' else ValueError +error = ValueError with assert_raises(error): with open(os.path.join(dir_path , "non_utf8.txt")) as f: eval(f.read()) From 20e7752e9b45599da6dcab8e56550dc09ee4b1a7 Mon Sep 17 00:00:00 2001 From: Moreal Date: Sat, 13 Aug 2022 07:29:27 +0900 Subject: [PATCH 04/10] Apply `cargo fmt` --- vm/src/stdlib/builtins.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 4c5579695f..0db174117b 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -248,7 +248,10 @@ mod builtins { #[cfg(feature = "rustpython-compiler")] #[pyfunction] fn eval( - source: Either, PyRef>, + source: Either< + Either, + PyRef, + >, scope: ScopeArgs, vm: &VirtualMachine, ) -> PyResult { @@ -261,12 +264,16 @@ mod builtins { }; for x in source { if *x == 0 { - return Err(vm.new_value_error("source code string cannot contain null bytes".to_owned())); + return Err(vm.new_value_error( + "source code string cannot contain null bytes".to_owned(), + )); } } - Ok(Either::A(vm.ctx.new_str(std::str::from_utf8(source).unwrap()))) - }, + Ok(Either::A( + vm.ctx.new_str(std::str::from_utf8(source).unwrap()), + )) + } Either::B(code) => Ok(Either::B(code)), }?; run_code(vm, code, scope, compile::Mode::Eval, "eval") From 7e88c8ff55017f0c11b807d14e97398a650aaad5 Mon Sep 17 00:00:00 2001 From: Moreal Date: Sat, 13 Aug 2022 07:30:32 +0900 Subject: [PATCH 05/10] Inline error type --- extra_tests/snippets/syntax_non_utf8.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/extra_tests/snippets/syntax_non_utf8.py b/extra_tests/snippets/syntax_non_utf8.py index 660532ad9d..247454eeec 100644 --- a/extra_tests/snippets/syntax_non_utf8.py +++ b/extra_tests/snippets/syntax_non_utf8.py @@ -5,7 +5,6 @@ dir_path = os.path.dirname(os.path.realpath(__file__)) -error = ValueError -with assert_raises(error): +with assert_raises(ValueError): with open(os.path.join(dir_path , "non_utf8.txt")) as f: eval(f.read()) From 6d017420c50ad18e4e4e5e876c5751e0733e5983 Mon Sep 17 00:00:00 2001 From: Moreal Date: Sat, 13 Aug 2022 08:39:58 +0900 Subject: [PATCH 06/10] Unmark resolved tests --- Lib/test/test_cmd_line.py | 2 -- Lib/test/test_subprocess.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 63a7b45d06..da4329048e 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -75,8 +75,6 @@ def test_verbose(self): rc, out, err = assert_python_ok('-vv') self.assertNotIn(b'stack overflow', err) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -E tests when PYTHON env vars are required.') def test_xoptions(self): diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 55f4f04dac..5ade946b57 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -761,8 +761,6 @@ def test_env(self): stdout, stderr = p.communicate() self.assertEqual(stdout, b"orange") - # TODO: RUSTPYTHON - @unittest.expectedFailure # Windows requires at least the SYSTEMROOT environment variable to start # Python @unittest.skipIf(sys.platform == 'win32', From 0bd702d287161c824dba5846740b4822dfbff6e7 Mon Sep 17 00:00:00 2001 From: Moreal Date: Sat, 13 Aug 2022 17:35:21 +0900 Subject: [PATCH 07/10] Use `ArgStrOrBytesLike` python argument type --- vm/src/stdlib/builtins.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 0db174117b..d3b99cf618 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -24,8 +24,8 @@ mod builtins { format::call_object_format, function::Either, function::{ - ArgBytesLike, ArgCallable, ArgIntoBool, ArgIterable, ArgMapping, FuncArgs, KwArgs, - OptionalArg, OptionalOption, PosArgs, PyArithmeticValue, + ArgBytesLike, ArgCallable, ArgIntoBool, ArgIterable, ArgMapping, ArgStrOrBytesLike, + FuncArgs, KwArgs, OptionalArg, OptionalOption, PosArgs, PyArithmeticValue, }, protocol::{PyIter, PyIterReturn}, py_io, @@ -248,20 +248,14 @@ mod builtins { #[cfg(feature = "rustpython-compiler")] #[pyfunction] fn eval( - source: Either< - Either, - PyRef, - >, + source: Either>, scope: ScopeArgs, vm: &VirtualMachine, ) -> PyResult { // source as string let code = match source { Either::A(either) => { - let source: &[u8] = match either { - Either::A(ref a) => a.as_str().as_bytes(), - Either::B(ref b) => b.as_bytes(), - }; + let source: &[u8] = &either.borrow_bytes(); for x in source { if *x == 0 { return Err(vm.new_value_error( From 714ce4d07d7c900e7ceb2dab9eae725f15f749e4 Mon Sep 17 00:00:00 2001 From: Moreal Date: Sun, 14 Aug 2022 02:27:38 +0900 Subject: [PATCH 08/10] Wrap unicode error as syntax error --- vm/src/stdlib/builtins.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index d3b99cf618..a03caa6f91 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -264,9 +264,17 @@ mod builtins { } } - Ok(Either::A( - vm.ctx.new_str(std::str::from_utf8(source).unwrap()), - )) + match std::str::from_utf8(source) { + Ok(s) => Ok(Either::A(vm.ctx.new_str(s))), + Err(err) => { + let msg = format!( + "(unicode error) 'utf-8' codec can't decode byte 0x{:x?} in position {}: invalid start byte", + source[err.valid_up_to()], + err.valid_up_to() + ); + Err(vm.new_exception_msg(vm.ctx.exceptions.syntax_error.to_owned(), msg)) + } + } } Either::B(code) => Ok(Either::B(code)), }?; From 879ed6e005cb98188051b13148d9599f26974f69 Mon Sep 17 00:00:00 2001 From: Lee Dogeon Date: Sun, 14 Aug 2022 03:37:16 +0900 Subject: [PATCH 09/10] Make loop simple with `slice::contains Co-authored-by: Jeong YunWon <69878+youknowone@users.noreply.github.com> --- vm/src/stdlib/builtins.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index a03caa6f91..688914ccad 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -256,12 +256,10 @@ mod builtins { let code = match source { Either::A(either) => { let source: &[u8] = &either.borrow_bytes(); - for x in source { - if *x == 0 { - return Err(vm.new_value_error( - "source code string cannot contain null bytes".to_owned(), - )); - } + if source.contains(&0) { + return Err(vm.new_value_error( + "source code string cannot contain null bytes".to_owned(), + )); } match std::str::from_utf8(source) { From c672d8f8b2217d41b680eeeb7dbe38b4f9a1119c Mon Sep 17 00:00:00 2001 From: Moreal Date: Sun, 14 Aug 2022 03:46:04 +0900 Subject: [PATCH 10/10] Make mapping error simple with `map_err` --- vm/src/stdlib/builtins.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 688914ccad..2679a9ab39 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -262,17 +262,16 @@ mod builtins { )); } - match std::str::from_utf8(source) { - Ok(s) => Ok(Either::A(vm.ctx.new_str(s))), - Err(err) => { - let msg = format!( - "(unicode error) 'utf-8' codec can't decode byte 0x{:x?} in position {}: invalid start byte", - source[err.valid_up_to()], - err.valid_up_to() - ); - Err(vm.new_exception_msg(vm.ctx.exceptions.syntax_error.to_owned(), msg)) - } - } + let source = std::str::from_utf8(source).map_err(|err| { + let msg = format!( + "(unicode error) 'utf-8' codec can't decode byte 0x{:x?} in position {}: invalid start byte", + source[err.valid_up_to()], + err.valid_up_to() + ); + + vm.new_exception_msg(vm.ctx.exceptions.syntax_error.to_owned(), msg) + })?; + Ok(Either::A(vm.ctx.new_str(source))) } Either::B(code) => Ok(Either::B(code)), }?;