diff --git a/Lib/codeop.py b/Lib/codeop.py index 96868047cb..eea6cbc701 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -65,14 +65,10 @@ def _maybe_compile(compiler, source, filename, symbol): try: compiler(source + "\n", filename, symbol) return None + except _IncompleteInputError as e: + return None except SyntaxError as e: - # XXX: RustPython; support multiline definitions in REPL - # See also: https://github.com/RustPython/RustPython/pull/5743 - strerr = str(e) - if source.endswith(":") and "expected an indented block" in strerr: - return None - elif "incomplete input" in str(e): - return None + pass # fallthrough return compiler(source, filename, symbol, incomplete_input=False) diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index e19162a6ab..09db151ad2 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -83,6 +83,8 @@ def test_inheritance(self): exc_set = set(e for e in exc_set if not e.startswith('_')) # RUSTPYTHON specific exc_set.discard("JitError") + # TODO: RUSTPYTHON; this will be officially introduced in Python 3.15 + exc_set.discard("IncompleteInputError") self.assertEqual(len(exc_set), 0, "%s not accounted for" % exc_set) interface_tests = ("length", "args", "str", "repr") diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index d61570ce10..e01ddcf0a8 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -664,6 +664,9 @@ def test_exceptions(self): BaseExceptionGroup, ExceptionGroup): continue + # TODO: RUSTPYTHON: fix name mapping for _IncompleteInputError + if exc is _IncompleteInputError: + continue if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), ('exceptions', 'OSError')) diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 390a2d5669..7647e19348 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -25,6 +25,7 @@ pub enum CompileErrorType { pub struct ParseError { #[source] pub error: parser::ParseErrorType, + pub raw_location: ruff_text_size::TextRange, pub location: SourceLocation, pub source_path: String, } @@ -48,6 +49,7 @@ impl CompileError { let location = source_code.source_location(error.location.start()); Self::Parse(ParseError { error: error.error, + raw_location: error.location, location, source_path: source_code.path.to_owned(), }) diff --git a/src/shell.rs b/src/shell.rs index cbe2c9efe0..801a989cf2 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -1,7 +1,8 @@ mod helper; use rustpython_compiler::{ - CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType, + CompileError, ParseError, parser::FStringErrorType, parser::LexicalErrorType, + parser::ParseErrorType, }; use rustpython_vm::{ AsObject, PyResult, VirtualMachine, @@ -14,7 +15,8 @@ use rustpython_vm::{ enum ShellExecResult { Ok, PyErr(PyBaseExceptionRef), - Continue, + ContinueBlock, + ContinueLine, } fn shell_exec( @@ -22,11 +24,17 @@ fn shell_exec( source: &str, scope: Scope, empty_line_given: bool, - continuing: bool, + continuing_block: bool, ) -> ShellExecResult { + // compiling expects only UNIX style line endings, and will replace windows line endings + // internally. Since we might need to analyze the source to determine if an error could be + // resolved by future input, we need the location from the error to match the source code that + // was actually compiled. + #[cfg(windows)] + let source = &source.replace("\r\n", "\n"); match vm.compile(source, compiler::Mode::Single, "".to_owned()) { Ok(code) => { - if empty_line_given || !continuing { + if empty_line_given || !continuing_block { // We want to execute the full code match vm.run_code_obj(code, scope) { Ok(_val) => ShellExecResult::Ok, @@ -40,8 +48,32 @@ fn shell_exec( Err(CompileError::Parse(ParseError { error: ParseErrorType::Lexical(LexicalErrorType::Eof), .. - })) => ShellExecResult::Continue, + })) => ShellExecResult::ContinueLine, + Err(CompileError::Parse(ParseError { + error: + ParseErrorType::Lexical(LexicalErrorType::FStringError( + FStringErrorType::UnterminatedTripleQuotedString, + )), + .. + })) => ShellExecResult::ContinueLine, Err(err) => { + // Check if the error is from an unclosed triple quoted string (which should always + // continue) + if let CompileError::Parse(ParseError { + error: ParseErrorType::Lexical(LexicalErrorType::UnclosedStringError), + raw_location, + .. + }) = err + { + let loc = raw_location.start().to_usize(); + let mut iter = source.chars(); + if let Some(quote) = iter.nth(loc) { + if iter.next() == Some(quote) && iter.next() == Some(quote) { + return ShellExecResult::ContinueLine; + } + } + }; + // bad_error == true if we are handling an error that should be thrown even if we are continuing // if its an indentation error, set to true if we are continuing and the error is on column 0, // since indentations errors on columns other than 0 should be ignored. @@ -50,10 +82,12 @@ fn shell_exec( let bad_error = match err { CompileError::Parse(ref p) => { match &p.error { - ParseErrorType::Lexical(LexicalErrorType::IndentationError) => continuing, // && p.location.is_some() + ParseErrorType::Lexical(LexicalErrorType::IndentationError) => { + continuing_block + } // && p.location.is_some() ParseErrorType::OtherError(msg) => { if msg.starts_with("Expected an indented block") { - continuing + continuing_block } else { true } @@ -68,7 +102,7 @@ fn shell_exec( if empty_line_given || bad_error { ShellExecResult::PyErr(vm.new_syntax_error(&err, Some(source))) } else { - ShellExecResult::Continue + ShellExecResult::ContinueBlock } } } @@ -93,10 +127,19 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { println!("No previous history."); } - let mut continuing = false; + // We might either be waiting to know if a block is complete, or waiting to know if a multiline + // statement is complete. In the former case, we need to ensure that we read one extra new line + // to know that the block is complete. In the latter, we can execute as soon as the statement is + // valid. + let mut continuing_block = false; + let mut continuing_line = false; loop { - let prompt_name = if continuing { "ps2" } else { "ps1" }; + let prompt_name = if continuing_block || continuing_line { + "ps2" + } else { + "ps1" + }; let prompt = vm .sys_module .get_attr(prompt_name, vm) @@ -105,6 +148,8 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { Ok(ref s) => s.as_str(), Err(_) => "", }; + + continuing_line = false; let result = match repl.readline(prompt) { ReadlineResult::Line(line) => { debug!("You entered {:?}", line); @@ -120,39 +165,44 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> { } full_input.push('\n'); - match shell_exec(vm, &full_input, scope.clone(), empty_line_given, continuing) { + match shell_exec( + vm, + &full_input, + scope.clone(), + empty_line_given, + continuing_block, + ) { ShellExecResult::Ok => { - if continuing { + if continuing_block { if empty_line_given { - // We should be exiting continue mode - continuing = false; + // We should exit continue mode since the block successfully executed + continuing_block = false; full_input.clear(); - Ok(()) - } else { - // We should stay in continue mode - continuing = true; - Ok(()) } } else { // We aren't in continue mode so proceed normally - continuing = false; full_input.clear(); - Ok(()) } + Ok(()) + } + // Continue, but don't change the mode + ShellExecResult::ContinueLine => { + continuing_line = true; + Ok(()) } - ShellExecResult::Continue => { - continuing = true; + ShellExecResult::ContinueBlock => { + continuing_block = true; Ok(()) } ShellExecResult::PyErr(err) => { - continuing = false; + continuing_block = false; full_input.clear(); Err(err) } } } ReadlineResult::Interrupt => { - continuing = false; + continuing_block = false; full_input.clear(); let keyboard_interrupt = vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned()); diff --git a/vm/src/compiler.rs b/vm/src/compiler.rs index b819fd9a42..84fea452d1 100644 --- a/vm/src/compiler.rs +++ b/vm/src/compiler.rs @@ -49,3 +49,10 @@ impl crate::convert::ToPyException for (CompileError, Option<&str>) { vm.new_syntax_error(&self.0, self.1) } } + +#[cfg(any(feature = "parser", feature = "codegen"))] +impl crate::convert::ToPyException for (CompileError, Option<&str>, bool) { + fn to_pyexception(&self, vm: &crate::VirtualMachine) -> crate::builtins::PyBaseExceptionRef { + vm.new_syntax_error_maybe_incomplete(&self.0, self.1, self.2) + } +} diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 6c4f97fe38..60b6d86704 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -495,6 +495,7 @@ pub struct ExceptionZoo { pub not_implemented_error: &'static Py, pub recursion_error: &'static Py, pub syntax_error: &'static Py, + pub incomplete_input_error: &'static Py, pub indentation_error: &'static Py, pub tab_error: &'static Py, pub system_error: &'static Py, @@ -743,6 +744,7 @@ impl ExceptionZoo { let recursion_error = PyRecursionError::init_builtin_type(); let syntax_error = PySyntaxError::init_builtin_type(); + let incomplete_input_error = PyIncompleteInputError::init_builtin_type(); let indentation_error = PyIndentationError::init_builtin_type(); let tab_error = PyTabError::init_builtin_type(); @@ -817,6 +819,7 @@ impl ExceptionZoo { not_implemented_error, recursion_error, syntax_error, + incomplete_input_error, indentation_error, tab_error, system_error, @@ -965,6 +968,7 @@ impl ExceptionZoo { "end_offset" => ctx.none(), "text" => ctx.none(), }); + extend_exception!(PyIncompleteInputError, ctx, excs.incomplete_input_error); extend_exception!(PyIndentationError, ctx, excs.indentation_error); extend_exception!(PyTabError, ctx, excs.tab_error); @@ -1623,6 +1627,28 @@ pub(super) mod types { } } + #[pyexception( + name = "_IncompleteInputError", + base = "PySyntaxError", + ctx = "incomplete_input_error" + )] + #[derive(Debug)] + pub struct PyIncompleteInputError {} + + #[pyexception] + impl PyIncompleteInputError { + #[pyslot] + #[pymethod(name = "__init__")] + pub(crate) fn slot_init( + zelf: PyObjectRef, + _args: FuncArgs, + vm: &VirtualMachine, + ) -> PyResult<()> { + zelf.set_attr("name", vm.ctx.new_str("SyntaxError"), vm)?; + Ok(()) + } + } + #[pyexception(name, base = "PySyntaxError", ctx = "indentation_error", impl)] #[derive(Debug)] pub struct PyIndentationError {} diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 13341c1b1e..95a1162f5b 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -245,6 +245,7 @@ pub(crate) fn parse( let top = parser::parse(source, mode.into()) .map_err(|parse_error| ParseError { error: parse_error.error, + raw_location: parse_error.location, location: text_range_to_source_range(&source_code, parse_error.location) .start .to_source_location(), @@ -295,8 +296,8 @@ pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400; // The following flags match the values from Include/cpython/compile.h // Caveat emptor: These flags are undocumented on purpose and depending // on their effect outside the standard library is **unsupported**. -const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200; -const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000; +pub const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200; +pub const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000; // __future__ flags - sync with Lib/__future__.py // TODO: These flags aren't being used in rust code diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 9a21dd34dd..60f6dd8dd9 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -186,6 +186,8 @@ mod builtins { return Err(vm.new_value_error("compile() unrecognized flags".to_owned())); } + let allow_incomplete = !(flags & ast::PY_CF_ALLOW_INCOMPLETE_INPUT).is_zero(); + if (flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() { #[cfg(not(feature = "compiler"))] { @@ -207,14 +209,17 @@ mod builtins { args.filename.to_string_lossy().into_owned(), opts, ) - .map_err(|err| (err, Some(source)).to_pyexception(vm))?; + .map_err(|err| { + (err, Some(source), allow_incomplete).to_pyexception(vm) + })?; Ok(code.into()) } } else { let mode = mode_str .parse::() .map_err(|err| vm.new_value_error(err.to_string()))?; - ast::parse(vm, source, mode).map_err(|e| (e, Some(source)).to_pyexception(vm)) + ast::parse(vm, source, mode) + .map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm)) } } } @@ -1056,6 +1061,7 @@ pub fn init_module(vm: &VirtualMachine, module: &Py) { "NotImplementedError" => ctx.exceptions.not_implemented_error.to_owned(), "RecursionError" => ctx.exceptions.recursion_error.to_owned(), "SyntaxError" => ctx.exceptions.syntax_error.to_owned(), + "_IncompleteInputError" => ctx.exceptions.incomplete_input_error.to_owned(), "IndentationError" => ctx.exceptions.indentation_error.to_owned(), "TabError" => ctx.exceptions.tab_error.to_owned(), "SystemError" => ctx.exceptions.system_error.to_owned(), diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs index 9a7a7fe748..978b694fc4 100644 --- a/vm/src/vm/vm_new.rs +++ b/vm/src/vm/vm_new.rs @@ -320,10 +320,11 @@ impl VirtualMachine { } #[cfg(any(feature = "parser", feature = "compiler"))] - pub fn new_syntax_error( + pub fn new_syntax_error_maybe_incomplete( &self, error: &crate::compiler::CompileError, source: Option<&str>, + allow_incomplete: bool, ) -> PyBaseExceptionRef { use crate::source::SourceLocation; @@ -343,12 +344,102 @@ impl VirtualMachine { .. }) => self.ctx.exceptions.indentation_error, #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::Eof, + ), + .. + }) => { + if allow_incomplete { + self.ctx.exceptions.incomplete_input_error + } else { + self.ctx.exceptions.syntax_error + } + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::FStringError( + ruff_python_parser::FStringErrorType::UnterminatedTripleQuotedString, + ), + ), + .. + }) => { + if allow_incomplete { + self.ctx.exceptions.incomplete_input_error + } else { + self.ctx.exceptions.syntax_error + } + } + #[cfg(feature = "parser")] + crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { + error: + ruff_python_parser::ParseErrorType::Lexical( + ruff_python_parser::LexicalErrorType::UnclosedStringError, + ), + raw_location, + .. + }) => { + if allow_incomplete { + let mut is_incomplete = false; + + if let Some(source) = source { + let loc = raw_location.start().to_usize(); + let mut iter = source.chars(); + if let Some(quote) = iter.nth(loc) { + if iter.next() == Some(quote) && iter.next() == Some(quote) { + is_incomplete = true; + } + } + } + + if is_incomplete { + self.ctx.exceptions.incomplete_input_error + } else { + self.ctx.exceptions.syntax_error + } + } else { + self.ctx.exceptions.syntax_error + } + } + #[cfg(feature = "parser")] crate::compiler::CompileError::Parse(rustpython_compiler::ParseError { error: ruff_python_parser::ParseErrorType::OtherError(s), + raw_location, .. }) => { if s.starts_with("Expected an indented block after") { - self.ctx.exceptions.indentation_error + if allow_incomplete { + // Check that all chars in the error are whitespace, if so, the source is + // incomplete. Otherwise, we've found code that might violates + // indentation rules. + let mut is_incomplete = true; + if let Some(source) = source { + let start = raw_location.start().to_usize(); + let end = raw_location.end().to_usize(); + let mut iter = source.chars(); + iter.nth(start); + for _ in start..end { + if let Some(c) = iter.next() { + if !c.is_ascii_whitespace() { + is_incomplete = false; + } + } else { + break; + } + } + } + + if is_incomplete { + self.ctx.exceptions.incomplete_input_error + } else { + self.ctx.exceptions.indentation_error + } + } else { + self.ctx.exceptions.indentation_error + } } else { self.ctx.exceptions.syntax_error } @@ -410,6 +501,15 @@ impl VirtualMachine { syntax_error } + #[cfg(any(feature = "parser", feature = "compiler"))] + pub fn new_syntax_error( + &self, + error: &crate::compiler::CompileError, + source: Option<&str>, + ) -> PyBaseExceptionRef { + self.new_syntax_error_maybe_incomplete(error, source, false) + } + pub fn new_import_error(&self, msg: String, name: PyStrRef) -> PyBaseExceptionRef { let import_error = self.ctx.exceptions.import_error.to_owned(); let exc = self.new_exception_msg(import_error, msg);