refactor parse_token_trees to not return unmatched_delims

rust-lang · matthiaskrgr · Feb 27, 2023 · Feb 27, 2023 · Feb 27, 2023 · Feb 27, 2023
commit f808877bbf8643cb9ea54a59cb5a61a631f545dd
diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs
@@ -1,10 +1,11 @@
 use crate::errors;
 use crate::lexer::unicode_chars::UNICODE_ARRAY;
+use crate::make_unclosed_delims_error;
 use rustc_ast::ast::{self, AttrStyle};
 use rustc_ast::token::{self, CommentKind, Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::util::unicode::contains_text_flow_control_chars;
-use rustc_errors::{error_code, Applicability, DiagnosticBuilder, PResult, StashKey};
+use rustc_errors::{error_code, Applicability, Diagnostic, DiagnosticBuilder, StashKey};
 use rustc_lexer::unescape::{self, Mode};
 use rustc_lexer::Cursor;
 use rustc_lexer::{Base, DocStyle, RawStrError};
@@ -44,7 +45,7 @@ pub(crate) fn parse_token_trees<'a>(
     mut src: &'a str,
     mut start_pos: BytePos,
     override_span: Option<Span>,
-) -> (PResult<'a, TokenStream>, Vec<UnmatchedDelim>) {
+) -> Result<TokenStream, Vec<Diagnostic>> {
     // Skip `#!`, if present.
     if let Some(shebang_len) = rustc_lexer::strip_shebang(src) {
         src = &src[shebang_len..];
@@ -61,7 +62,29 @@ pub(crate) fn parse_token_trees<'a>(
         override_span,
         nbsp_is_whitespace: false,
     };
-    tokentrees::TokenTreesReader::parse_all_token_trees(string_reader)
+    let (token_trees, unmatched_delims) =
+        tokentrees::TokenTreesReader::parse_all_token_trees(string_reader);
+    match token_trees {
+        Ok(stream) if unmatched_delims.is_empty() => Ok(stream),
+        _ => {
+            // Return error if there are unmatched delimiters or unclosng delimiters.
+            // We emit delimiter mismatch errors first, then emit the unclosing delimiter mismatch
+            // because the delimiter mismatch is more likely to be the root cause of error
+
+            let mut buffer = Vec::with_capacity(1);
+            // Not using `emit_unclosed_delims` to use `db.buffer`
+            for unmatched in unmatched_delims {
+                if let Some(err) = make_unclosed_delims_error(unmatched, &sess) {
+                    err.buffer(&mut buffer);
+                }
+            }
+            if let Err(err) = token_trees {
+                // Add unclosing delimiter error
+                err.buffer(&mut buffer);
+            }
+            Err(buffer)
+        }
+    }
 }
 
 struct StringReader<'a> {

diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs
@@ -161,7 +161,6 @@ impl<'a> TokenTreesReader<'a> {
                         }
                     }
                     let (tok, _) = self.diag_info.open_braces.pop().unwrap();
-                    debug!("anan now: open {:#?} close {:#?}", open_delim, close_delim);
                     self.diag_info.unmatched_delims.push(UnmatchedDelim {
                         expected_delim: tok,
                         found_delim: Some(close_delim),

diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs
@@ -196,30 +196,7 @@ pub fn maybe_file_to_stream(
         ));
     });
 
-    let (token_trees, unmatched_delims) =
-        lexer::parse_token_trees(sess, src.as_str(), source_file.start_pos, override_span);
-
-    match token_trees {
-        Ok(stream) if unmatched_delims.is_empty() => Ok(stream),
-        _ => {
-            // Return error if there are unmatched delimiters or unclosng delimiters.
-            // We emit delimiter mismatch errors first, then emit the unclosing delimiter mismatch
-            // because the delimiter mismatch is more likely to be the root cause of the
-
-            let mut buffer = Vec::with_capacity(1);
-            // Not using `emit_unclosed_delims` to use `db.buffer`
-            for unmatched in unmatched_delims {
-                if let Some(err) = make_unclosed_delims_error(unmatched, &sess) {
-                    err.buffer(&mut buffer);
-                }
-            }
-            if let Err(err) = token_trees {
-                // Add unclosing delimiter error
-                err.buffer(&mut buffer);
-            }
-            Err(buffer)
-        }
-    }
+    lexer::parse_token_trees(sess, src.as_str(), source_file.start_pos, override_span)
 }
 
 /// Given a stream and the `ParseSess`, produces a parser.

diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -19,9 +19,9 @@ use crate::errors::{
 };
 
 use crate::fluent_generated as fluent;
-use rustc_ast as ast;
-use crate::parser;
 use crate::lexer::UnmatchedDelim;
+use crate::parser;
+use rustc_ast as ast;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Lit, LitKind, TokenKind};
 use rustc_ast::util::parser::AssocOp;