From 04886d825d296299683399e3907c84634430b488 Mon Sep 17 00:00:00 2001 From: Padraic Fanning Date: Tue, 1 Feb 2022 19:01:04 -0500 Subject: [PATCH 1/4] Allow brackets in f-string format spec --- parser/src/fstring.rs | 70 ++++++++++--------- ..._fstring_parse_selfdocumenting_format.snap | 22 ++++-- ...ing__tests__parse_fstring_nested_spec.snap | 61 ++++++++++++---- ..._tests__parse_fstring_not_nested_spec.snap | 21 ++++-- 4 files changed, 119 insertions(+), 55 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index cec5a72e3a..85428505dd 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -91,52 +91,56 @@ impl<'a> FStringParser<'a> { } ':' if delims.is_empty() => { - let mut nested = false; let mut in_nested = false; - let mut spec_expression = String::new(); + let mut spec_constructor = Vec::new(); + let mut constant_piece = String::new(); + let mut formatted_value_piece = String::new(); while let Some(&next) = self.chars.peek() { match next { + '{' if in_nested => return Err(ExpressionNestedTooDeeply), + '}' if in_nested => { + in_nested = false; + spec_constructor.push(self.expr(ExprKind::FormattedValue { + value: + Box::new( + parse_fstring_expr(&formatted_value_piece).map_err( + |e| InvalidExpression(Box::new(e.error)), + )?, + ), + conversion: None, + format_spec: None, + })); + formatted_value_piece.clear(); + } + _ if in_nested => { + formatted_value_piece.push(next); + } '{' => { - if in_nested { - return Err(ExpressionNestedTooDeeply); - } in_nested = true; - nested = true; - self.chars.next(); - continue; + spec_constructor.push(self.expr(ExprKind::Constant { + value: constant_piece.to_owned().into(), + kind: None, + })); + constant_piece.clear(); } - '}' => { - if in_nested { - in_nested = false; - self.chars.next(); - } - break; + '}' => break, + _ => { + constant_piece.push(next); } - _ => (), } - spec_expression.push(next); self.chars.next(); } + spec_constructor.push(self.expr(ExprKind::Constant { + value: constant_piece.to_owned().into(), + kind: None, + })); + constant_piece.clear(); if in_nested { return Err(UnclosedLbrace); } - spec = Some(if nested { - Box::new( - self.expr(ExprKind::FormattedValue { - value: Box::new( - parse_fstring_expr(&spec_expression) - .map_err(|e| InvalidExpression(Box::new(e.error)))?, - ), - conversion: None, - format_spec: None, - }), - ) - } else { - Box::new(self.expr(ExprKind::Constant { - value: spec_expression.to_owned().into(), - kind: None, - })) - }) + spec = Some(Box::new(self.expr(ExprKind::JoinedStr { + values: spec_constructor, + }))) } '(' | '{' | '[' => { expression.push(ch); diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap index e0713f5edd..9ba81d0fb6 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap @@ -1,6 +1,7 @@ --- source: parser/src/fstring.rs expression: parse_ast + --- Located { location: Location { @@ -62,11 +63,22 @@ Located { column: 0, }, custom: (), - node: Constant { - value: Str( - ">10", - ), - kind: None, + node: JoinedStr { + values: [ + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + ">10", + ), + kind: None, + }, + }, + ], }, }, ), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap index bba192e305..3656ef8b04 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap @@ -37,20 +37,57 @@ Located { column: 0, }, custom: (), - node: FormattedValue { - value: Located { - location: Location { - row: 1, - column: 2, + node: JoinedStr { + values: [ + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "", + ), + kind: None, + }, }, - custom: (), - node: Name { - id: "spec", - ctx: Load, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "spec", + ctx: Load, + }, + }, + conversion: None, + format_spec: None, + }, }, - }, - conversion: None, - format_spec: None, + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "", + ), + kind: None, + }, + }, + ], }, }, ), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap index ed40d483e8..682465de8c 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap @@ -37,11 +37,22 @@ Located { column: 0, }, custom: (), - node: Constant { - value: Str( - "spec", - ), - kind: None, + node: JoinedStr { + values: [ + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: Constant { + value: Str( + "spec", + ), + kind: None, + }, + }, + ], }, }, ), From a1dffe671e4199f2604d47fefe0516ef317098ef Mon Sep 17 00:00:00 2001 From: Padraic Fanning Date: Tue, 1 Feb 2022 20:20:11 -0500 Subject: [PATCH 2/4] Allow replacement fields in format specs --- parser/src/fstring.rs | 21 ++++++------ ...ing__tests__parse_fstring_nested_spec.snap | 32 ++++++++++++++++--- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 85428505dd..ec861e3777 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -100,16 +100,19 @@ impl<'a> FStringParser<'a> { '{' if in_nested => return Err(ExpressionNestedTooDeeply), '}' if in_nested => { in_nested = false; - spec_constructor.push(self.expr(ExprKind::FormattedValue { - value: - Box::new( - parse_fstring_expr(&formatted_value_piece).map_err( - |e| InvalidExpression(Box::new(e.error)), - )?, + spec_constructor.push( + self.expr(ExprKind::FormattedValue { + value: Box::new( + FStringParser::new( + &format!("{{{}}}", formatted_value_piece), + Location::default(), + ) + .parse()?, ), - conversion: None, - format_spec: None, - })); + conversion: None, + format_spec: None, + }), + ); formatted_value_piece.clear(); } _ if in_nested => { diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap index 3656ef8b04..6bcd55732f 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap +++ b/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap @@ -61,13 +61,35 @@ Located { node: FormattedValue { value: Located { location: Location { - row: 1, - column: 2, + row: 0, + column: 0, }, custom: (), - node: Name { - id: "spec", - ctx: Load, + node: JoinedStr { + values: [ + Located { + location: Location { + row: 0, + column: 0, + }, + custom: (), + node: FormattedValue { + value: Located { + location: Location { + row: 1, + column: 2, + }, + custom: (), + node: Name { + id: "spec", + ctx: Load, + }, + }, + conversion: None, + format_spec: None, + }, + }, + ], }, }, conversion: None, From f2257a93b00f0b66f8886a8d97dd81f78e2ce16e Mon Sep 17 00:00:00 2001 From: Padraic Fanning Date: Tue, 1 Feb 2022 22:46:18 -0500 Subject: [PATCH 3/4] Allow proper recursion in f-string format specs --- parser/src/fstring.rs | 56 +++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index ec861e3777..e78cf39ccf 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -11,13 +11,15 @@ use self::FStringErrorType::*; struct FStringParser<'a> { chars: iter::Peekable>, str_location: Location, + recurse_lvl: u8, } impl<'a> FStringParser<'a> { - fn new(source: &'a str, str_location: Location) -> Self { + fn new(source: &'a str, str_location: Location, recurse_lvl: u8) -> Self { Self { chars: source.chars().peekable(), str_location, + recurse_lvl, } } @@ -95,25 +97,35 @@ impl<'a> FStringParser<'a> { let mut spec_constructor = Vec::new(); let mut constant_piece = String::new(); let mut formatted_value_piece = String::new(); + let mut spec_delims = Vec::new(); while let Some(&next) = self.chars.peek() { match next { - '{' if in_nested => return Err(ExpressionNestedTooDeeply), + '{' if in_nested => { + spec_delims.push(next); + formatted_value_piece.push(next); + } '}' if in_nested => { - in_nested = false; - spec_constructor.push( - self.expr(ExprKind::FormattedValue { - value: Box::new( - FStringParser::new( - &format!("{{{}}}", formatted_value_piece), - Location::default(), - ) - .parse()?, - ), - conversion: None, - format_spec: None, - }), - ); - formatted_value_piece.clear(); + if spec_delims.is_empty() { + in_nested = false; + spec_constructor.push( + self.expr(ExprKind::FormattedValue { + value: Box::new( + FStringParser::new( + &format!("{{{}}}", formatted_value_piece), + Location::default(), + &self.recurse_lvl + 1, + ) + .parse()?, + ), + conversion: None, + format_spec: None, + }), + ); + formatted_value_piece.clear(); + } else { + spec_delims.pop(); + formatted_value_piece.push(next); + } } _ if in_nested => { formatted_value_piece.push(next); @@ -223,6 +235,10 @@ impl<'a> FStringParser<'a> { } fn parse(mut self) -> Result { + if self.recurse_lvl >= 2 { + return Err(ExpressionNestedTooDeeply); + } + let mut content = String::new(); let mut values = vec![]; @@ -276,7 +292,7 @@ fn parse_fstring_expr(source: &str) -> Result { /// Parse an fstring from a string, located at a certain position in the sourcecode. /// In case of errors, we will get the location and the error returned. pub fn parse_located_fstring(source: &str, location: Location) -> Result { - FStringParser::new(source, location) + FStringParser::new(source, location, 0) .parse() .map_err(|error| FStringError { error, location }) } @@ -286,7 +302,7 @@ mod tests { use super::*; fn parse_fstring(source: &str) -> Result { - FStringParser::new(source, Location::default()).parse() + FStringParser::new(source, Location::default(), 0).parse() } #[test] @@ -354,7 +370,7 @@ mod tests { assert_eq!(parse_fstring("{5!}"), Err(InvalidConversionFlag)); assert_eq!(parse_fstring("{5!x}"), Err(InvalidConversionFlag)); - assert_eq!(parse_fstring("{a:{a:{b}}"), Err(ExpressionNestedTooDeeply)); + assert_eq!(parse_fstring("{a:{a:{b}}}"), Err(ExpressionNestedTooDeeply)); assert_eq!(parse_fstring("{a:b}}"), Err(UnopenedRbrace)); assert_eq!(parse_fstring("}"), Err(UnopenedRbrace)); From a936c367071995cc5d472a632a65f3fe1a5898e4 Mon Sep 17 00:00:00 2001 From: Padraic Fanning Date: Tue, 1 Feb 2022 23:41:00 -0500 Subject: [PATCH 4/4] Add format-spec string test snippets --- extra_tests/snippets/fstrings.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/extra_tests/snippets/fstrings.py b/extra_tests/snippets/fstrings.py index 66303f1146..c874f196dc 100644 --- a/extra_tests/snippets/fstrings.py +++ b/extra_tests/snippets/fstrings.py @@ -47,6 +47,15 @@ spec = "0>+#10x" assert f"{16:{spec}}{foo}" == '00000+0x10bar' +part_spec = ">+#10x" +assert f"{16:0{part_spec}}{foo}" == '00000+0x10bar' + +# TODO: RUSTPYTHON, delete the next block once `test_fstring.py` can successfully parse +assert f'{10:#{1}0x}' == ' 0xa' +assert f'{10:{"#"}1{0}{"x"}}' == ' 0xa' +assert f'{-10:-{"#"}1{0}x}' == ' -0xa' +assert f'{-10:{"-"}#{1}0{"x"}}' == ' -0xa' + # TODO: # spec = "bla" # assert_raises(ValueError, lambda: f"{16:{spec}}")