diff --git a/grammars/MagicPython.cson b/grammars/MagicPython.cson index 6486dba4..d837d6a3 100644 --- a/grammars/MagicPython.cson +++ b/grammars/MagicPython.cson @@ -308,6 +308,18 @@ repository: name: "storage.type.class.python" match: "\\b(?= | <= | < | >) (?# 5) + | (:=) (?# 6) + ''' captures: "1": @@ -631,6 +645,8 @@ repository: name: "keyword.operator.arithmetic.python" "5": name: "keyword.operator.comparison.python" + "6": + name: "keyword.operator.assignment.python" punctuation: patterns: [ { @@ -781,18 +797,6 @@ repository: { include: "#regexp-double-one-line" } - { - include: "#fregexp-single-three-line" - } - { - include: "#fregexp-double-three-line" - } - { - include: "#fregexp-single-one-line" - } - { - include: "#fregexp-double-one-line" - } ] string: patterns: [ @@ -1255,6 +1259,10 @@ repository: name: "punctuation.section.function.lambda.begin.python" contentName: "meta.function.lambda.parameters.python" patterns: [ + { + name: "keyword.operator.positional.parameter.python" + match: "/" + } { name: "keyword.operator.unpacking.parameter.python" match: "(\\*\\*|\\*)" @@ -1402,6 +1410,10 @@ repository: "1": name: "punctuation.definition.parameters.end.python" patterns: [ + { + name: "keyword.operator.positional.parameter.python" + match: "/" + } { name: "keyword.operator.unpacking.parameter.python" match: "(\\*\\*|\\*)" @@ -1826,14 +1838,14 @@ repository: match: ''' (?x) (?) + (?!\\{\\{) + (?= \\{ ( + \\w*? (?!(['"])|((?) - - ''' - end: "(\\)|(?=\\'\\'\\'))" - beginCaptures: - "1": - name: "punctuation.parenthesis.named.begin.regexp support.other.parenthesis.regexp" - "2": - name: "entity.name.tag.named.group.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.named.end.regexp support.other.parenthesis.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#single-three-fregexp-expression" - } - { - include: "#comments-string-single-three" - } - ] - "single-three-fregexp-lookahead": - begin: "(\\()\\?=" - end: "(\\)|(?=\\'\\'\\'))" - beginCaptures: - "0": - name: "keyword.operator.lookahead.regexp" - "1": - name: "punctuation.parenthesis.lookahead.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#single-three-fregexp-expression" - } - { - include: "#comments-string-single-three" - } - ] - "single-three-fregexp-lookahead-negative": - begin: "(\\()\\?!" - end: "(\\)|(?=\\'\\'\\'))" - beginCaptures: - "0": - name: "keyword.operator.lookahead.negative.regexp" - "1": - name: "punctuation.parenthesis.lookahead.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.negative.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#single-three-fregexp-expression" - } - { - include: "#comments-string-single-three" - } - ] - "single-three-fregexp-lookbehind": - begin: "(\\()\\?<=" - end: "(\\)|(?=\\'\\'\\'))" - beginCaptures: - "0": - name: "keyword.operator.lookbehind.regexp" - "1": - name: "punctuation.parenthesis.lookbehind.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookbehind.end.regexp keyword.operator.lookbehind.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#single-three-fregexp-expression" - } - { - include: "#comments-string-single-three" - } - ] - "single-three-fregexp-lookbehind-negative": - begin: "(\\()\\?) - - ''' - end: "(\\)|(?=\"))|((?=(?) - - ''' - end: "(\\)|(?=\"\"\"))" - beginCaptures: - "1": - name: "punctuation.parenthesis.named.begin.regexp support.other.parenthesis.regexp" - "2": - name: "entity.name.tag.named.group.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.named.end.regexp support.other.parenthesis.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#double-three-fregexp-expression" - } - { - include: "#comments-string-double-three" - } - ] - "double-three-fregexp-lookahead": - begin: "(\\()\\?=" - end: "(\\)|(?=\"\"\"))" - beginCaptures: - "0": - name: "keyword.operator.lookahead.regexp" - "1": - name: "punctuation.parenthesis.lookahead.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#double-three-fregexp-expression" - } - { - include: "#comments-string-double-three" - } - ] - "double-three-fregexp-lookahead-negative": - begin: "(\\()\\?!" - end: "(\\)|(?=\"\"\"))" - beginCaptures: - "0": - name: "keyword.operator.lookahead.negative.regexp" - "1": - name: "punctuation.parenthesis.lookahead.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.negative.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#double-three-fregexp-expression" - } - { - include: "#comments-string-double-three" - } - ] - "double-three-fregexp-lookbehind": - begin: "(\\()\\?<=" - end: "(\\)|(?=\"\"\"))" - beginCaptures: - "0": - name: "keyword.operator.lookbehind.regexp" - "1": - name: "punctuation.parenthesis.lookbehind.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookbehind.end.regexp keyword.operator.lookbehind.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#double-three-fregexp-expression" - } - { - include: "#comments-string-double-three" - } - ] - "double-three-fregexp-lookbehind-negative": - begin: "(\\()\\?=^]? [-+ ]? \\#? \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )(?=}) @@ -4741,7 +3868,7 @@ repository: } ] "fstring-terminator-single-tail": - begin: "(![rsa])?(:)(?=.*?{)" + begin: "((?:=?)(?:![rsa])?)(:)(?=.*?{)" end: "(?=})|(?=\\n)" beginCaptures: "1": @@ -4850,7 +3977,7 @@ repository: ] "fstring-raw-quoted-multi-line": name: "meta.fstring.python" - begin: "(\\b(?:[R][fF]|[fF][R]))('''|\"\"\")" + begin: "(\\b(?:[rR][fF]|[fF][rR]))('''|\"\"\")" end: "(\\2)" beginCaptures: "1": @@ -4944,12 +4071,16 @@ repository: patterns: [ { name: "storage.type.format.python" - match: "(![rsa])(?=})" + match: "(=(![rsa])?)(?=})" + } + { + name: "storage.type.format.python" + match: "(=?![rsa])(?=})" } { match: ''' (?x) - (![rsa])? + ( (?: =?) (?: ![rsa])? ) ( : \\w? [<>=^]? [-+ ]? \\#? \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )(?=}) @@ -4965,7 +4096,7 @@ repository: } ] "fstring-terminator-multi-tail": - begin: "(![rsa])?(:)(?=.*?{)" + begin: "((?:=?)(?:![rsa])?)(:)(?=.*?{)" end: "(?=})" beginCaptures: "1": diff --git a/grammars/MagicPython.tmLanguage b/grammars/MagicPython.tmLanguage index 5a07f545..a1c3f8e7 100644 --- a/grammars/MagicPython.tmLanguage +++ b/grammars/MagicPython.tmLanguage @@ -471,6 +471,22 @@ it's probably control flow like: match \b(?<!\.)(class)\b + + match + (?x) + ^\s*( + case | match + )(?=\s*([-+\w\d(\[{'":#]|$))\b + + captures + + 1 + + name + keyword.control.flow.python + + + expression-bare @@ -933,6 +949,8 @@ E.g. "arr[idx](args)" | (\*\* | \* | \+ | - | % | // | / | @) (?# 4) | (!= | == | >= | <= | < | >) (?# 5) + + | (:=) (?# 6) captures @@ -961,6 +979,11 @@ E.g. "arr[idx](args)" name keyword.operator.comparison.python + 6 + + name + keyword.operator.assignment.python + punctuation @@ -1190,22 +1213,6 @@ E.g. "arr[idx](args)" include #regexp-double-one-line - - include - #fregexp-single-three-line - - - include - #fregexp-double-three-line - - - include - #fregexp-single-one-line - - - include - #fregexp-double-one-line - string @@ -1936,6 +1943,12 @@ E.g. "arr[idx](args)" meta.function.lambda.parameters.python patterns + + name + keyword.operator.positional.parameter.python + match + / + name keyword.operator.unpacking.parameter.python @@ -2184,6 +2197,12 @@ correctly identify the "in" as a control flow keyword. patterns + + name + keyword.operator.positional.parameter.python + match + / + name keyword.operator.unpacking.parameter.python @@ -2849,14 +2868,14 @@ correctly identify the "in" as a control flow keyword. match (?x) (?<!\.) \b( - __import__ | abs | all | any | ascii | bin | breakpoint | callable - | chr | compile | copyright | credits | delattr | dir | divmod - | enumerate | eval | exec | exit | filter | format | getattr - | globals | hasattr | hash | help | hex | id | input - | isinstance | issubclass | iter | len | license | locals | map - | max | memoryview | min | next | oct | open | ord | pow | print - | quit | range | reload | repr | reversed | round - | setattr | sorted | sum | vars | zip + __import__ | abs | aiter | all | any | anext | ascii | bin + | breakpoint | callable | chr | compile | copyright | credits + | delattr | dir | divmod | enumerate | eval | exec | exit + | filter | format | getattr | globals | hasattr | hash | help + | hex | id | input | isinstance | issubclass | iter | len + | license | locals | map | max | memoryview | min | next + | oct | open | ord | pow | print | quit | range | reload | repr + | reversed | round | setattr | sorted | sum | vars | zip )\b @@ -2901,24 +2920,29 @@ indirectly through syntactic constructs (?x) \b( __(?: - abs | add | aenter | aexit | aiter | and | anext | await - | bool | call | ceil | cmp | coerce | complex | contains - | copy | deepcopy | del | delattr | delete | delitem - | delslice | dir | div | divmod | enter | eq | exit | float - | floor | floordiv | format | ge | get | getattr - | getattribute | getinitargs | getitem | getnewargs - | getslice | getstate | gt | hash | hex | iadd | iand | idiv - | ifloordiv | ilshift | imod | imul | index | init - | instancecheck | int | invert | ior | ipow | irshift | isub - | iter | itruediv | ixor | le | len | long | lshift | lt - | missing | mod | mul | ne | neg | new | next | nonzero | oct | or - | pos | pow | radd | rand | rdiv | rdivmod | reduce - | reduce_ex | repr | reversed | rfloordiv | rlshift | rmod - | rmul | ror | round | rpow | rrshift | rshift | rsub - | rtruediv | rxor | set | setattr | setitem | setslice - | setstate | sizeof | str | sub | subclasscheck | truediv - | trunc | unicode | xor | matmul | rmatmul | imatmul - | init_subclass | set_name | fspath | bytes | prepare + abs | add | aenter | aexit | aiter | and | anext + | await | bool | call | ceil | class_getitem + | cmp | coerce | complex | contains | copy + | deepcopy | del | delattr | delete | delitem + | delslice | dir | div | divmod | enter | eq + | exit | float | floor | floordiv | format | ge + | get | getattr | getattribute | getinitargs + | getitem | getnewargs | getslice | getstate | gt + | hash | hex | iadd | iand | idiv | ifloordiv | + | ilshift | imod | imul | index | init + | instancecheck | int | invert | ior | ipow + | irshift | isub | iter | itruediv | ixor | le + | len | long | lshift | lt | missing | mod | mul + | ne | neg | new | next | nonzero | oct | or | pos + | pow | radd | rand | rdiv | rdivmod | reduce + | reduce_ex | repr | reversed | rfloordiv | + | rlshift | rmod | rmul | ror | round | rpow + | rrshift | rshift | rsub | rtruediv | rxor | set + | setattr | setitem | set_name | setslice + | setstate | sizeof | str | sub | subclasscheck + | truediv | trunc | unicode | xor | matmul + | rmatmul | imatmul | init_subclass | set_name + | fspath | bytes | prepare | length_hint )__ )\b @@ -2939,12 +2963,12 @@ indirectly through syntactic constructs (?x) \b( __(?: - all | bases | builtins | class | class_getitem | code | debug - | defaults | dict | doc | file | func | kwdefaults | members - | metaclass | methods | module | mro | mro_entries | name - | qualname | post_init | self | signature | slots | subclasses - | version | weakref | wrapped | annotations | classcell - | spec | path | package | future | traceback + all | annotations | bases | builtins | class + | closure | code | debug | defaults | dict | doc | file | func + | globals | kwdefaults | match_args | members | metaclass | methods + | module | mro | mro_entries | name | qualname | post_init | self + | signature | slots | subclasses | version | weakref | wrapped + | classcell | spec | path | package | future | traceback )__ )\b @@ -5627,85 +5651,30 @@ indirectly through syntactic constructs - single-one-fregexp-expression - - patterns - - - include - #fregexp-base-expression - - - include - #single-one-regexp-character-set - - - include - #single-one-regexp-comments - - - include - #regexp-flags - - - include - #single-one-regexp-named-group - - - include - #regexp-backreference - - - include - #single-one-fregexp-lookahead - - - include - #single-one-fregexp-lookahead-negative - - - include - #single-one-fregexp-lookbehind - - - include - #single-one-fregexp-lookbehind-negative - - - include - #single-one-fregexp-conditional - - - include - #single-one-fregexp-parentheses-non-capturing - - - include - #single-one-fregexp-parentheses - - - - single-one-fregexp-named-group + string-raw-quoted-single-line name - meta.named.regexp + string.quoted.raw.single.python begin - (?x) - (\() (\?P <\w+(?:\s+[[:alnum:]]+)?>) - + \b(([uU]R)|(R))((['"])) end - (\)|(?=\'))|((?=(?<!\\)\n)) + (\4)|((?<!\\)\n) beginCaptures - 1 + 2 name - support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp + invalid.deprecated.prefix.python - 2 + 3 name - entity.name.tag.named.group.regexp + storage.type.string.python + + 4 + + name + punctuation.definition.string.begin.python endCaptures @@ -5713,7 +5682,7 @@ indirectly through syntactic constructs 1 name - support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp + punctuation.definition.string.end.python 2 @@ -5725,27 +5694,37 @@ indirectly through syntactic constructs include - #single-one-fregexp-expression + #string-single-bad-brace1-formatting-raw + + + include + #string-single-bad-brace2-formatting-raw + + + include + #string-raw-guts - single-one-fregexp-lookahead + string-bin-quoted-single-line + name + string.quoted.binary.single.python begin - (\()\?= + (\b[bB])((['"])) end - (\)|(?=\'))|((?=(?<!\\)\n)) + (\2)|((?<!\\)\n) beginCaptures - 0 + 1 name - keyword.operator.lookahead.regexp + storage.type.string.python - 1 + 2 name - punctuation.parenthesis.lookahead.begin.regexp + punctuation.definition.string.begin.python endCaptures @@ -5753,7 +5732,7 @@ indirectly through syntactic constructs 1 name - keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp + punctuation.definition.string.end.python 2 @@ -5765,27 +5744,29 @@ indirectly through syntactic constructs include - #single-one-fregexp-expression + #string-entity - single-one-fregexp-lookahead-negative + string-raw-bin-quoted-single-line + name + string.quoted.raw.binary.single.python begin - (\()\?! + (\b(?:R[bB]|[bB]R))((['"])) end - (\)|(?=\'))|((?=(?<!\\)\n)) + (\2)|((?<!\\)\n) beginCaptures - 0 + 1 name - keyword.operator.lookahead.negative.regexp + storage.type.string.python - 1 + 2 name - punctuation.parenthesis.lookahead.begin.regexp + punctuation.definition.string.begin.python endCaptures @@ -5793,7 +5774,7 @@ indirectly through syntactic constructs 1 name - keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp + punctuation.definition.string.end.python 2 @@ -5805,27 +5786,34 @@ indirectly through syntactic constructs include - #single-one-fregexp-expression + #string-raw-bin-guts - single-one-fregexp-lookbehind + string-quoted-single-line + name + string.quoted.single.python begin - (\()\?<= + (?:\b([rR])(?=[uU]))?([uU])?((['"])) end - (\)|(?=\'))|((?=(?<!\\)\n)) + (\3)|((?<!\\)\n) beginCaptures - 0 + 1 name - keyword.operator.lookbehind.regexp + invalid.illegal.prefix.python - 1 + 2 name - punctuation.parenthesis.lookbehind.begin.regexp + storage.type.string.python + + 3 + + name + punctuation.definition.string.begin.python endCaptures @@ -5833,7 +5821,7 @@ indirectly through syntactic constructs 1 name - keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp + punctuation.definition.string.end.python 2 @@ -5845,1814 +5833,82 @@ indirectly through syntactic constructs include - #single-one-fregexp-expression + #string-single-bad-brace1-formatting-unicode + + + include + #string-single-bad-brace2-formatting-unicode + + + include + #string-unicode-guts - single-one-fregexp-lookbehind-negative + string-single-bad-brace1-formatting-unicode + comment + template using {% ... %} begin - (\()\?<! + (?x) + (?= \{% + ( .*? (?!(['"])|((?<!\\)\n)) ) + %\} + ) + end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.negative.regexp - - 1 + (?=(['"])|((?<!\\)\n)) + patterns + - name - punctuation.parenthesis.lookbehind.begin.regexp + include + #escape-sequence-unicode - - endCaptures - - 1 - name - keyword.operator.lookbehind.negative.regexp punctuation.parenthesis.lookbehind.end.regexp + include + #escape-sequence - 2 - name - invalid.illegal.newline.python + include + #string-line-continuation - + + + string-single-bad-brace1-formatting-raw + + comment + template using {% ... %} + begin + (?x) + (?= \{% + ( .*? (?!(['"])|((?<!\\)\n)) ) + %\} + ) + + end + (?=(['"])|((?<!\\)\n)) patterns include - #single-one-fregexp-expression + #string-consume-escape - single-one-fregexp-conditional + string-single-bad-brace2-formatting-unicode + comment + odd format or format-like syntax begin - (\()\?\((\w+(?:\s+[[:alnum:]]+)?|\d+)\) - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.conditional.regexp - - 1 - - name - punctuation.parenthesis.conditional.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.conditional.negative.regexp punctuation.parenthesis.conditional.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-one-fregexp-parentheses-non-capturing - - begin - \(\?: - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-one-fregexp-parentheses - - begin - \( - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-three-fregexp-expression - - patterns - - - include - #fregexp-base-expression - - - include - #single-three-regexp-character-set - - - include - #single-three-regexp-comments - - - include - #regexp-flags - - - include - #single-three-regexp-named-group - - - include - #regexp-backreference - - - include - #single-three-fregexp-lookahead - - - include - #single-three-fregexp-lookahead-negative - - - include - #single-three-fregexp-lookbehind - - - include - #single-three-fregexp-lookbehind-negative - - - include - #single-three-fregexp-conditional - - - include - #single-three-fregexp-parentheses-non-capturing - - - include - #single-three-fregexp-parentheses - - - include - #comments-string-single-three - - - - single-three-fregexp-named-group - - name - meta.named.regexp - begin - (?x) - (\() (\?P <\w+(?:\s+[[:alnum:]]+)?>) - - end - (\)|(?=\'\'\')) - beginCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp - - 2 - - name - entity.name.tag.named.group.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-lookahead - - begin - (\()\?= - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - keyword.operator.lookahead.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-lookahead-negative - - begin - (\()\?! - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - keyword.operator.lookahead.negative.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-lookbehind - - begin - (\()\?<= - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-lookbehind-negative - - begin - (\()\?<! - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.negative.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.negative.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-conditional - - begin - (\()\?\((\w+(?:\s+[[:alnum:]]+)?|\d+)\) - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - keyword.operator.conditional.regexp - - 1 - - name - punctuation.parenthesis.conditional.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.conditional.negative.regexp punctuation.parenthesis.conditional.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-parentheses-non-capturing - - begin - \(\?: - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-parentheses - - begin - \( - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - double-one-fregexp-expression - - patterns - - - include - #fregexp-base-expression - - - include - #double-one-regexp-character-set - - - include - #double-one-regexp-comments - - - include - #regexp-flags - - - include - #double-one-regexp-named-group - - - include - #regexp-backreference - - - include - #double-one-fregexp-lookahead - - - include - #double-one-fregexp-lookahead-negative - - - include - #double-one-fregexp-lookbehind - - - include - #double-one-fregexp-lookbehind-negative - - - include - #double-one-fregexp-conditional - - - include - #double-one-fregexp-parentheses-non-capturing - - - include - #double-one-fregexp-parentheses - - - - double-one-fregexp-named-group - - name - meta.named.regexp - begin - (?x) - (\() (\?P <\w+(?:\s+[[:alnum:]]+)?>) - - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp - - 2 - - name - entity.name.tag.named.group.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-fregexp-expression - - - - double-one-fregexp-lookahead - - begin - (\()\?= - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookahead.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-fregexp-expression - - - - double-one-fregexp-lookahead-negative - - begin - (\()\?! - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookahead.negative.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-fregexp-expression - - - - double-one-fregexp-lookbehind - - begin - (\()\?<= - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-fregexp-expression - - - - double-one-fregexp-lookbehind-negative - - begin - (\()\?<! - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.negative.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.negative.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-fregexp-expression - - - - double-one-fregexp-conditional - - begin - (\()\?\((\w+(?:\s+[[:alnum:]]+)?|\d+)\) - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.conditional.regexp - - 1 - - name - punctuation.parenthesis.conditional.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.conditional.negative.regexp punctuation.parenthesis.conditional.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-fregexp-expression - - - - double-one-fregexp-parentheses-non-capturing - - begin - \(\?: - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-fregexp-expression - - - - double-one-fregexp-parentheses - - begin - \( - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-fregexp-expression - - - - double-three-fregexp-expression - - patterns - - - include - #fregexp-base-expression - - - include - #double-three-regexp-character-set - - - include - #double-three-regexp-comments - - - include - #regexp-flags - - - include - #double-three-regexp-named-group - - - include - #regexp-backreference - - - include - #double-three-fregexp-lookahead - - - include - #double-three-fregexp-lookahead-negative - - - include - #double-three-fregexp-lookbehind - - - include - #double-three-fregexp-lookbehind-negative - - - include - #double-three-fregexp-conditional - - - include - #double-three-fregexp-parentheses-non-capturing - - - include - #double-three-fregexp-parentheses - - - include - #comments-string-double-three - - - - double-three-fregexp-named-group - - name - meta.named.regexp - begin - (?x) - (\() (\?P <\w+(?:\s+[[:alnum:]]+)?>) - - end - (\)|(?=""")) - beginCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp - - 2 - - name - entity.name.tag.named.group.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-fregexp-expression - - - include - #comments-string-double-three - - - - double-three-fregexp-lookahead - - begin - (\()\?= - end - (\)|(?=""")) - beginCaptures - - 0 - - name - keyword.operator.lookahead.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-fregexp-expression - - - include - #comments-string-double-three - - - - double-three-fregexp-lookahead-negative - - begin - (\()\?! - end - (\)|(?=""")) - beginCaptures - - 0 - - name - keyword.operator.lookahead.negative.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-fregexp-expression - - - include - #comments-string-double-three - - - - double-three-fregexp-lookbehind - - begin - (\()\?<= - end - (\)|(?=""")) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-fregexp-expression - - - include - #comments-string-double-three - - - - double-three-fregexp-lookbehind-negative - - begin - (\()\?<! - end - (\)|(?=""")) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.negative.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.negative.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-fregexp-expression - - - include - #comments-string-double-three - - - - double-three-fregexp-conditional - - begin - (\()\?\((\w+(?:\s+[[:alnum:]]+)?|\d+)\) - end - (\)|(?=""")) - beginCaptures - - 0 - - name - keyword.operator.conditional.regexp - - 1 - - name - punctuation.parenthesis.conditional.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.conditional.negative.regexp punctuation.parenthesis.conditional.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-fregexp-expression - - - include - #comments-string-double-three - - - - double-three-fregexp-parentheses-non-capturing - - begin - \(\?: - end - (\)|(?=""")) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-fregexp-expression - - - include - #comments-string-double-three - - - - double-three-fregexp-parentheses - - begin - \( - end - (\)|(?=""")) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-fregexp-expression - - - include - #comments-string-double-three - - - - fregexp-single-one-line - - name - string.interpolated.python string.regexp.quoted.single.python - begin - \b(([uU]r)|([fF]r)|(r[fF]?))(\') - end - (\')|(?<!\\)(\n) - beginCaptures - - 2 - - name - invalid.deprecated.prefix.python - - 3 - - name - storage.type.string.python - - 4 - - name - storage.type.string.python - - 5 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - fregexp-single-three-line - - name - string.interpolated.python string.regexp.quoted.multi.python - begin - \b(([uU]r)|([fF]r)|(r[fF]?))(\'\'\') - end - (\'\'\') - beginCaptures - - 2 - - name - invalid.deprecated.prefix.python - - 3 - - name - storage.type.string.python - - 4 - - name - storage.type.string.python - - 5 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - - fregexp-double-one-line - - name - string.interpolated.python string.regexp.quoted.single.python - begin - \b(([uU]r)|([fF]r)|(r[fF]?))(") - end - (")|(?<!\\)(\n) - beginCaptures - - 2 - - name - invalid.deprecated.prefix.python - - 3 - - name - storage.type.string.python - - 4 - - name - storage.type.string.python - - 5 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-fregexp-expression - - - - fregexp-double-three-line - - name - string.interpolated.python string.regexp.quoted.multi.python - begin - \b(([uU]r)|([fF]r)|(r[fF]?))(""") - end - (""") - beginCaptures - - 2 - - name - invalid.deprecated.prefix.python - - 3 - - name - storage.type.string.python - - 4 - - name - storage.type.string.python - - 5 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-fregexp-expression - - - - string-raw-quoted-single-line - - name - string.quoted.raw.single.python - begin - \b(([uU]R)|(R))((['"])) - end - (\4)|((?<!\\)\n) - beginCaptures - - 2 - - name - invalid.deprecated.prefix.python - - 3 - - name - storage.type.string.python - - 4 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #string-single-bad-brace1-formatting-raw - - - include - #string-single-bad-brace2-formatting-raw - - - include - #string-raw-guts - - - - string-bin-quoted-single-line - - name - string.quoted.binary.single.python - begin - (\b[bB])((['"])) - end - (\2)|((?<!\\)\n) - beginCaptures - - 1 - - name - storage.type.string.python - - 2 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #string-entity - - - - string-raw-bin-quoted-single-line - - name - string.quoted.raw.binary.single.python - begin - (\b(?:R[bB]|[bB]R))((['"])) - end - (\2)|((?<!\\)\n) - beginCaptures - - 1 - - name - storage.type.string.python - - 2 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #string-raw-bin-guts - - - - string-quoted-single-line - - name - string.quoted.single.python - begin - (?:\b([rR])(?=[uU]))?([uU])?((['"])) - end - (\3)|((?<!\\)\n) - beginCaptures - - 1 - - name - invalid.illegal.prefix.python - - 2 - - name - storage.type.string.python - - 3 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #string-single-bad-brace1-formatting-unicode - - - include - #string-single-bad-brace2-formatting-unicode - - - include - #string-unicode-guts - - - - string-single-bad-brace1-formatting-unicode - - comment - template using {% ... %} - begin - (?x) - (?= \{% - ( .*? (?!(['"])|((?<!\\)\n)) ) - %\} - ) - - end - (?=(['"])|((?<!\\)\n)) - patterns - - - include - #escape-sequence-unicode - - - include - #escape-sequence - - - include - #string-line-continuation - - - - string-single-bad-brace1-formatting-raw - - comment - template using {% ... %} - begin - (?x) - (?= \{% - ( .*? (?!(['"])|((?<!\\)\n)) ) - %\} - ) - - end - (?=(['"])|((?<!\\)\n)) - patterns - - - include - #string-consume-escape - - - - string-single-bad-brace2-formatting-unicode - - comment - odd format or format-like syntax - begin - (?x) - (?!\{\{) - (?= \{ ( - \w*? (?!(['"])|((?<!\\)\n)) [^!:\.\[}\w] - ) - .*?(?!(['"])|((?<!\\)\n)) - \} - ) - + (?x) + (?!\{\{) + (?= \{ ( + \w*? (?!(['"])|((?<!\\)\n)) [^!:\.\[}\w] + ) + .*?(?!(['"])|((?<!\\)\n)) + \} + ) + end (?=(['"])|((?<!\\)\n)) patterns @@ -8118,7 +6374,7 @@ indirectly through syntactic constructs name meta.fstring.python begin - (\b(?:[R][fF]|[fF][R]))((['"])) + (\b(?:[rR][fF]|[fF][rR]))((['"])) end (\2)|((?<!\\)\n) beginCaptures @@ -8259,12 +6515,18 @@ indirectly through syntactic constructs name storage.type.format.python match - (![rsa])(?=}) + (=(![rsa])?)(?=}) + + + name + storage.type.format.python + match + (=?![rsa])(?=}) match (?x) - (![rsa])? + ( (?: =?) (?: ![rsa])? ) ( : \w? [<>=^]? [-+ ]? \#? \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )(?=}) @@ -8291,7 +6553,7 @@ indirectly through syntactic constructs fstring-terminator-single-tail begin - (![rsa])?(:)(?=.*?{) + ((?:=?)(?:![rsa])?)(:)(?=.*?{) end (?=})|(?=\n) beginCaptures @@ -8490,7 +6752,7 @@ indirectly through syntactic constructs name meta.fstring.python begin - (\b(?:[R][fF]|[fF][R]))('''|""") + (\b(?:[rR][fF]|[fF][rR]))('''|""") end (\2) beginCaptures @@ -8631,12 +6893,18 @@ indirectly through syntactic constructs name storage.type.format.python match - (![rsa])(?=}) + (=(![rsa])?)(?=}) + + + name + storage.type.format.python + match + (=?![rsa])(?=}) match (?x) - (![rsa])? + ( (?: =?) (?: ![rsa])? ) ( : \w? [<>=^]? [-+ ]? \#? \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )(?=}) @@ -8663,7 +6931,7 @@ indirectly through syntactic constructs fstring-terminator-multi-tail begin - (![rsa])?(:)(?=.*?{) + ((?:=?)(?:![rsa])?)(:)(?=.*?{) end (?=}) beginCaptures diff --git a/grammars/src/MagicPython.syntax.yaml b/grammars/src/MagicPython.syntax.yaml index 9c80c8b3..ebfccdaf 100644 --- a/grammars/src/MagicPython.syntax.yaml +++ b/grammars/src/MagicPython.syntax.yaml @@ -103,82 +103,6 @@ repository: bquote: '(""")' equote: '(""")' - - file: 'regexp.inc.syntax.yaml' - vars: - prefix: 'single-one-' - basename: 'fregexp' - marker: "|(?=\\')" - nested: '' - guard: "|((?=(?= | <= | < | >) (?# 5) + | (:=) (?# 6) + captures: '1': {name: keyword.operator.logical.python} '2': {name: keyword.control.flow.python} '3': {name: keyword.operator.bitwise.python} '4': {name: keyword.operator.arithmetic.python} '5': {name: keyword.operator.comparison.python} + '6': {name: keyword.operator.assignment.python} punctuation: patterns: @@ -697,10 +631,6 @@ repository: - include: '#regexp-double-three-line' - include: '#regexp-single-one-line' - include: '#regexp-double-one-line' - - include: '#fregexp-single-three-line' - - include: '#fregexp-double-three-line' - - include: '#fregexp-single-one-line' - - include: '#fregexp-double-one-line' string: patterns: @@ -1002,6 +932,8 @@ repository: contentName: meta.function.lambda.parameters.python patterns: + - name: keyword.operator.positional.parameter.python + match: / - name: keyword.operator.unpacking.parameter.python match: (\*\*|\*) - include: '#lambda-nested-incomplete' @@ -1096,6 +1028,8 @@ repository: '1': {name: punctuation.definition.parameters.end.python} patterns: + - name: keyword.operator.positional.parameter.python + match: / - name: keyword.operator.unpacking.parameter.python match: (\*\*|\*) - include: '#lambda-incomplete' @@ -1357,14 +1291,14 @@ repository: match: | (?x) (?=^]? [-+ ]? \#? \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )(?=}) captures: @@ -148,7 +150,7 @@ fstring-terminator-${line}: - include: '#fstring-terminator-${line}-tail' fstring-terminator-${line}-tail: - begin: (![rsa])?(:)(?=.*?{) + begin: ((?:=?)(?:![rsa])?)(:)(?=.*?{) end: (?=})${fguard} beginCaptures: '1': {name: storage.type.format.python} diff --git a/misc/example.py b/misc/example.py index 6b820e63..718f3074 100644 --- a/misc/example.py +++ b/misc/example.py @@ -1,7 +1,7 @@ import asyncio -def showcase(): +def showcase(a, b, /, c, d, *, e, f): """Some code to showcase the syntax. Docstrings are recognized and have an additional scope. @@ -21,7 +21,9 @@ async def coroutine(db:aio_db.DatabaseConnection) -> List[str]: async with db.transaction(): result = await db.query(...) - print(f'Result: {result!r}') + print(f'Result: {result!r} {a=} {b=!r}') + print(Rf'data: {c=}') + print(rf'data: {c=}') mapping = None # type: Dict[int, Any] # PEP 484 @@ -41,8 +43,14 @@ async def coroutine(db:aio_db.DatabaseConnection) -> List[str]: # NOTE Numbers with leading zeros are invalid in Python 3, # use 0o... answer = func(0xdeadbeef + 0b00100001 + 0123 + 0o123 + - 1_005_123 + # PEP 515 + 1_005_123 + # PEP 515 # complex numbers .10e12 + 2j) @ mat + # walrus operator + filtered_data = [y for x in data if (y := f(x)) is not None] + + # position-only params + bar = lambda q, w, /, e, r: (q + w + e + r) + return R'''No escapes '\' in this \one''' diff --git a/misc/scopes b/misc/scopes index 12785af1..8eec651e 100644 --- a/misc/scopes +++ b/misc/scopes @@ -56,6 +56,7 @@ keyword.operator.lookahead.regexp keyword.operator.lookbehind.negative.regexp keyword.operator.lookbehind.regexp keyword.operator.negation.regexp +keyword.operator.positional.parameter.python keyword.operator.python keyword.operator.quantifier.regexp keyword.operator.unpacking.arguments.python diff --git a/test/atom-spec/python-spec.js b/test/atom-spec/python-spec.js index 032c3682..79edf0a0 100644 --- a/test/atom-spec/python-spec.js +++ b/test/atom-spec/python-spec.js @@ -134,7 +134,7 @@ describe("Grammar Tests", function() { it("test/builtins/builtins3.py", function() { - tokens = grammar.tokenizeLines("__all__ = ['bar', 'baz']\nsome.__bases__\nsome.__class__\nassert __debug__\n__builtins__\n__builtins__.len\nprint(__builtins__)\nsome.__dict__\nsome.__doc__\nsome.__file__\nsome.__members__\nsome.__metaclass__\nsome.__methods__\nsome.__module__\nsome.__mro__\nsome.__name__\nsome.__slots__\nsome.__subclasses__\nsome.__version__\nsome.__weakref__\nsome.__qualname__\nsome.__code__\nsome.__wrapped__\nsome.__signature__\nsome.__defaults__\nsome.__func__\nsome.__self__\nsome.__kwdefaults__\nsome.__matmul__\nsome.__imatmul__\nsome.__rmatmul__\nsome.__annotations__\nsome.__init_subclass__\nsome.__set_name__\nsome.__fspath__\nsome.__classcell__\nsome.__bytes__\nsome.__spec__\nsome.__path__\nsome.__prepare__\nsome.__package__\nsome.__traceback__\nsome.__notspecial__") + tokens = grammar.tokenizeLines("__all__ = ['bar', 'baz']\nsome.__bases__\nsome.__class__\nassert __debug__\n__builtins__\n__builtins__.len\nprint(__builtins__)\nsome.__dict__\nsome.__doc__\nsome.__file__\nsome.__members__\nsome.__metaclass__\nsome.__methods__\nsome.__module__\nsome.__mro__\nsome.__name__\nsome.__slots__\nsome.__subclasses__\nsome.__version__\nsome.__weakref__\nsome.__qualname__\nsome.__code__\nsome.__wrapped__\nsome.__signature__\nsome.__defaults__\nsome.__func__\nsome.__self__\nsome.__kwdefaults__\nsome.__matmul__\nsome.__imatmul__\nsome.__rmatmul__\nsome.__annotations__\nsome.__init_subclass__\nsome.__set_name__\nsome.__fspath__\nsome.__classcell__\nsome.__bytes__\nsome.__spec__\nsome.__path__\nsome.__prepare__\nsome.__package__\nsome.__traceback__\nsome.__closure__\nsome.__globals__\nsome.__match_args__\nsome.__notspecial__") expect(tokens[0][0].value).toBe("__all__"); expect(tokens[0][0].scopes).toEqual(["source.python","support.variable.magic.python"]); expect(tokens[0][1].value).toBe(" "); @@ -411,8 +411,26 @@ describe("Grammar Tests", function() { expect(tokens[42][0].scopes).toEqual(["source.python"]); expect(tokens[42][1].value).toBe("."); expect(tokens[42][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); - expect(tokens[42][2].value).toBe("__notspecial__"); - expect(tokens[42][2].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[42][2].value).toBe("__closure__"); + expect(tokens[42][2].scopes).toEqual(["source.python","meta.member.access.python","support.variable.magic.python"]); + expect(tokens[43][0].value).toBe("some"); + expect(tokens[43][0].scopes).toEqual(["source.python"]); + expect(tokens[43][1].value).toBe("."); + expect(tokens[43][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[43][2].value).toBe("__globals__"); + expect(tokens[43][2].scopes).toEqual(["source.python","meta.member.access.python","support.variable.magic.python"]); + expect(tokens[44][0].value).toBe("some"); + expect(tokens[44][0].scopes).toEqual(["source.python"]); + expect(tokens[44][1].value).toBe("."); + expect(tokens[44][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[44][2].value).toBe("__match_args__"); + expect(tokens[44][2].scopes).toEqual(["source.python","meta.member.access.python","support.variable.magic.python"]); + expect(tokens[45][0].value).toBe("some"); + expect(tokens[45][0].scopes).toEqual(["source.python"]); + expect(tokens[45][1].value).toBe("."); + expect(tokens[45][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[45][2].value).toBe("__notspecial__"); + expect(tokens[45][2].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); }); it("test/builtins/builtins4.py", @@ -516,13 +534,25 @@ describe("Grammar Tests", function() { it("test/builtins/builtins7.py", function() { - tokens = grammar.tokenizeLines("breakpoint()") + tokens = grammar.tokenizeLines("breakpoint()\naiter()\nanext()") expect(tokens[0][0].value).toBe("breakpoint"); expect(tokens[0][0].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); expect(tokens[0][1].value).toBe("("); expect(tokens[0][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); expect(tokens[0][2].value).toBe(")"); expect(tokens[0][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][0].value).toBe("aiter"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[1][1].value).toBe("("); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][2].value).toBe(")"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[2][0].value).toBe("anext"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[2][1].value).toBe("("); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][2].value).toBe(")"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); }); it("test/calls/call1.py", @@ -2056,7 +2086,7 @@ describe("Grammar Tests", function() { it("test/classes/class9.py", function() { - tokens = grammar.tokenizeLines("class Foo:\n __slots__ = ()") + tokens = grammar.tokenizeLines("class Foo:\n __slots__ = ()\n __match_args__ = ('key', 'name')") expect(tokens[0][0].value).toBe("class"); expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); expect(tokens[0][1].value).toBe(" "); @@ -2079,6 +2109,36 @@ describe("Grammar Tests", function() { expect(tokens[1][5].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); expect(tokens[1][6].value).toBe(")"); expect(tokens[1][6].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("__match_args__"); + expect(tokens[2][1].scopes).toEqual(["source.python","support.variable.magic.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe("="); + expect(tokens[2][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("("); + expect(tokens[2][5].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[2][6].value).toBe("'"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][7].value).toBe("key"); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][8].value).toBe("'"); + expect(tokens[2][8].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][9].value).toBe(","); + expect(tokens[2][9].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][10].value).toBe(" "); + expect(tokens[2][10].scopes).toEqual(["source.python"]); + expect(tokens[2][11].value).toBe("'"); + expect(tokens[2][11].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][12].value).toBe("name"); + expect(tokens[2][12].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][13].value).toBe("'"); + expect(tokens[2][13].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][14].value).toBe(")"); + expect(tokens[2][14].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); }); it("test/classes/super1.py", @@ -5800,6 +5860,134 @@ describe("Grammar Tests", function() { expect(tokens[5][4].scopes).toEqual(["source.python"]); }); + it("test/expressions/expr21.py", + function() { + tokens = grammar.tokenizeLines("while chunk := file.read(8192):\n process(chunk)\n y0 = (y1 := f(x))") + expect(tokens[0][0].value).toBe("while"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("chunk"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe(":="); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("file"); + expect(tokens[0][6].scopes).toEqual(["source.python","variable.legacy.builtin.python"]); + expect(tokens[0][7].value).toBe("."); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][8].value).toBe("read"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][9].value).toBe("("); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][10].value).toBe("8192"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][11].value).toBe(")"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][12].value).toBe(":"); + expect(tokens[0][12].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("process"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][2].value).toBe("("); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][3].value).toBe("chunk"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("y0"); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe("="); + expect(tokens[2][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("("); + expect(tokens[2][5].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[2][6].value).toBe("y1"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe(":="); + expect(tokens[2][8].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[2][10].value).toBe("f"); + expect(tokens[2][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[2][11].value).toBe("("); + expect(tokens[2][11].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][12].value).toBe("x"); + expect(tokens[2][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[2][13].value).toBe(")"); + expect(tokens[2][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[2][14].value).toBe(")"); + expect(tokens[2][14].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + }); + + it("test/expressions/expr22.py", + function() { + tokens = grammar.tokenizeLines("match = 13\ncase = 12\nif case == 4:\n return match * 5") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("13"); + expect(tokens[0][4].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][0].value).toBe("case"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("12"); + expect(tokens[1][4].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][0].value).toBe("if"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("case"); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("=="); + expect(tokens[2][4].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("4"); + expect(tokens[2][6].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][7].value).toBe(":"); + expect(tokens[2][7].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("return"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("match"); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[3][5].value).toBe("*"); + expect(tokens[3][5].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[3][6].value).toBe(" "); + expect(tokens[3][6].scopes).toEqual(["source.python"]); + expect(tokens[3][7].value).toBe("5"); + expect(tokens[3][7].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + it("test/expressions/expr3.py", function() { tokens = grammar.tokenizeLines("(a, *rest, b) = range(5)") @@ -6269,7 +6457,7 @@ describe("Grammar Tests", function() { expect(tokens[1][1].value).toBe(" "); expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); expect(tokens[1][2].value).toBe("__class_getitem__"); - expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","support.variable.magic.python"]); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","support.function.magic.python"]); expect(tokens[1][3].value).toBe("("); expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); expect(tokens[1][4].value).toBe(")"); @@ -6539,350 +6727,48 @@ describe("Grammar Tests", function() { function() { tokens = grammar.tokenizeLines("rf\"{} { }\"\nrf\"\"\"{}\n{ }\n\"\"\"") expect(tokens[0][0].value).toBe("rf"); - expect(tokens[0][0].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); expect(tokens[0][1].value).toBe("\""); - expect(tokens[0][1].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); expect(tokens[0][2].value).toBe("{"); - expect(tokens[0][2].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); expect(tokens[0][3].value).toBe("}"); - expect(tokens[0][3].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); expect(tokens[0][4].value).toBe(" "); - expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); expect(tokens[0][5].value).toBe("{"); - expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); expect(tokens[0][6].value).toBe(" "); - expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","invalid.illegal.brace.python"]); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.brace.python"]); expect(tokens[0][7].value).toBe("}"); - expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); expect(tokens[0][8].value).toBe("\""); - expect(tokens[0][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); expect(tokens[1][0].value).toBe("rf"); - expect(tokens[1][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","storage.type.string.python"]); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.multi.python string.interpolated.python"]); expect(tokens[1][1].value).toBe("\"\"\""); - expect(tokens[1][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python punctuation.definition.string.begin.python"]); expect(tokens[1][2].value).toBe("{"); - expect(tokens[1][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); expect(tokens[1][3].value).toBe("}"); - expect(tokens[1][3].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe(""); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); expect(tokens[2][0].value).toBe("{"); - expect(tokens[2][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); expect(tokens[2][1].value).toBe(" "); - expect(tokens[2][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","invalid.illegal.brace.python"]); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.brace.python"]); expect(tokens[2][2].value).toBe("}"); - expect(tokens[2][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][3].value).toBe(""); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); expect(tokens[3][0].value).toBe("\"\"\""); - expect(tokens[3][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","punctuation.definition.string.end.python"]); - }); - - it("test/fstrings/nested1.py", - function() { - tokens = grammar.tokenizeLines("f'''\n prefix {\n foo(f\"\"\"\n inner prefix\n { bar[\"q\"] + f'insane{42 + 9000}stuff{def aaa(): pass}111'}\n inner suffix\n \"\"\")\n } suffix\n'''") - expect(tokens[0][0].value).toBe("f"); - expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); - expect(tokens[0][1].value).toBe("'''"); - expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[0][2].value).toBe(""); - expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[1][0].value).toBe(" prefix "); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[1][1].value).toBe("{"); - expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[2][0].value).toBe(" "); - expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[2][1].value).toBe("foo"); - expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.generic.python"]); - expect(tokens[2][2].value).toBe("("); - expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); - expect(tokens[2][3].value).toBe("f"); - expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); - expect(tokens[2][4].value).toBe("\"\"\""); - expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[2][5].value).toBe(""); - expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[3][0].value).toBe(" inner prefix"); - expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[4][0].value).toBe(" "); - expect(tokens[4][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[4][1].value).toBe("{"); - expect(tokens[4][1].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[4][2].value).toBe(" "); - expect(tokens[4][2].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); - expect(tokens[4][3].value).toBe("bar"); - expect(tokens[4][3].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.indexed-name.python"]); - expect(tokens[4][4].value).toBe("["); - expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); - expect(tokens[4][5].value).toBe("\""); - expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.item-access.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[4][6].value).toBe("q"); - expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.item-access.arguments.python","string.quoted.single.python"]); - expect(tokens[4][7].value).toBe("\""); - expect(tokens[4][7].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.item-access.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); - expect(tokens[4][8].value).toBe("]"); - expect(tokens[4][8].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); - expect(tokens[4][9].value).toBe(" "); - expect(tokens[4][9].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); - expect(tokens[4][10].value).toBe("+"); - expect(tokens[4][10].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","keyword.operator.arithmetic.python"]); - expect(tokens[4][11].value).toBe(" "); - expect(tokens[4][11].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); - expect(tokens[4][12].value).toBe("f"); - expect(tokens[4][12].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[4][13].value).toBe("'"); - expect(tokens[4][13].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[4][14].value).toBe("insane"); - expect(tokens[4][14].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[4][15].value).toBe("{"); - expect(tokens[4][15].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[4][16].value).toBe("42"); - expect(tokens[4][16].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.numeric.dec.python"]); - expect(tokens[4][17].value).toBe(" "); - expect(tokens[4][17].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); - expect(tokens[4][18].value).toBe("+"); - expect(tokens[4][18].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","keyword.operator.arithmetic.python"]); - expect(tokens[4][19].value).toBe(" "); - expect(tokens[4][19].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); - expect(tokens[4][20].value).toBe("9000"); - expect(tokens[4][20].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.numeric.dec.python"]); - expect(tokens[4][21].value).toBe("}"); - expect(tokens[4][21].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[4][22].value).toBe("stuff"); - expect(tokens[4][22].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[4][23].value).toBe("{"); - expect(tokens[4][23].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[4][24].value).toBe("def"); - expect(tokens[4][24].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","keyword.control.flow.python"]); - expect(tokens[4][25].value).toBe(" "); - expect(tokens[4][25].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); - expect(tokens[4][26].value).toBe("aaa"); - expect(tokens[4][26].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","meta.function-call.python","meta.function-call.generic.python"]); - expect(tokens[4][27].value).toBe("("); - expect(tokens[4][27].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); - expect(tokens[4][28].value).toBe(")"); - expect(tokens[4][28].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); - expect(tokens[4][29].value).toBe(":"); - expect(tokens[4][29].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","punctuation.separator.colon.python"]); - expect(tokens[4][30].value).toBe(" "); - expect(tokens[4][30].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); - expect(tokens[4][31].value).toBe("pass"); - expect(tokens[4][31].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","keyword.control.flow.python"]); - expect(tokens[4][32].value).toBe("}"); - expect(tokens[4][32].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[4][33].value).toBe("111"); - expect(tokens[4][33].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[4][34].value).toBe("'"); - expect(tokens[4][34].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[4][35].value).toBe("}"); - expect(tokens[4][35].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[4][36].value).toBe(""); - expect(tokens[4][36].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[5][0].value).toBe(" inner suffix"); - expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[6][0].value).toBe(" "); - expect(tokens[6][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[6][1].value).toBe("\"\"\""); - expect(tokens[6][1].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[6][2].value).toBe(")"); - expect(tokens[6][2].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); - expect(tokens[7][0].value).toBe(" "); - expect(tokens[7][0].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[7][1].value).toBe("}"); - expect(tokens[7][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[7][2].value).toBe(" suffix"); - expect(tokens[7][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[8][0].value).toBe("'''"); - expect(tokens[8][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python punctuation.definition.string.end.python"]); }); - it("test/fstrings/nested2.py", + it("test/fstrings/fraw1.py", function() { - tokens = grammar.tokenizeLines("f\"result: {value:{60}.{16!s:2}{'qwerty'[2]}}\"") - expect(tokens[0][0].value).toBe("f"); - expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[0][1].value).toBe("\""); - expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[0][2].value).toBe("result: "); - expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[0][3].value).toBe("{"); - expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][4].value).toBe("value"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[0][5].value).toBe(":"); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); - expect(tokens[0][6].value).toBe("{"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][7].value).toBe("60"); - expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); - expect(tokens[0][8].value).toBe("}"); - expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][9].value).toBe("."); - expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[0][10].value).toBe("{"); - expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][11].value).toBe("16"); - expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); - expect(tokens[0][12].value).toBe("!s"); - expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); - expect(tokens[0][13].value).toBe(":2"); - expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); - expect(tokens[0][14].value).toBe("}"); - expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][15].value).toBe("{"); - expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][16].value).toBe("'"); - expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][17].value).toBe("qwerty"); - expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python"]); - expect(tokens[0][18].value).toBe("'"); - expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.end.python"]); - expect(tokens[0][19].value).toBe("["); - expect(tokens[0][19].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.begin.python"]); - expect(tokens[0][20].value).toBe("2"); - expect(tokens[0][20].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); - expect(tokens[0][21].value).toBe("]"); - expect(tokens[0][21].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.end.python"]); - expect(tokens[0][22].value).toBe("}"); - expect(tokens[0][22].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][23].value).toBe("}"); - expect(tokens[0][23].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][24].value).toBe("\""); - expect(tokens[0][24].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); - }); - - it("test/fstrings/nested3.py", - function() { - tokens = grammar.tokenizeLines("f\"result: {value:{60}.{16!s:2}{'qwerty'\n[2]}}\"\n# comment") - expect(tokens[0][0].value).toBe("f"); - expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[0][1].value).toBe("\""); - expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[0][2].value).toBe("result: "); - expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[0][3].value).toBe("{"); - expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][4].value).toBe("value"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[0][5].value).toBe(":"); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); - expect(tokens[0][6].value).toBe("{"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][7].value).toBe("60"); - expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); - expect(tokens[0][8].value).toBe("}"); - expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][9].value).toBe("."); - expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[0][10].value).toBe("{"); - expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][11].value).toBe("16"); - expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); - expect(tokens[0][12].value).toBe("!s"); - expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); - expect(tokens[0][13].value).toBe(":2"); - expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); - expect(tokens[0][14].value).toBe("}"); - expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][15].value).toBe("{"); - expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][16].value).toBe("'"); - expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][17].value).toBe("qwerty"); - expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python"]); - expect(tokens[0][18].value).toBe("'"); - expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.end.python"]); - expect(tokens[0][19].value).toBe(""); - expect(tokens[0][19].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.newline.python"]); - expect(tokens[1][0].value).toBe("["); - expect(tokens[1][0].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); - expect(tokens[1][1].value).toBe("2"); - expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - expect(tokens[1][2].value).toBe("]"); - expect(tokens[1][2].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); - expect(tokens[1][3].value).toBe("}}"); - expect(tokens[1][3].scopes).toEqual(["source.python"]); - expect(tokens[1][4].value).toBe("\""); - expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[1][5].value).toBe(""); - expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python","invalid.illegal.newline.python"]); - expect(tokens[2][0].value).toBe("#"); - expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[2][1].value).toBe(" comment"); - expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - }); - - it("test/fstrings/nested4.py", - function() { - tokens = grammar.tokenizeLines("f\"\"\"result: {value:{60}.{16!s:2}{'qwerty'\n[2]}}\"\"\"\ndef foo(): pass") - expect(tokens[0][0].value).toBe("f"); - expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); - expect(tokens[0][1].value).toBe("\"\"\""); - expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[0][2].value).toBe("result: "); - expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[0][3].value).toBe("{"); - expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][4].value).toBe("value"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[0][5].value).toBe(":"); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); - expect(tokens[0][6].value).toBe("{"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][7].value).toBe("60"); - expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); - expect(tokens[0][8].value).toBe("}"); - expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][9].value).toBe("."); - expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[0][10].value).toBe("{"); - expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][11].value).toBe("16"); - expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); - expect(tokens[0][12].value).toBe("!s"); - expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); - expect(tokens[0][13].value).toBe(":2"); - expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); - expect(tokens[0][14].value).toBe("}"); - expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][15].value).toBe("{"); - expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][16].value).toBe("'"); - expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][17].value).toBe("qwerty"); - expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python"]); - expect(tokens[0][18].value).toBe("'"); - expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.end.python"]); - expect(tokens[1][0].value).toBe("["); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.begin.python"]); - expect(tokens[1][1].value).toBe("2"); - expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); - expect(tokens[1][2].value).toBe("]"); - expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.end.python"]); - expect(tokens[1][3].value).toBe("}"); - expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[1][4].value).toBe("}"); - expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[1][5].value).toBe("\"\"\""); - expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[2][0].value).toBe("def"); - expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); - expect(tokens[2][1].value).toBe(" "); - expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); - expect(tokens[2][2].value).toBe("foo"); - expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); - expect(tokens[2][3].value).toBe("("); - expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); - expect(tokens[2][4].value).toBe(")"); - expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); - expect(tokens[2][5].value).toBe(":"); - expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); - expect(tokens[2][6].value).toBe(" "); - expect(tokens[2][6].scopes).toEqual(["source.python"]); - expect(tokens[2][7].value).toBe("pass"); - expect(tokens[2][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); - }); - - it("test/fstrings/prefixes1.py", - function() { - tokens = grammar.tokenizeLines("a = f's t r'\na = f\"s t r\"\na = F's t r'\na = F\"s t r\"\na = f'''s t r'''\na = F\"\"\"s t r\"\"\"") + tokens = grammar.tokenizeLines("a = fr'[a-z]'\na = Fr'[a-z]'\na = rf'[a-z]'\na = rF'[a-z]'") expect(tokens[0][0].value).toBe("a"); expect(tokens[0][0].scopes).toEqual(["source.python"]); expect(tokens[0][1].value).toBe(" "); @@ -6891,14 +6777,14 @@ describe("Grammar Tests", function() { expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); expect(tokens[0][3].value).toBe(" "); expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("f"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][4].value).toBe("fr"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); expect(tokens[0][5].value).toBe("'"); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[0][6].value).toBe("s t r"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("[a-z]"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); expect(tokens[0][7].value).toBe("'"); - expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); expect(tokens[1][0].value).toBe("a"); expect(tokens[1][0].scopes).toEqual(["source.python"]); expect(tokens[1][1].value).toBe(" "); @@ -6907,14 +6793,14 @@ describe("Grammar Tests", function() { expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); expect(tokens[1][3].value).toBe(" "); expect(tokens[1][3].scopes).toEqual(["source.python"]); - expect(tokens[1][4].value).toBe("f"); - expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[1][5].value).toBe("\""); - expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[1][6].value).toBe("s t r"); - expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[1][7].value).toBe("\""); - expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][4].value).toBe("Fr"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("[a-z]"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); expect(tokens[2][0].value).toBe("a"); expect(tokens[2][0].scopes).toEqual(["source.python"]); expect(tokens[2][1].value).toBe(" "); @@ -6923,14 +6809,14 @@ describe("Grammar Tests", function() { expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); expect(tokens[2][3].value).toBe(" "); expect(tokens[2][3].scopes).toEqual(["source.python"]); - expect(tokens[2][4].value).toBe("F"); - expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][4].value).toBe("rf"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); expect(tokens[2][5].value).toBe("'"); - expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[2][6].value).toBe("s t r"); - expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("[a-z]"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); expect(tokens[2][7].value).toBe("'"); - expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); expect(tokens[3][0].value).toBe("a"); expect(tokens[3][0].scopes).toEqual(["source.python"]); expect(tokens[3][1].value).toBe(" "); @@ -6939,271 +6825,863 @@ describe("Grammar Tests", function() { expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); expect(tokens[3][3].value).toBe(" "); expect(tokens[3][3].scopes).toEqual(["source.python"]); - expect(tokens[3][4].value).toBe("F"); - expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[3][5].value).toBe("\""); - expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[3][6].value).toBe("s t r"); - expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[3][7].value).toBe("\""); - expect(tokens[3][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[4][0].value).toBe("a"); - expect(tokens[4][0].scopes).toEqual(["source.python"]); - expect(tokens[4][1].value).toBe(" "); - expect(tokens[4][1].scopes).toEqual(["source.python"]); - expect(tokens[4][2].value).toBe("="); - expect(tokens[4][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[4][3].value).toBe(" "); - expect(tokens[4][3].scopes).toEqual(["source.python"]); - expect(tokens[4][4].value).toBe("f"); - expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); - expect(tokens[4][5].value).toBe("'''"); - expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[4][6].value).toBe("s t r"); - expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[4][7].value).toBe("'''"); - expect(tokens[4][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[5][0].value).toBe("a"); - expect(tokens[5][0].scopes).toEqual(["source.python"]); - expect(tokens[5][1].value).toBe(" "); - expect(tokens[5][1].scopes).toEqual(["source.python"]); - expect(tokens[5][2].value).toBe("="); - expect(tokens[5][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[5][3].value).toBe(" "); - expect(tokens[5][3].scopes).toEqual(["source.python"]); - expect(tokens[5][4].value).toBe("F"); - expect(tokens[5][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); - expect(tokens[5][5].value).toBe("\"\"\""); - expect(tokens[5][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[5][6].value).toBe("s t r"); - expect(tokens[5][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); - expect(tokens[5][7].value).toBe("\"\"\""); - expect(tokens[5][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][4].value).toBe("rF"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][5].value).toBe("'"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[3][6].value).toBe("[a-z]"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][7].value).toBe("'"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); }); - it("test/fstrings/prefixes2.py", + it("test/fstrings/fraw2.py", function() { - tokens = grammar.tokenizeLines("f'some {obj}'\nF'some {obj}'\nrf'some {obj}'\nrF'some {obj}'\nRf'some {obj}'\nRF'some {obj}'") - expect(tokens[0][0].value).toBe("f"); - expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + tokens = grammar.tokenizeLines("rf'fo{{2}}'\nrf\"fo{{2}}\"\nrf'''fo{{2}}'''\nrf\"\"\"fo{{2}}\"\"\"") + expect(tokens[0][0].value).toBe("rf"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); expect(tokens[0][1].value).toBe("'"); - expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[0][2].value).toBe("some "); - expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[0][3].value).toBe("{"); - expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][4].value).toBe("obj"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[0][5].value).toBe("}"); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][6].value).toBe("'"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[1][0].value).toBe("F"); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[1][1].value).toBe("'"); - expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[1][2].value).toBe("some "); - expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[1][3].value).toBe("{"); - expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[1][4].value).toBe("obj"); - expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[1][5].value).toBe("}"); - expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[1][6].value).toBe("'"); - expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("fo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][4].value).toBe("2"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("}}"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("rf"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("\""); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("fo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][3].value).toBe("{{"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[1][4].value).toBe("2"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][5].value).toBe("}}"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[1][6].value).toBe("\""); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); expect(tokens[2][0].value).toBe("rf"); - expect(tokens[2][0].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[2][1].value).toBe("'"); - expect(tokens[2][1].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[2][2].value).toBe("some "); - expect(tokens[2][2].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[2][3].value).toBe("{obj}"); - expect(tokens[2][3].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[2][4].value).toBe("'"); - expect(tokens[2][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[3][0].value).toBe("rF"); - expect(tokens[3][0].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[3][1].value).toBe("'"); - expect(tokens[3][1].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[3][2].value).toBe("some "); - expect(tokens[3][2].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[3][3].value).toBe("{obj}"); - expect(tokens[3][3].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[3][4].value).toBe("'"); - expect(tokens[3][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[4][0].value).toBe("Rf"); - expect(tokens[4][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); - expect(tokens[4][1].value).toBe("'"); - expect(tokens[4][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); - expect(tokens[4][2].value).toBe("some "); - expect(tokens[4][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); - expect(tokens[4][3].value).toBe("{"); - expect(tokens[4][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[4][4].value).toBe("obj"); - expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[4][5].value).toBe("}"); - expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[4][6].value).toBe("'"); - expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[5][0].value).toBe("RF"); - expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); - expect(tokens[5][1].value).toBe("'"); - expect(tokens[5][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); - expect(tokens[5][2].value).toBe("some "); - expect(tokens[5][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); - expect(tokens[5][3].value).toBe("{"); - expect(tokens[5][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[5][4].value).toBe("obj"); - expect(tokens[5][4].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[5][5].value).toBe("}"); - expect(tokens[5][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[5][6].value).toBe("'"); - expect(tokens[5][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("'''"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("fo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[2][3].value).toBe("{{"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[2][4].value).toBe("2"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[2][5].value).toBe("}}"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[2][6].value).toBe("'''"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("rf"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("\"\"\""); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe("fo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[3][3].value).toBe("{{"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[3][4].value).toBe("2"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[3][5].value).toBe("}}"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[3][6].value).toBe("\"\"\""); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python punctuation.definition.string.end.python"]); }); - it("test/fstrings/prefixes3.py", + it("test/fstrings/fraw3.py", function() { - tokens = grammar.tokenizeLines("fr'some {obj}'\nFr'some {obj}'\nfR'some {obj}'\nFR'some {obj}'") - expect(tokens[0][0].value).toBe("fr"); - expect(tokens[0][0].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); + tokens = grammar.tokenizeLines("rf'fo{2}'\nrf\"fo{2}\"\nrf'''fo{2}'''\nrf\"\"\"fo{2}\"\"\"") + expect(tokens[0][0].value).toBe("rf"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); expect(tokens[0][1].value).toBe("'"); - expect(tokens[0][1].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][2].value).toBe("some "); - expect(tokens[0][2].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[0][3].value).toBe("{obj}"); - expect(tokens[0][3].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[0][4].value).toBe("'"); - expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[1][0].value).toBe("Fr"); - expect(tokens[1][0].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[1][1].value).toBe("'"); - expect(tokens[1][1].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[1][2].value).toBe("some "); - expect(tokens[1][2].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[1][3].value).toBe("{obj}"); - expect(tokens[1][3].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[1][4].value).toBe("'"); - expect(tokens[1][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[2][0].value).toBe("fR"); - expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); - expect(tokens[2][1].value).toBe("'"); - expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); - expect(tokens[2][2].value).toBe("some "); - expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("fo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("2"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][5].value).toBe("}"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("rf"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("\""); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("fo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][3].value).toBe("{"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe("2"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[1][5].value).toBe("}"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][6].value).toBe("\""); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("rf"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("'''"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("fo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); expect(tokens[2][3].value).toBe("{"); expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[2][4].value).toBe("obj"); - expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][4].value).toBe("2"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); expect(tokens[2][5].value).toBe("}"); expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[2][6].value).toBe("'"); - expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[3][0].value).toBe("FR"); - expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); - expect(tokens[3][1].value).toBe("'"); - expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); - expect(tokens[3][2].value).toBe("some "); - expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][6].value).toBe("'''"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("rf"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("\"\"\""); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe("fo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); expect(tokens[3][3].value).toBe("{"); expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[3][4].value).toBe("obj"); - expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[3][4].value).toBe("2"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); expect(tokens[3][5].value).toBe("}"); expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[3][6].value).toBe("'"); - expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][6].value).toBe("\"\"\""); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python punctuation.definition.string.end.python"]); }); - it("test/fstrings/prefixes4.py", + it("test/fstrings/fraw4.py", function() { - tokens = grammar.tokenizeLines("fu'some {obj}'\nFu'some {obj}'\nfU'some {obj}'\nFU'some {obj}'\n\nuf'some {obj}'\nuF'some {obj}'\nUf'some {obj}'\nUF'some {obj}'") + tokens = grammar.tokenizeLines("a = rf'fo{{{2}}}'\na = rf'fo{{{bar}}}'\na = rf'fo{{2}}'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("rf"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("fo"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][7].value).toBe("{{"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][8].value).toBe("{"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("2"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][10].value).toBe("}"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("}}"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][12].value).toBe("'"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("rf"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("fo"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][7].value).toBe("{{"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[1][8].value).toBe("{"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][9].value).toBe("bar"); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][10].value).toBe("}"); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][11].value).toBe("}}"); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[1][12].value).toBe("'"); + expect(tokens[1][12].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("rf"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][5].value).toBe("'"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("fo"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][7].value).toBe("{{"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[2][8].value).toBe("2"); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][9].value).toBe("}}"); + expect(tokens[2][9].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[2][10].value).toBe("'"); + expect(tokens[2][10].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/nested1.py", + function() { + tokens = grammar.tokenizeLines("f'''\n prefix {\n foo(f\"\"\"\n inner prefix\n { bar[\"q\"] + f'insane{42 + 9000}stuff{def aaa(): pass}111'}\n inner suffix\n \"\"\")\n } suffix\n'''") expect(tokens[0][0].value).toBe("f"); - expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[0][1].value).toBe("u"); - expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); - expect(tokens[0][2].value).toBe("'"); - expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[0][3].value).toBe("some "); - expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[0][4].value).toBe("{"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][5].value).toBe("obj"); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[0][6].value).toBe("}"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[0][7].value).toBe("'"); - expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[1][0].value).toBe("F"); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[1][1].value).toBe("u"); - expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); - expect(tokens[1][2].value).toBe("'"); - expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[1][3].value).toBe("some "); - expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[1][4].value).toBe("{"); - expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[1][5].value).toBe("obj"); - expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[1][6].value).toBe("}"); - expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[1][7].value).toBe("'"); - expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[2][0].value).toBe("f"); - expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[2][1].value).toBe("U"); - expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); - expect(tokens[2][2].value).toBe("'"); - expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[2][3].value).toBe("some "); - expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[2][4].value).toBe("{"); - expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[2][5].value).toBe("obj"); - expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[2][6].value).toBe("}"); - expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[2][7].value).toBe("'"); - expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[3][0].value).toBe("F"); - expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[3][1].value).toBe("U"); - expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); - expect(tokens[3][2].value).toBe("'"); - expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); - expect(tokens[3][3].value).toBe("some "); - expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[3][4].value).toBe("{"); - expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[3][5].value).toBe("obj"); - expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[3][6].value).toBe("}"); - expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[3][7].value).toBe("'"); - expect(tokens[3][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[4][0].value).toBe(""); - expect(tokens[4][0].scopes).toEqual(["source.python"]); - expect(tokens[5][0].value).toBe("u"); - expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); - expect(tokens[5][1].value).toBe("f"); - expect(tokens[5][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); - expect(tokens[5][2].value).toBe("'"); - expect(tokens[5][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); - expect(tokens[5][3].value).toBe("some "); - expect(tokens[5][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); - expect(tokens[5][4].value).toBe("{"); - expect(tokens[5][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[5][5].value).toBe("obj"); - expect(tokens[5][5].scopes).toEqual(["source.python","meta.fstring.python"]); - expect(tokens[5][6].value).toBe("}"); - expect(tokens[5][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); - expect(tokens[5][7].value).toBe("'"); - expect(tokens[5][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); - expect(tokens[6][0].value).toBe("u"); - expect(tokens[6][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); - expect(tokens[6][1].value).toBe("F"); - expect(tokens[6][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'''"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe(""); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][0].value).toBe(" prefix "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("{"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][1].value).toBe("foo"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[2][2].value).toBe("("); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe("f"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[2][4].value).toBe("\"\"\""); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[2][5].value).toBe(""); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[3][0].value).toBe(" inner prefix"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[4][1].value).toBe("{"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[4][3].value).toBe("bar"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.indexed-name.python"]); + expect(tokens[4][4].value).toBe("["); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][5].value).toBe("\""); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.item-access.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][6].value).toBe("q"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.item-access.arguments.python","string.quoted.single.python"]); + expect(tokens[4][7].value).toBe("\""); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.item-access.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][8].value).toBe("]"); + expect(tokens[4][8].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); + expect(tokens[4][9].value).toBe(" "); + expect(tokens[4][9].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[4][10].value).toBe("+"); + expect(tokens[4][10].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[4][11].value).toBe(" "); + expect(tokens[4][11].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[4][12].value).toBe("f"); + expect(tokens[4][12].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][13].value).toBe("'"); + expect(tokens[4][13].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[4][14].value).toBe("insane"); + expect(tokens[4][14].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][15].value).toBe("{"); + expect(tokens[4][15].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][16].value).toBe("42"); + expect(tokens[4][16].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[4][17].value).toBe(" "); + expect(tokens[4][17].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); + expect(tokens[4][18].value).toBe("+"); + expect(tokens[4][18].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[4][19].value).toBe(" "); + expect(tokens[4][19].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); + expect(tokens[4][20].value).toBe("9000"); + expect(tokens[4][20].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[4][21].value).toBe("}"); + expect(tokens[4][21].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][22].value).toBe("stuff"); + expect(tokens[4][22].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][23].value).toBe("{"); + expect(tokens[4][23].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][24].value).toBe("def"); + expect(tokens[4][24].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","keyword.control.flow.python"]); + expect(tokens[4][25].value).toBe(" "); + expect(tokens[4][25].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); + expect(tokens[4][26].value).toBe("aaa"); + expect(tokens[4][26].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[4][27].value).toBe("("); + expect(tokens[4][27].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][28].value).toBe(")"); + expect(tokens[4][28].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[4][29].value).toBe(":"); + expect(tokens[4][29].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","punctuation.separator.colon.python"]); + expect(tokens[4][30].value).toBe(" "); + expect(tokens[4][30].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); + expect(tokens[4][31].value).toBe("pass"); + expect(tokens[4][31].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","keyword.control.flow.python"]); + expect(tokens[4][32].value).toBe("}"); + expect(tokens[4][32].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][33].value).toBe("111"); + expect(tokens[4][33].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][34].value).toBe("'"); + expect(tokens[4][34].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][35].value).toBe("}"); + expect(tokens[4][35].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][36].value).toBe(""); + expect(tokens[4][36].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[5][0].value).toBe(" inner suffix"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[6][1].value).toBe("\"\"\""); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][2].value).toBe(")"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[7][1].value).toBe("}"); + expect(tokens[7][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[7][2].value).toBe(" suffix"); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[8][0].value).toBe("'''"); + expect(tokens[8][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/nested2.py", + function() { + tokens = grammar.tokenizeLines("f\"result: {value:{60}.{16!s:2}{'qwerty'[2]}}\"") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("\""); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("result: "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("value"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][6].value).toBe("{"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("60"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe("}"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("."); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][10].value).toBe("{"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("16"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][12].value).toBe("!s"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][13].value).toBe(":2"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][14].value).toBe("}"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][15].value).toBe("{"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][16].value).toBe("'"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][17].value).toBe("qwerty"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python"]); + expect(tokens[0][18].value).toBe("'"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][19].value).toBe("["); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.begin.python"]); + expect(tokens[0][20].value).toBe("2"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][21].value).toBe("]"); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.end.python"]); + expect(tokens[0][22].value).toBe("}"); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][23].value).toBe("}"); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][24].value).toBe("\""); + expect(tokens[0][24].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/nested3.py", + function() { + tokens = grammar.tokenizeLines("f\"result: {value:{60}.{16!s:2}{'qwerty'\n[2]}}\"\n# comment") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("\""); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("result: "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("value"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][6].value).toBe("{"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("60"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe("}"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("."); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][10].value).toBe("{"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("16"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][12].value).toBe("!s"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][13].value).toBe(":2"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][14].value).toBe("}"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][15].value).toBe("{"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][16].value).toBe("'"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][17].value).toBe("qwerty"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python"]); + expect(tokens[0][18].value).toBe("'"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][19].value).toBe(""); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.newline.python"]); + expect(tokens[1][0].value).toBe("["); + expect(tokens[1][0].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[1][1].value).toBe("2"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][2].value).toBe("]"); + expect(tokens[1][2].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[1][3].value).toBe("}}"); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("\""); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][5].value).toBe(""); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python","invalid.illegal.newline.python"]); + expect(tokens[2][0].value).toBe("#"); + expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][1].value).toBe(" comment"); + expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/fstrings/nested4.py", + function() { + tokens = grammar.tokenizeLines("f\"\"\"result: {value:{60}.{16!s:2}{'qwerty'\n[2]}}\"\"\"\ndef foo(): pass") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("\"\"\""); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("result: "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("value"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][6].value).toBe("{"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("60"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe("}"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("."); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][10].value).toBe("{"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("16"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][12].value).toBe("!s"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][13].value).toBe(":2"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][14].value).toBe("}"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][15].value).toBe("{"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][16].value).toBe("'"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][17].value).toBe("qwerty"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python"]); + expect(tokens[0][18].value).toBe("'"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("["); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.begin.python"]); + expect(tokens[1][1].value).toBe("2"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[1][2].value).toBe("]"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.end.python"]); + expect(tokens[1][3].value).toBe("}"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe("}"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][5].value).toBe("\"\"\""); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe(")"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][5].value).toBe(":"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[2][6].value).toBe(" "); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe("pass"); + expect(tokens[2][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/fstrings/prefixes1.py", + function() { + tokens = grammar.tokenizeLines("a = f's t r'\na = f\"s t r\"\na = F's t r'\na = F\"s t r\"\na = f'''s t r'''\na = F\"\"\"s t r\"\"\"") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("s t r"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("f"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][5].value).toBe("\""); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("s t r"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][7].value).toBe("\""); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("F"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][5].value).toBe("'"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("s t r"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("="); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("F"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[3][5].value).toBe("\""); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[3][6].value).toBe("s t r"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[3][7].value).toBe("\""); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe("a"); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("="); + expect(tokens[4][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[4][3].value).toBe(" "); + expect(tokens[4][3].scopes).toEqual(["source.python"]); + expect(tokens[4][4].value).toBe("f"); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[4][5].value).toBe("'''"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[4][6].value).toBe("s t r"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[4][7].value).toBe("'''"); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("a"); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe("="); + expect(tokens[5][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python"]); + expect(tokens[5][4].value).toBe("F"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[5][5].value).toBe("\"\"\""); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[5][6].value).toBe("s t r"); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[5][7].value).toBe("\"\"\""); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/prefixes2.py", + function() { + tokens = grammar.tokenizeLines("f'some {obj}'\nF'some {obj}'\nrf'some {obj}'\nrF'some {obj}'\nRf'some {obj}'\nRF'some {obj}'") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("some "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("obj"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe("}"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("F"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("'"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("some "); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][3].value).toBe("{"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe("obj"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][5].value).toBe("}"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][6].value).toBe("'"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("rf"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("'"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("some "); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][3].value).toBe("{"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][4].value).toBe("obj"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][5].value).toBe("}"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][6].value).toBe("'"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("rF"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("'"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe("some "); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][3].value).toBe("{"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][4].value).toBe("obj"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[3][5].value).toBe("}"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][6].value).toBe("'"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe("Rf"); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[4][1].value).toBe("'"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[4][2].value).toBe("some "); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[4][3].value).toBe("{"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][4].value).toBe("obj"); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[4][5].value).toBe("}"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][6].value).toBe("'"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("RF"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[5][1].value).toBe("'"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[5][2].value).toBe("some "); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[5][3].value).toBe("{"); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[5][4].value).toBe("obj"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[5][5].value).toBe("}"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[5][6].value).toBe("'"); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/prefixes3.py", + function() { + tokens = grammar.tokenizeLines("fr'some {obj}'\nFr'some {obj}'\nfR'some {obj}'\nFR'some {obj}'") + expect(tokens[0][0].value).toBe("fr"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("some "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("obj"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe("}"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("Fr"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("'"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("some "); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][3].value).toBe("{"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe("obj"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][5].value).toBe("}"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][6].value).toBe("'"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("fR"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("'"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("some "); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][3].value).toBe("{"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][4].value).toBe("obj"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][5].value).toBe("}"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][6].value).toBe("'"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("FR"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("'"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe("some "); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][3].value).toBe("{"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][4].value).toBe("obj"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[3][5].value).toBe("}"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][6].value).toBe("'"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/prefixes4.py", + function() { + tokens = grammar.tokenizeLines("fu'some {obj}'\nFu'some {obj}'\nfU'some {obj}'\nFU'some {obj}'\n\nuf'some {obj}'\nuF'some {obj}'\nUf'some {obj}'\nUF'some {obj}'") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("u"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[0][2].value).toBe("'"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][3].value).toBe("some "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][4].value).toBe("{"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][5].value).toBe("obj"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][6].value).toBe("}"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("F"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("u"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[1][2].value).toBe("'"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[1][3].value).toBe("some "); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][4].value).toBe("{"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][5].value).toBe("obj"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][6].value).toBe("}"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("f"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("U"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[2][2].value).toBe("'"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[2][3].value).toBe("some "); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][4].value).toBe("{"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][5].value).toBe("obj"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][6].value).toBe("}"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("F"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("U"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[3][2].value).toBe("'"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[3][3].value).toBe("some "); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[3][4].value).toBe("{"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][5].value).toBe("obj"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[3][6].value).toBe("}"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][7].value).toBe("'"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[5][0].value).toBe("u"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[5][1].value).toBe("f"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[5][2].value).toBe("'"); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); + expect(tokens[5][3].value).toBe("some "); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[5][4].value).toBe("{"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[5][5].value).toBe("obj"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[5][6].value).toBe("}"); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[5][7].value).toBe("'"); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe("u"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[6][1].value).toBe("F"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); expect(tokens[6][2].value).toBe("'"); expect(tokens[6][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); expect(tokens[6][3].value).toBe("some "); @@ -7490,6 +7968,37 @@ describe("Grammar Tests", function() { expect(tokens[2][20].scopes).toEqual(["source.python","comment.line.number-sign.python"]); }); + it("test/fstrings/simple10.py", + function() { + tokens = grammar.tokenizeLines("f'values: {a=} {b=!r}'") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("values: "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("a"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe("="); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][6].value).toBe("}"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][8].value).toBe("{"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("b"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][10].value).toBe("=!r"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][11].value).toBe("}"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][12].value).toBe("'"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + it("test/fstrings/simple2.py", function() { tokens = grammar.tokenizeLines("a = f\"normal {{ normal }} normal } {10!r} normal {fo.__add__!s}\"") @@ -8501,6 +9010,67 @@ describe("Grammar Tests", function() { expect(tokens[14][9].scopes).toEqual(["source.python","constant.numeric.dec.python"]); }); + it("test/functions/decl15.py", + function() { + tokens = grammar.tokenizeLines("def showcase(a, b, /, c, d, *, e, f):\n return") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("showcase"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe("a"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][5].value).toBe(","); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][7].value).toBe("b"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][8].value).toBe(","); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][10].value).toBe("/"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.positional.parameter.python"]); + expect(tokens[0][11].value).toBe(", "); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][12].value).toBe("c"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][13].value).toBe(","); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][14].value).toBe(" "); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][15].value).toBe("d"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][16].value).toBe(","); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][17].value).toBe(" "); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][18].value).toBe("*"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[0][19].value).toBe(", "); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][20].value).toBe("e"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][21].value).toBe(","); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][22].value).toBe(" "); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][23].value).toBe("f"); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][24].value).toBe(")"); + expect(tokens[0][24].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][25].value).toBe(":"); + expect(tokens[0][25].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("return"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + it("test/functions/decl2.py", function() { tokens = grammar.tokenizeLines("def result_annot(lambda, lambda=) -> qqq[None]:\n pass") @@ -9575,254 +10145,67 @@ describe("Grammar Tests", function() { expect(tokens[8][6].value).toBe(" "); expect(tokens[8][6].scopes).toEqual(["source.python"]); expect(tokens[8][7].value).toBe("pass"); - expect(tokens[8][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); - }); - - it("test/functions/decorators8.py", - function() { - tokens = grammar.tokenizeLines("@deco().abc # SyntaxError: invalid syntax\ndef foo(): pass") - expect(tokens[0][0].value).toBe("@"); - expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); - expect(tokens[0][1].value).toBe("deco"); - expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); - expect(tokens[0][2].value).toBe("("); - expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); - expect(tokens[0][3].value).toBe(")"); - expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); - expect(tokens[0][4].value).toBe(".abc"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.decorator.python","invalid.illegal.decorator.python"]); - expect(tokens[0][5].value).toBe(" "); - expect(tokens[0][5].scopes).toEqual(["source.python"]); - expect(tokens[0][6].value).toBe("#"); - expect(tokens[0][6].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[0][7].value).toBe(" SyntaxError: invalid syntax"); - expect(tokens[0][7].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - expect(tokens[1][0].value).toBe("def"); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); - expect(tokens[1][1].value).toBe(" "); - expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); - expect(tokens[1][2].value).toBe("foo"); - expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); - expect(tokens[1][3].value).toBe("("); - expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); - expect(tokens[1][4].value).toBe(")"); - expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); - expect(tokens[1][5].value).toBe(":"); - expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); - expect(tokens[1][6].value).toBe(" "); - expect(tokens[1][6].scopes).toEqual(["source.python"]); - expect(tokens[1][7].value).toBe("pass"); - expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); - }); - - it("test/functions/lambda1.py", - function() { - tokens = grammar.tokenizeLines("# That's how we want it:\nlll(lambda)") - expect(tokens[0][0].value).toBe("#"); - expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[0][1].value).toBe(" That's how we want it:"); - expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - expect(tokens[1][0].value).toBe("lll"); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); - expect(tokens[1][1].value).toBe("("); - expect(tokens[1][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); - expect(tokens[1][2].value).toBe("lambda"); - expect(tokens[1][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","storage.type.function.lambda.python"]); - expect(tokens[1][3].value).toBe(")"); - expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); - }); - - it("test/functions/lambda2.py", - function() { - tokens = grammar.tokenizeLines("lll(lambda=1)") - expect(tokens[0][0].value).toBe("lll"); - expect(tokens[0][0].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); - expect(tokens[0][1].value).toBe("("); - expect(tokens[0][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); - expect(tokens[0][2].value).toBe("lambda"); - expect(tokens[0][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.control.flow.python"]); - expect(tokens[0][3].value).toBe("="); - expect(tokens[0][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); - expect(tokens[0][4].value).toBe("1"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); - expect(tokens[0][5].value).toBe(")"); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); - }); - - it("test/functions/lambda3.py", - function() { - tokens = grammar.tokenizeLines("anon = lambda lambda: 42") - expect(tokens[0][0].value).toBe("anon"); - expect(tokens[0][0].scopes).toEqual(["source.python"]); - expect(tokens[0][1].value).toBe(" "); - expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("="); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("lambda"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); - expect(tokens[0][5].value).toBe(" "); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][6].value).toBe("lambda"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","storage.type.function.lambda.python"]); - expect(tokens[0][7].value).toBe(":"); - expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); - expect(tokens[0][8].value).toBe(" "); - expect(tokens[0][8].scopes).toEqual(["source.python"]); - expect(tokens[0][9].value).toBe("42"); - expect(tokens[0][9].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - }); - - it("test/functions/lambda4.py", - function() { - tokens = grammar.tokenizeLines("lambda a : 1") - expect(tokens[0][0].value).toBe("lambda"); - expect(tokens[0][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); - expect(tokens[0][1].value).toBe(" "); - expect(tokens[0][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][2].value).toBe("a"); - expect(tokens[0][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][4].value).toBe(":"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); - expect(tokens[0][5].value).toBe(" "); - expect(tokens[0][5].scopes).toEqual(["source.python"]); - expect(tokens[0][6].value).toBe("1"); - expect(tokens[0][6].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - }); - - it("test/functions/lambda5.py", - function() { - tokens = grammar.tokenizeLines("anon = lambda -> qqq[None]: None\ndef f(): return 1 # this line should not break") - expect(tokens[0][0].value).toBe("anon"); - expect(tokens[0][0].scopes).toEqual(["source.python"]); - expect(tokens[0][1].value).toBe(" "); - expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("="); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("lambda"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); - expect(tokens[0][5].value).toBe(" "); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][6].value).toBe("->"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.illegal.annotation.python"]); - expect(tokens[0][7].value).toBe(" qqq[None]"); - expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][8].value).toBe(":"); - expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); - expect(tokens[0][9].value).toBe(" "); - expect(tokens[0][9].scopes).toEqual(["source.python"]); - expect(tokens[0][10].value).toBe("None"); - expect(tokens[0][10].scopes).toEqual(["source.python","constant.language.python"]); - expect(tokens[1][0].value).toBe("def"); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); - expect(tokens[1][1].value).toBe(" "); - expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); - expect(tokens[1][2].value).toBe("f"); - expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); - expect(tokens[1][3].value).toBe("("); - expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); - expect(tokens[1][4].value).toBe(")"); - expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); - expect(tokens[1][5].value).toBe(":"); - expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); - expect(tokens[1][6].value).toBe(" "); - expect(tokens[1][6].scopes).toEqual(["source.python"]); - expect(tokens[1][7].value).toBe("return"); - expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); - expect(tokens[1][8].value).toBe(" "); - expect(tokens[1][8].scopes).toEqual(["source.python"]); - expect(tokens[1][9].value).toBe("1"); - expect(tokens[1][9].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - expect(tokens[1][10].value).toBe(" "); - expect(tokens[1][10].scopes).toEqual(["source.python"]); - expect(tokens[1][11].value).toBe("#"); - expect(tokens[1][11].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[1][12].value).toBe(" this line should not break"); - expect(tokens[1][12].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - }); - - it("test/functions/lambda6.py", - function() { - tokens = grammar.tokenizeLines("anon = lambda a=123, c={'': 555}, \\\n d=toow(24): None") - expect(tokens[0][0].value).toBe("anon"); - expect(tokens[0][0].scopes).toEqual(["source.python"]); - expect(tokens[0][1].value).toBe(" "); - expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("="); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("lambda"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); - expect(tokens[0][5].value).toBe(" "); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][6].value).toBe("a"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); - expect(tokens[0][7].value).toBe("="); - expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); - expect(tokens[0][8].value).toBe("123"); - expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","constant.numeric.dec.python"]); - expect(tokens[0][9].value).toBe(","); - expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); - expect(tokens[0][10].value).toBe(" "); - expect(tokens[0][10].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][11].value).toBe("c"); - expect(tokens[0][11].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); - expect(tokens[0][12].value).toBe("="); - expect(tokens[0][12].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); - expect(tokens[0][13].value).toBe("{"); - expect(tokens[0][13].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.begin.python"]); - expect(tokens[0][14].value).toBe("'"); - expect(tokens[0][14].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][15].value).toBe("'"); - expect(tokens[0][15].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); - expect(tokens[0][16].value).toBe(":"); - expect(tokens[0][16].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.dict.python"]); - expect(tokens[0][17].value).toBe(" "); - expect(tokens[0][17].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][18].value).toBe("555"); - expect(tokens[0][18].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","constant.numeric.dec.python"]); - expect(tokens[0][19].value).toBe("}"); - expect(tokens[0][19].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.end.python"]); - expect(tokens[0][20].value).toBe(","); - expect(tokens[0][20].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); - expect(tokens[0][21].value).toBe(" "); - expect(tokens[0][21].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][22].value).toBe("\\"); - expect(tokens[0][22].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.continuation.line.python"]); - expect(tokens[0][23].value).toBe(""); - expect(tokens[0][23].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[1][0].value).toBe(" "); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[1][1].value).toBe("d"); - expect(tokens[1][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); - expect(tokens[1][2].value).toBe("="); - expect(tokens[1][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); - expect(tokens[1][3].value).toBe("toow"); - expect(tokens[1][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","meta.function-call.generic.python"]); - expect(tokens[1][4].value).toBe("("); - expect(tokens[1][4].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); - expect(tokens[1][5].value).toBe("24"); - expect(tokens[1][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); - expect(tokens[1][6].value).toBe(")"); - expect(tokens[1][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); - expect(tokens[1][7].value).toBe(":"); - expect(tokens[1][7].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); - expect(tokens[1][8].value).toBe(" "); - expect(tokens[1][8].scopes).toEqual(["source.python"]); - expect(tokens[1][9].value).toBe("None"); - expect(tokens[1][9].scopes).toEqual(["source.python","constant.language.python"]); + expect(tokens[8][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); }); - it("test/functions/lambda7.py", + it("test/functions/decorators8.py", function() { - tokens = grammar.tokenizeLines("anon = lambda a, c={'key':\n\n 555}, e=fff: None") - expect(tokens[0][0].value).toBe("anon"); + tokens = grammar.tokenizeLines("@deco().abc # SyntaxError: invalid syntax\ndef foo(): pass") + expect(tokens[0][0].value).toBe("@"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[0][1].value).toBe("deco"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][2].value).toBe("("); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][3].value).toBe(")"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][4].value).toBe(".abc"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.decorator.python","invalid.illegal.decorator.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("#"); + expect(tokens[0][6].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][7].value).toBe(" SyntaxError: invalid syntax"); + expect(tokens[0][7].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/lambda1.py", + function() { + tokens = grammar.tokenizeLines("# That's how we want it:\nlll(lambda)") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" That's how we want it:"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("lll"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][1].value).toBe("("); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][2].value).toBe("lambda"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","storage.type.function.lambda.python"]); + expect(tokens[1][3].value).toBe(")"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/functions/lambda10.py", + function() { + tokens = grammar.tokenizeLines("showcase = lambda a, /, b, *, c: (a + b + c)") + expect(tokens[0][0].value).toBe("showcase"); expect(tokens[0][0].scopes).toEqual(["source.python"]); expect(tokens[0][1].value).toBe(" "); expect(tokens[0][1].scopes).toEqual(["source.python"]); @@ -9840,49 +10223,70 @@ describe("Grammar Tests", function() { expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); expect(tokens[0][8].value).toBe(" "); expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][9].value).toBe("c"); - expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); - expect(tokens[0][10].value).toBe("="); - expect(tokens[0][10].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); - expect(tokens[0][11].value).toBe("{"); - expect(tokens[0][11].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.begin.python"]); - expect(tokens[0][12].value).toBe("'"); - expect(tokens[0][12].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][13].value).toBe("key"); - expect(tokens[0][13].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python"]); - expect(tokens[0][14].value).toBe("'"); - expect(tokens[0][14].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); - expect(tokens[0][15].value).toBe(":"); - expect(tokens[0][15].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.dict.python"]); - expect(tokens[1][0].value).toBe(""); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[2][0].value).toBe(" "); - expect(tokens[2][0].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[2][1].value).toBe("555"); - expect(tokens[2][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","constant.numeric.dec.python"]); - expect(tokens[2][2].value).toBe("}"); - expect(tokens[2][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.end.python"]); - expect(tokens[2][3].value).toBe(","); - expect(tokens[2][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); - expect(tokens[2][4].value).toBe(" "); - expect(tokens[2][4].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[2][5].value).toBe("e"); - expect(tokens[2][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); - expect(tokens[2][6].value).toBe("="); - expect(tokens[2][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); - expect(tokens[2][7].value).toBe("fff"); - expect(tokens[2][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[2][8].value).toBe(":"); - expect(tokens[2][8].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); - expect(tokens[2][9].value).toBe(" "); - expect(tokens[2][9].scopes).toEqual(["source.python"]); - expect(tokens[2][10].value).toBe("None"); - expect(tokens[2][10].scopes).toEqual(["source.python","constant.language.python"]); + expect(tokens[0][9].value).toBe("/"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.positional.parameter.python"]); + expect(tokens[0][10].value).toBe(", "); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][11].value).toBe("b"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][12].value).toBe(","); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][14].value).toBe("*"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[0][15].value).toBe(", "); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][16].value).toBe("c"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][17].value).toBe(":"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[0][18].value).toBe(" "); + expect(tokens[0][18].scopes).toEqual(["source.python"]); + expect(tokens[0][19].value).toBe("("); + expect(tokens[0][19].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][20].value).toBe("a"); + expect(tokens[0][20].scopes).toEqual(["source.python"]); + expect(tokens[0][21].value).toBe(" "); + expect(tokens[0][21].scopes).toEqual(["source.python"]); + expect(tokens[0][22].value).toBe("+"); + expect(tokens[0][22].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][23].value).toBe(" "); + expect(tokens[0][23].scopes).toEqual(["source.python"]); + expect(tokens[0][24].value).toBe("b"); + expect(tokens[0][24].scopes).toEqual(["source.python"]); + expect(tokens[0][25].value).toBe(" "); + expect(tokens[0][25].scopes).toEqual(["source.python"]); + expect(tokens[0][26].value).toBe("+"); + expect(tokens[0][26].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][27].value).toBe(" "); + expect(tokens[0][27].scopes).toEqual(["source.python"]); + expect(tokens[0][28].value).toBe("c"); + expect(tokens[0][28].scopes).toEqual(["source.python"]); + expect(tokens[0][29].value).toBe(")"); + expect(tokens[0][29].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); }); - it("test/functions/lambda8.py", + it("test/functions/lambda2.py", function() { - tokens = grammar.tokenizeLines("anon = lambda a,\n d=1: None\ndef foo(): pass") + tokens = grammar.tokenizeLines("lll(lambda=1)") + expect(tokens[0][0].value).toBe("lll"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][1].value).toBe("("); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][2].value).toBe("lambda"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.control.flow.python"]); + expect(tokens[0][3].value).toBe("="); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[0][4].value).toBe("1"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][5].value).toBe(")"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/functions/lambda3.py", + function() { + tokens = grammar.tokenizeLines("anon = lambda lambda: 42") expect(tokens[0][0].value).toBe("anon"); expect(tokens[0][0].scopes).toEqual(["source.python"]); expect(tokens[0][1].value).toBe(" "); @@ -9895,159 +10299,39 @@ describe("Grammar Tests", function() { expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); expect(tokens[0][5].value).toBe(" "); expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][6].value).toBe("a"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); - expect(tokens[0][7].value).toBe(","); - expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); - expect(tokens[0][8].value).toBe(""); - expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python"]); - expect(tokens[1][0].value).toBe(" "); - expect(tokens[1][0].scopes).toEqual(["source.python"]); - expect(tokens[1][1].value).toBe("d"); - expect(tokens[1][1].scopes).toEqual(["source.python"]); - expect(tokens[1][2].value).toBe("="); - expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[1][3].value).toBe("1"); - expect(tokens[1][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - expect(tokens[1][4].value).toBe(":"); - expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); - expect(tokens[1][5].value).toBe(" "); - expect(tokens[1][5].scopes).toEqual(["source.python"]); - expect(tokens[1][6].value).toBe("None"); - expect(tokens[1][6].scopes).toEqual(["source.python","constant.language.python"]); - expect(tokens[2][0].value).toBe("def"); - expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); - expect(tokens[2][1].value).toBe(" "); - expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); - expect(tokens[2][2].value).toBe("foo"); - expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); - expect(tokens[2][3].value).toBe("("); - expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); - expect(tokens[2][4].value).toBe(")"); - expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); - expect(tokens[2][5].value).toBe(":"); - expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); - expect(tokens[2][6].value).toBe(" "); - expect(tokens[2][6].scopes).toEqual(["source.python"]); - expect(tokens[2][7].value).toBe("pass"); - expect(tokens[2][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); - }); - - it("test/functions/lambda9.py", - function() { - tokens = grammar.tokenizeLines("lambda *, b: b\nlambda *b: b\nlambda **b: b") - expect(tokens[0][0].value).toBe("lambda"); - expect(tokens[0][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); - expect(tokens[0][1].value).toBe(" "); - expect(tokens[0][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][2].value).toBe("*"); - expect(tokens[0][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); - expect(tokens[0][3].value).toBe(", "); - expect(tokens[0][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][4].value).toBe("b"); - expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); - expect(tokens[0][5].value).toBe(":"); - expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); - expect(tokens[0][6].value).toBe(" "); - expect(tokens[0][6].scopes).toEqual(["source.python"]); - expect(tokens[0][7].value).toBe("b"); - expect(tokens[0][7].scopes).toEqual(["source.python"]); - expect(tokens[1][0].value).toBe("lambda"); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); - expect(tokens[1][1].value).toBe(" "); - expect(tokens[1][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[1][2].value).toBe("*"); - expect(tokens[1][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); - expect(tokens[1][3].value).toBe("b"); - expect(tokens[1][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); - expect(tokens[1][4].value).toBe(":"); - expect(tokens[1][4].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); - expect(tokens[1][5].value).toBe(" "); - expect(tokens[1][5].scopes).toEqual(["source.python"]); - expect(tokens[1][6].value).toBe("b"); - expect(tokens[1][6].scopes).toEqual(["source.python"]); - expect(tokens[2][0].value).toBe("lambda"); - expect(tokens[2][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); - expect(tokens[2][1].value).toBe(" "); - expect(tokens[2][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[2][2].value).toBe("**"); - expect(tokens[2][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); - expect(tokens[2][3].value).toBe("b"); - expect(tokens[2][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); - expect(tokens[2][4].value).toBe(":"); - expect(tokens[2][4].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); - expect(tokens[2][5].value).toBe(" "); - expect(tokens[2][5].scopes).toEqual(["source.python"]); - expect(tokens[2][6].value).toBe("b"); - expect(tokens[2][6].scopes).toEqual(["source.python"]); - }); - - it("test/illegals/backticks1.py", - function() { - tokens = grammar.tokenizeLines("a = `(1, `2`)`") - expect(tokens[0][0].value).toBe("a"); - expect(tokens[0][0].scopes).toEqual(["source.python"]); - expect(tokens[0][1].value).toBe(" "); - expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("="); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("`"); - expect(tokens[0][4].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); - expect(tokens[0][5].value).toBe("("); - expect(tokens[0][5].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.parenthesis.begin.python"]); - expect(tokens[0][6].value).toBe("1"); - expect(tokens[0][6].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); - expect(tokens[0][7].value).toBe(","); - expect(tokens[0][7].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.separator.element.python"]); - expect(tokens[0][8].value).toBe(" "); - expect(tokens[0][8].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); - expect(tokens[0][9].value).toBe("`"); - expect(tokens[0][9].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","invalid.deprecated.backtick.python"]); - expect(tokens[0][10].value).toBe("2"); - expect(tokens[0][10].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); - expect(tokens[0][11].value).toBe("`"); - expect(tokens[0][11].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","invalid.deprecated.backtick.python"]); - expect(tokens[0][12].value).toBe(")"); - expect(tokens[0][12].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.parenthesis.end.python"]); - expect(tokens[0][13].value).toBe("`"); - expect(tokens[0][13].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][6].value).toBe("lambda"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","storage.type.function.lambda.python"]); + expect(tokens[0][7].value).toBe(":"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("42"); + expect(tokens[0][9].scopes).toEqual(["source.python","constant.numeric.dec.python"]); }); - it("test/illegals/backticks2.py", + it("test/functions/lambda4.py", function() { - tokens = grammar.tokenizeLines("a = `\\\n123`\nprint a") - expect(tokens[0][0].value).toBe("a"); - expect(tokens[0][0].scopes).toEqual(["source.python"]); + tokens = grammar.tokenizeLines("lambda a : 1") + expect(tokens[0][0].value).toBe("lambda"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); expect(tokens[0][1].value).toBe(" "); - expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("="); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][2].value).toBe("a"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("`"); - expect(tokens[0][4].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); - expect(tokens[0][5].value).toBe("\\"); - expect(tokens[0][5].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.separator.continuation.line.python"]); - expect(tokens[0][6].value).toBe(""); - expect(tokens[0][6].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); - expect(tokens[1][0].value).toBe("123"); - expect(tokens[1][0].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); - expect(tokens[1][1].value).toBe("`"); - expect(tokens[1][1].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); - expect(tokens[2][0].value).toBe("print"); - expect(tokens[2][0].scopes).toEqual(["source.python","support.function.builtin.python"]); - expect(tokens[2][1].value).toBe(" "); - expect(tokens[2][1].scopes).toEqual(["source.python"]); - expect(tokens[2][2].value).toBe("a"); - expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][4].value).toBe(":"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("1"); + expect(tokens[0][6].scopes).toEqual(["source.python","constant.numeric.dec.python"]); }); - it("test/illegals/backticks3.py", + it("test/functions/lambda5.py", function() { - tokens = grammar.tokenizeLines("a = lambda `123`\n# comment") - expect(tokens[0][0].value).toBe("a"); + tokens = grammar.tokenizeLines("anon = lambda -> qqq[None]: None\ndef f(): return 1 # this line should not break") + expect(tokens[0][0].value).toBe("anon"); expect(tokens[0][0].scopes).toEqual(["source.python"]); expect(tokens[0][1].value).toBe(" "); expect(tokens[0][1].scopes).toEqual(["source.python"]); @@ -10059,25 +10343,16 @@ describe("Grammar Tests", function() { expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); expect(tokens[0][5].value).toBe(" "); expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); - expect(tokens[0][6].value).toBe("`"); - expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python"]); - expect(tokens[0][7].value).toBe("123"); - expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); - expect(tokens[0][8].value).toBe("`"); - expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python"]); - expect(tokens[0][9].value).toBe(""); - expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python"]); - expect(tokens[1][0].value).toBe("#"); - expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[1][1].value).toBe(" comment"); - expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - }); - - it("test/illegals/illegal1.py", - function() { - tokens = grammar.tokenizeLines("->\ndef f(): pass\n$\n?\na=$('.class').fuuuu(baz=1)\n# we recover just fine\nb = !some_ruby?\n# hey ;)") - expect(tokens[0][0].value).toBe("->"); - expect(tokens[0][0].scopes).toEqual(["source.python","invalid.illegal.annotation.python"]); + expect(tokens[0][6].value).toBe("->"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.illegal.annotation.python"]); + expect(tokens[0][7].value).toBe(" qqq[None]"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][8].value).toBe(":"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("None"); + expect(tokens[0][10].scopes).toEqual(["source.python","constant.language.python"]); expect(tokens[1][0].value).toBe("def"); expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); expect(tokens[1][1].value).toBe(" "); @@ -10092,772 +10367,768 @@ describe("Grammar Tests", function() { expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); expect(tokens[1][6].value).toBe(" "); expect(tokens[1][6].scopes).toEqual(["source.python"]); - expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].value).toBe("return"); expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); - expect(tokens[2][0].value).toBe("$"); - expect(tokens[2][0].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); - expect(tokens[3][0].value).toBe("?"); - expect(tokens[3][0].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); - expect(tokens[4][0].value).toBe("a"); - expect(tokens[4][0].scopes).toEqual(["source.python"]); - expect(tokens[4][1].value).toBe("="); - expect(tokens[4][1].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[4][2].value).toBe("$"); - expect(tokens[4][2].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); - expect(tokens[4][3].value).toBe("("); - expect(tokens[4][3].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); - expect(tokens[4][4].value).toBe("'"); - expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[4][5].value).toBe(".class"); - expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.single.python"]); - expect(tokens[4][6].value).toBe("'"); - expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); - expect(tokens[4][7].value).toBe(")"); - expect(tokens[4][7].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); - expect(tokens[4][8].value).toBe("."); - expect(tokens[4][8].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); - expect(tokens[4][9].value).toBe("fuuuu"); - expect(tokens[4][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); - expect(tokens[4][10].value).toBe("("); - expect(tokens[4][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); - expect(tokens[4][11].value).toBe("baz"); - expect(tokens[4][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); - expect(tokens[4][12].value).toBe("="); - expect(tokens[4][12].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); - expect(tokens[4][13].value).toBe("1"); - expect(tokens[4][13].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); - expect(tokens[4][14].value).toBe(")"); - expect(tokens[4][14].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); - expect(tokens[5][0].value).toBe("#"); - expect(tokens[5][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[5][1].value).toBe(" we recover just fine"); - expect(tokens[5][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - expect(tokens[6][0].value).toBe("b"); - expect(tokens[6][0].scopes).toEqual(["source.python"]); - expect(tokens[6][1].value).toBe(" "); - expect(tokens[6][1].scopes).toEqual(["source.python"]); - expect(tokens[6][2].value).toBe("="); - expect(tokens[6][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[6][3].value).toBe(" "); - expect(tokens[6][3].scopes).toEqual(["source.python"]); - expect(tokens[6][4].value).toBe("!"); - expect(tokens[6][4].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); - expect(tokens[6][5].value).toBe("some_ruby"); - expect(tokens[6][5].scopes).toEqual(["source.python"]); - expect(tokens[6][6].value).toBe("?"); - expect(tokens[6][6].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); - expect(tokens[7][0].value).toBe("#"); - expect(tokens[7][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[7][1].value).toBe(" hey ;)"); - expect(tokens[7][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][8].value).toBe(" "); + expect(tokens[1][8].scopes).toEqual(["source.python"]); + expect(tokens[1][9].value).toBe("1"); + expect(tokens[1][9].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][10].value).toBe(" "); + expect(tokens[1][10].scopes).toEqual(["source.python"]); + expect(tokens[1][11].value).toBe("#"); + expect(tokens[1][11].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][12].value).toBe(" this line should not break"); + expect(tokens[1][12].scopes).toEqual(["source.python","comment.line.number-sign.python"]); }); - it("test/illegals/illegal2.py", + it("test/functions/lambda6.py", function() { - tokens = grammar.tokenizeLines("a&&b||c\na &&= a\nb ||= c") - expect(tokens[0][0].value).toBe("a"); + tokens = grammar.tokenizeLines("anon = lambda a=123, c={'': 555}, \\\n d=toow(24): None") + expect(tokens[0][0].value).toBe("anon"); expect(tokens[0][0].scopes).toEqual(["source.python"]); - expect(tokens[0][1].value).toBe("&&"); - expect(tokens[0][1].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); - expect(tokens[0][2].value).toBe("b"); - expect(tokens[0][2].scopes).toEqual(["source.python"]); - expect(tokens[0][3].value).toBe("||"); - expect(tokens[0][3].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); - expect(tokens[0][4].value).toBe("c"); - expect(tokens[0][4].scopes).toEqual(["source.python"]); - expect(tokens[1][0].value).toBe("a"); - expect(tokens[1][0].scopes).toEqual(["source.python"]); - expect(tokens[1][1].value).toBe(" "); - expect(tokens[1][1].scopes).toEqual(["source.python"]); - expect(tokens[1][2].value).toBe("&&"); - expect(tokens[1][2].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); - expect(tokens[1][3].value).toBe("="); - expect(tokens[1][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[1][4].value).toBe(" "); - expect(tokens[1][4].scopes).toEqual(["source.python"]); - expect(tokens[1][5].value).toBe("a"); - expect(tokens[1][5].scopes).toEqual(["source.python"]); - expect(tokens[2][0].value).toBe("b"); - expect(tokens[2][0].scopes).toEqual(["source.python"]); - expect(tokens[2][1].value).toBe(" "); - expect(tokens[2][1].scopes).toEqual(["source.python"]); - expect(tokens[2][2].value).toBe("||"); - expect(tokens[2][2].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); - expect(tokens[2][3].value).toBe("="); - expect(tokens[2][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[2][4].value).toBe(" "); - expect(tokens[2][4].scopes).toEqual(["source.python"]); - expect(tokens[2][5].value).toBe("c"); - expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][7].value).toBe("="); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[0][8].value).toBe("123"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][9].value).toBe(","); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][11].value).toBe("c"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][12].value).toBe("="); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[0][13].value).toBe("{"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.begin.python"]); + expect(tokens[0][14].value).toBe("'"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][15].value).toBe("'"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][16].value).toBe(":"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.dict.python"]); + expect(tokens[0][17].value).toBe(" "); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][18].value).toBe("555"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][19].value).toBe("}"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.end.python"]); + expect(tokens[0][20].value).toBe(","); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][21].value).toBe(" "); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][22].value).toBe("\\"); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][23].value).toBe(""); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[1][1].value).toBe("d"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[1][3].value).toBe("toow"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][4].value).toBe("("); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][5].value).toBe("24"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[1][6].value).toBe(")"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][7].value).toBe(":"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[1][8].value).toBe(" "); + expect(tokens[1][8].scopes).toEqual(["source.python"]); + expect(tokens[1][9].value).toBe("None"); + expect(tokens[1][9].scopes).toEqual(["source.python","constant.language.python"]); }); - it("test/linecont/linecont1.py", + it("test/functions/lambda7.py", function() { - tokens = grammar.tokenizeLines("1 + \\ sdgfsdf\ndef foo(): pass") - expect(tokens[0][0].value).toBe("1"); - expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + tokens = grammar.tokenizeLines("anon = lambda a, c={'key':\n\n 555}, e=fff: None") + expect(tokens[0][0].value).toBe("anon"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); expect(tokens[0][1].value).toBe(" "); expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("+"); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); expect(tokens[0][3].value).toBe(" "); expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("\\"); - expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); expect(tokens[0][5].value).toBe(" "); - expect(tokens[0][5].scopes).toEqual(["source.python"]); - expect(tokens[0][6].value).toBe("sdgfsdf"); - expect(tokens[0][6].scopes).toEqual(["source.python","invalid.illegal.line.continuation.python"]); - expect(tokens[1][0].value).toBe("def"); - expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); - expect(tokens[1][1].value).toBe(" "); - expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); - expect(tokens[1][2].value).toBe("foo"); - expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); - expect(tokens[1][3].value).toBe("("); - expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); - expect(tokens[1][4].value).toBe(")"); - expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); - expect(tokens[1][5].value).toBe(":"); - expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); - expect(tokens[1][6].value).toBe(" "); - expect(tokens[1][6].scopes).toEqual(["source.python"]); - expect(tokens[1][7].value).toBe("pass"); - expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][9].value).toBe("c"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][10].value).toBe("="); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[0][11].value).toBe("{"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.begin.python"]); + expect(tokens[0][12].value).toBe("'"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][13].value).toBe("key"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python"]); + expect(tokens[0][14].value).toBe("'"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][15].value).toBe(":"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.dict.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][1].value).toBe("555"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","constant.numeric.dec.python"]); + expect(tokens[2][2].value).toBe("}"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.end.python"]); + expect(tokens[2][3].value).toBe(","); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][5].value).toBe("e"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][6].value).toBe("="); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[2][7].value).toBe("fff"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][8].value).toBe(":"); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[2][10].value).toBe("None"); + expect(tokens[2][10].scopes).toEqual(["source.python","constant.language.python"]); }); - it("test/linecont/linecont2.py", + it("test/functions/lambda8.py", function() { - tokens = grammar.tokenizeLines("1 + \\\n 3") - expect(tokens[0][0].value).toBe("1"); - expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + tokens = grammar.tokenizeLines("anon = lambda a,\n d=1: None\ndef foo(): pass") + expect(tokens[0][0].value).toBe("anon"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); expect(tokens[0][1].value).toBe(" "); expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("+"); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); expect(tokens[0][3].value).toBe(" "); expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("\\"); - expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); - expect(tokens[0][5].value).toBe(""); - expect(tokens[0][5].scopes).toEqual(["source.python"]); - expect(tokens[1][0].value).toBe(" "); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][8].value).toBe(""); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python"]); + expect(tokens[1][0].value).toBe(" "); expect(tokens[1][0].scopes).toEqual(["source.python"]); - expect(tokens[1][1].value).toBe("3"); - expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - }); - - it("test/numbers/complex1.py", - function() { - tokens = grammar.tokenizeLines("3.14j\n10.j\n10j\n.001j\n1e100j\n3.14e-10j") - expect(tokens[0][0].value).toBe("3.14"); - expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[0][1].value).toBe("j"); - expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); - expect(tokens[1][0].value).toBe("10."); - expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[1][1].value).toBe("j"); - expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); - expect(tokens[2][0].value).toBe("10"); - expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - expect(tokens[2][1].value).toBe("j"); - expect(tokens[2][1].scopes).toEqual(["source.python","constant.numeric.dec.python","storage.type.imaginary.number.python"]); - expect(tokens[3][0].value).toBe(".001"); - expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[3][1].value).toBe("j"); - expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); - expect(tokens[4][0].value).toBe("1e100"); - expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[4][1].value).toBe("j"); - expect(tokens[4][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); - expect(tokens[5][0].value).toBe("3.14e-10"); - expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[5][1].value).toBe("j"); - expect(tokens[5][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); - }); - - it("test/numbers/complex2.py", - function() { - tokens = grammar.tokenizeLines("3.141_592j\n10_000.j\n10_000j\n.001_123j\n1e10_000j\n3.141_592e-10_000j") - expect(tokens[0][0].value).toBe("3.141_592"); - expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[0][1].value).toBe("j"); - expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); - expect(tokens[1][0].value).toBe("10_000."); - expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[1][1].value).toBe("j"); - expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); - expect(tokens[2][0].value).toBe("10_000"); - expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - expect(tokens[2][1].value).toBe("j"); - expect(tokens[2][1].scopes).toEqual(["source.python","constant.numeric.dec.python","storage.type.imaginary.number.python"]); - expect(tokens[3][0].value).toBe(".001_123"); - expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[3][1].value).toBe("j"); - expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); - expect(tokens[4][0].value).toBe("1e10_000"); - expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[4][1].value).toBe("j"); - expect(tokens[4][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); - expect(tokens[5][0].value).toBe("3.141_592e-10_000"); - expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[5][1].value).toBe("j"); - expect(tokens[5][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); - }); - - it("test/numbers/dec.py", - function() { - tokens = grammar.tokenizeLines("123\n000\n123_456\n1_2_3_4_5_6") - expect(tokens[0][0].value).toBe("123"); - expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - expect(tokens[1][0].value).toBe("000"); - expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - expect(tokens[2][0].value).toBe("123_456"); - expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - expect(tokens[3][0].value).toBe("1_2_3_4_5_6"); - expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][1].value).toBe("d"); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe("1"); + expect(tokens[1][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][4].value).toBe(":"); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("None"); + expect(tokens[1][6].scopes).toEqual(["source.python","constant.language.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe(")"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][5].value).toBe(":"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[2][6].value).toBe(" "); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe("pass"); + expect(tokens[2][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); }); - it("test/numbers/floats1.py", + it("test/functions/lambda9.py", function() { - tokens = grammar.tokenizeLines("123.456\n0.456\n000.0001\n.01234\n123e5\n123e-5\n000123e-005\n123.456e+5\n0.456e-5") - expect(tokens[0][0].value).toBe("123.456"); - expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[1][0].value).toBe("0.456"); - expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[2][0].value).toBe("000.0001"); - expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[3][0].value).toBe(".01234"); - expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[4][0].value).toBe("123e5"); - expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[5][0].value).toBe("123e-5"); - expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[6][0].value).toBe("000123e-005"); - expect(tokens[6][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[7][0].value).toBe("123.456e+5"); - expect(tokens[7][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[8][0].value).toBe("0.456e-5"); - expect(tokens[8][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + tokens = grammar.tokenizeLines("lambda *, b: b\nlambda *b: b\nlambda **b: b") + expect(tokens[0][0].value).toBe("lambda"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][2].value).toBe("*"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[0][3].value).toBe(", "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][4].value).toBe("b"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("b"); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("lambda"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[1][2].value).toBe("*"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[1][3].value).toBe("b"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][4].value).toBe(":"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("b"); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("lambda"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][2].value).toBe("**"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[2][3].value).toBe("b"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][4].value).toBe(":"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("b"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); }); - it("test/numbers/floats2.py", + it("test/illegals/backticks1.py", function() { - tokens = grammar.tokenizeLines("1_234.567_890\n0.456_789\n000.000_1\n.012_34\n1_234e5_000\n1_234e-5_000\n000_123e-000_5\n1_234.567_8e+5_000\n0.456_78e-5_000") - expect(tokens[0][0].value).toBe("1_234.567_890"); - expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[1][0].value).toBe("0.456_789"); - expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[2][0].value).toBe("000.000_1"); - expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[3][0].value).toBe(".012_34"); - expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[4][0].value).toBe("1_234e5_000"); - expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[5][0].value).toBe("1_234e-5_000"); - expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[6][0].value).toBe("000_123e-000_5"); - expect(tokens[6][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[7][0].value).toBe("1_234.567_8e+5_000"); - expect(tokens[7][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); - expect(tokens[8][0].value).toBe("0.456_78e-5_000"); - expect(tokens[8][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + tokens = grammar.tokenizeLines("a = `(1, `2`)`") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("`"); + expect(tokens[0][4].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][5].value).toBe("("); + expect(tokens[0][5].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][6].value).toBe("1"); + expect(tokens[0][6].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.separator.element.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][9].value).toBe("`"); + expect(tokens[0][9].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][10].value).toBe("2"); + expect(tokens[0][10].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); + expect(tokens[0][11].value).toBe("`"); + expect(tokens[0][11].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][12].value).toBe(")"); + expect(tokens[0][12].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.parenthesis.end.python"]); + expect(tokens[0][13].value).toBe("`"); + expect(tokens[0][13].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); }); - it("test/numbers/invalid.py", + it("test/illegals/backticks2.py", function() { - tokens = grammar.tokenizeLines("0123\n0123l\n123f\n123d\n123A\n123__456\n123_") - expect(tokens[0][0].value).toBe("0"); - expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); - expect(tokens[0][1].value).toBe("123"); - expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.dec.python","invalid.illegal.dec.python"]); - expect(tokens[1][0].value).toBe("0123l"); - expect(tokens[1][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[2][0].value).toBe("123f"); - expect(tokens[2][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[3][0].value).toBe("123d"); - expect(tokens[3][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[4][0].value).toBe("123A"); - expect(tokens[4][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[5][0].value).toBe("123__456"); - expect(tokens[5][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[6][0].value).toBe("123_"); - expect(tokens[6][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + tokens = grammar.tokenizeLines("a = `\\\n123`\nprint a") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("`"); + expect(tokens[0][4].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][5].value).toBe("\\"); + expect(tokens[0][5].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][6].value).toBe(""); + expect(tokens[0][6].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[1][0].value).toBe("123"); + expect(tokens[1][0].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); + expect(tokens[1][1].value).toBe("`"); + expect(tokens[1][1].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[2][0].value).toBe("print"); + expect(tokens[2][0].scopes).toEqual(["source.python","support.function.builtin.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("a"); + expect(tokens[2][2].scopes).toEqual(["source.python"]); }); - it("test/numbers/nondec.py", + it("test/illegals/backticks3.py", function() { - tokens = grammar.tokenizeLines("0o1234567\n0O1234567\n0o1_234_567\n0b0000\n0B0001\n0b_0011_1001\n0xabcdef01234567890\n0XFF12\n0xab_cd_ef_01_23_45_67_89_00\n\n0o\n0b\n0x\n0z\n\n0b02\n0x0z\n0o90") - expect(tokens[0][0].value).toBe("0o"); - expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.oct.python","storage.type.number.python"]); - expect(tokens[0][1].value).toBe("1234567"); - expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.oct.python"]); - expect(tokens[1][0].value).toBe("0O"); - expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.oct.python","storage.type.number.python"]); - expect(tokens[1][1].value).toBe("1234567"); - expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.oct.python"]); - expect(tokens[2][0].value).toBe("0o"); - expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.oct.python","storage.type.number.python"]); - expect(tokens[2][1].value).toBe("1_234_567"); - expect(tokens[2][1].scopes).toEqual(["source.python","constant.numeric.oct.python"]); - expect(tokens[3][0].value).toBe("0b"); - expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); - expect(tokens[3][1].value).toBe("0000"); - expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.bin.python"]); - expect(tokens[4][0].value).toBe("0B"); - expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); - expect(tokens[4][1].value).toBe("0001"); - expect(tokens[4][1].scopes).toEqual(["source.python","constant.numeric.bin.python"]); - expect(tokens[5][0].value).toBe("0b"); - expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); - expect(tokens[5][1].value).toBe("_0011_1001"); - expect(tokens[5][1].scopes).toEqual(["source.python","constant.numeric.bin.python"]); - expect(tokens[6][0].value).toBe("0x"); - expect(tokens[6][0].scopes).toEqual(["source.python","constant.numeric.hex.python","storage.type.number.python"]); - expect(tokens[6][1].value).toBe("abcdef01234567890"); - expect(tokens[6][1].scopes).toEqual(["source.python","constant.numeric.hex.python"]); - expect(tokens[7][0].value).toBe("0X"); - expect(tokens[7][0].scopes).toEqual(["source.python","constant.numeric.hex.python","storage.type.number.python"]); - expect(tokens[7][1].value).toBe("FF12"); - expect(tokens[7][1].scopes).toEqual(["source.python","constant.numeric.hex.python"]); - expect(tokens[8][0].value).toBe("0x"); - expect(tokens[8][0].scopes).toEqual(["source.python","constant.numeric.hex.python","storage.type.number.python"]); - expect(tokens[8][1].value).toBe("ab_cd_ef_01_23_45_67_89_00"); - expect(tokens[8][1].scopes).toEqual(["source.python","constant.numeric.hex.python"]); - expect(tokens[9][0].value).toBe(""); - expect(tokens[9][0].scopes).toEqual(["source.python"]); - expect(tokens[10][0].value).toBe("0o"); - expect(tokens[10][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[11][0].value).toBe("0b"); - expect(tokens[11][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[12][0].value).toBe("0x"); - expect(tokens[12][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[13][0].value).toBe("0z"); - expect(tokens[13][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[14][0].value).toBe(""); - expect(tokens[14][0].scopes).toEqual(["source.python"]); - expect(tokens[15][0].value).toBe("0b02"); - expect(tokens[15][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[16][0].value).toBe("0x0z"); - expect(tokens[16][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); - expect(tokens[17][0].value).toBe("0o90"); - expect(tokens[17][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + tokens = grammar.tokenizeLines("a = lambda `123`\n# comment") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("`"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][7].value).toBe("123"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe("`"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][9].value).toBe(""); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python"]); + expect(tokens[1][0].value).toBe("#"); + expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][1].value).toBe(" comment"); + expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/illegals/illegal1.py", + function() { + tokens = grammar.tokenizeLines("->\ndef f(): pass\n$\n?\na=$('.class').fuuuu(baz=1)\n# we recover just fine\nb = !some_ruby?\n# hey ;)") + expect(tokens[0][0].value).toBe("->"); + expect(tokens[0][0].scopes).toEqual(["source.python","invalid.illegal.annotation.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("f"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][0].value).toBe("$"); + expect(tokens[2][0].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[3][0].value).toBe("?"); + expect(tokens[3][0].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[4][0].value).toBe("a"); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("="); + expect(tokens[4][1].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[4][2].value).toBe("$"); + expect(tokens[4][2].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[4][3].value).toBe("("); + expect(tokens[4][3].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[4][4].value).toBe("'"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][5].value).toBe(".class"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][6].value).toBe("'"); + expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][7].value).toBe(")"); + expect(tokens[4][7].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[4][8].value).toBe("."); + expect(tokens[4][8].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[4][9].value).toBe("fuuuu"); + expect(tokens[4][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[4][10].value).toBe("("); + expect(tokens[4][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][11].value).toBe("baz"); + expect(tokens[4][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[4][12].value).toBe("="); + expect(tokens[4][12].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[4][13].value).toBe("1"); + expect(tokens[4][13].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[4][14].value).toBe(")"); + expect(tokens[4][14].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][0].value).toBe("#"); + expect(tokens[5][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[5][1].value).toBe(" we recover just fine"); + expect(tokens[5][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[6][0].value).toBe("b"); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python"]); + expect(tokens[6][2].value).toBe("="); + expect(tokens[6][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[6][3].value).toBe(" "); + expect(tokens[6][3].scopes).toEqual(["source.python"]); + expect(tokens[6][4].value).toBe("!"); + expect(tokens[6][4].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[6][5].value).toBe("some_ruby"); + expect(tokens[6][5].scopes).toEqual(["source.python"]); + expect(tokens[6][6].value).toBe("?"); + expect(tokens[6][6].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[7][0].value).toBe("#"); + expect(tokens[7][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[7][1].value).toBe(" hey ;)"); + expect(tokens[7][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); }); - it("test/numbers/python2.py", + it("test/illegals/illegal2.py", function() { - tokens = grammar.tokenizeLines("123l\n123L\n1_234l") - expect(tokens[0][0].value).toBe("123"); - expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.bin.python"]); - expect(tokens[0][1].value).toBe("l"); - expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); - expect(tokens[1][0].value).toBe("123"); - expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.bin.python"]); - expect(tokens[1][1].value).toBe("L"); - expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); - expect(tokens[2][0].value).toBe("1_234l"); - expect(tokens[2][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + tokens = grammar.tokenizeLines("a&&b||c\na &&= a\nb ||= c") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe("&&"); + expect(tokens[0][1].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[0][2].value).toBe("b"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe("||"); + expect(tokens[0][3].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[0][4].value).toBe("c"); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("&&"); + expect(tokens[1][2].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[1][3].value).toBe("="); + expect(tokens[1][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[1][5].value).toBe("a"); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("b"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("||"); + expect(tokens[2][2].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[2][3].value).toBe("="); + expect(tokens[2][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("c"); + expect(tokens[2][5].scopes).toEqual(["source.python"]); }); - it("test/regexp/comments1.py", + it("test/linecont/linecont1.py", function() { - tokens = grammar.tokenizeLines("a = r'''foo[abc] # comment'''") - expect(tokens[0][0].value).toBe("a"); - expect(tokens[0][0].scopes).toEqual(["source.python"]); + tokens = grammar.tokenizeLines("1 + \\ sdgfsdf\ndef foo(): pass") + expect(tokens[0][0].value).toBe("1"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); expect(tokens[0][1].value).toBe(" "); expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("="); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][2].value).toBe("+"); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); expect(tokens[0][3].value).toBe(" "); expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("r"); - expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","storage.type.string.python"]); - expect(tokens[0][5].value).toBe("'''"); - expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][6].value).toBe("foo"); - expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); - expect(tokens[0][7].value).toBe("["); - expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); - expect(tokens[0][8].value).toBe("a"); - expect(tokens[0][8].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[0][9].value).toBe("b"); - expect(tokens[0][9].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[0][10].value).toBe("c"); - expect(tokens[0][10].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[0][11].value).toBe("]"); - expect(tokens[0][11].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); - expect(tokens[0][12].value).toBe(" "); - expect(tokens[0][12].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); - expect(tokens[0][13].value).toBe("#"); - expect(tokens[0][13].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[0][14].value).toBe(" comment"); - expect(tokens[0][14].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); - expect(tokens[0][15].value).toBe("'''"); - expect(tokens[0][15].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[0][4].value).toBe("\\"); + expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("sdgfsdf"); + expect(tokens[0][6].scopes).toEqual(["source.python","invalid.illegal.line.continuation.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); }); - it("test/regexp/fregexp1.py", + it("test/linecont/linecont2.py", function() { - tokens = grammar.tokenizeLines("a = fr'[a-z]'\na = Fr'[a-z]'\na = rf'[a-z]'\na = rF'[a-z]'") - expect(tokens[0][0].value).toBe("a"); - expect(tokens[0][0].scopes).toEqual(["source.python"]); + tokens = grammar.tokenizeLines("1 + \\\n 3") + expect(tokens[0][0].value).toBe("1"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); expect(tokens[0][1].value).toBe(" "); expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("="); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][2].value).toBe("+"); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); expect(tokens[0][3].value).toBe(" "); expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("fr"); - expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[0][5].value).toBe("'"); - expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][6].value).toBe("["); - expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); - expect(tokens[0][7].value).toBe("a"); - expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[0][8].value).toBe("-"); - expect(tokens[0][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[0][9].value).toBe("z"); - expect(tokens[0][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[0][10].value).toBe("]"); - expect(tokens[0][10].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); - expect(tokens[0][11].value).toBe("'"); - expect(tokens[0][11].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[1][0].value).toBe("a"); + expect(tokens[0][4].value).toBe("\\"); + expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][5].value).toBe(""); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe(" "); expect(tokens[1][0].scopes).toEqual(["source.python"]); - expect(tokens[1][1].value).toBe(" "); - expect(tokens[1][1].scopes).toEqual(["source.python"]); - expect(tokens[1][2].value).toBe("="); - expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[1][3].value).toBe(" "); - expect(tokens[1][3].scopes).toEqual(["source.python"]); - expect(tokens[1][4].value).toBe("Fr"); - expect(tokens[1][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[1][5].value).toBe("'"); - expect(tokens[1][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[1][6].value).toBe("["); - expect(tokens[1][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); - expect(tokens[1][7].value).toBe("a"); - expect(tokens[1][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[1][8].value).toBe("-"); - expect(tokens[1][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[1][9].value).toBe("z"); - expect(tokens[1][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[1][10].value).toBe("]"); - expect(tokens[1][10].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); - expect(tokens[1][11].value).toBe("'"); - expect(tokens[1][11].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[2][0].value).toBe("a"); - expect(tokens[2][0].scopes).toEqual(["source.python"]); - expect(tokens[2][1].value).toBe(" "); - expect(tokens[2][1].scopes).toEqual(["source.python"]); - expect(tokens[2][2].value).toBe("="); - expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[2][3].value).toBe(" "); - expect(tokens[2][3].scopes).toEqual(["source.python"]); - expect(tokens[2][4].value).toBe("rf"); - expect(tokens[2][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[2][5].value).toBe("'"); - expect(tokens[2][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[2][6].value).toBe("["); - expect(tokens[2][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); - expect(tokens[2][7].value).toBe("a"); - expect(tokens[2][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[2][8].value).toBe("-"); - expect(tokens[2][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[2][9].value).toBe("z"); - expect(tokens[2][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[2][10].value).toBe("]"); - expect(tokens[2][10].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); - expect(tokens[2][11].value).toBe("'"); - expect(tokens[2][11].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[3][0].value).toBe("a"); - expect(tokens[3][0].scopes).toEqual(["source.python"]); - expect(tokens[3][1].value).toBe(" "); - expect(tokens[3][1].scopes).toEqual(["source.python"]); - expect(tokens[3][2].value).toBe("="); - expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[3][3].value).toBe(" "); - expect(tokens[3][3].scopes).toEqual(["source.python"]); - expect(tokens[3][4].value).toBe("rF"); - expect(tokens[3][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[3][5].value).toBe("'"); - expect(tokens[3][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[3][6].value).toBe("["); - expect(tokens[3][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); - expect(tokens[3][7].value).toBe("a"); - expect(tokens[3][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[3][8].value).toBe("-"); - expect(tokens[3][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[3][9].value).toBe("z"); - expect(tokens[3][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.character.set.regexp"]); - expect(tokens[3][10].value).toBe("]"); - expect(tokens[3][10].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); - expect(tokens[3][11].value).toBe("'"); - expect(tokens[3][11].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); + expect(tokens[1][1].value).toBe("3"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); }); - it("test/regexp/fregexp2.py", + it("test/numbers/complex1.py", function() { - tokens = grammar.tokenizeLines("rf'fo{{2}}'\nrf\"fo{{2}}\"\nrf'''fo{{2}}'''\nrf\"\"\"fo{{2}}\"\"\"") - expect(tokens[0][0].value).toBe("rf"); - expect(tokens[0][0].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[0][1].value).toBe("'"); - expect(tokens[0][1].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][2].value).toBe("fo"); - expect(tokens[0][2].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[0][3].value).toBe("{{2}}"); - expect(tokens[0][3].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","keyword.operator.quantifier.regexp"]); - expect(tokens[0][4].value).toBe("'"); - expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[1][0].value).toBe("rf"); - expect(tokens[1][0].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[1][1].value).toBe("\""); - expect(tokens[1][1].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[1][2].value).toBe("fo"); - expect(tokens[1][2].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[1][3].value).toBe("{{2}}"); - expect(tokens[1][3].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","keyword.operator.quantifier.regexp"]); - expect(tokens[1][4].value).toBe("\""); - expect(tokens[1][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[2][0].value).toBe("rf"); - expect(tokens[2][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[2][1].value).toBe("'''"); - expect(tokens[2][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[2][2].value).toBe("fo"); - expect(tokens[2][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python"]); - expect(tokens[2][3].value).toBe("{{2}}"); - expect(tokens[2][3].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","keyword.operator.quantifier.regexp"]); - expect(tokens[2][4].value).toBe("'''"); - expect(tokens[2][4].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[3][0].value).toBe("rf"); - expect(tokens[3][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[3][1].value).toBe("\"\"\""); - expect(tokens[3][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[3][2].value).toBe("fo"); - expect(tokens[3][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python"]); - expect(tokens[3][3].value).toBe("{{2}}"); - expect(tokens[3][3].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","keyword.operator.quantifier.regexp"]); - expect(tokens[3][4].value).toBe("\"\"\""); - expect(tokens[3][4].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","punctuation.definition.string.end.python"]); + tokens = grammar.tokenizeLines("3.14j\n10.j\n10j\n.001j\n1e100j\n3.14e-10j") + expect(tokens[0][0].value).toBe("3.14"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[0][1].value).toBe("j"); + expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[1][0].value).toBe("10."); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[1][1].value).toBe("j"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[2][0].value).toBe("10"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][1].value).toBe("j"); + expect(tokens[2][1].scopes).toEqual(["source.python","constant.numeric.dec.python","storage.type.imaginary.number.python"]); + expect(tokens[3][0].value).toBe(".001"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[3][1].value).toBe("j"); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[4][0].value).toBe("1e100"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[4][1].value).toBe("j"); + expect(tokens[4][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[5][0].value).toBe("3.14e-10"); + expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[5][1].value).toBe("j"); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + }); + + it("test/numbers/complex2.py", + function() { + tokens = grammar.tokenizeLines("3.141_592j\n10_000.j\n10_000j\n.001_123j\n1e10_000j\n3.141_592e-10_000j") + expect(tokens[0][0].value).toBe("3.141_592"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[0][1].value).toBe("j"); + expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[1][0].value).toBe("10_000."); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[1][1].value).toBe("j"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[2][0].value).toBe("10_000"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][1].value).toBe("j"); + expect(tokens[2][1].scopes).toEqual(["source.python","constant.numeric.dec.python","storage.type.imaginary.number.python"]); + expect(tokens[3][0].value).toBe(".001_123"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[3][1].value).toBe("j"); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[4][0].value).toBe("1e10_000"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[4][1].value).toBe("j"); + expect(tokens[4][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[5][0].value).toBe("3.141_592e-10_000"); + expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[5][1].value).toBe("j"); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + }); + + it("test/numbers/dec.py", + function() { + tokens = grammar.tokenizeLines("123\n000\n123_456\n1_2_3_4_5_6") + expect(tokens[0][0].value).toBe("123"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][0].value).toBe("000"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][0].value).toBe("123_456"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[3][0].value).toBe("1_2_3_4_5_6"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); }); - it("test/regexp/fregexp3.py", + it("test/numbers/floats1.py", function() { - tokens = grammar.tokenizeLines("rf'fo{2}'\nrf\"fo{2}\"\nrf'''fo{2}'''\nrf\"\"\"fo{2}\"\"\"") - expect(tokens[0][0].value).toBe("rf"); - expect(tokens[0][0].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[0][1].value).toBe("'"); - expect(tokens[0][1].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][2].value).toBe("fo"); - expect(tokens[0][2].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[0][3].value).toBe("{2}"); - expect(tokens[0][3].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[0][4].value).toBe("'"); - expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[1][0].value).toBe("rf"); - expect(tokens[1][0].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[1][1].value).toBe("\""); - expect(tokens[1][1].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[1][2].value).toBe("fo"); - expect(tokens[1][2].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[1][3].value).toBe("{2}"); - expect(tokens[1][3].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[1][4].value).toBe("\""); - expect(tokens[1][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[2][0].value).toBe("rf"); - expect(tokens[2][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[2][1].value).toBe("'''"); - expect(tokens[2][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[2][2].value).toBe("fo"); - expect(tokens[2][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python"]); - expect(tokens[2][3].value).toBe("{2}"); - expect(tokens[2][3].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python"]); - expect(tokens[2][4].value).toBe("'''"); - expect(tokens[2][4].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[3][0].value).toBe("rf"); - expect(tokens[3][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[3][1].value).toBe("\"\"\""); - expect(tokens[3][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[3][2].value).toBe("fo"); - expect(tokens[3][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python"]); - expect(tokens[3][3].value).toBe("{2}"); - expect(tokens[3][3].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python"]); - expect(tokens[3][4].value).toBe("\"\"\""); - expect(tokens[3][4].scopes).toEqual(["source.python","string.regexp.quoted.multi.python string.interpolated.python","punctuation.definition.string.end.python"]); + tokens = grammar.tokenizeLines("123.456\n0.456\n000.0001\n.01234\n123e5\n123e-5\n000123e-005\n123.456e+5\n0.456e-5") + expect(tokens[0][0].value).toBe("123.456"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[1][0].value).toBe("0.456"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[2][0].value).toBe("000.0001"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[3][0].value).toBe(".01234"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[4][0].value).toBe("123e5"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[5][0].value).toBe("123e-5"); + expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[6][0].value).toBe("000123e-005"); + expect(tokens[6][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[7][0].value).toBe("123.456e+5"); + expect(tokens[7][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[8][0].value).toBe("0.456e-5"); + expect(tokens[8][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); }); - it("test/regexp/fregexp4.py", + it("test/numbers/floats2.py", function() { - tokens = grammar.tokenizeLines("a = rf'fo{{2}}'\na = r'fo{{2}}'\na = r'fo{2}'") - expect(tokens[0][0].value).toBe("a"); - expect(tokens[0][0].scopes).toEqual(["source.python"]); - expect(tokens[0][1].value).toBe(" "); - expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("="); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("rf"); - expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[0][5].value).toBe("'"); - expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][6].value).toBe("fo"); - expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[0][7].value).toBe("{{2}}"); - expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","keyword.operator.quantifier.regexp"]); - expect(tokens[0][8].value).toBe("'"); - expect(tokens[0][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[1][0].value).toBe("a"); - expect(tokens[1][0].scopes).toEqual(["source.python"]); - expect(tokens[1][1].value).toBe(" "); - expect(tokens[1][1].scopes).toEqual(["source.python"]); - expect(tokens[1][2].value).toBe("="); - expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[1][3].value).toBe(" "); - expect(tokens[1][3].scopes).toEqual(["source.python"]); - expect(tokens[1][4].value).toBe("r"); - expect(tokens[1][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); - expect(tokens[1][5].value).toBe("'"); - expect(tokens[1][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[1][6].value).toBe("fo{"); - expect(tokens[1][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); - expect(tokens[1][7].value).toBe("{2}"); - expect(tokens[1][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","keyword.operator.quantifier.regexp"]); - expect(tokens[1][8].value).toBe("}"); - expect(tokens[1][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); - expect(tokens[1][9].value).toBe("'"); - expect(tokens[1][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); - expect(tokens[2][0].value).toBe("a"); - expect(tokens[2][0].scopes).toEqual(["source.python"]); - expect(tokens[2][1].value).toBe(" "); - expect(tokens[2][1].scopes).toEqual(["source.python"]); - expect(tokens[2][2].value).toBe("="); - expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[2][3].value).toBe(" "); - expect(tokens[2][3].scopes).toEqual(["source.python"]); - expect(tokens[2][4].value).toBe("r"); - expect(tokens[2][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); - expect(tokens[2][5].value).toBe("'"); - expect(tokens[2][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[2][6].value).toBe("fo"); - expect(tokens[2][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); - expect(tokens[2][7].value).toBe("{2}"); - expect(tokens[2][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","keyword.operator.quantifier.regexp"]); - expect(tokens[2][8].value).toBe("'"); - expect(tokens[2][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + tokens = grammar.tokenizeLines("1_234.567_890\n0.456_789\n000.000_1\n.012_34\n1_234e5_000\n1_234e-5_000\n000_123e-000_5\n1_234.567_8e+5_000\n0.456_78e-5_000") + expect(tokens[0][0].value).toBe("1_234.567_890"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[1][0].value).toBe("0.456_789"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[2][0].value).toBe("000.000_1"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[3][0].value).toBe(".012_34"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[4][0].value).toBe("1_234e5_000"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[5][0].value).toBe("1_234e-5_000"); + expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[6][0].value).toBe("000_123e-000_5"); + expect(tokens[6][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[7][0].value).toBe("1_234.567_8e+5_000"); + expect(tokens[7][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[8][0].value).toBe("0.456_78e-5_000"); + expect(tokens[8][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); }); - it("test/regexp/fregexp5.py", + it("test/numbers/invalid.py", function() { - tokens = grammar.tokenizeLines("a = rf'{{foo}}'\na = r'\\{foo\\}'") - expect(tokens[0][0].value).toBe("a"); - expect(tokens[0][0].scopes).toEqual(["source.python"]); - expect(tokens[0][1].value).toBe(" "); - expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("="); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("rf"); - expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[0][5].value).toBe("'"); - expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][6].value).toBe("{{"); - expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","constant.character.escape.python"]); - expect(tokens[0][7].value).toBe("foo"); - expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[0][8].value).toBe("}}"); - expect(tokens[0][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","constant.character.escape.python"]); - expect(tokens[0][9].value).toBe("'"); - expect(tokens[0][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[1][0].value).toBe("a"); - expect(tokens[1][0].scopes).toEqual(["source.python"]); - expect(tokens[1][1].value).toBe(" "); - expect(tokens[1][1].scopes).toEqual(["source.python"]); - expect(tokens[1][2].value).toBe("="); - expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[1][3].value).toBe(" "); - expect(tokens[1][3].scopes).toEqual(["source.python"]); - expect(tokens[1][4].value).toBe("r"); - expect(tokens[1][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); - expect(tokens[1][5].value).toBe("'"); - expect(tokens[1][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[1][6].value).toBe("\\{"); - expect(tokens[1][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python","constant.character.escape.regexp"]); - expect(tokens[1][7].value).toBe("foo"); - expect(tokens[1][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); - expect(tokens[1][8].value).toBe("\\}"); - expect(tokens[1][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python","constant.character.escape.regexp"]); - expect(tokens[1][9].value).toBe("'"); - expect(tokens[1][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + tokens = grammar.tokenizeLines("0123\n0123l\n123f\n123d\n123A\n123__456\n123_") + expect(tokens[0][0].value).toBe("0"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][1].value).toBe("123"); + expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.dec.python","invalid.illegal.dec.python"]); + expect(tokens[1][0].value).toBe("0123l"); + expect(tokens[1][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[2][0].value).toBe("123f"); + expect(tokens[2][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[3][0].value).toBe("123d"); + expect(tokens[3][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[4][0].value).toBe("123A"); + expect(tokens[4][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[5][0].value).toBe("123__456"); + expect(tokens[5][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[6][0].value).toBe("123_"); + expect(tokens[6][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); }); - it("test/regexp/fregexp6.py", + it("test/numbers/nondec.py", function() { - tokens = grammar.tokenizeLines("a = rf'fo{{{2}}}'\na = rf'fo{{{bar}}}'\na = rf'fo{{2}}'") - expect(tokens[0][0].value).toBe("a"); - expect(tokens[0][0].scopes).toEqual(["source.python"]); - expect(tokens[0][1].value).toBe(" "); - expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("="); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("rf"); - expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[0][5].value).toBe("'"); - expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[0][6].value).toBe("fo"); - expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[0][7].value).toBe("{{"); - expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","constant.character.escape.python"]); - expect(tokens[0][8].value).toBe("{2}"); - expect(tokens[0][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[0][9].value).toBe("}}"); - expect(tokens[0][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","constant.character.escape.python"]); - expect(tokens[0][10].value).toBe("'"); - expect(tokens[0][10].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[1][0].value).toBe("a"); - expect(tokens[1][0].scopes).toEqual(["source.python"]); - expect(tokens[1][1].value).toBe(" "); - expect(tokens[1][1].scopes).toEqual(["source.python"]); - expect(tokens[1][2].value).toBe("="); - expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[1][3].value).toBe(" "); - expect(tokens[1][3].scopes).toEqual(["source.python"]); - expect(tokens[1][4].value).toBe("rf"); - expect(tokens[1][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[1][5].value).toBe("'"); - expect(tokens[1][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[1][6].value).toBe("fo"); - expect(tokens[1][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[1][7].value).toBe("{{"); - expect(tokens[1][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","constant.character.escape.python"]); - expect(tokens[1][8].value).toBe("{bar}"); - expect(tokens[1][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[1][9].value).toBe("}}"); - expect(tokens[1][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","constant.character.escape.python"]); - expect(tokens[1][10].value).toBe("'"); - expect(tokens[1][10].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); - expect(tokens[2][0].value).toBe("a"); - expect(tokens[2][0].scopes).toEqual(["source.python"]); - expect(tokens[2][1].value).toBe(" "); - expect(tokens[2][1].scopes).toEqual(["source.python"]); - expect(tokens[2][2].value).toBe("="); - expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); - expect(tokens[2][3].value).toBe(" "); - expect(tokens[2][3].scopes).toEqual(["source.python"]); - expect(tokens[2][4].value).toBe("rf"); - expect(tokens[2][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","storage.type.string.python"]); - expect(tokens[2][5].value).toBe("'"); - expect(tokens[2][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.begin.python"]); - expect(tokens[2][6].value).toBe("fo"); - expect(tokens[2][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python"]); - expect(tokens[2][7].value).toBe("{{2}}"); - expect(tokens[2][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","keyword.operator.quantifier.regexp"]); - expect(tokens[2][8].value).toBe("'"); - expect(tokens[2][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python string.interpolated.python","punctuation.definition.string.end.python"]); + tokens = grammar.tokenizeLines("0o1234567\n0O1234567\n0o1_234_567\n0b0000\n0B0001\n0b_0011_1001\n0xabcdef01234567890\n0XFF12\n0xab_cd_ef_01_23_45_67_89_00\n\n0o\n0b\n0x\n0z\n\n0b02\n0x0z\n0o90") + expect(tokens[0][0].value).toBe("0o"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.oct.python","storage.type.number.python"]); + expect(tokens[0][1].value).toBe("1234567"); + expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.oct.python"]); + expect(tokens[1][0].value).toBe("0O"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.oct.python","storage.type.number.python"]); + expect(tokens[1][1].value).toBe("1234567"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.oct.python"]); + expect(tokens[2][0].value).toBe("0o"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.oct.python","storage.type.number.python"]); + expect(tokens[2][1].value).toBe("1_234_567"); + expect(tokens[2][1].scopes).toEqual(["source.python","constant.numeric.oct.python"]); + expect(tokens[3][0].value).toBe("0b"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); + expect(tokens[3][1].value).toBe("0000"); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.bin.python"]); + expect(tokens[4][0].value).toBe("0B"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); + expect(tokens[4][1].value).toBe("0001"); + expect(tokens[4][1].scopes).toEqual(["source.python","constant.numeric.bin.python"]); + expect(tokens[5][0].value).toBe("0b"); + expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); + expect(tokens[5][1].value).toBe("_0011_1001"); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.numeric.bin.python"]); + expect(tokens[6][0].value).toBe("0x"); + expect(tokens[6][0].scopes).toEqual(["source.python","constant.numeric.hex.python","storage.type.number.python"]); + expect(tokens[6][1].value).toBe("abcdef01234567890"); + expect(tokens[6][1].scopes).toEqual(["source.python","constant.numeric.hex.python"]); + expect(tokens[7][0].value).toBe("0X"); + expect(tokens[7][0].scopes).toEqual(["source.python","constant.numeric.hex.python","storage.type.number.python"]); + expect(tokens[7][1].value).toBe("FF12"); + expect(tokens[7][1].scopes).toEqual(["source.python","constant.numeric.hex.python"]); + expect(tokens[8][0].value).toBe("0x"); + expect(tokens[8][0].scopes).toEqual(["source.python","constant.numeric.hex.python","storage.type.number.python"]); + expect(tokens[8][1].value).toBe("ab_cd_ef_01_23_45_67_89_00"); + expect(tokens[8][1].scopes).toEqual(["source.python","constant.numeric.hex.python"]); + expect(tokens[9][0].value).toBe(""); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[10][0].value).toBe("0o"); + expect(tokens[10][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[11][0].value).toBe("0b"); + expect(tokens[11][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[12][0].value).toBe("0x"); + expect(tokens[12][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[13][0].value).toBe("0z"); + expect(tokens[13][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[14][0].value).toBe(""); + expect(tokens[14][0].scopes).toEqual(["source.python"]); + expect(tokens[15][0].value).toBe("0b02"); + expect(tokens[15][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[16][0].value).toBe("0x0z"); + expect(tokens[16][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[17][0].value).toBe("0o90"); + expect(tokens[17][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + }); + + it("test/numbers/python2.py", + function() { + tokens = grammar.tokenizeLines("123l\n123L\n1_234l") + expect(tokens[0][0].value).toBe("123"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.bin.python"]); + expect(tokens[0][1].value).toBe("l"); + expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); + expect(tokens[1][0].value).toBe("123"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.bin.python"]); + expect(tokens[1][1].value).toBe("L"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); + expect(tokens[2][0].value).toBe("1_234l"); + expect(tokens[2][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + }); + + it("test/regexp/comments1.py", + function() { + tokens = grammar.tokenizeLines("a = r'''foo[abc] # comment'''") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("r"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("'''"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("foo"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[0][7].value).toBe("["); + expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[0][8].value).toBe("a"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][9].value).toBe("b"); + expect(tokens[0][9].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][10].value).toBe("c"); + expect(tokens[0][10].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][11].value).toBe("]"); + expect(tokens[0][11].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[0][13].value).toBe("#"); + expect(tokens[0][13].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][14].value).toBe(" comment"); + expect(tokens[0][14].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[0][15].value).toBe("'''"); + expect(tokens[0][15].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.definition.string.end.python"]); }); it("test/regexp/python1.py", @@ -12546,478 +12817,1559 @@ describe("Grammar Tests", function() { expect(tokens[1][9].scopes).toEqual(["source.python"]); }); - it("test/statements/import2.py", + it("test/statements/import2.py", + function() { + tokens = grammar.tokenizeLines("from .... import a\nfrom ... import b\nfrom .. import c") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("...."); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("import"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("from"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("..."); + expect(tokens[1][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("import"); + expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("b"); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("from"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe(".."); + expect(tokens[2][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("import"); + expect(tokens[2][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("c"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import3.py", + function() { + tokens = grammar.tokenizeLines("from....import a\nfrom...import b\nfrom..import c\nfrom.import d") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe("...."); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("import"); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("a"); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("from"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe("..."); + expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[1][2].value).toBe("import"); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("b"); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("from"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][1].value).toBe(".."); + expect(tokens[2][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[2][2].value).toBe("import"); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("c"); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("from"); + expect(tokens[3][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[3][1].value).toBe("."); + expect(tokens[3][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[3][2].value).toBe("import"); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("d"); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import4.py", + function() { + tokens = grammar.tokenizeLines("from....foo import a\nfrom...foo import b\nfrom..foo import c\nfrom.foo import d") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe("...."); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("import"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("from"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe("..."); + expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("import"); + expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("b"); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("from"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][1].value).toBe(".."); + expect(tokens[2][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("import"); + expect(tokens[2][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("c"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("from"); + expect(tokens[3][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[3][1].value).toBe("."); + expect(tokens[3][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[3][2].value).toBe("foo"); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("import"); + expect(tokens[3][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[3][5].value).toBe(" "); + expect(tokens[3][5].scopes).toEqual(["source.python"]); + expect(tokens[3][6].value).toBe("d"); + expect(tokens[3][6].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import5.py", + function() { + tokens = grammar.tokenizeLines("from.foo import d\nfrom.import a\n\nfoo.import\n\nraise Exception from Foo\n\ndef bar():\n yield from baz") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe("."); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("import"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("d"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("from"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe("."); + expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[1][2].value).toBe("import"); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("a"); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("foo"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("."); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[3][2].value).toBe("import"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.member.access.python","keyword.control.import.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[5][0].value).toBe("raise"); + expect(tokens[5][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe("Exception"); + expect(tokens[5][2].scopes).toEqual(["source.python","support.type.exception.python"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python"]); + expect(tokens[5][4].value).toBe("from"); + expect(tokens[5][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][5].value).toBe(" "); + expect(tokens[5][5].scopes).toEqual(["source.python"]); + expect(tokens[5][6].value).toBe("Foo"); + expect(tokens[5][6].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe(""); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[7][0].value).toBe("def"); + expect(tokens[7][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[7][1].value).toBe(" "); + expect(tokens[7][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[7][2].value).toBe("bar"); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[7][3].value).toBe("("); + expect(tokens[7][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[7][4].value).toBe(")"); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[7][5].value).toBe(":"); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("yield from"); + expect(tokens[8][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][2].value).toBe(" "); + expect(tokens[8][2].scopes).toEqual(["source.python"]); + expect(tokens[8][3].value).toBe("baz"); + expect(tokens[8][3].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import6.py", function() { - tokens = grammar.tokenizeLines("from .... import a\nfrom ... import b\nfrom .. import c") + tokens = grammar.tokenizeLines("from __future__ import generator_stop") expect(tokens[0][0].value).toBe("from"); expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); expect(tokens[0][1].value).toBe(" "); expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("...."); - expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("__future__"); + expect(tokens[0][2].scopes).toEqual(["source.python","support.variable.magic.python"]); expect(tokens[0][3].value).toBe(" "); expect(tokens[0][3].scopes).toEqual(["source.python"]); expect(tokens[0][4].value).toBe("import"); expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); expect(tokens[0][5].value).toBe(" "); expect(tokens[0][5].scopes).toEqual(["source.python"]); - expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].value).toBe("generator_stop"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import7.py", + function() { + tokens = grammar.tokenizeLines("from .importing import *\nfrom importing import *") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("."); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][3].value).toBe("importing"); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("import"); + expect(tokens[0][5].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][6].value).toBe(" "); expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("*"); + expect(tokens[0][7].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); expect(tokens[1][0].value).toBe("from"); expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); expect(tokens[1][1].value).toBe(" "); expect(tokens[1][1].scopes).toEqual(["source.python"]); - expect(tokens[1][2].value).toBe("..."); - expect(tokens[1][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[1][2].value).toBe("importing"); + expect(tokens[1][2].scopes).toEqual(["source.python"]); expect(tokens[1][3].value).toBe(" "); expect(tokens[1][3].scopes).toEqual(["source.python"]); expect(tokens[1][4].value).toBe("import"); expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.import.python"]); expect(tokens[1][5].value).toBe(" "); expect(tokens[1][5].scopes).toEqual(["source.python"]); - expect(tokens[1][6].value).toBe("b"); - expect(tokens[1][6].scopes).toEqual(["source.python"]); - expect(tokens[2][0].value).toBe("from"); - expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[2][1].value).toBe(" "); + expect(tokens[1][6].value).toBe("*"); + expect(tokens[1][6].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + }); + + it("test/statements/import8.py", + function() { + tokens = grammar.tokenizeLines("try:\n import \\\n time as ham, \\\n datetime \\\n # XXX: comment at the end of import\nexcept Exception as exc:\n pass") + expect(tokens[0][0].value).toBe("try"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(":"); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("import"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("\\"); + expect(tokens[1][3].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[1][4].value).toBe(""); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("time"); expect(tokens[2][1].scopes).toEqual(["source.python"]); - expect(tokens[2][2].value).toBe(".."); - expect(tokens[2][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[2][3].value).toBe(" "); - expect(tokens[2][3].scopes).toEqual(["source.python"]); - expect(tokens[2][4].value).toBe("import"); - expect(tokens[2][4].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe("as"); + expect(tokens[2][3].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("ham"); expect(tokens[2][5].scopes).toEqual(["source.python"]); - expect(tokens[2][6].value).toBe("c"); - expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe(","); + expect(tokens[2][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe("\\"); + expect(tokens[2][8].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[2][9].value).toBe(""); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("datetime"); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("\\"); + expect(tokens[3][3].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[3][4].value).toBe(""); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("#"); + expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][3].value).toBe("XXX"); + expect(tokens[4][3].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[4][4].value).toBe(": comment at the end of import"); + expect(tokens[4][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[5][0].value).toBe("except"); + expect(tokens[5][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe("Exception"); + expect(tokens[5][2].scopes).toEqual(["source.python","support.type.exception.python"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python"]); + expect(tokens[5][4].value).toBe("as"); + expect(tokens[5][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][5].value).toBe(" "); + expect(tokens[5][5].scopes).toEqual(["source.python"]); + expect(tokens[5][6].value).toBe("exc"); + expect(tokens[5][6].scopes).toEqual(["source.python"]); + expect(tokens[5][7].value).toBe(":"); + expect(tokens[5][7].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("pass"); + expect(tokens[6][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); }); - it("test/statements/import3.py", + it("test/statements/import9.py", function() { - tokens = grammar.tokenizeLines("from....import a\nfrom...import b\nfrom..import c\nfrom.import d") + tokens = grammar.tokenizeLines("from . . . foo import \\\n (\n # XXX: legal comment inside import\n time as bar,\n # another comment\n baz,\n datetime as ham\n )\nraise Exception('!') from None") expect(tokens[0][0].value).toBe("from"); expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[0][1].value).toBe("...."); - expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[0][2].value).toBe("import"); - expect(tokens[0][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("."); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); expect(tokens[0][3].value).toBe(" "); expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("a"); - expect(tokens[0][4].scopes).toEqual(["source.python"]); - expect(tokens[1][0].value).toBe("from"); - expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[1][1].value).toBe("..."); - expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[1][2].value).toBe("import"); - expect(tokens[1][2].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[1][3].value).toBe(" "); - expect(tokens[1][3].scopes).toEqual(["source.python"]); - expect(tokens[1][4].value).toBe("b"); - expect(tokens[1][4].scopes).toEqual(["source.python"]); - expect(tokens[2][0].value).toBe("from"); - expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[2][1].value).toBe(".."); - expect(tokens[2][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[2][2].value).toBe("import"); - expect(tokens[2][2].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[2][3].value).toBe(" "); - expect(tokens[2][3].scopes).toEqual(["source.python"]); - expect(tokens[2][4].value).toBe("c"); - expect(tokens[2][4].scopes).toEqual(["source.python"]); - expect(tokens[3][0].value).toBe("from"); - expect(tokens[3][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[3][1].value).toBe("."); - expect(tokens[3][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[3][2].value).toBe("import"); - expect(tokens[3][2].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[3][3].value).toBe(" "); - expect(tokens[3][3].scopes).toEqual(["source.python"]); - expect(tokens[3][4].value).toBe("d"); + expect(tokens[0][4].value).toBe("."); + expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("."); + expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("foo"); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("import"); + expect(tokens[0][10].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe("\\"); + expect(tokens[0][12].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][13].value).toBe(""); + expect(tokens[0][13].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("("); + expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("#"); + expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][3].value).toBe("XXX"); + expect(tokens[2][3].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[2][4].value).toBe(": legal comment inside import"); + expect(tokens[2][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("time"); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("as"); + expect(tokens[3][3].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[3][4].value).toBe(" "); expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[3][5].value).toBe("bar"); + expect(tokens[3][5].scopes).toEqual(["source.python"]); + expect(tokens[3][6].value).toBe(","); + expect(tokens[3][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("#"); + expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[4][2].value).toBe(" another comment"); + expect(tokens[4][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("baz"); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe(","); + expect(tokens[5][2].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("datetime"); + expect(tokens[6][1].scopes).toEqual(["source.python"]); + expect(tokens[6][2].value).toBe(" "); + expect(tokens[6][2].scopes).toEqual(["source.python"]); + expect(tokens[6][3].value).toBe("as"); + expect(tokens[6][3].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[6][4].value).toBe(" "); + expect(tokens[6][4].scopes).toEqual(["source.python"]); + expect(tokens[6][5].value).toBe("ham"); + expect(tokens[6][5].scopes).toEqual(["source.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe(")"); + expect(tokens[7][1].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[8][0].value).toBe("raise"); + expect(tokens[8][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][1].value).toBe(" "); + expect(tokens[8][1].scopes).toEqual(["source.python"]); + expect(tokens[8][2].value).toBe("Exception"); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.function-call.python","support.type.exception.python"]); + expect(tokens[8][3].value).toBe("("); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[8][4].value).toBe("'"); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[8][5].value).toBe("!"); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[8][6].value).toBe("'"); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[8][7].value).toBe(")"); + expect(tokens[8][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[8][8].value).toBe(" "); + expect(tokens[8][8].scopes).toEqual(["source.python"]); + expect(tokens[8][9].value).toBe("from"); + expect(tokens[8][9].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][10].value).toBe(" "); + expect(tokens[8][10].scopes).toEqual(["source.python"]); + expect(tokens[8][11].value).toBe("None"); + expect(tokens[8][11].scopes).toEqual(["source.python","constant.language.python"]); }); - it("test/statements/import4.py", + it("test/statements/match1.py", function() { - tokens = grammar.tokenizeLines("from....foo import a\nfrom...foo import b\nfrom..foo import c\nfrom.foo import d") - expect(tokens[0][0].value).toBe("from"); - expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[0][1].value).toBe("...."); - expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + tokens = grammar.tokenizeLines("def foo(status):\n match status:\n case 404:\n return \"Not found\"\n case 401 | 403:\n return \"Not allowed\"\n case _:\n return \"Something's wrong with the internet\"") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); expect(tokens[0][2].value).toBe("foo"); - expect(tokens[0][2].scopes).toEqual(["source.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("import"); - expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[0][5].value).toBe(" "); - expect(tokens[0][5].scopes).toEqual(["source.python"]); - expect(tokens[0][6].value).toBe("a"); - expect(tokens[0][6].scopes).toEqual(["source.python"]); - expect(tokens[1][0].value).toBe("from"); - expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[1][1].value).toBe("..."); - expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[1][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe("status"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][5].value).toBe(")"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][6].value).toBe(":"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("match"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][2].value).toBe(" "); expect(tokens[1][2].scopes).toEqual(["source.python"]); - expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].value).toBe("status"); expect(tokens[1][3].scopes).toEqual(["source.python"]); - expect(tokens[1][4].value).toBe("import"); - expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[1][5].value).toBe(" "); - expect(tokens[1][5].scopes).toEqual(["source.python"]); - expect(tokens[1][6].value).toBe("b"); - expect(tokens[1][6].scopes).toEqual(["source.python"]); - expect(tokens[2][0].value).toBe("from"); - expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[2][1].value).toBe(".."); - expect(tokens[2][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[2][2].value).toBe("foo"); + expect(tokens[1][4].value).toBe(":"); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("case"); + expect(tokens[2][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][2].value).toBe(" "); expect(tokens[2][2].scopes).toEqual(["source.python"]); - expect(tokens[2][3].value).toBe(" "); - expect(tokens[2][3].scopes).toEqual(["source.python"]); - expect(tokens[2][4].value).toBe("import"); - expect(tokens[2][4].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[2][5].value).toBe(" "); - expect(tokens[2][5].scopes).toEqual(["source.python"]); - expect(tokens[2][6].value).toBe("c"); - expect(tokens[2][6].scopes).toEqual(["source.python"]); - expect(tokens[3][0].value).toBe("from"); - expect(tokens[3][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[3][1].value).toBe("."); - expect(tokens[3][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[3][2].value).toBe("foo"); + expect(tokens[2][3].value).toBe("404"); + expect(tokens[2][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][4].value).toBe(":"); + expect(tokens[2][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("return"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); expect(tokens[3][2].scopes).toEqual(["source.python"]); - expect(tokens[3][3].value).toBe(" "); - expect(tokens[3][3].scopes).toEqual(["source.python"]); - expect(tokens[3][4].value).toBe("import"); - expect(tokens[3][4].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[3][5].value).toBe(" "); - expect(tokens[3][5].scopes).toEqual(["source.python"]); - expect(tokens[3][6].value).toBe("d"); - expect(tokens[3][6].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("\""); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][4].value).toBe("Not found"); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[3][5].value).toBe("\""); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("case"); + expect(tokens[4][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python"]); + expect(tokens[4][3].value).toBe("401"); + expect(tokens[4][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[4][4].value).toBe(" "); + expect(tokens[4][4].scopes).toEqual(["source.python"]); + expect(tokens[4][5].value).toBe("|"); + expect(tokens[4][5].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[4][6].value).toBe(" "); + expect(tokens[4][6].scopes).toEqual(["source.python"]); + expect(tokens[4][7].value).toBe("403"); + expect(tokens[4][7].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[4][8].value).toBe(":"); + expect(tokens[4][8].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("return"); + expect(tokens[5][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("\""); + expect(tokens[5][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[5][4].value).toBe("Not allowed"); + expect(tokens[5][4].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[5][5].value).toBe("\""); + expect(tokens[5][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("case"); + expect(tokens[6][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[6][2].value).toBe(" "); + expect(tokens[6][2].scopes).toEqual(["source.python"]); + expect(tokens[6][3].value).toBe("_"); + expect(tokens[6][3].scopes).toEqual(["source.python"]); + expect(tokens[6][4].value).toBe(":"); + expect(tokens[6][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("return"); + expect(tokens[7][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("\""); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][4].value).toBe("Something's wrong with the internet"); + expect(tokens[7][4].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[7][5].value).toBe("\""); + expect(tokens[7][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); }); - it("test/statements/import5.py", + it("test/statements/match2.py", function() { - tokens = grammar.tokenizeLines("from.foo import d\nfrom.import a\n\nfoo.import\n\nraise Exception from Foo\n\ndef bar():\n yield from baz") - expect(tokens[0][0].value).toBe("from"); - expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[0][1].value).toBe("."); - expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[0][2].value).toBe("foo"); + tokens = grammar.tokenizeLines("match point:\n case Point(x=0, y=0):\n print(\"Origin is the point's location.\")\n case Point(x=0, y=y):\n print(f\"The point is on the y-axis.\")\n case Point(x=x, y=0):\n print(f\"The point is on the x-axis.\")\n case Point():\n print(\"The point is located somewhere else on the plane.\")\n case _:\n print(\"Not a point\")") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("point"); expect(tokens[0][2].scopes).toEqual(["source.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("import"); - expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[0][5].value).toBe(" "); - expect(tokens[0][5].scopes).toEqual(["source.python"]); - expect(tokens[0][6].value).toBe("d"); - expect(tokens[0][6].scopes).toEqual(["source.python"]); - expect(tokens[1][0].value).toBe("from"); - expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[1][1].value).toBe("."); - expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[1][2].value).toBe("import"); - expect(tokens[1][2].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[1][3].value).toBe(" "); - expect(tokens[1][3].scopes).toEqual(["source.python"]); - expect(tokens[1][4].value).toBe("a"); - expect(tokens[1][4].scopes).toEqual(["source.python"]); - expect(tokens[2][0].value).toBe(""); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("case"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("Point"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][4].value).toBe("("); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][5].value).toBe("x"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[1][6].value).toBe("="); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[1][7].value).toBe("0"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[1][8].value).toBe(","); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[1][9].value).toBe(" "); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][10].value).toBe("y"); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[1][11].value).toBe("="); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[1][12].value).toBe("0"); + expect(tokens[1][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[1][13].value).toBe(")"); + expect(tokens[1][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][14].value).toBe(":"); + expect(tokens[1][14].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("print"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[2][2].value).toBe("("); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe("\""); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][4].value).toBe("Origin is the point's location."); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[2][5].value).toBe("\""); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][6].value).toBe(")"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("case"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("Point"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[3][4].value).toBe("("); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][5].value).toBe("x"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[3][6].value).toBe("="); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[3][7].value).toBe("0"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][8].value).toBe(","); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[3][9].value).toBe(" "); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][10].value).toBe("y"); + expect(tokens[3][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[3][11].value).toBe("="); + expect(tokens[3][11].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[3][12].value).toBe("y"); + expect(tokens[3][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][13].value).toBe(")"); + expect(tokens[3][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][14].value).toBe(":"); + expect(tokens[3][14].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("print"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][3].value).toBe("f"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][4].value).toBe("\""); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[4][5].value).toBe("The point is on the y-axis."); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][6].value).toBe("\""); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][7].value).toBe(")"); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("case"); + expect(tokens[5][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("Point"); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[5][4].value).toBe("("); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][5].value).toBe("x"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[5][6].value).toBe("="); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[5][7].value).toBe("x"); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][8].value).toBe(","); + expect(tokens[5][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[5][9].value).toBe(" "); + expect(tokens[5][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][10].value).toBe("y"); + expect(tokens[5][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[5][11].value).toBe("="); + expect(tokens[5][11].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[5][12].value).toBe("0"); + expect(tokens[5][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[5][13].value).toBe(")"); + expect(tokens[5][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][14].value).toBe(":"); + expect(tokens[5][14].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("print"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[6][2].value).toBe("("); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[6][3].value).toBe("f"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][4].value).toBe("\""); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[6][5].value).toBe("The point is on the x-axis."); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][6].value).toBe("\""); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][7].value).toBe(")"); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("case"); + expect(tokens[7][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("Point"); + expect(tokens[7][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[7][4].value).toBe("("); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[7][5].value).toBe(")"); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][6].value).toBe(":"); + expect(tokens[7][6].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("print"); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[8][2].value).toBe("("); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[8][3].value).toBe("\""); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[8][4].value).toBe("The point is located somewhere else on the plane."); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[8][5].value).toBe("\""); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[8][6].value).toBe(")"); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("case"); + expect(tokens[9][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[9][2].value).toBe(" "); + expect(tokens[9][2].scopes).toEqual(["source.python"]); + expect(tokens[9][3].value).toBe("_"); + expect(tokens[9][3].scopes).toEqual(["source.python"]); + expect(tokens[9][4].value).toBe(":"); + expect(tokens[9][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("print"); + expect(tokens[10][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[10][2].value).toBe("("); + expect(tokens[10][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[10][3].value).toBe("\""); + expect(tokens[10][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][4].value).toBe("Not a point"); + expect(tokens[10][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[10][5].value).toBe("\""); + expect(tokens[10][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[10][6].value).toBe(")"); + expect(tokens[10][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/statements/match3.py", + function() { + tokens = grammar.tokenizeLines("match points:\n case []:\n print(\"No points in the list.\")\n case [Point(0, 0)]:\n print(\"The origin is the only point in the list.\")\n case [Point(x, y)]:\n print(f\"A single point is in the list.\")\n case [Point(0, y1), Point(0, y2)]:\n print(f\"Two points on the Y axis are in the list.\")\n case _:\n print(\"Something else is found in the list.\")") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("points"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("case"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("["); + expect(tokens[1][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[1][4].value).toBe("]"); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[2][0].value).toBe(" "); expect(tokens[2][0].scopes).toEqual(["source.python"]); - expect(tokens[3][0].value).toBe("foo"); + expect(tokens[2][1].value).toBe("print"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[2][2].value).toBe("("); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe("\""); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][4].value).toBe("No points in the list."); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[2][5].value).toBe("\""); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][6].value).toBe(")"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][0].value).toBe(" "); expect(tokens[3][0].scopes).toEqual(["source.python"]); - expect(tokens[3][1].value).toBe("."); - expect(tokens[3][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); - expect(tokens[3][2].value).toBe("import"); - expect(tokens[3][2].scopes).toEqual(["source.python","meta.member.access.python","keyword.control.import.python"]); - expect(tokens[4][0].value).toBe(""); + expect(tokens[3][1].value).toBe("case"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("["); + expect(tokens[3][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[3][4].value).toBe("Point"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[3][5].value).toBe("("); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][6].value).toBe("0"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][7].value).toBe(","); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[3][8].value).toBe(" "); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][9].value).toBe("0"); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][10].value).toBe(")"); + expect(tokens[3][10].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][11].value).toBe("]"); + expect(tokens[3][11].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[3][12].value).toBe(":"); + expect(tokens[3][12].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[4][0].value).toBe(" "); expect(tokens[4][0].scopes).toEqual(["source.python"]); - expect(tokens[5][0].value).toBe("raise"); - expect(tokens[5][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); - expect(tokens[5][1].value).toBe(" "); - expect(tokens[5][1].scopes).toEqual(["source.python"]); - expect(tokens[5][2].value).toBe("Exception"); - expect(tokens[5][2].scopes).toEqual(["source.python","support.type.exception.python"]); - expect(tokens[5][3].value).toBe(" "); - expect(tokens[5][3].scopes).toEqual(["source.python"]); - expect(tokens[5][4].value).toBe("from"); - expect(tokens[5][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); - expect(tokens[5][5].value).toBe(" "); - expect(tokens[5][5].scopes).toEqual(["source.python"]); - expect(tokens[5][6].value).toBe("Foo"); - expect(tokens[5][6].scopes).toEqual(["source.python"]); - expect(tokens[6][0].value).toBe(""); + expect(tokens[4][1].value).toBe("print"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][3].value).toBe("\""); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][4].value).toBe("The origin is the only point in the list."); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[4][5].value).toBe("\""); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][6].value).toBe(")"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("case"); + expect(tokens[5][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("["); + expect(tokens[5][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[5][4].value).toBe("Point"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[5][5].value).toBe("("); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][6].value).toBe("x"); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][7].value).toBe(","); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[5][8].value).toBe(" "); + expect(tokens[5][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][9].value).toBe("y"); + expect(tokens[5][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][10].value).toBe(")"); + expect(tokens[5][10].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][11].value).toBe("]"); + expect(tokens[5][11].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[5][12].value).toBe(":"); + expect(tokens[5][12].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[6][0].value).toBe(" "); expect(tokens[6][0].scopes).toEqual(["source.python"]); - expect(tokens[7][0].value).toBe("def"); - expect(tokens[7][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); - expect(tokens[7][1].value).toBe(" "); - expect(tokens[7][1].scopes).toEqual(["source.python","meta.function.python"]); - expect(tokens[7][2].value).toBe("bar"); - expect(tokens[7][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); - expect(tokens[7][3].value).toBe("("); - expect(tokens[7][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); - expect(tokens[7][4].value).toBe(")"); - expect(tokens[7][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); - expect(tokens[7][5].value).toBe(":"); - expect(tokens[7][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); - expect(tokens[8][0].value).toBe(" "); + expect(tokens[6][1].value).toBe("print"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[6][2].value).toBe("("); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[6][3].value).toBe("f"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][4].value).toBe("\""); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[6][5].value).toBe("A single point is in the list."); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][6].value).toBe("\""); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][7].value).toBe(")"); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("case"); + expect(tokens[7][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("["); + expect(tokens[7][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[7][4].value).toBe("Point"); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[7][5].value).toBe("("); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[7][6].value).toBe("0"); + expect(tokens[7][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[7][7].value).toBe(","); + expect(tokens[7][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[7][8].value).toBe(" "); + expect(tokens[7][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[7][9].value).toBe("y1"); + expect(tokens[7][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[7][10].value).toBe(")"); + expect(tokens[7][10].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][11].value).toBe(","); + expect(tokens[7][11].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[7][12].value).toBe(" "); + expect(tokens[7][12].scopes).toEqual(["source.python"]); + expect(tokens[7][13].value).toBe("Point"); + expect(tokens[7][13].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[7][14].value).toBe("("); + expect(tokens[7][14].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[7][15].value).toBe("0"); + expect(tokens[7][15].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[7][16].value).toBe(","); + expect(tokens[7][16].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[7][17].value).toBe(" "); + expect(tokens[7][17].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[7][18].value).toBe("y2"); + expect(tokens[7][18].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[7][19].value).toBe(")"); + expect(tokens[7][19].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][20].value).toBe("]"); + expect(tokens[7][20].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[7][21].value).toBe(":"); + expect(tokens[7][21].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[8][0].value).toBe(" "); expect(tokens[8][0].scopes).toEqual(["source.python"]); - expect(tokens[8][1].value).toBe("yield from"); - expect(tokens[8][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); - expect(tokens[8][2].value).toBe(" "); - expect(tokens[8][2].scopes).toEqual(["source.python"]); - expect(tokens[8][3].value).toBe("baz"); - expect(tokens[8][3].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("print"); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[8][2].value).toBe("("); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[8][3].value).toBe("f"); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[8][4].value).toBe("\""); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[8][5].value).toBe("Two points on the Y axis are in the list."); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[8][6].value).toBe("\""); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[8][7].value).toBe(")"); + expect(tokens[8][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("case"); + expect(tokens[9][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[9][2].value).toBe(" "); + expect(tokens[9][2].scopes).toEqual(["source.python"]); + expect(tokens[9][3].value).toBe("_"); + expect(tokens[9][3].scopes).toEqual(["source.python"]); + expect(tokens[9][4].value).toBe(":"); + expect(tokens[9][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("print"); + expect(tokens[10][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[10][2].value).toBe("("); + expect(tokens[10][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[10][3].value).toBe("\""); + expect(tokens[10][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][4].value).toBe("Something else is found in the list."); + expect(tokens[10][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[10][5].value).toBe("\""); + expect(tokens[10][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[10][6].value).toBe(")"); + expect(tokens[10][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); }); - it("test/statements/import6.py", + it("test/statements/match4.py", function() { - tokens = grammar.tokenizeLines("from __future__ import generator_stop") - expect(tokens[0][0].value).toBe("from"); - expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + tokens = grammar.tokenizeLines("match point:\n case Point(x, y) if x == y:\n print(f\"The point is located on the diagonal Y=X.\")\n case Point(x, y):\n print(f\"Point is not on the diagonal.\")") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); expect(tokens[0][1].value).toBe(" "); expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("__future__"); - expect(tokens[0][2].scopes).toEqual(["source.python","support.variable.magic.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("import"); - expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[0][5].value).toBe(" "); - expect(tokens[0][5].scopes).toEqual(["source.python"]); - expect(tokens[0][6].value).toBe("generator_stop"); - expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("point"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("case"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("Point"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][4].value).toBe("("); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][5].value).toBe("x"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][6].value).toBe(","); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][8].value).toBe("y"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][9].value).toBe(")"); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][10].value).toBe(" "); + expect(tokens[1][10].scopes).toEqual(["source.python"]); + expect(tokens[1][11].value).toBe("if"); + expect(tokens[1][11].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][12].value).toBe(" "); + expect(tokens[1][12].scopes).toEqual(["source.python"]); + expect(tokens[1][13].value).toBe("x"); + expect(tokens[1][13].scopes).toEqual(["source.python"]); + expect(tokens[1][14].value).toBe(" "); + expect(tokens[1][14].scopes).toEqual(["source.python"]); + expect(tokens[1][15].value).toBe("=="); + expect(tokens[1][15].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[1][16].value).toBe(" "); + expect(tokens[1][16].scopes).toEqual(["source.python"]); + expect(tokens[1][17].value).toBe("y"); + expect(tokens[1][17].scopes).toEqual(["source.python"]); + expect(tokens[1][18].value).toBe(":"); + expect(tokens[1][18].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("print"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[2][2].value).toBe("("); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe("f"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][4].value).toBe("\""); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[2][5].value).toBe("The point is located on the diagonal Y=X."); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][6].value).toBe("\""); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][7].value).toBe(")"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("case"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("Point"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[3][4].value).toBe("("); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][5].value).toBe("x"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][6].value).toBe(","); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[3][7].value).toBe(" "); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][8].value).toBe("y"); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][9].value).toBe(")"); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][10].value).toBe(":"); + expect(tokens[3][10].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("print"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][3].value).toBe("f"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][4].value).toBe("\""); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[4][5].value).toBe("Point is not on the diagonal."); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][6].value).toBe("\""); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][7].value).toBe(")"); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); }); - it("test/statements/import7.py", + it("test/statements/match5.py", function() { - tokens = grammar.tokenizeLines("from .importing import *\nfrom importing import *") - expect(tokens[0][0].value).toBe("from"); - expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + tokens = grammar.tokenizeLines("match command.split() if command else ['default']:\n ... # Other cases\n case [\"north\"] | [\"go\", \"north\"]:\n ... # handle case\n case [\"get\", obj] | [\"pick\", \"up\", *other] | [\"pick\", obj, \"up\"]:\n ... # handle case") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); expect(tokens[0][1].value).toBe(" "); expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("."); - expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[0][3].value).toBe("importing"); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe(" "); - expect(tokens[0][4].scopes).toEqual(["source.python"]); - expect(tokens[0][5].value).toBe("import"); - expect(tokens[0][5].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[0][6].value).toBe(" "); - expect(tokens[0][6].scopes).toEqual(["source.python"]); - expect(tokens[0][7].value).toBe("*"); - expect(tokens[0][7].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); - expect(tokens[1][0].value).toBe("from"); - expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[1][1].value).toBe(" "); - expect(tokens[1][1].scopes).toEqual(["source.python"]); - expect(tokens[1][2].value).toBe("importing"); + expect(tokens[0][2].value).toBe("command"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe("."); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][4].value).toBe("split"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][5].value).toBe("("); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][6].value).toBe(")"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("if"); + expect(tokens[0][8].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("command"); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe("else"); + expect(tokens[0][12].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python"]); + expect(tokens[0][14].value).toBe("["); + expect(tokens[0][14].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[0][15].value).toBe("'"); + expect(tokens[0][15].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][16].value).toBe("default"); + expect(tokens[0][16].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][17].value).toBe("'"); + expect(tokens[0][17].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][18].value).toBe("]"); + expect(tokens[0][18].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[0][19].value).toBe(":"); + expect(tokens[0][19].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("..."); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[1][2].value).toBe(" "); expect(tokens[1][2].scopes).toEqual(["source.python"]); - expect(tokens[1][3].value).toBe(" "); - expect(tokens[1][3].scopes).toEqual(["source.python"]); - expect(tokens[1][4].value).toBe("import"); - expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[1][5].value).toBe(" "); - expect(tokens[1][5].scopes).toEqual(["source.python"]); - expect(tokens[1][6].value).toBe("*"); - expect(tokens[1][6].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[1][3].value).toBe("#"); + expect(tokens[1][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][4].value).toBe(" Other cases"); + expect(tokens[1][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("case"); + expect(tokens[2][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe("["); + expect(tokens[2][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[2][4].value).toBe("\""); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][5].value).toBe("north"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][6].value).toBe("\""); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][7].value).toBe("]"); + expect(tokens[2][7].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[2][8].value).toBe(" "); + expect(tokens[2][8].scopes).toEqual(["source.python"]); + expect(tokens[2][9].value).toBe("|"); + expect(tokens[2][9].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[2][10].value).toBe(" "); + expect(tokens[2][10].scopes).toEqual(["source.python"]); + expect(tokens[2][11].value).toBe("["); + expect(tokens[2][11].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[2][12].value).toBe("\""); + expect(tokens[2][12].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][13].value).toBe("go"); + expect(tokens[2][13].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][14].value).toBe("\""); + expect(tokens[2][14].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][15].value).toBe(","); + expect(tokens[2][15].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][16].value).toBe(" "); + expect(tokens[2][16].scopes).toEqual(["source.python"]); + expect(tokens[2][17].value).toBe("\""); + expect(tokens[2][17].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][18].value).toBe("north"); + expect(tokens[2][18].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][19].value).toBe("\""); + expect(tokens[2][19].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][20].value).toBe("]"); + expect(tokens[2][20].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[2][21].value).toBe(":"); + expect(tokens[2][21].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("..."); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("#"); + expect(tokens[3][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][4].value).toBe(" handle case"); + expect(tokens[3][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("case"); + expect(tokens[4][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python"]); + expect(tokens[4][3].value).toBe("["); + expect(tokens[4][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[4][4].value).toBe("\""); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][5].value).toBe("get"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][6].value).toBe("\""); + expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][7].value).toBe(","); + expect(tokens[4][7].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][8].value).toBe(" "); + expect(tokens[4][8].scopes).toEqual(["source.python"]); + expect(tokens[4][9].value).toBe("obj"); + expect(tokens[4][9].scopes).toEqual(["source.python"]); + expect(tokens[4][10].value).toBe("]"); + expect(tokens[4][10].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[4][11].value).toBe(" "); + expect(tokens[4][11].scopes).toEqual(["source.python"]); + expect(tokens[4][12].value).toBe("|"); + expect(tokens[4][12].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[4][13].value).toBe(" "); + expect(tokens[4][13].scopes).toEqual(["source.python"]); + expect(tokens[4][14].value).toBe("["); + expect(tokens[4][14].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[4][15].value).toBe("\""); + expect(tokens[4][15].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][16].value).toBe("pick"); + expect(tokens[4][16].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][17].value).toBe("\""); + expect(tokens[4][17].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][18].value).toBe(","); + expect(tokens[4][18].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][19].value).toBe(" "); + expect(tokens[4][19].scopes).toEqual(["source.python"]); + expect(tokens[4][20].value).toBe("\""); + expect(tokens[4][20].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][21].value).toBe("up"); + expect(tokens[4][21].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][22].value).toBe("\""); + expect(tokens[4][22].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][23].value).toBe(","); + expect(tokens[4][23].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][24].value).toBe(" "); + expect(tokens[4][24].scopes).toEqual(["source.python"]); + expect(tokens[4][25].value).toBe("*"); + expect(tokens[4][25].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[4][26].value).toBe("other"); + expect(tokens[4][26].scopes).toEqual(["source.python"]); + expect(tokens[4][27].value).toBe("]"); + expect(tokens[4][27].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[4][28].value).toBe(" "); + expect(tokens[4][28].scopes).toEqual(["source.python"]); + expect(tokens[4][29].value).toBe("|"); + expect(tokens[4][29].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[4][30].value).toBe(" "); + expect(tokens[4][30].scopes).toEqual(["source.python"]); + expect(tokens[4][31].value).toBe("["); + expect(tokens[4][31].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[4][32].value).toBe("\""); + expect(tokens[4][32].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][33].value).toBe("pick"); + expect(tokens[4][33].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][34].value).toBe("\""); + expect(tokens[4][34].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][35].value).toBe(","); + expect(tokens[4][35].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][36].value).toBe(" "); + expect(tokens[4][36].scopes).toEqual(["source.python"]); + expect(tokens[4][37].value).toBe("obj"); + expect(tokens[4][37].scopes).toEqual(["source.python"]); + expect(tokens[4][38].value).toBe(","); + expect(tokens[4][38].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][39].value).toBe(" "); + expect(tokens[4][39].scopes).toEqual(["source.python"]); + expect(tokens[4][40].value).toBe("\""); + expect(tokens[4][40].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][41].value).toBe("up"); + expect(tokens[4][41].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][42].value).toBe("\""); + expect(tokens[4][42].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][43].value).toBe("]"); + expect(tokens[4][43].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[4][44].value).toBe(":"); + expect(tokens[4][44].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("..."); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("#"); + expect(tokens[5][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[5][4].value).toBe(" handle case"); + expect(tokens[5][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); }); - it("test/statements/import8.py", + it("test/statements/match6.py", function() { - tokens = grammar.tokenizeLines("try:\n import \\\n time as ham, \\\n datetime \\\n # XXX: comment at the end of import\nexcept Exception as exc:\n pass") - expect(tokens[0][0].value).toBe("try"); + tokens = grammar.tokenizeLines("match (foo + bar):\n ... # cases\nmatch [foo, bar]:\n ... # cases\nmatch {foo, bar}:\n ... # cases") + expect(tokens[0][0].value).toBe("match"); expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); - expect(tokens[0][1].value).toBe(":"); - expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("("); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][3].value).toBe("foo"); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("+"); + expect(tokens[0][5].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("bar"); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe(")"); + expect(tokens[0][8].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[0][9].value).toBe(":"); + expect(tokens[0][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); expect(tokens[1][0].value).toBe(" "); expect(tokens[1][0].scopes).toEqual(["source.python"]); - expect(tokens[1][1].value).toBe("import"); - expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe("..."); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); expect(tokens[1][2].value).toBe(" "); expect(tokens[1][2].scopes).toEqual(["source.python"]); - expect(tokens[1][3].value).toBe("\\"); - expect(tokens[1][3].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); - expect(tokens[1][4].value).toBe(""); - expect(tokens[1][4].scopes).toEqual(["source.python"]); - expect(tokens[2][0].value).toBe(" "); - expect(tokens[2][0].scopes).toEqual(["source.python"]); - expect(tokens[2][1].value).toBe("time"); + expect(tokens[1][3].value).toBe("#"); + expect(tokens[1][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][4].value).toBe(" cases"); + expect(tokens[1][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe("match"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(" "); expect(tokens[2][1].scopes).toEqual(["source.python"]); - expect(tokens[2][2].value).toBe(" "); - expect(tokens[2][2].scopes).toEqual(["source.python"]); - expect(tokens[2][3].value).toBe("as"); - expect(tokens[2][3].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[2][4].value).toBe(" "); - expect(tokens[2][4].scopes).toEqual(["source.python"]); - expect(tokens[2][5].value).toBe("ham"); + expect(tokens[2][2].value).toBe("["); + expect(tokens[2][2].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[2][3].value).toBe("foo"); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe(","); + expect(tokens[2][4].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][5].value).toBe(" "); expect(tokens[2][5].scopes).toEqual(["source.python"]); - expect(tokens[2][6].value).toBe(","); - expect(tokens[2][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); - expect(tokens[2][7].value).toBe(" "); - expect(tokens[2][7].scopes).toEqual(["source.python"]); - expect(tokens[2][8].value).toBe("\\"); - expect(tokens[2][8].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); - expect(tokens[2][9].value).toBe(""); - expect(tokens[2][9].scopes).toEqual(["source.python"]); - expect(tokens[3][0].value).toBe(" "); + expect(tokens[2][6].value).toBe("bar"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe("]"); + expect(tokens[2][7].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[2][8].value).toBe(":"); + expect(tokens[2][8].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); expect(tokens[3][0].scopes).toEqual(["source.python"]); - expect(tokens[3][1].value).toBe("datetime"); - expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("..."); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); expect(tokens[3][2].value).toBe(" "); expect(tokens[3][2].scopes).toEqual(["source.python"]); - expect(tokens[3][3].value).toBe("\\"); - expect(tokens[3][3].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); - expect(tokens[3][4].value).toBe(""); - expect(tokens[3][4].scopes).toEqual(["source.python"]); - expect(tokens[4][0].value).toBe(" "); - expect(tokens[4][0].scopes).toEqual(["source.python"]); - expect(tokens[4][1].value).toBe("#"); - expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[4][2].value).toBe(" "); - expect(tokens[4][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - expect(tokens[4][3].value).toBe("XXX"); - expect(tokens[4][3].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); - expect(tokens[4][4].value).toBe(": comment at the end of import"); - expect(tokens[4][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - expect(tokens[5][0].value).toBe("except"); - expect(tokens[5][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); - expect(tokens[5][1].value).toBe(" "); - expect(tokens[5][1].scopes).toEqual(["source.python"]); - expect(tokens[5][2].value).toBe("Exception"); - expect(tokens[5][2].scopes).toEqual(["source.python","support.type.exception.python"]); - expect(tokens[5][3].value).toBe(" "); - expect(tokens[5][3].scopes).toEqual(["source.python"]); - expect(tokens[5][4].value).toBe("as"); - expect(tokens[5][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); - expect(tokens[5][5].value).toBe(" "); - expect(tokens[5][5].scopes).toEqual(["source.python"]); - expect(tokens[5][6].value).toBe("exc"); - expect(tokens[5][6].scopes).toEqual(["source.python"]); - expect(tokens[5][7].value).toBe(":"); - expect(tokens[5][7].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); - expect(tokens[6][0].value).toBe(" "); - expect(tokens[6][0].scopes).toEqual(["source.python"]); - expect(tokens[6][1].value).toBe("pass"); - expect(tokens[6][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][3].value).toBe("#"); + expect(tokens[3][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][4].value).toBe(" cases"); + expect(tokens[3][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe("match"); + expect(tokens[4][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("{"); + expect(tokens[4][2].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]); + expect(tokens[4][3].value).toBe("foo"); + expect(tokens[4][3].scopes).toEqual(["source.python"]); + expect(tokens[4][4].value).toBe(","); + expect(tokens[4][4].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][5].value).toBe(" "); + expect(tokens[4][5].scopes).toEqual(["source.python"]); + expect(tokens[4][6].value).toBe("bar"); + expect(tokens[4][6].scopes).toEqual(["source.python"]); + expect(tokens[4][7].value).toBe("}"); + expect(tokens[4][7].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]); + expect(tokens[4][8].value).toBe(":"); + expect(tokens[4][8].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("..."); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("#"); + expect(tokens[5][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[5][4].value).toBe(" cases"); + expect(tokens[5][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); }); - it("test/statements/import9.py", + it("test/statements/match7.py", function() { - tokens = grammar.tokenizeLines("from . . . foo import \\\n (\n # XXX: legal comment inside import\n time as bar,\n # another comment\n baz,\n datetime as ham\n )\nraise Exception('!') from None") - expect(tokens[0][0].value).toBe("from"); - expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + tokens = grammar.tokenizeLines("match 'prefix' + foo:\n ... # cases\nmatch \"prefix\" + foo:\n ... # cases\nmatch f'prefix{foo}':\n ... # cases\nmatch f\"prefix{foo}\":\n ... # cases\nmatch -foo:\n ... # cases\nmatch not foo:\n ... # cases") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); expect(tokens[0][1].value).toBe(" "); expect(tokens[0][1].scopes).toEqual(["source.python"]); - expect(tokens[0][2].value).toBe("."); - expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); - expect(tokens[0][3].value).toBe(" "); - expect(tokens[0][3].scopes).toEqual(["source.python"]); - expect(tokens[0][4].value).toBe("."); - expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("'"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][3].value).toBe("prefix"); + expect(tokens[0][3].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); expect(tokens[0][5].value).toBe(" "); expect(tokens[0][5].scopes).toEqual(["source.python"]); - expect(tokens[0][6].value).toBe("."); - expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][6].value).toBe("+"); + expect(tokens[0][6].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); expect(tokens[0][7].value).toBe(" "); expect(tokens[0][7].scopes).toEqual(["source.python"]); expect(tokens[0][8].value).toBe("foo"); expect(tokens[0][8].scopes).toEqual(["source.python"]); - expect(tokens[0][9].value).toBe(" "); - expect(tokens[0][9].scopes).toEqual(["source.python"]); - expect(tokens[0][10].value).toBe("import"); - expect(tokens[0][10].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[0][11].value).toBe(" "); - expect(tokens[0][11].scopes).toEqual(["source.python"]); - expect(tokens[0][12].value).toBe("\\"); - expect(tokens[0][12].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); - expect(tokens[0][13].value).toBe(""); - expect(tokens[0][13].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe(":"); + expect(tokens[0][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); expect(tokens[1][0].value).toBe(" "); expect(tokens[1][0].scopes).toEqual(["source.python"]); - expect(tokens[1][1].value).toBe("("); - expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); - expect(tokens[2][0].value).toBe(" "); - expect(tokens[2][0].scopes).toEqual(["source.python"]); - expect(tokens[2][1].value).toBe("#"); - expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[2][2].value).toBe(" "); - expect(tokens[2][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - expect(tokens[2][3].value).toBe("XXX"); - expect(tokens[2][3].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); - expect(tokens[2][4].value).toBe(": legal comment inside import"); - expect(tokens[2][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - expect(tokens[3][0].value).toBe(" "); + expect(tokens[1][1].value).toBe("..."); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("#"); + expect(tokens[1][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][4].value).toBe(" cases"); + expect(tokens[1][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe("match"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("\""); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][3].value).toBe("prefix"); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][4].value).toBe("\""); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("+"); + expect(tokens[2][6].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe("foo"); + expect(tokens[2][8].scopes).toEqual(["source.python"]); + expect(tokens[2][9].value).toBe(":"); + expect(tokens[2][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); expect(tokens[3][0].scopes).toEqual(["source.python"]); - expect(tokens[3][1].value).toBe("time"); - expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("..."); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); expect(tokens[3][2].value).toBe(" "); expect(tokens[3][2].scopes).toEqual(["source.python"]); - expect(tokens[3][3].value).toBe("as"); - expect(tokens[3][3].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[3][4].value).toBe(" "); - expect(tokens[3][4].scopes).toEqual(["source.python"]); - expect(tokens[3][5].value).toBe("bar"); - expect(tokens[3][5].scopes).toEqual(["source.python"]); - expect(tokens[3][6].value).toBe(","); - expect(tokens[3][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); - expect(tokens[4][0].value).toBe(" "); - expect(tokens[4][0].scopes).toEqual(["source.python"]); - expect(tokens[4][1].value).toBe("#"); - expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); - expect(tokens[4][2].value).toBe(" another comment"); - expect(tokens[4][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); - expect(tokens[5][0].value).toBe(" "); + expect(tokens[3][3].value).toBe("#"); + expect(tokens[3][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][4].value).toBe(" cases"); + expect(tokens[3][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe("match"); + expect(tokens[4][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("f"); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][3].value).toBe("'"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[4][4].value).toBe("prefix"); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][5].value).toBe("{"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][6].value).toBe("foo"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[4][7].value).toBe("}"); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][8].value).toBe("'"); + expect(tokens[4][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][9].value).toBe(":"); + expect(tokens[4][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); expect(tokens[5][0].scopes).toEqual(["source.python"]); - expect(tokens[5][1].value).toBe("baz"); - expect(tokens[5][1].scopes).toEqual(["source.python"]); - expect(tokens[5][2].value).toBe(","); - expect(tokens[5][2].scopes).toEqual(["source.python","punctuation.separator.element.python"]); - expect(tokens[6][0].value).toBe(" "); - expect(tokens[6][0].scopes).toEqual(["source.python"]); - expect(tokens[6][1].value).toBe("datetime"); + expect(tokens[5][1].value).toBe("..."); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("#"); + expect(tokens[5][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[5][4].value).toBe(" cases"); + expect(tokens[5][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[6][0].value).toBe("match"); + expect(tokens[6][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[6][1].value).toBe(" "); expect(tokens[6][1].scopes).toEqual(["source.python"]); - expect(tokens[6][2].value).toBe(" "); - expect(tokens[6][2].scopes).toEqual(["source.python"]); - expect(tokens[6][3].value).toBe("as"); - expect(tokens[6][3].scopes).toEqual(["source.python","keyword.control.import.python"]); - expect(tokens[6][4].value).toBe(" "); - expect(tokens[6][4].scopes).toEqual(["source.python"]); - expect(tokens[6][5].value).toBe("ham"); - expect(tokens[6][5].scopes).toEqual(["source.python"]); + expect(tokens[6][2].value).toBe("f"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][3].value).toBe("\""); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[6][4].value).toBe("prefix"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][5].value).toBe("{"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[6][6].value).toBe("foo"); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[6][7].value).toBe("}"); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[6][8].value).toBe("\""); + expect(tokens[6][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][9].value).toBe(":"); + expect(tokens[6][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); expect(tokens[7][0].value).toBe(" "); expect(tokens[7][0].scopes).toEqual(["source.python"]); - expect(tokens[7][1].value).toBe(")"); - expect(tokens[7][1].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); - expect(tokens[8][0].value).toBe("raise"); + expect(tokens[7][1].value).toBe("..."); + expect(tokens[7][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("#"); + expect(tokens[7][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[7][4].value).toBe(" cases"); + expect(tokens[7][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[8][0].value).toBe("match"); expect(tokens[8][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); expect(tokens[8][1].value).toBe(" "); expect(tokens[8][1].scopes).toEqual(["source.python"]); - expect(tokens[8][2].value).toBe("Exception"); - expect(tokens[8][2].scopes).toEqual(["source.python","meta.function-call.python","support.type.exception.python"]); - expect(tokens[8][3].value).toBe("("); - expect(tokens[8][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); - expect(tokens[8][4].value).toBe("'"); - expect(tokens[8][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); - expect(tokens[8][5].value).toBe("!"); - expect(tokens[8][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); - expect(tokens[8][6].value).toBe("'"); - expect(tokens[8][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); - expect(tokens[8][7].value).toBe(")"); - expect(tokens[8][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); - expect(tokens[8][8].value).toBe(" "); - expect(tokens[8][8].scopes).toEqual(["source.python"]); - expect(tokens[8][9].value).toBe("from"); - expect(tokens[8][9].scopes).toEqual(["source.python","keyword.control.flow.python"]); - expect(tokens[8][10].value).toBe(" "); - expect(tokens[8][10].scopes).toEqual(["source.python"]); - expect(tokens[8][11].value).toBe("None"); - expect(tokens[8][11].scopes).toEqual(["source.python","constant.language.python"]); + expect(tokens[8][2].value).toBe("-"); + expect(tokens[8][2].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[8][3].value).toBe("foo"); + expect(tokens[8][3].scopes).toEqual(["source.python"]); + expect(tokens[8][4].value).toBe(":"); + expect(tokens[8][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("..."); + expect(tokens[9][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[9][2].value).toBe(" "); + expect(tokens[9][2].scopes).toEqual(["source.python"]); + expect(tokens[9][3].value).toBe("#"); + expect(tokens[9][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[9][4].value).toBe(" cases"); + expect(tokens[9][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[10][0].value).toBe("match"); + expect(tokens[10][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[10][1].value).toBe(" "); + expect(tokens[10][1].scopes).toEqual(["source.python"]); + expect(tokens[10][2].value).toBe("not"); + expect(tokens[10][2].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[10][3].value).toBe(" "); + expect(tokens[10][3].scopes).toEqual(["source.python"]); + expect(tokens[10][4].value).toBe("foo"); + expect(tokens[10][4].scopes).toEqual(["source.python"]); + expect(tokens[10][5].value).toBe(":"); + expect(tokens[10][5].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[11][0].value).toBe(" "); + expect(tokens[11][0].scopes).toEqual(["source.python"]); + expect(tokens[11][1].value).toBe("..."); + expect(tokens[11][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[11][2].value).toBe(" "); + expect(tokens[11][2].scopes).toEqual(["source.python"]); + expect(tokens[11][3].value).toBe("#"); + expect(tokens[11][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[11][4].value).toBe(" cases"); + expect(tokens[11][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); }); it("test/statements/nonlocal1.py", diff --git a/test/builtins/builtins3.py b/test/builtins/builtins3.py index 181eacd1..f1f90d9c 100644 --- a/test/builtins/builtins3.py +++ b/test/builtins/builtins3.py @@ -40,6 +40,9 @@ some.__prepare__ some.__package__ some.__traceback__ +some.__closure__ +some.__globals__ +some.__match_args__ some.__notspecial__ @@ -183,4 +186,13 @@ __traceback__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python . : meta.member.access.python, punctuation.separator.period.python, source.python +__closure__ : meta.member.access.python, source.python, support.variable.magic.python +some : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__globals__ : meta.member.access.python, source.python, support.variable.magic.python +some : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__match_args__ : meta.member.access.python, source.python, support.variable.magic.python +some : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python __notspecial__ : meta.attribute.python, meta.member.access.python, source.python diff --git a/test/builtins/builtins7.py b/test/builtins/builtins7.py index eb02d7fb..6d23b2b7 100644 --- a/test/builtins/builtins7.py +++ b/test/builtins/builtins7.py @@ -1,4 +1,6 @@ breakpoint() +aiter() +anext() @@ -6,3 +8,9 @@ breakpoint : meta.function-call.python, source.python, support.function.builtin.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +aiter : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +anext : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/classes/class9.py b/test/classes/class9.py index bbf24feb..a4ea943f 100644 --- a/test/classes/class9.py +++ b/test/classes/class9.py @@ -1,5 +1,6 @@ class Foo: __slots__ = () + __match_args__ = ('key', 'name') @@ -13,4 +14,19 @@ class : meta.class.python, source.python, storage.type.class.python = : keyword.operator.assignment.python, source.python : source.python ( : punctuation.parenthesis.begin.python, source.python +) : punctuation.parenthesis.end.python, source.python + : source.python +__match_args__ : source.python, support.variable.magic.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +( : punctuation.parenthesis.begin.python, source.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +key : source.python, string.quoted.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +name : source.python, string.quoted.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.single.python ) : punctuation.parenthesis.end.python, source.python diff --git a/test/expressions/expr21.py b/test/expressions/expr21.py new file mode 100644 index 00000000..59556710 --- /dev/null +++ b/test/expressions/expr21.py @@ -0,0 +1,39 @@ +while chunk := file.read(8192): + process(chunk) + y0 = (y1 := f(x)) + + + +while : keyword.control.flow.python, source.python + : source.python +chunk : source.python + : source.python +:= : keyword.operator.assignment.python, source.python + : source.python +file : source.python, variable.legacy.builtin.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +read : meta.function-call.generic.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +8192 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +process : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +chunk : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +y0 : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +( : punctuation.parenthesis.begin.python, source.python +y1 : source.python + : source.python +:= : keyword.operator.assignment.python, source.python + : source.python +f : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +) : punctuation.parenthesis.end.python, source.python diff --git a/test/expressions/expr22.py b/test/expressions/expr22.py new file mode 100644 index 00000000..7c03f542 --- /dev/null +++ b/test/expressions/expr22.py @@ -0,0 +1,33 @@ +match = 13 +case = 12 +if case == 4: + return match * 5 + + + +match : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +13 : constant.numeric.dec.python, source.python +case : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +12 : constant.numeric.dec.python, source.python +if : keyword.control.flow.python, source.python + : source.python +case : source.python + : source.python +== : keyword.operator.comparison.python, source.python + : source.python +4 : constant.numeric.dec.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +return : keyword.control.flow.python, source.python + : source.python +match : source.python + : source.python +* : keyword.operator.arithmetic.python, source.python + : source.python +5 : constant.numeric.dec.python, source.python diff --git a/test/expressions/special2.py b/test/expressions/special2.py index 2f360985..727d664f 100644 --- a/test/expressions/special2.py +++ b/test/expressions/special2.py @@ -7,7 +7,7 @@ def __class_getitem__(): pass __post_init__ : source.python, support.variable.magic.python def : meta.function.python, source.python, storage.type.function.python : meta.function.python, source.python -__class_getitem__ : meta.function.python, source.python, support.variable.magic.python +__class_getitem__ : meta.function.python, source.python, support.function.magic.python ( : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.begin.python, source.python ) : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.end.python, source.python : : meta.function.python, punctuation.section.function.begin.python, source.python diff --git a/test/fstrings/empty2.py b/test/fstrings/empty2.py index 77dc9087..9a007e14 100644 --- a/test/fstrings/empty2.py +++ b/test/fstrings/empty2.py @@ -7,20 +7,22 @@ -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -{ : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.single.python - : source.python, string.interpolated.python, string.regexp.quoted.single.python -{ : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.single.python - : invalid.illegal.brace.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -{ : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -} : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -{ : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python - : invalid.illegal.brace.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -} : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : invalid.illegal.brace.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.multi.python +""" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : invalid.illegal.brace.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +""" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.multi.python diff --git a/test/fstrings/fraw1.py b/test/fstrings/fraw1.py new file mode 100644 index 00000000..31e2196e --- /dev/null +++ b/test/fstrings/fraw1.py @@ -0,0 +1,40 @@ +a = fr'[a-z]' +a = Fr'[a-z]' +a = rf'[a-z]' +a = rF'[a-z]' + + + + +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +fr : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +[a-z] : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +Fr : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +[a-z] : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +[a-z] : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +rF : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +[a-z] : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python diff --git a/test/fstrings/fraw2.py b/test/fstrings/fraw2.py new file mode 100644 index 00000000..df0a2fed --- /dev/null +++ b/test/fstrings/fraw2.py @@ -0,0 +1,36 @@ +rf'fo{{2}}' +rf"fo{{2}}" +rf'''fo{{2}}''' +rf"""fo{{2}}""" + + + + +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +2 : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +}} : constant.character.escape.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +2 : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +}} : constant.character.escape.python, meta.fstring.python, source.python +" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.multi.python +''' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +2 : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +}} : constant.character.escape.python, meta.fstring.python, source.python +''' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.multi.python +""" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +2 : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +}} : constant.character.escape.python, meta.fstring.python, source.python +""" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.multi.python diff --git a/test/fstrings/fraw3.py b/test/fstrings/fraw3.py new file mode 100644 index 00000000..da1a6aff --- /dev/null +++ b/test/fstrings/fraw3.py @@ -0,0 +1,36 @@ +rf'fo{2}' +rf"fo{2}" +rf'''fo{2}''' +rf"""fo{2}""" + + + + +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +2 : constant.numeric.dec.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +2 : constant.numeric.dec.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.multi.python +''' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +2 : constant.numeric.dec.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +''' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.multi.python +""" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +2 : constant.numeric.dec.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +""" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.multi.python diff --git a/test/fstrings/fraw4.py b/test/fstrings/fraw4.py new file mode 100644 index 00000000..2bf2d465 --- /dev/null +++ b/test/fstrings/fraw4.py @@ -0,0 +1,45 @@ +a = rf'fo{{{2}}}' +a = rf'fo{{{bar}}}' +a = rf'fo{{2}}' + + + + + +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +2 : constant.numeric.dec.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +}} : constant.character.escape.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +bar : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +}} : constant.character.escape.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +2 : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +}} : constant.character.escape.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python diff --git a/test/fstrings/prefixes2.py b/test/fstrings/prefixes2.py index 861d2b66..78ee4246 100644 --- a/test/fstrings/prefixes2.py +++ b/test/fstrings/prefixes2.py @@ -22,16 +22,20 @@ obj : meta.fstring.python, source.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python ' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -some : source.python, string.interpolated.python, string.regexp.quoted.single.python -{obj} : source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rF : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -some : source.python, string.interpolated.python, string.regexp.quoted.single.python -{obj} : source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +obj : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rF : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +obj : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python Rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python ' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python diff --git a/test/fstrings/prefixes3.py b/test/fstrings/prefixes3.py index 200fa896..46d12ffb 100644 --- a/test/fstrings/prefixes3.py +++ b/test/fstrings/prefixes3.py @@ -6,16 +6,20 @@ -fr : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -some : source.python, string.interpolated.python, string.regexp.quoted.single.python -{obj} : source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -Fr : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -some : source.python, string.interpolated.python, string.regexp.quoted.single.python -{obj} : source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python +fr : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +obj : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +Fr : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +obj : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python fR : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python ' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python diff --git a/test/fstrings/simple10.py b/test/fstrings/simple10.py new file mode 100644 index 00000000..ae793021 --- /dev/null +++ b/test/fstrings/simple10.py @@ -0,0 +1,18 @@ +f'values: {a=} {b=!r}' + + + + +f : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +values: : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +a : meta.fstring.python, source.python += : meta.fstring.python, source.python, storage.type.format.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +b : meta.fstring.python, source.python +=!r : meta.fstring.python, source.python, storage.type.format.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python diff --git a/test/functions/decl15.py b/test/functions/decl15.py new file mode 100644 index 00000000..7f5c4ecc --- /dev/null +++ b/test/functions/decl15.py @@ -0,0 +1,34 @@ +def showcase(a, b, /, c, d, *, e, f): + return + + + + +def : meta.function.python, source.python, storage.type.function.python + : meta.function.python, source.python +showcase : entity.name.function.python, meta.function.python, source.python +( : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.begin.python, source.python +a : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +, : meta.function.parameters.python, meta.function.python, punctuation.separator.parameters.python, source.python + : meta.function.parameters.python, meta.function.python, source.python +b : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +, : meta.function.parameters.python, meta.function.python, punctuation.separator.parameters.python, source.python + : meta.function.parameters.python, meta.function.python, source.python +/ : keyword.operator.positional.parameter.python, meta.function.parameters.python, meta.function.python, source.python +, : meta.function.parameters.python, meta.function.python, source.python +c : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +, : meta.function.parameters.python, meta.function.python, punctuation.separator.parameters.python, source.python + : meta.function.parameters.python, meta.function.python, source.python +d : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +, : meta.function.parameters.python, meta.function.python, punctuation.separator.parameters.python, source.python + : meta.function.parameters.python, meta.function.python, source.python +* : keyword.operator.unpacking.parameter.python, meta.function.parameters.python, meta.function.python, source.python +, : meta.function.parameters.python, meta.function.python, source.python +e : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +, : meta.function.parameters.python, meta.function.python, punctuation.separator.parameters.python, source.python + : meta.function.parameters.python, meta.function.python, source.python +f : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +) : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.end.python, source.python +: : meta.function.python, punctuation.section.function.begin.python, source.python + : source.python +return : keyword.control.flow.python, source.python diff --git a/test/functions/lambda10.py b/test/functions/lambda10.py new file mode 100644 index 00000000..c033ac71 --- /dev/null +++ b/test/functions/lambda10.py @@ -0,0 +1,35 @@ +showcase = lambda a, /, b, *, c: (a + b + c) + + + + +showcase : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +lambda : meta.lambda-function.python, source.python, storage.type.function.lambda.python + : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +a : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python, variable.parameter.function.language.python +, : meta.function.lambda.parameters.python, meta.lambda-function.python, punctuation.separator.parameters.python, source.python + : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +/ : keyword.operator.positional.parameter.python, meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +, : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +b : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python, variable.parameter.function.language.python +, : meta.function.lambda.parameters.python, meta.lambda-function.python, punctuation.separator.parameters.python, source.python + : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +* : keyword.operator.unpacking.parameter.python, meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +, : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +c : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python, variable.parameter.function.language.python +: : meta.lambda-function.python, punctuation.section.function.lambda.begin.python, source.python + : source.python +( : punctuation.parenthesis.begin.python, source.python +a : source.python + : source.python ++ : keyword.operator.arithmetic.python, source.python + : source.python +b : source.python + : source.python ++ : keyword.operator.arithmetic.python, source.python + : source.python +c : source.python +) : punctuation.parenthesis.end.python, source.python diff --git a/test/regexp/fregexp1.py b/test/regexp/fregexp1.py deleted file mode 100644 index 9e77d66b..00000000 --- a/test/regexp/fregexp1.py +++ /dev/null @@ -1,56 +0,0 @@ -a = fr'[a-z]' -a = Fr'[a-z]' -a = rf'[a-z]' -a = rF'[a-z]' - - - - -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -fr : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -[ : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.begin.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -- : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -z : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -] : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.end.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -Fr : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -[ : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.begin.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -- : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -z : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -] : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.end.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -[ : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.begin.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -- : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -z : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -] : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.end.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rF : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -[ : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.begin.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -- : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -z : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -] : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.end.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python diff --git a/test/regexp/fregexp2.py b/test/regexp/fregexp2.py deleted file mode 100644 index 72eccc28..00000000 --- a/test/regexp/fregexp2.py +++ /dev/null @@ -1,28 +0,0 @@ -rf'fo{{2}}' -rf"fo{{2}}" -rf'''fo{{2}}''' -rf"""fo{{2}}""" - - - - -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.multi.python -''' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -fo : source.python, string.interpolated.python, string.regexp.quoted.multi.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.multi.python -''' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -fo : source.python, string.interpolated.python, string.regexp.quoted.multi.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python diff --git a/test/regexp/fregexp3.py b/test/regexp/fregexp3.py deleted file mode 100644 index ede8a6c3..00000000 --- a/test/regexp/fregexp3.py +++ /dev/null @@ -1,28 +0,0 @@ -rf'fo{2}' -rf"fo{2}" -rf'''fo{2}''' -rf"""fo{2}""" - - - - -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{2} : source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{2} : source.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.multi.python -''' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -fo : source.python, string.interpolated.python, string.regexp.quoted.multi.python -{2} : source.python, string.interpolated.python, string.regexp.quoted.multi.python -''' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -fo : source.python, string.interpolated.python, string.regexp.quoted.multi.python -{2} : source.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python diff --git a/test/regexp/fregexp4.py b/test/regexp/fregexp4.py deleted file mode 100644 index 5b707ad3..00000000 --- a/test/regexp/fregexp4.py +++ /dev/null @@ -1,36 +0,0 @@ -a = rf'fo{{2}}' -a = r'fo{{2}}' -a = r'fo{2}' - - - - - -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -r : source.python, storage.type.string.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.regexp.quoted.single.python -fo{ : source.python, string.regexp.quoted.single.python -{2} : keyword.operator.quantifier.regexp, source.python, string.regexp.quoted.single.python -} : source.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -r : source.python, storage.type.string.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.regexp.quoted.single.python -fo : source.python, string.regexp.quoted.single.python -{2} : keyword.operator.quantifier.regexp, source.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.regexp.quoted.single.python diff --git a/test/regexp/fregexp5.py b/test/regexp/fregexp5.py deleted file mode 100644 index 0d339e27..00000000 --- a/test/regexp/fregexp5.py +++ /dev/null @@ -1,26 +0,0 @@ -a = rf'{{foo}}' -a = r'\{foo\}' - - - - -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -{{ : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -foo : source.python, string.interpolated.python, string.regexp.quoted.single.python -}} : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -r : source.python, storage.type.string.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.regexp.quoted.single.python -\{ : constant.character.escape.regexp, source.python, string.regexp.quoted.single.python -foo : source.python, string.regexp.quoted.single.python -\} : constant.character.escape.regexp, source.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.regexp.quoted.single.python diff --git a/test/regexp/fregexp6.py b/test/regexp/fregexp6.py deleted file mode 100644 index a51311d0..00000000 --- a/test/regexp/fregexp6.py +++ /dev/null @@ -1,39 +0,0 @@ -a = rf'fo{{{2}}}' -a = rf'fo{{{bar}}}' -a = rf'fo{{2}}' - - - - - -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{ : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -{2} : source.python, string.interpolated.python, string.regexp.quoted.single.python -}} : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{ : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -{bar} : source.python, string.interpolated.python, string.regexp.quoted.single.python -}} : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python diff --git a/test/statements/match1.py b/test/statements/match1.py new file mode 100644 index 00000000..f393066a --- /dev/null +++ b/test/statements/match1.py @@ -0,0 +1,61 @@ +def foo(status): + match status: + case 404: + return "Not found" + case 401 | 403: + return "Not allowed" + case _: + return "Something's wrong with the internet" + + + + +def : meta.function.python, source.python, storage.type.function.python + : meta.function.python, source.python +foo : entity.name.function.python, meta.function.python, source.python +( : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.begin.python, source.python +status : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +) : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.end.python, source.python +: : meta.function.python, punctuation.section.function.begin.python, source.python + : source.python +match : keyword.control.flow.python, source.python + : source.python +status : source.python +: : punctuation.separator.colon.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +404 : constant.numeric.dec.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +return : keyword.control.flow.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Not found : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +401 : constant.numeric.dec.python, source.python + : source.python +| : keyword.operator.bitwise.python, source.python + : source.python +403 : constant.numeric.dec.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +return : keyword.control.flow.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Not allowed : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +_ : source.python +: : punctuation.separator.colon.python, source.python + : source.python +return : keyword.control.flow.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Something's wrong with the internet : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python diff --git a/test/statements/match2.py b/test/statements/match2.py new file mode 100644 index 00000000..a88ba055 --- /dev/null +++ b/test/statements/match2.py @@ -0,0 +1,112 @@ +match point: + case Point(x=0, y=0): + print("Origin is the point's location.") + case Point(x=0, y=y): + print(f"The point is on the y-axis.") + case Point(x=x, y=0): + print(f"The point is on the x-axis.") + case Point(): + print("The point is located somewhere else on the plane.") + case _: + print("Not a point") + + + +match : keyword.control.flow.python, source.python + : source.python +point : source.python +: : punctuation.separator.colon.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Origin is the point's location. : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +The point is on the y-axis. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +The point is on the x-axis. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +The point is located somewhere else on the plane. : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +_ : source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Not a point : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/statements/match3.py b/test/statements/match3.py new file mode 100644 index 00000000..ac8e176c --- /dev/null +++ b/test/statements/match3.py @@ -0,0 +1,114 @@ +match points: + case []: + print("No points in the list.") + case [Point(0, 0)]: + print("The origin is the only point in the list.") + case [Point(x, y)]: + print(f"A single point is in the list.") + case [Point(0, y1), Point(0, y2)]: + print(f"Two points on the Y axis are in the list.") + case _: + print("Something else is found in the list.") + + + +match : keyword.control.flow.python, source.python + : source.python +points : source.python +: : punctuation.separator.colon.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +No points in the list. : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +The origin is the only point in the list. : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +A single point is in the list. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y1 : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +, : punctuation.separator.element.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y2 : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +Two points on the Y axis are in the list. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +_ : source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Something else is found in the list. : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/statements/match4.py b/test/statements/match4.py new file mode 100644 index 00000000..fc4857f5 --- /dev/null +++ b/test/statements/match4.py @@ -0,0 +1,59 @@ +match point: + case Point(x, y) if x == y: + print(f"The point is located on the diagonal Y=X.") + case Point(x, y): + print(f"Point is not on the diagonal.") + + + + +match : keyword.control.flow.python, source.python + : source.python +point : source.python +: : punctuation.separator.colon.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +if : keyword.control.flow.python, source.python + : source.python +x : source.python + : source.python +== : keyword.operator.comparison.python, source.python + : source.python +y : source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +The point is located on the diagonal Y=X. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +Point is not on the diagonal. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/statements/match5.py b/test/statements/match5.py new file mode 100644 index 00000000..905ef8ba --- /dev/null +++ b/test/statements/match5.py @@ -0,0 +1,112 @@ +match command.split() if command else ['default']: + ... # Other cases + case ["north"] | ["go", "north"]: + ... # handle case + case ["get", obj] | ["pick", "up", *other] | ["pick", obj, "up"]: + ... # handle case + + + + +match : keyword.control.flow.python, source.python + : source.python +command : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +split : meta.function-call.generic.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python + : source.python +if : keyword.control.flow.python, source.python + : source.python +command : source.python + : source.python +else : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +default : source.python, string.quoted.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.single.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + Other cases : comment.line.number-sign.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +north : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +] : punctuation.definition.list.end.python, source.python + : source.python +| : keyword.operator.bitwise.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +go : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +north : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + handle case : comment.line.number-sign.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +get : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +obj : source.python +] : punctuation.definition.list.end.python, source.python + : source.python +| : keyword.operator.bitwise.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +pick : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +up : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +* : keyword.operator.arithmetic.python, source.python +other : source.python +] : punctuation.definition.list.end.python, source.python + : source.python +| : keyword.operator.bitwise.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +pick : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +obj : source.python +, : punctuation.separator.element.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +up : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + handle case : comment.line.number-sign.python, source.python diff --git a/test/statements/match6.py b/test/statements/match6.py new file mode 100644 index 00000000..b74491f7 --- /dev/null +++ b/test/statements/match6.py @@ -0,0 +1,53 @@ +match (foo + bar): + ... # cases +match [foo, bar]: + ... # cases +match {foo, bar}: + ... # cases + + + + +match : keyword.control.flow.python, source.python + : source.python +( : punctuation.parenthesis.begin.python, source.python +foo : source.python + : source.python ++ : keyword.operator.arithmetic.python, source.python + : source.python +bar : source.python +) : punctuation.parenthesis.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +foo : source.python +, : punctuation.separator.element.python, source.python + : source.python +bar : source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +{ : punctuation.definition.dict.begin.python, source.python +foo : source.python +, : punctuation.separator.element.python, source.python + : source.python +bar : source.python +} : punctuation.definition.dict.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python \ No newline at end of file diff --git a/test/statements/match7.py b/test/statements/match7.py new file mode 100644 index 00000000..878fa813 --- /dev/null +++ b/test/statements/match7.py @@ -0,0 +1,97 @@ +match 'prefix' + foo: + ... # cases +match "prefix" + foo: + ... # cases +match f'prefix{foo}': + ... # cases +match f"prefix{foo}": + ... # cases +match -foo: + ... # cases +match not foo: + ... # cases + + + + +match : keyword.control.flow.python, source.python + : source.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +prefix : source.python, string.quoted.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.single.python + : source.python ++ : keyword.operator.arithmetic.python, source.python + : source.python +foo : source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +prefix : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python + : source.python ++ : keyword.operator.arithmetic.python, source.python + : source.python +foo : source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +f : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +prefix : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +foo : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +f : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +prefix : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +foo : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +- : keyword.operator.arithmetic.python, source.python +foo : source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +not : keyword.operator.logical.python, source.python + : source.python +foo : source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python \ No newline at end of file