From c512f0a1a33b87d8d04618d1a1b005e6dec1cd19 Mon Sep 17 00:00:00 2001
From: Dongbo Wang <dongbow@microsoft.com>
Date: Mon, 28 Feb 2022 15:37:23 -0800
Subject: [PATCH 1/4] Fix the parsing code for method generic arguments

---
 .../engine/parser/Parser.cs                   | 46 +++++++---
 .../engine/parser/ast.cs                      | 92 +++----------------
 .../Parser/MethodInvocation.Tests.ps1         | 57 +++++++++---
 3 files changed, 92 insertions(+), 103 deletions(-)

diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs
index a855be0b0a7..c2f29053df0 100644
--- a/src/System.Management.Automation/engine/parser/Parser.cs
+++ b/src/System.Management.Automation/engine/parser/Parser.cs
@@ -7760,9 +7760,10 @@ private ExpressionAst MemberAccessRule(ExpressionAst targetExpr, Token operatorT
             }
             else if (_ungotToken == null)
             {
-                // Member name may be an incomplete token like `$a.$(Command-Name`; we do not look for generic args or
-                // invocation token(s) if the member name token is recognisably incomplete.
-                genericTypeArguments = GenericMethodArgumentsRule(out Token rBracket);
+                // Member name may be an incomplete token like `$a.$(Command-Name`, in which case, '_ungotToken != null'.
+                // We do not look for generic args or invocation token if the member name token is recognisably incomplete.
+                int resyncIndex = _tokenizer.GetRestorePoint();
+                genericTypeArguments = GenericMethodArgumentsRule(resyncIndex, out Token rBracket);
                 Token lParen = NextInvokeMemberToken();
 
                 if (lParen != null)
@@ -7778,6 +7779,13 @@ private ExpressionAst MemberAccessRule(ExpressionAst targetExpr, Token operatorT
                         "member and paren must be adjacent when the method is not generic");
                     return MemberInvokeRule(targetExpr, lParen, operatorToken, member, genericTypeArguments);
                 }
+                else if (rBracket != null)
+                {
+                    // We had a legit section of generic arguments but no 'lParen' following that, so this is not a method
+                    // invocation, but an invalid indexing operation. Resync the tokenizer back to before the generic arg
+                    // parsing and then continue.
+                    Resync(resyncIndex);
+                }
             }
 
             return new MemberExpressionAst(
@@ -7785,15 +7793,13 @@ private ExpressionAst MemberAccessRule(ExpressionAst targetExpr, Token operatorT
                 targetExpr,
                 member,
                 @static: operatorToken.Kind == TokenKind.ColonColon,
-                nullConditional: operatorToken.Kind == TokenKind.QuestionDot,
-                genericTypeArguments);
+                nullConditional: operatorToken.Kind == TokenKind.QuestionDot);
         }
 
-        private List<ITypeName> GenericMethodArgumentsRule(out Token rBracketToken)
+        private List<ITypeName> GenericMethodArgumentsRule(int resyncIndex, out Token rBracketToken)
         {
             List<ITypeName> genericTypes = null;
 
-            int resyncIndex = _tokenizer.GetRestorePoint();
             Token lBracket = NextToken();
             rBracketToken = null;
 
@@ -7817,7 +7823,23 @@ private List<ITypeName> GenericMethodArgumentsRule(out Token rBracketToken)
 
                 SkipNewlines();
                 Token firstToken = NextToken();
-                if (firstToken.Kind == TokenKind.Identifier || firstToken.Kind == TokenKind.LBracket)
+
+                // For method generic arguments, we only support the syntax `$var.Method[TypeName1 <, TypeName2 ...>]`,
+                // not the syntax `$var.Method[[TypeName1] <, [TypeName2] ...>]`.
+                // The latter syntax has been supported for type expression since the beginning, but it's ambiguous in
+                // this scenario because we could be looking at an indexing operation on a property like:
+                //    `$var.Property[<expression>]`
+                // and the '<expression>' could start with a type expression like `[TypeName]::Method()`, or even just
+                // a single type expression acting as a key to a hashtable property. Such cases will cause ambiguities.
+                //
+                // It could be possible to write code that sorts out the ambiguity and continue to support the latter
+                // syntax for method generic arguments, and thus to allow assembly-qualified type names. But we choose
+                // to not do so becuase:
+                //   1. that will definitely increase the complexity of the parsing code and also make it fragile;
+                //   2. the latter syntax hurts readability due to the number of opening/closing brackets.
+                // The downside is that the assembly-qualified type names won't be supported for method generic args,
+                // but that's likely not a problem in practice, and we can revisit if it turns out otherwise.
+                if (firstToken.Kind == TokenKind.Identifier)
                 {
                     resyncIndex = -1;
                     genericTypes = GenericTypeArgumentsRule(firstToken, out rBracketToken);
@@ -7836,11 +7858,11 @@ private List<ITypeName> GenericMethodArgumentsRule(out Token rBracketToken)
             finally
             {
                 SetTokenizerMode(oldTokenizerMode);
-            }
 
-            if (resyncIndex > 0)
-            {
-                Resync(resyncIndex);
+                if (resyncIndex > 0)
+                {
+                    Resync(resyncIndex);
+                }
             }
 
             return genericTypes;
diff --git a/src/System.Management.Automation/engine/parser/ast.cs b/src/System.Management.Automation/engine/parser/ast.cs
index 8671f3c58f2..bb817ec3d78 100644
--- a/src/System.Management.Automation/engine/parser/ast.cs
+++ b/src/System.Management.Automation/engine/parser/ast.cs
@@ -8012,16 +8012,10 @@ public class MemberExpressionAst : ExpressionAst, ISupportsAssignment
         /// <param name="static">True if the '::' operator was used, false if '.' is used.
         /// True if the member access is for a static member, using '::', false if accessing a member on an instance using '.'.
         /// </param>
-        /// <param name="genericTypes">The generic type arguments passed to the member.</param>
         /// <exception cref="PSArgumentNullException">
         /// If <paramref name="extent"/>, <paramref name="expression"/>, or <paramref name="member"/> is null.
         /// </exception>
-        public MemberExpressionAst(
-            IScriptExtent extent,
-            ExpressionAst expression,
-            CommandElementAst member,
-            bool @static,
-            IList<ITypeName> genericTypes)
+        public MemberExpressionAst(IScriptExtent extent, ExpressionAst expression, CommandElementAst member, bool @static)
             : base(extent)
         {
             if (expression == null || member == null)
@@ -8034,35 +8028,6 @@ public MemberExpressionAst(
             this.Member = member;
             SetParent(member);
             this.Static = @static;
-
-            if (genericTypes is not null && genericTypes.Count > 0)
-            {
-                this.GenericTypeArguments = new ReadOnlyCollection<ITypeName>(genericTypes);
-            }
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="MemberExpressionAst"/> class.
-        /// </summary>
-        /// <param name="extent">
-        /// The extent of the expression, starting with the expression before the operator '.' or '::' and ending after
-        /// membername or expression naming the member.
-        /// </param>
-        /// <param name="expression">The expression before the member access operator '.' or '::'.</param>
-        /// <param name="member">The name or expression naming the member to access.</param>
-        /// <param name="static">True if the '::' operator was used, false if '.' is used.
-        /// True if the member access is for a static member, using '::', false if accessing a member on an instance using '.'.
-        /// </param>
-        /// <exception cref="PSArgumentNullException">
-        /// If <paramref name="extent"/>, <paramref name="expression"/>, or <paramref name="member"/> is null.
-        /// </exception>
-        public MemberExpressionAst(
-            IScriptExtent extent,
-            ExpressionAst expression,
-            CommandElementAst member,
-            bool @static)
-            : this(extent, expression, member, @static, genericTypes: null)
-        {
         }
 
         /// <summary>
@@ -8076,46 +8041,15 @@ public MemberExpressionAst(
         /// <param name="member">The name or expression naming the member to access.</param>
         /// <param name="static">True if the '::' operator was used, false if '.' or '?.' is used.</param>
         /// <param name="nullConditional">True if '?.' used.</param>
-        /// <param name="genericTypes">The generic type arguments passed to the member.</param>
         /// <exception cref="PSArgumentNullException">
         /// If <paramref name="extent"/>, <paramref name="expression"/>, or <paramref name="member"/> is null.
         /// </exception>
-        public MemberExpressionAst(
-            IScriptExtent extent,
-            ExpressionAst expression,
-            CommandElementAst member,
-            bool @static,
-            bool nullConditional,
-            IList<ITypeName> genericTypes)
-            : this(extent, expression, member, @static, genericTypes)
+        public MemberExpressionAst(IScriptExtent extent, ExpressionAst expression, CommandElementAst member, bool @static, bool nullConditional)
+            : this(extent, expression, member, @static)
         {
             this.NullConditional = nullConditional;
         }
 
-        /// <summary>
-        /// Initializes a new instance of the <see cref="MemberExpressionAst"/> class.
-        /// </summary>
-        /// <param name="extent">
-        /// The extent of the expression, starting with the expression before the operator '.', '::' or '?.' and ending after
-        /// membername or expression naming the member.
-        /// </param>
-        /// <param name="expression">The expression before the member access operator '.', '::' or '?.'.</param>
-        /// <param name="member">The name or expression naming the member to access.</param>
-        /// <param name="static">True if the '::' operator was used, false if '.' or '?.' is used.</param>
-        /// <param name="nullConditional">True if '?.' used.</param>
-        /// <exception cref="PSArgumentNullException">
-        /// If <paramref name="extent"/>, <paramref name="expression"/>, or <paramref name="member"/> is null.
-        /// </exception>
-        public MemberExpressionAst(
-            IScriptExtent extent,
-            ExpressionAst expression,
-            CommandElementAst member,
-            bool @static,
-            bool nullConditional)
-            : this(extent, expression, member, @static, nullConditional, genericTypes: null)
-        {
-        }
-
         /// <summary>
         /// The expression that produces the value to retrieve the member from.  This property is never null.
         /// </summary>
@@ -8136,11 +8070,6 @@ public MemberExpressionAst(
         /// </summary>
         public bool NullConditional { get; protected set; }
 
-        /// <summary>
-        /// Gets a list of generic type arguments passed to this member.
-        /// </summary>
-        public ReadOnlyCollection<ITypeName> GenericTypeArguments { get; }
-
         /// <summary>
         /// Copy the MemberExpressionAst instance.
         /// </summary>
@@ -8154,8 +8083,7 @@ public override Ast Copy()
                 newExpression,
                 newMember,
                 this.Static,
-                this.NullConditional,
-                this.GenericTypeArguments);
+                this.NullConditional);
         }
 
         #region Visitors
@@ -8214,13 +8142,18 @@ public InvokeMemberExpressionAst(
             IEnumerable<ExpressionAst> arguments,
             bool @static,
             IList<ITypeName> genericTypes)
-            : base(extent, expression, method, @static, genericTypes)
+            : base(extent, expression, method, @static)
         {
             if (arguments != null && arguments.Any())
             {
                 this.Arguments = new ReadOnlyCollection<ExpressionAst>(arguments.ToArray());
                 SetParents(Arguments);
             }
+
+            if (genericTypes != null && genericTypes.Count > 0)
+            {
+                this.GenericTypeArguments = new ReadOnlyCollection<ITypeName>(genericTypes);
+            }
         }
 
         /// <summary>
@@ -8308,6 +8241,11 @@ public InvokeMemberExpressionAst(
         {
         }
 
+        /// <summary>
+        /// Gets a list of generic type arguments passed to this method invocation.
+        /// </summary>
+        public ReadOnlyCollection<ITypeName> GenericTypeArguments { get; }
+
         /// <summary>
         /// The non-empty collection of arguments to pass when invoking the method, or null if no arguments were specified.
         /// </summary>
diff --git a/test/powershell/Language/Parser/MethodInvocation.Tests.ps1 b/test/powershell/Language/Parser/MethodInvocation.Tests.ps1
index e60f8a2b148..ff7e49105a2 100644
--- a/test/powershell/Language/Parser/MethodInvocation.Tests.ps1
+++ b/test/powershell/Language/Parser/MethodInvocation.Tests.ps1
@@ -13,9 +13,18 @@ Describe 'Generic Method invocation' -Tags 'CI' {
                 Script       = '[Array]::Empty[System.Collections.Generic.Dictionary[System.Numerics.BigInteger, System.Collections.Generic.List[string[,]]]]()'
                 ExpectedType = [System.Collections.Generic.Dictionary[System.Numerics.BigInteger, System.Collections.Generic.List[string[, ]]][]]
             }
+        )
+
+        $IndexingAProperty = @(
+            @{
+                Script       = '[object]::Property[[type]]'
+                IndexType    = 'System.Management.Automation.Language.TypeExpressionAst'
+                IndexString  = '[type]'
+            }
             @{
-                Script       = '[Array]::$("Empty")[[System.Collections.Generic.Dictionary`2[[System.String, System.Private.CoreLib],[System.Numerics.BigInteger, System.Runtime.Numerics]], System.Private.CoreLib]]()'
-                ExpectedType = [System.Collections.Generic.Dictionary`2[[System.String, System.Private.CoreLib], [System.Numerics.BigInteger, System.Runtime.Numerics]][], System.Private.CoreLib]
+                Script       = '$object.IPSubnet[[Array]::IndexOf($_.IPAddress, $_.IPAddress[0])]'
+                IndexType    = 'System.Management.Automation.Language.InvokeMemberExpressionAst'
+                IndexString  = '[Array]::IndexOf($_.IPAddress, $_.IPAddress[0])'
             }
         )
 
@@ -37,8 +46,8 @@ Describe 'Generic Method invocation' -Tags 'CI' {
             }
             @{
                 Script         = '[array]::empty[type]]()'
-                ExpectedErrors = @('UnexpectedToken', 'ExpectedExpression')
-                ErrorCount     = 2
+                ExpectedErrors = @('MissingArrayIndexExpression', 'UnexpectedToken', 'ExpectedExpression')
+                ErrorCount     = 3
             }
             @{
                 Script         = '$object.Method[type,]()'
@@ -70,6 +79,21 @@ Describe 'Generic Method invocation' -Tags 'CI' {
                 ExpectedErrors = @('EndSquareBracketExpectedAtEndOfType', 'UnexpectedToken')
                 ErrorCount     = 2
             }
+            @{
+                Script         = '$object.Method[[type]]()'
+                ExpectedErrors = @('UnexpectedToken', 'ExpectedExpression')
+                ErrorCount     = 2
+            }
+            @{
+                Script         = '[Array]::Empty[[type]]()'
+                ExpectedErrors = @('UnexpectedToken', 'ExpectedExpression')
+                ErrorCount     = 2
+            }
+            @{
+                Script         = '$object.Property[type]'
+                ExpectedErrors = @('MissingArrayIndexExpression', 'UnexpectedToken')
+                ErrorCount     = 2
+            }
         )
     }
 
@@ -79,6 +103,21 @@ Describe 'Generic Method invocation' -Tags 'CI' {
         { [scriptblock]::Create($script) } | Should -Not -Throw
     }
 
+    It "parses fine for indexing a property" -TestCases $IndexingAProperty {
+        param($Script, $IndexType, $IndexString)
+
+        $parseErrors = $null
+
+        $ast = [System.Management.Automation.Language.Parser]::ParseInput($Script, [ref]$null, [ref]$parseErrors)
+        $parseErrors | Should -BeNullOrEmpty
+
+        $cmdExpr = $ast.EndBlock.Statements[0].PipelineElements[0]
+        $cmdExpr | Should -BeOfType 'System.Management.Automation.Language.CommandExpressionAst'
+        $cmdExpr.Expression | Should -BeOfType 'System.Management.Automation.Language.IndexExpressionAst'
+        $cmdExpr.Expression.Index | Should -BeOfType $IndexType
+        $cmdExpr.Expression.Index.ToString() | Should -BeExactly $IndexString
+    }
+
     It 'reports a parse error for "<Script>"' -TestCases $ExpectedParseErrors {
         param($Script, $ExpectedErrors, $ErrorCount)
 
@@ -164,16 +203,6 @@ Describe 'Generic Method invocation' -Tags 'CI' {
         }
     }
 
-    It 'lists the same method group information with and without type parameters' {
-        $dict = [System.Collections.Concurrent.ConcurrentDictionary[string, int]]::new()
-        $noTypeParams = $dict.AddOrUpdate
-        $typeParams = $dict.AddOrUpdate[string]
-        $extraTypeParams = $dict.AddOrUpdate[string, int]
-
-        $typeParams.OverloadDefinitions | Should -BeExactly $noTypeParams.OverloadDefinitions
-        $extraTypeParams.OverloadDefinitions | Should -BeExactly $typeParams.OverloadDefinitions
-    }
-
     It 'successfully invokes common Linq generic methods' {
         [System.Collections.Generic.List[int]]$list = @( 1, 2, 3, 4, 5 )
         $result = [System.Linq.Enumerable]::Select[int, float](

From baadc88a012f2b89f514f3fae127bf0eaa1a8468 Mon Sep 17 00:00:00 2001
From: Dongbo Wang <dongbow@microsoft.com>
Date: Mon, 28 Feb 2022 15:40:43 -0800
Subject: [PATCH 2/4] Minor change

---
 src/System.Management.Automation/engine/parser/Parser.cs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs
index c2f29053df0..d370b5a7b0a 100644
--- a/src/System.Management.Automation/engine/parser/Parser.cs
+++ b/src/System.Management.Automation/engine/parser/Parser.cs
@@ -7745,7 +7745,6 @@ private ExpressionAst MemberAccessRule(ExpressionAst targetExpr, Token operatorT
             // On entry, we've verified that operatorToken is not preceded by whitespace.
 
             CommandElementAst member = MemberNameRule();
-            List<ITypeName> genericTypeArguments = null;
 
             if (member == null)
             {
@@ -7763,7 +7762,7 @@ private ExpressionAst MemberAccessRule(ExpressionAst targetExpr, Token operatorT
                 // Member name may be an incomplete token like `$a.$(Command-Name`, in which case, '_ungotToken != null'.
                 // We do not look for generic args or invocation token if the member name token is recognisably incomplete.
                 int resyncIndex = _tokenizer.GetRestorePoint();
-                genericTypeArguments = GenericMethodArgumentsRule(resyncIndex, out Token rBracket);
+                List<ITypeName> genericTypeArguments = GenericMethodArgumentsRule(resyncIndex, out Token rBracket);
                 Token lParen = NextInvokeMemberToken();
 
                 if (lParen != null)

From 89e951d329b5b9bd0e664ef487489bcbe48faea3 Mon Sep 17 00:00:00 2001
From: Dongbo Wang <dongbow@microsoft.com>
Date: Mon, 28 Feb 2022 15:59:40 -0800
Subject: [PATCH 3/4] Update comments

---
 src/System.Management.Automation/engine/parser/Parser.cs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs
index d370b5a7b0a..1d3f299f304 100644
--- a/src/System.Management.Automation/engine/parser/Parser.cs
+++ b/src/System.Management.Automation/engine/parser/Parser.cs
@@ -7828,14 +7828,14 @@ private List<ITypeName> GenericMethodArgumentsRule(int resyncIndex, out Token rB
                 // The latter syntax has been supported for type expression since the beginning, but it's ambiguous in
                 // this scenario because we could be looking at an indexing operation on a property like:
                 //    `$var.Property[<expression>]`
-                // and the '<expression>' could start with a type expression like `[TypeName]::Method()`, or even just
+                // and the `<expression>` could start with a type expression like `[TypeName]::Method()`, or even just
                 // a single type expression acting as a key to a hashtable property. Such cases will cause ambiguities.
                 //
                 // It could be possible to write code that sorts out the ambiguity and continue to support the latter
                 // syntax for method generic arguments, and thus to allow assembly-qualified type names. But we choose
-                // to not do so becuase:
+                // not to do so because:
                 //   1. that will definitely increase the complexity of the parsing code and also make it fragile;
-                //   2. the latter syntax hurts readability due to the number of opening/closing brackets.
+                //   2. the latter syntax hurts readability a lot due to the number of opening/closing brackets.
                 // The downside is that the assembly-qualified type names won't be supported for method generic args,
                 // but that's likely not a problem in practice, and we can revisit if it turns out otherwise.
                 if (firstToken.Kind == TokenKind.Identifier)

From 2892d6ec737556029e5d3eb0fba96644c81147c6 Mon Sep 17 00:00:00 2001
From: Dongbo Wang <dongbow@microsoft.com>
Date: Tue, 1 Mar 2022 09:36:08 -0800
Subject: [PATCH 4/4] Add 2 test cases

---
 .../Parser/MethodInvocation.Tests.ps1         | 33 ++++++++++++++++---
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/test/powershell/Language/Parser/MethodInvocation.Tests.ps1 b/test/powershell/Language/Parser/MethodInvocation.Tests.ps1
index ff7e49105a2..ca4db39bd66 100644
--- a/test/powershell/Language/Parser/MethodInvocation.Tests.ps1
+++ b/test/powershell/Language/Parser/MethodInvocation.Tests.ps1
@@ -26,6 +26,28 @@ Describe 'Generic Method invocation' -Tags 'CI' {
                 IndexType    = 'System.Management.Automation.Language.InvokeMemberExpressionAst'
                 IndexString  = '[Array]::IndexOf($_.IPAddress, $_.IPAddress[0])'
             }
+            @{
+                Script       = @'
+                    [IPAddress]::Parse(
+                        $_.IPSubnet[
+                            [Array]::IndexOf($_.IPAddress, $_.IPAddress[0])
+                        ]
+                    )
+'@
+                IndexType    = 'System.Management.Automation.Language.InvokeMemberExpressionAst'
+                IndexString  = '[Array]::IndexOf($_.IPAddress, $_.IPAddress[0])'
+            }
+            @{
+                Script       = @'
+                    [IPAddress]::Parse(
+                        $_.IPSubnet[
+                            ([Array]::IndexOf($_.IPAddress, $_.IPAddress[0]))
+                        ]
+                    )
+'@
+                IndexType    = 'System.Management.Automation.Language.ParenExpressionAst'
+                IndexString  = '([Array]::IndexOf($_.IPAddress, $_.IPAddress[0]))'
+            }
         )
 
         $ExpectedParseErrors = @(
@@ -103,7 +125,7 @@ Describe 'Generic Method invocation' -Tags 'CI' {
         { [scriptblock]::Create($script) } | Should -Not -Throw
     }
 
-    It "parses fine for indexing a property" -TestCases $IndexingAProperty {
+    It "parses fine for indexing a property: <Script>" -TestCases $IndexingAProperty {
         param($Script, $IndexType, $IndexString)
 
         $parseErrors = $null
@@ -113,9 +135,12 @@ Describe 'Generic Method invocation' -Tags 'CI' {
 
         $cmdExpr = $ast.EndBlock.Statements[0].PipelineElements[0]
         $cmdExpr | Should -BeOfType 'System.Management.Automation.Language.CommandExpressionAst'
-        $cmdExpr.Expression | Should -BeOfType 'System.Management.Automation.Language.IndexExpressionAst'
-        $cmdExpr.Expression.Index | Should -BeOfType $IndexType
-        $cmdExpr.Expression.Index.ToString() | Should -BeExactly $IndexString
+
+        $indexExpr = $cmdExpr.Expression -is [System.Management.Automation.Language.InvokeMemberExpressionAst] ? $cmdExpr.Expression.Arguments[0] : $cmdExpr.Expression
+
+        $indexExpr | Should -BeOfType 'System.Management.Automation.Language.IndexExpressionAst'
+        $indexExpr.Index | Should -BeOfType $IndexType
+        $indexExpr.Index.ToString() | Should -BeExactly $IndexString
     }
 
     It 'reports a parse error for "<Script>"' -TestCases $ExpectedParseErrors {