Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 3170969

Browse files
CTrandoCTrando
authored
Improving how arg set handles specialized function parameters (microsoft#1263)
* Making specialized functions more granular in the way that we can specify their parameters and have them handled just like normal functions, fixing test cases related to that, fixing documentation for TypeVar related to keyword only arguments
1 parent 3d5e6f9 commit 3170969

File tree

16 files changed

+230
-32
lines changed

16 files changed

+230
-32
lines changed

‎src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Constants.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ public IPythonInstance GetConstantFromLiteral(Expression expr, LookupOptions opt
3232
return new PythonAsciiString(b, Interpreter);
3333
case int integer:
3434
return new PythonConstant(integer, Interpreter.GetBuiltinType(BuiltinTypeId.Int));
35+
case bool b:
36+
return new PythonConstant(b, Interpreter.GetBuiltinType(BuiltinTypeId.Bool));
3537
}
3638
}
3739

‎src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ public void HandleWith(WithStatement node) {
3535
// If fetching context from __enter__ failed, annotation in the stub may be using
3636
// type from typing that we haven't specialized yet or there may be an issue in
3737
// the stub itself, such as type or incorrect type. Try using context manager then.
38-
context = context ?? contextManager;
38+
context = context.IsUnknown() ? contextManager : context;
3939
}
4040

4141
if (item.Variable is NameExpression nex && !string.IsNullOrEmpty(nex.Name)) {

‎src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
// See the Apache Version 2.0 License for specific language governing
1414
// permissions and limitations under the License.
1515

16+
using System.Collections.Generic;
1617
using System.Linq;
1718
using Microsoft.Python.Analysis.Analyzer;
1819
using Microsoft.Python.Analysis.Types;
@@ -39,10 +40,13 @@ public static void SpecializeFunction(this IDocumentAnalysis analysis, string na
3940
/// <summary>
4041
/// Provides ability to dynamically calculate function return type.
4142
/// </summary>
42-
public static void SpecializeFunction(this IDocumentAnalysis analysis, string name, ReturnValueProvider returnTypeCallback, string[] dependencies = null) {
43+
internal static void SpecializeFunction(this IDocumentAnalysis analysis, string name, ReturnValueProvider returnTypeCallback, IReadOnlyList<ParameterInfo> parameters = null, string[] dependencies = null) {
4344
var f = analysis.GetOrCreateFunction(name);
4445
if (f != null) {
4546
foreach (var o in f.Overloads.OfType<PythonFunctionOverload>()) {
47+
if (parameters != null) {
48+
o.SetParameters(parameters);
49+
}
4650
o.SetReturnValueProvider(returnTypeCallback);
4751
}
4852
f.Specialize(dependencies);

‎src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
using Microsoft.Python.Core;
2323
using Microsoft.Python.Core.IO;
2424
using Microsoft.Python.Parsing;
25+
using Microsoft.Python.Parsing.Ast;
2526

2627
namespace Microsoft.Python.Analysis.Modules {
2728
/// <summary>
@@ -43,7 +44,7 @@ public BuiltinsPythonModule(string moduleName, string filePath, IServiceContaine
4344
public override IEnumerable<string> GetMemberNames() => base.GetMemberNames().Except(_hiddenNames).ToArray();
4445

4546
protected override string[] GetScrapeArguments(IPythonInterpreter interpreter)
46-
=> !InstallPath.TryGetFile("scrape_module.py", out var sb) ? null : new [] { "-W", "ignore", "-B", "-E", sb };
47+
=> !InstallPath.TryGetFile("scrape_module.py", out var sb) ? null : new[] { "-W", "ignore", "-B", "-E", sb };
4748

4849
protected override void OnAnalysisComplete() {
4950
SpecializeTypes();
@@ -158,7 +159,9 @@ private void SpecializeFunctions() {
158159
Analysis.SpecializeFunction("max", BuiltinsSpecializations.Identity);
159160
Analysis.SpecializeFunction("min", BuiltinsSpecializations.Identity);
160161
Analysis.SpecializeFunction("next", BuiltinsSpecializations.Next);
161-
Analysis.SpecializeFunction("open", BuiltinsSpecializations.Open, new[] { "io" });
162+
163+
Analysis.SpecializeFunction("open", BuiltinsSpecializations.Open, OpenConstructor(), new[] { "io" });
164+
162165
Analysis.SpecializeFunction("ord", Interpreter.GetBuiltinType(BuiltinTypeId.Int));
163166
Analysis.SpecializeFunction("pow", BuiltinsSpecializations.Identity);
164167
Analysis.SpecializeFunction("range", BuiltinsSpecializations.Range);
@@ -170,5 +173,26 @@ private void SpecializeFunctions() {
170173
//SpecializeFunction(_builtinName, "sorted", ReturnsListOfInputIterable);
171174
//SpecializeFunction(_builtinName, "super", SpecialSuper);
172175
}
176+
177+
private IReadOnlyList<ParameterInfo> OpenConstructor() {
178+
if (Interpreter.LanguageVersion.Is2x()) {
179+
return new[] {
180+
new ParameterInfo("name", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, null),
181+
new ParameterInfo("mode", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, new PythonConstant("r", Interpreter.GetBuiltinType(BuiltinTypeId.Str))),
182+
new ParameterInfo("buffering", Interpreter.GetBuiltinType(BuiltinTypeId.Int), ParameterKind.Normal, new PythonConstant(-1, Interpreter.GetBuiltinType(BuiltinTypeId.Int))),
183+
};
184+
} else {
185+
return new[] {
186+
new ParameterInfo("file", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, null),
187+
new ParameterInfo("mode", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, new PythonConstant("r", Interpreter.GetBuiltinType(BuiltinTypeId.Str))),
188+
new ParameterInfo("buffering", Interpreter.GetBuiltinType(BuiltinTypeId.Int), ParameterKind.Normal, new PythonConstant(-1, Interpreter.GetBuiltinType(BuiltinTypeId.Int))),
189+
new ParameterInfo("encoding", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, new PythonConstant(null, Interpreter.GetBuiltinType(BuiltinTypeId.NoneType))),
190+
new ParameterInfo("errors", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, new PythonConstant(null, Interpreter.GetBuiltinType(BuiltinTypeId.NoneType))),
191+
new ParameterInfo("newline", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, new PythonConstant(null, Interpreter.GetBuiltinType(BuiltinTypeId.NoneType))),
192+
new ParameterInfo("closefd", Interpreter.GetBuiltinType(BuiltinTypeId.Bool), ParameterKind.Normal, new PythonConstant(true, Interpreter.GetBuiltinType(BuiltinTypeId.Bool))),
193+
new ParameterInfo("opener", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, new PythonConstant(null, Interpreter.GetBuiltinType(BuiltinTypeId.Str)))
194+
};
195+
}
196+
}
173197
}
174198
}

‎src/Analysis/Ast/Impl/Resources.resx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -204,4 +204,4 @@
204204
<data name="GenericNotAllUnique" xml:space="preserve">
205205
<value>Arguments to Generic must all be unique.</value>
206206
</data>
207-
</root>
207+
</root>

‎src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,8 +135,11 @@ public static IMember GetAttr(IPythonModule module, IPythonFunctionOverload over
135135
def = args[2];
136136
}
137137

138+
// second argument to getattr was not a string, which is a runtime error
139+
// getattr(a, 3.14)
138140
if (name == null) {
139-
return def;
141+
// TODO diagnostic error when second arg of getattr is not a string
142+
return module.Interpreter.UnknownType;
140143
}
141144

142145
return o?.GetPythonType().GetMember(name) ?? def;

‎src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,31 +35,43 @@ public GenericTypeParameter(string name, IPythonModule declaringModule, IReadOnl
3535

3636

3737
public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declaringModule, IndexSpan location = default) {
38-
var args = argSet.Values<IMember>();
38+
var args = argSet.Arguments;
39+
var constraintArgs = argSet.ListArgument?.Values ?? Array.Empty<IMember>();
40+
3941
if (args.Count == 0) {
4042
// TODO: report that at least one argument is required.
4143
return declaringModule.Interpreter.UnknownType;
4244
}
4345

44-
var name = (args[0] as IPythonConstant)?.GetString();
46+
var name = (args[0].Value as IPythonConstant)?.GetString();
4547
if (string.IsNullOrEmpty(name)) {
4648
// TODO: report that type name is not a string.
4749
return declaringModule.Interpreter.UnknownType;
4850
}
4951

50-
var constraints = args.Skip(1).Select(a => {
52+
var constraints = constraintArgs.Select(a => {
5153
// Type constraints may be specified as type name strings.
5254
var typeString = (a as IPythonConstant)?.GetString();
5355
return !string.IsNullOrEmpty(typeString) ? argSet.Eval.GetTypeFromString(typeString) : a.GetPythonType();
54-
}).ToArray();
56+
}).ToArray() ?? Array.Empty<IPythonType>();
57+
5558
if (constraints.Any(c => c.IsUnknown())) {
5659
// TODO: report that some constraints could not be resolved.
5760
}
5861

59-
var docArgs = new[] { $"'{name}'" }.Concat(constraints.Select(c => c.IsUnknown() ? "?" : c.Name));
60-
var documentation = CodeFormatter.FormatSequence("TypeVar", '(', docArgs);
61-
62+
var documentation = GetDocumentation(args, constraints);
6263
return new GenericTypeParameter(name, declaringModule, constraints, documentation, location);
6364
}
65+
66+
private static string GetDocumentation(IReadOnlyList<IArgument> args, IReadOnlyList<IPythonType> constraints) {
67+
var name = (args[0].Value as IPythonConstant).GetString();
68+
var keyWordArgs = args.Skip(1)
69+
.Where(x => !x.ValueIsDefault)
70+
.Select(x => $"{x.Name}={(x.Value as IPythonConstant)?.Value}");
71+
72+
var docArgs = constraints.Select(c => c.IsUnknown() ? "?" : c.Name).Concat(keyWordArgs).Prepend($"'{name}'");
73+
var documentation = CodeFormatter.FormatSequence("TypeVar", '(', docArgs);
74+
return documentation;
75+
}
6476
}
6577
}

‎src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
using Microsoft.Python.Analysis.Values;
2424
using Microsoft.Python.Core;
2525
using Microsoft.Python.Parsing;
26+
using Microsoft.Python.Parsing.Ast;
2627

2728
namespace Microsoft.Python.Analysis.Specializations.Typing {
2829
internal sealed class TypingModule : SpecializedModule {
@@ -49,6 +50,14 @@ private void SpecializeMembers() {
4950
// TypeVar
5051
var fn = PythonFunctionType.Specialize("TypeVar", this, GetMemberDocumentation("TypeVar"));
5152
var o = new PythonFunctionOverload(fn.Name, location);
53+
o.SetParameters(new List<ParameterInfo> {
54+
new ParameterInfo("name", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.Normal, null),
55+
new ParameterInfo("constraints", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.List, null),
56+
new ParameterInfo("bound", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.KeywordOnly, new PythonConstant(null, Interpreter.GetBuiltinType(BuiltinTypeId.NoneType))),
57+
new ParameterInfo("covariant", Interpreter.GetBuiltinType(BuiltinTypeId.Bool), ParameterKind.KeywordOnly, new PythonConstant(false, Interpreter.GetBuiltinType(BuiltinTypeId.Bool))),
58+
new ParameterInfo("contravariant", Interpreter.GetBuiltinType(BuiltinTypeId.Bool), ParameterKind.KeywordOnly, new PythonConstant(false, Interpreter.GetBuiltinType(BuiltinTypeId.Bool)))
59+
});
60+
5261
// When called, create generic parameter type. For documentation
5362
// use original TypeVar declaration so it appear as a tooltip.
5463
o.SetReturnValueProvider((interpreter, overload, args)

‎src/Analysis/Ast/Impl/Types/ArgumentSet.cs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,13 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in
108108
var overload = fn.Overloads[overloadIndex];
109109
var fd = overload.FunctionDefinition;
110110

111-
if (fn.IsSpecialized) {
112-
// Typically specialized function, like TypeVar() that does not actually have AST definition.
113-
// Make the arguments from the call expression. If argument does not have name,
114-
// try using name from the function definition based on the argument position.
111+
// Some specialized functions have more complicated definitions, so we pass
112+
// parameters to those, TypeVar() is an example, so we allow the latter logic to handle
113+
// argument instatiation. For simple specialized functions, it is enough to handle here.
114+
if (fn.IsSpecialized && overload.Parameters.Count == 0) {
115+
// Specialized functions typically don't have AST definitions.
116+
// We construct the arguments from the call expression. If an argument does not have a name,
117+
// we try using name from the function definition based on the argument's position.
115118
_arguments = new List<Argument>();
116119
for (var i = 0; i < callExpr.Args.Count; i++) {
117120
var name = callExpr.Args[i].Name;

‎src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,16 @@ public interface IArgument {
4343
/// Argument annotation type, if any.
4444
/// </summary>
4545
IPythonType Type { get; }
46-
46+
4747
/// <summary>
4848
/// Parameter location in the AST.
4949
/// </summary>
5050
Node Location { get; }
51+
52+
/// <summary>
53+
/// Returns true if this value of the argument is default
54+
/// </summary>
55+
bool ValueIsDefault { get; }
5156
}
5257

5358
/// <summary>
@@ -58,7 +63,7 @@ public interface IListArgument {
5863
/// Argument name.
5964
/// </summary>
6065
string Name { get; }
61-
66+
6267
/// <summary>
6368
/// ValueExpression that evaluates to the value of the argument.
6469
/// Function call parameter.

0 commit comments

Comments
 (0)