diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index 26dfceb52..3f87fb444 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -21,7 +21,7 @@ Python is a requirement for the language server to run. In VS Code, an interpret _must_ be selected in order for the language server to properly initialize. If your language server fails to start, be sure that you have selected an interpreter. -The language server can only run on platforms where the .NET Core can run. This rougly means: +The language server can only run on platforms where the .NET Core can run. This roughly means: - Windows, 32/64 bit - macOS, 64 bit diff --git a/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs b/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs index 2b5b22c83..937634006 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ActivityTracker.cs @@ -19,48 +19,21 @@ namespace Microsoft.Python.Analysis.Analyzer { internal static class ActivityTracker { - private static readonly Dictionary _modules = new Dictionary(); + private static readonly HashSet _modules = new HashSet(); private static readonly object _lock = new object(); private static bool _tracking; private static Stopwatch _sw; - private struct AnalysisState { - public int Count; - public bool IsComplete; - } - public static void OnEnqueueModule(string path) { if (string.IsNullOrEmpty(path)) { return; } lock (_lock) { - if (!_modules.TryGetValue(path, out var st)) { - _modules[path] = default; - } else { - st.IsComplete = false; - } - } - } - - public static void OnModuleAnalysisComplete(string path) { - lock (_lock) { - if (_modules.TryGetValue(path, out var st)) { - st.Count++; - st.IsComplete = true; - } + _modules.Add(path); } } - public static bool IsAnalysisComplete { - get { - lock (_lock) { - return _modules.All(m => m.Value.IsComplete); - } - } - } - - public static void StartTracking() { lock (_lock) { if (!_tracking) { @@ -71,22 +44,15 @@ public static void StartTracking() { } } - public static void EndTracking() { + public static (int modulesCount, double totalMilliseconds) EndTracking() { lock (_lock) { if (_tracking) { _sw?.Stop(); _tracking = false; } - } - } - public static int ModuleCount { - get { - lock (_lock) { - return _modules.Count; - } + return (_modules.Count, _sw?.Elapsed.TotalMilliseconds ?? 0); } } - public static double MillisecondsElapsed => _sw?.Elapsed.TotalMilliseconds ?? 0; } } diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs index 0365fad92..d62f11fe2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs @@ -15,31 +15,48 @@ using System; using System.Diagnostics; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Analyzer { [DebuggerDisplay("{Name} : {FilePath}")] - internal struct AnalysisModuleKey : IEquatable { + internal readonly struct AnalysisModuleKey : IEquatable { + private enum KeyType { Default, Typeshed, LibraryAsDocument } + + private readonly KeyType _type; public string Name { get; } public string FilePath { get; } - public bool IsTypeshed { get; } + public bool IsTypeshed => _type == KeyType.Typeshed; + public bool IsLibraryAsDocument => _type == KeyType.LibraryAsDocument; public AnalysisModuleKey(IPythonModule module) { Name = module.Name; FilePath = module.ModuleType == ModuleType.CompiledBuiltin ? null : module.FilePath; - IsTypeshed = module is StubPythonModule stub && stub.IsTypeshed; + _type = module is StubPythonModule stub && stub.IsTypeshed + ? KeyType.Typeshed + : module.ModuleType == ModuleType.Library && module is IDocument document && document.IsOpen + ? KeyType.LibraryAsDocument + : KeyType.Default; } public AnalysisModuleKey(string name, string filePath, bool isTypeshed) { Name = name; FilePath = filePath; - IsTypeshed = isTypeshed; + _type = isTypeshed ? KeyType.Typeshed : KeyType.Default; + } + + private AnalysisModuleKey(string name, string filePath, KeyType type) { + Name = name; + FilePath = filePath; + _type = type; } + public AnalysisModuleKey GetLibraryAsDocumentKey() => new AnalysisModuleKey(Name, FilePath, KeyType.LibraryAsDocument); + public bool Equals(AnalysisModuleKey other) - => Name.EqualsOrdinal(other.Name) && FilePath.PathEquals(other.FilePath) && IsTypeshed == other.IsTypeshed; + => Name.EqualsOrdinal(other.Name) && FilePath.PathEquals(other.FilePath) && _type == other._type; public override bool Equals(object obj) => obj is AnalysisModuleKey other && Equals(other); @@ -47,7 +64,7 @@ public override int GetHashCode() { unchecked { var hashCode = (Name != null ? Name.GetHashCode() : 0); hashCode = (hashCode * 397) ^ (FilePath != null ? FilePath.GetPathHashCode() : 0); - hashCode = (hashCode * 397) ^ IsTypeshed.GetHashCode(); + hashCode = (hashCode * 397) ^ _type.GetHashCode(); return hashCode; } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index bb1665b8d..8d4b9751d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -26,6 +26,7 @@ internal interface IAnalyzable { /// /// Notifies document that its analysis is now complete. /// - void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass); + /// Document analysis + void NotifyAnalysisComplete(IDocumentAnalysis analysis); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs index 122992414..970c4bb3f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs @@ -20,6 +20,7 @@ using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core.Collections; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public interface IPythonAnalyzer { @@ -27,7 +28,7 @@ public interface IPythonAnalyzer { /// /// Schedules module for analysis. Module will be scheduled if version of AST is greater than the one used to get previous analysis /// - void EnqueueDocumentForAnalysis(IPythonModule module, int version); + void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int version); /// /// Schedules module for analysis for its existing AST, but with new dependencies. diff --git a/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs index b073f20b2..f5d836b0c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs @@ -30,7 +30,7 @@ public EmptyAnalysis(IServiceContainer services, IDocument document) { Document = document ?? throw new ArgumentNullException(nameof(document)); GlobalScope = new EmptyGlobalScope(document); Ast = AstUtilities.MakeEmptyAst(document.Uri); - ExpressionEvaluator = new ExpressionEval(services, document); + ExpressionEvaluator = new ExpressionEval(services, document, Ast); } public IDocument Document { get; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index 657ab23d2..863fb1af4 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -89,11 +89,11 @@ public IMember GetValueFromLambda(LambdaExpression expr) { public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr) { SymbolTable.Evaluate(cls.ClassDefinition); // Determine argument types - var args = ArgumentSet.Empty; + var args = ArgumentSet.Empty(expr, this); var init = cls.GetMember(@"__init__"); if (init != null) { using (OpenScope(cls.DeclaringModule, cls.ClassDefinition, out _)) { - var a = new ArgumentSet(init, 0, new PythonInstance(cls), expr, Module, this); + var a = new ArgumentSet(init, 0, new PythonInstance(cls), expr, this); if (a.Errors.Count > 0) { // AddDiagnostics(Module.Uri, a.Errors); } @@ -108,7 +108,7 @@ private IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) { case IPythonFunctionType fn: return GetValueFromFunctionType(fn, t.Self, expr); case IPythonPropertyType p: - return GetValueFromProperty(p, t.Self); + return GetValueFromProperty(p, t.Self, expr); case IPythonIteratorType _ when t.Self is IPythonCollection seq: return seq.GetIterator(); } @@ -201,10 +201,10 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance return UnknownType; } - private IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance) { + private IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance, CallExpression expr) { // Function may not have been walked yet. Do it now. SymbolTable.Evaluate(p.FunctionDefinition); - return instance.Call(p.Name, ArgumentSet.Empty); + return instance.Call(p.Name, ArgumentSet.Empty(expr, this)); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs index e9ff23df2..514d1e735 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs @@ -16,12 +16,15 @@ using System; using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Specializations.Typing.Types; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; +using ErrorCodes = Microsoft.Python.Analysis.Diagnostics.ErrorCodes; namespace Microsoft.Python.Analysis.Analyzer.Evaluation { internal sealed partial class ExpressionEval { @@ -59,40 +62,58 @@ private IMember GetValueFromGeneric(IMember target, Expression expr) { } /// - /// Given generic type and list of indices in the expression like - /// Generic[T1, T2, ...] or List[str] creates generic class base - /// (if the former) on specific type (if the latter). + /// Returns whether the arguments to Generic are valid /// - private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList indices, Expression expr) { - // See which ones are generic parameters as defined by TypeVar() - // and which are specific types. Normally there should not be a mix. - var genericTypeArgs = indices.OfType().ToArray(); - var specificTypes = indices.Where(i => !(i is IGenericTypeDefinition)).OfType().ToArray(); - - if (genericTypeArgs.Length > 0 && genericTypeArgs.Length != indices.Count) { - // TODO: report that some type arguments are not declared with TypeVar. + private bool GenericClassParameterValid(IReadOnlyList genericTypeArgs, IReadOnlyList args, Expression expr) { + // All arguments to Generic must be type parameters + // e.g. Generic[T, str] throws a runtime error + if (genericTypeArgs.Count != args.Count) { + ReportDiagnostics(Module.Uri, new DiagnosticsEntry( + Resources.GenericNotAllTypeParameters, + GetLocation(expr).Span, + ErrorCodes.TypingGenericArguments, + Severity.Error, + DiagnosticSource.Analysis)); + return false; } - if (specificTypes.Length > 0 && specificTypes.Length != indices.Count) { - // TODO: report that arguments are not specific types or are not declared. + + // All arguments to Generic must be distinct + if (genericTypeArgs.Distinct().Count() != genericTypeArgs.Count) { + ReportDiagnostics(Module.Uri, new DiagnosticsEntry( + Resources.GenericNotAllUnique, + GetLocation(expr).Span, + ErrorCodes.TypingGenericArguments, + Severity.Error, + DiagnosticSource.Analysis)); + return false; } + return true; + } + + /// + /// Given generic type and list of arguments in the expression like + /// Generic[T1, T2, ...] or List[str] creates generic class base + /// (if the former) on specific type (if the latter). + /// + private IMember CreateSpecificTypeFromIndex(IGenericType gt, IReadOnlyList args, Expression expr) { + var genericTypeArgs = args.OfType().ToArray(); + if (gt.Name.EqualsOrdinal("Generic")) { - // Generic[T1, T2, ...] expression. Create generic base for the class. - if (genericTypeArgs.Length > 0) { - return new GenericClassParameter(genericTypeArgs, Module); - } else { - // TODO: report too few type arguments for Generic[]. + if (!GenericClassParameterValid(genericTypeArgs, args, expr)) { return UnknownType; } + + // Generic[T1, T2, ...] expression. Create generic base for the class. + return new GenericClassParameter(genericTypeArgs, Module); } // For other types just use supplied arguments - if (indices.Count > 0) { - return gt.CreateSpecificType(new ArgumentSet(indices)); + if (args.Count > 0) { + return gt.CreateSpecificType(new ArgumentSet(args, expr, this)); } - // TODO: report too few type arguments for the generic expression. - return UnknownType; + return UnknownType; } private IReadOnlyList EvaluateIndex(IndexExpression expr) { @@ -104,7 +125,10 @@ private IReadOnlyList EvaluateIndex(IndexExpression expr) { } } else { var index = GetValueFromExpression(expr.Index); - indices.Add(index ?? UnknownType); + // Don't count null indexes as arguments + if (index != null) { + indices.Add(index); + } } return indices; } @@ -132,7 +156,7 @@ private IMember CreateClassInstance(PythonClassType cls, IReadOnlyList var argSet = initOverload != null ? new ArgumentSet(initFunc, 0, null, callExpr, this) - : new ArgumentSet(constructorArguments); + : new ArgumentSet(constructorArguments, callExpr, this); argSet.Evaluate(); var specificType = cls.CreateSpecificType(argSet); @@ -194,7 +218,7 @@ private static IReadOnlyList GetSpecificTypeFromArgumentValue(objec var itemType = iter.GetIterator().Next.GetPythonType(); if (!itemType.IsUnknown()) { specificTypes.Add(itemType); - } else if(argumentValue is IPythonInstance inst) { + } else if (argumentValue is IPythonInstance inst) { specificTypes.Add(inst.GetPythonType()); } break; diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs index 16f89c794..71a709bc1 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs @@ -13,13 +13,15 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; +using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Types.Collections; using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; +using ErrorCodes = Microsoft.Python.Analysis.Diagnostics.ErrorCodes; namespace Microsoft.Python.Analysis.Analyzer.Evaluation { internal sealed partial class ExpressionEval { @@ -47,7 +49,7 @@ private IMember GetValueFromUnaryOp(UnaryExpression expr, string op) { var fn = instance.GetPythonType()?.GetMember(op); // Process functions declared in code modules. Scraped/compiled/stub modules do not actually perform any operations. if (fn?.DeclaringModule != null && (fn.DeclaringModule.ModuleType == ModuleType.User || fn.DeclaringModule.ModuleType == ModuleType.Library)) { - var result = fn.Call(instance, op, ArgumentSet.Empty); + var result = fn.Call(instance, op, ArgumentSet.Empty(expr, this)); if (!result.IsUnknown()) { return result; } @@ -122,6 +124,9 @@ private IMember GetValueFromBinaryOp(Expression expr) { if (leftIsSupported && rightIsSupported) { if (TryGetValueFromBuiltinBinaryOp(op, leftTypeId, rightTypeId, Interpreter.LanguageVersion.Is3x(), out var member)) { + if (member.IsUnknown()) { + ReportOperatorDiagnostics(expr, leftType, rightType, op); + } return member; } } @@ -132,9 +137,9 @@ private IMember GetValueFromBinaryOp(Expression expr) { if (op.IsComparison()) { // If the op is a comparison, and the thing on the left is the builtin, // flip the operation and call it instead. - ret = CallOperator(op.InvertComparison(), right, rightType, left, leftType, tryRight: false); + ret = CallOperator(op.InvertComparison(), right, rightType, left, leftType, expr, tryRight: false); } else { - ret = CallOperator(op, left, leftType, right, rightType, tryLeft: false); + ret = CallOperator(op, left, leftType, right, rightType, expr, tryLeft: false); } if (!ret.IsUnknown()) { @@ -146,7 +151,7 @@ private IMember GetValueFromBinaryOp(Expression expr) { if (rightIsSupported) { // Try calling the function on the left side, otherwise just return right. - var ret = CallOperator(op, left, leftType, right, rightType, tryRight: false); + var ret = CallOperator(op, left, leftType, right, rightType, expr, tryRight: false); if (!ret.IsUnknown()) { return ret; @@ -155,13 +160,13 @@ private IMember GetValueFromBinaryOp(Expression expr) { return op.IsComparison() ? Interpreter.GetBuiltinType(BuiltinTypeId.Bool) : right; } - var callRet = CallOperator(op, left, leftType, right, rightType); + var callRet = CallOperator(op, left, leftType, right, rightType, expr); if (!callRet.IsUnknown()) { return callRet; } if (op.IsComparison()) { - callRet = CallOperator(op.InvertComparison(), right, rightType, left, leftType); + callRet = CallOperator(op.InvertComparison(), right, rightType, left, leftType, expr); if (!callRet.IsUnknown()) { return callRet; @@ -197,18 +202,18 @@ private IMember GetValueFromBinaryOp(Expression expr) { return left.IsUnknown() ? right : left; } - private IMember CallOperator(PythonOperator op, IMember left, IPythonType leftType, IMember right, IPythonType rightType, bool tryLeft = true, bool tryRight = true) { + private IMember CallOperator(PythonOperator op, IMember left, IPythonType leftType, IMember right, IPythonType rightType, Expression expr, bool tryLeft = true, bool tryRight = true) { var (funcName, swappedFuncName) = OpMethodName(op); if (tryLeft && funcName != null && left is IPythonInstance lpi) { - var ret = leftType.Call(lpi, funcName, new ArgumentSet(new[] { right })); + var ret = leftType.Call(lpi, funcName, new ArgumentSet(new[] { right }, expr, this)); if (!ret.IsUnknown()) { return ret; } } if (tryRight && swappedFuncName != null && right is IPythonInstance rpi) { - var ret = rightType.Call(rpi, swappedFuncName, new ArgumentSet(new[] { left })); + var ret = rightType.Call(rpi, swappedFuncName, new ArgumentSet(new[] { left }, expr, this)); if (!ret.IsUnknown()) { return ret; } @@ -435,5 +440,14 @@ private static (string name, string swappedName) OpMethodName(PythonOperator op) return (null, null); } + private void ReportOperatorDiagnostics(Expression expr, IPythonType leftType, IPythonType rightType, PythonOperator op) { + ReportDiagnostics(Module.Uri, new DiagnosticsEntry( + Resources.UnsupporedOperandType.FormatInvariant(op.ToCodeString(), leftType.Name, rightType.Name), + GetLocation(expr).Span, + ErrorCodes.UnsupportedOperandType, + Severity.Error, + DiagnosticSource.Analysis)); + } + } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index c5baaa8ec..bd5760289 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -41,7 +41,7 @@ public void DeclareVariable(string name, IMember value, VariableSource source) => DeclareVariable(name, value, source, default(Location)); public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module) - => DeclareVariable(name, value, source, new Location(module, default)); + => DeclareVariable(name, value, source, new Location(module)); public void DeclareVariable(string name, IMember value, VariableSource source, Node location, bool overwrite = false) => DeclareVariable(name, value, source, GetLocationOfName(location), overwrite); diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 6bf57caf0..f66170650 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -38,15 +38,12 @@ internal sealed partial class ExpressionEval : IExpressionEvaluator { private readonly object _lock = new object(); private readonly List _diagnostics = new List(); - public ExpressionEval(IServiceContainer services, IPythonModule module) - : this(services, module, new GlobalScope(module)) { } - - public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalScope gs) { + public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) { Services = services ?? throw new ArgumentNullException(nameof(services)); Module = module ?? throw new ArgumentNullException(nameof(module)); - Ast = module.GetAst(); + Ast = ast ?? throw new ArgumentNullException(nameof(ast)); - GlobalScope = gs; + GlobalScope = new GlobalScope(module, ast); CurrentScope = GlobalScope; DefaultLocation = new Location(module); //Log = services.GetService(); @@ -61,7 +58,7 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalSc public Location DefaultLocation { get; } public IPythonModule BuiltinsModule => Interpreter.ModuleResolution.BuiltinsModule; - public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty; + public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; public Location GetLocationOfName(Node node) { if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) { @@ -104,7 +101,7 @@ public Location GetLocationOfName(Node node) { public IServiceContainer Services { get; } IScope IExpressionEvaluator.CurrentScope => CurrentScope; IGlobalScope IExpressionEvaluator.GlobalScope => GlobalScope; - public LocationInfo GetLocation(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty; + public LocationInfo GetLocation(Node node) => node?.GetLocation(this) ?? LocationInfo.Empty; public IEnumerable Diagnostics => _diagnostics; public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) { @@ -188,6 +185,10 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options = L case NamedExpression namedExpr: m = GetValueFromExpression(namedExpr.Value); break; + // indexing with nothing, e.g Generic[] + case ErrorExpression error: + m = null; + break; default: m = GetValueFromBinaryOp(expr) ?? GetConstantFromLiteral(expr, options); break; @@ -272,7 +273,7 @@ private IMember GetValueFromMember(MemberExpression expr) { case IPythonClassType _: return value; case IPythonPropertyType prop: - return prop.Call(instance, prop.Name, ArgumentSet.Empty); + return prop.Call(instance, prop.Name, ArgumentSet.Empty(expr, this)); case IPythonType p: return new PythonBoundType(p, instance); case null: diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs index 1626e8c5d..5837a9d0c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs @@ -116,7 +116,7 @@ private void HandleTypedVariable(IPythonType variableType, IMember value, Expres instance = value; } } - instance = instance ?? variableType?.CreateInstance(variableType.Name, ArgumentSet.Empty) ?? Eval.UnknownType; + instance = instance ?? variableType?.CreateInstance(variableType.Name, ArgumentSet.Empty(expr, Eval)) ?? Eval.UnknownType; if (expr is NameExpression ne) { Eval.DeclareVariable(ne.Name, instance, VariableSource.Declaration, ne); diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index cba333e73..6c330bfaa 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -125,8 +125,8 @@ private void SpecializeFuture(FromImportStatement node) { var printNameExpression = node.Names.FirstOrDefault(n => n?.Name == "print_function"); if (printNameExpression != null) { - var fn = new PythonFunctionType("print", new Location(Module, default), null, string.Empty); - var o = new PythonFunctionOverload(fn.Name, new Location(Module, default)); + var fn = new PythonFunctionType("print", new Location(Module), null, string.Empty); + var o = new PythonFunctionOverload(fn.Name, new Location(Module)); var parameters = new List { new ParameterInfo("*values", Interpreter.GetBuiltinType(BuiltinTypeId.Object), ParameterKind.List, null), new ParameterInfo("sep", Interpreter.GetBuiltinType(BuiltinTypeId.Str), ParameterKind.KeywordOnly, null), diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index 3562c7f12..341db6291 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -56,19 +56,22 @@ private void HandleImport(ModuleName moduleImportExpression, NameExpression asNa // import_module('fob.oar.baz') var importNames = ImmutableArray.Empty; var variableModule = default(PythonVariableModule); + var importedNamesCount = 0; foreach (var nameExpression in moduleImportExpression.Names) { importNames = importNames.Add(nameExpression.Name); var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, forceAbsolute); if (!HandleImportSearchResult(imports, variableModule, asNameExpression, moduleImportExpression, out variableModule)) { - return; + break; } + + importedNamesCount++; } // "import fob.oar.baz as baz" is handled as baz = import_module('fob.oar.baz') // "import fob.oar.baz" is handled as fob = import_module('fob') - if (!string.IsNullOrEmpty(asNameExpression?.Name)) { + if (!string.IsNullOrEmpty(asNameExpression?.Name) && importedNamesCount == moduleImportExpression.Names.Count) { Eval.DeclareVariable(asNameExpression.Name, variableModule, VariableSource.Import, asNameExpression); - } else if (importNames.Count > 0 && !string.IsNullOrEmpty(importNames[0]) && _variableModules.TryGetValue(importNames[0], out variableModule)) { + } else if (importedNamesCount > 0 && !string.IsNullOrEmpty(importNames[0]) && _variableModules.TryGetValue(importNames[0], out variableModule)) { var firstName = moduleImportExpression.Names[0]; Eval.DeclareVariable(importNames[0], variableModule, VariableSource.Import, firstName); } @@ -87,7 +90,7 @@ private bool HandleImportSearchResult(in IImportSearchResult imports, in PythonV case RelativeImportBeyondTopLevel importBeyondTopLevel: var message = Resources.ErrorRelativeImportBeyondTopLevel.FormatInvariant(importBeyondTopLevel.RelativeImportName); Eval.ReportDiagnostics(Eval.Module.Uri, - new DiagnosticsEntry(message, location.GetLocation(Eval.Module).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); + new DiagnosticsEntry(message, location.GetLocation(Eval).Span, ErrorCodes.UnresolvedImport, Severity.Warning, DiagnosticSource.Analysis)); variableModule = default; return false; case ImportNotFound importNotFound: diff --git a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs index 42caa54da..bf8f7df93 100644 --- a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs @@ -29,7 +29,7 @@ namespace Microsoft.Python.Analysis.Analyzer { /// Analysis of a library code. /// internal sealed class LibraryAnalysis : IDocumentAnalysis { - public LibraryAnalysis(IDocument document, int version, IServiceContainer services, GlobalScope globalScope, IReadOnlyList starImportMemberNames) { + public LibraryAnalysis(IDocument document, int version, IGlobalScope globalScope, IExpressionEvaluator eval, IReadOnlyList starImportMemberNames) { Check.ArgumentNotNull(nameof(document), document); Check.ArgumentNotNull(nameof(globalScope), globalScope); @@ -37,14 +37,7 @@ public LibraryAnalysis(IDocument document, int version, IServiceContainer servic Version = version; GlobalScope = globalScope; - var ast = Document.GetAst(); - ast.Reduce(x => x is ImportStatement || x is FromImportStatement); - var c = (IAstNodeContainer)Document; - c.ClearContent(); - c.ClearAst(); - c.AddAstNode(document, ast); - - ExpressionEvaluator = new ExpressionEval(services, document, globalScope); + ExpressionEvaluator = eval; StarImportMemberNames = starImportMemberNames; } diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 427845247..87834d78d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -37,8 +37,8 @@ internal class ModuleWalker : AnalysisWalker { private int _allReferencesCount; private bool _allIsUsable = true; - public ModuleWalker(IServiceContainer services, IPythonModule module) - : base(new ExpressionEval(services, module)) { + public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast) + : base(new ExpressionEval(services, module, ast)) { _stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null; } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index d04f03978..0face4bd1 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Caching; @@ -31,6 +32,7 @@ using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Services; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable { @@ -70,9 +72,9 @@ public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = d => _analysisCompleteEvent.WaitAsync(cancellationToken); public async Task GetAnalysisAsync(IPythonModule module, int waitTime, CancellationToken cancellationToken) { - var key = new AnalysisModuleKey(module); PythonAnalyzerEntry entry; lock (_syncObj) { + var key = new AnalysisModuleKey(module); if (!_analysisEntries.TryGetValue(key, out entry)) { var emptyAnalysis = new EmptyAnalysis(_services, (IDocument)module); entry = new PythonAnalyzerEntry(emptyAnalysis); @@ -122,11 +124,13 @@ public void InvalidateAnalysis(IPythonModule module) { } public void RemoveAnalysis(IPythonModule module) { + AnalysisModuleKey key; lock (_syncObj) { - _analysisEntries.Remove(new AnalysisModuleKey(module)); + key = new AnalysisModuleKey(module); + _analysisEntries.Remove(key); } - _dependencyResolver.Remove(new AnalysisModuleKey(module)); + _dependencyResolver.Remove(key); } public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray analysisDependencies) { @@ -146,16 +150,28 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray= bufferVersion) { return; } + + // It is possible that parsing request for the library has been started when document is open, + // but it is closed at the moment of analysis and then become open again. + // In this case, we still need to analyze the document, but using correct entry. + var libraryAsDocumentKey = key.GetLibraryAsDocumentKey(); + if (entry.PreviousAnalysis is LibraryAnalysis && _analysisEntries.TryGetValue(libraryAsDocumentKey, out var documentEntry)) { + key = libraryAsDocumentKey; + entry = documentEntry; + } + } else { entry = new PythonAnalyzerEntry(new EmptyAnalysis(_services, (IDocument)module)); _analysisEntries[key] = entry; @@ -163,7 +179,7 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, int bufferVersion) } } - if (entry.Invalidate(module, bufferVersion, version, out var dependencies)) { + if (entry.Invalidate(module, ast, bufferVersion, version, out var dependencies)) { AnalyzeDocument(key, entry, dependencies); } } @@ -207,12 +223,12 @@ public IReadOnlyList LoadedModules { internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) => AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); - private void AnalyzeDocument(AnalysisModuleKey key, PythonAnalyzerEntry entry, ImmutableArray dependencies) { + private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry entry, in ImmutableArray dependencies) { _analysisCompleteEvent.Reset(); ActivityTracker.StartTracking(); _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) queued"); - var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserModule, dependencies); + var graphVersion = _dependencyResolver.ChangeValue(key, entry, entry.IsUserOrBuiltin || key.IsLibraryAsDocument, dependencies); lock (_syncObj) { if (_version > graphVersion) { @@ -228,7 +244,7 @@ private void AnalyzeDocument(AnalysisModuleKey key, PythonAnalyzerEntry entry, I } } - private bool TryCreateSession(int graphVersion, PythonAnalyzerEntry entry, out PythonAnalyzerSession session) { + private bool TryCreateSession(in int graphVersion, in PythonAnalyzerEntry entry, out PythonAnalyzerSession session) { var analyzeUserModuleOutOfOrder = false; lock (_syncObj) { if (_currentSession != null) { @@ -246,11 +262,6 @@ private bool TryCreateSession(int graphVersion, PythonAnalyzerEntry entry, out P } if (!_dependencyResolver.TryCreateWalker(graphVersion, 2, out var walker)) { - if (entry.Module.ModuleType == ModuleType.Builtins) { - session = CreateSession(null, entry); - return true; - } - session = null; return false; } @@ -292,7 +303,7 @@ private void StartNextSession(Task task) { session.Start(false); } - private PythonAnalyzerSession CreateSession(IDependencyChainWalker walker, PythonAnalyzerEntry entry) + private PythonAnalyzerSession CreateSession(in IDependencyChainWalker walker, in PythonAnalyzerEntry entry) => new PythonAnalyzerSession(_services, _progress, _analysisCompleteEvent, _startNextSession, _disposeToken.CancellationToken, walker, _version, entry); private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray missingKeys) { diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs index 9e41c7ef4..cb00d93c7 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerEntry.cs @@ -20,6 +20,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.DependencyResolution; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; @@ -32,13 +33,13 @@ internal sealed class PythonAnalyzerEntry { private readonly object _syncObj = new object(); private TaskCompletionSource _analysisTcs; private IPythonModule _module; - private bool _isUserModule; + private ModuleType _moduleType; + private PythonAst _ast; private IDocumentAnalysis _previousAnalysis; private HashSet _parserDependencies; private HashSet _analysisDependencies; private int _bufferVersion; private int _analysisVersion; - private int _depth; public IPythonModule Module { get { @@ -48,10 +49,18 @@ public IPythonModule Module { } } + public bool IsUserOrBuiltin { + get { + lock (_syncObj) { + return _moduleType == ModuleType.User || _moduleType == ModuleType.Builtins; + } + } + } + public bool IsUserModule { get { lock (_syncObj) { - return _isUserModule; + return _moduleType == ModuleType.User; } } } @@ -74,21 +83,12 @@ public int AnalysisVersion { } } - public int Depth { - get { - lock (_syncObj) { - return _depth; - } - } - } - public bool NotAnalyzed => PreviousAnalysis is EmptyAnalysis; public PythonAnalyzerEntry(EmptyAnalysis emptyAnalysis) { _previousAnalysis = emptyAnalysis; _module = emptyAnalysis.Document; - _isUserModule = emptyAnalysis.Document.ModuleType == ModuleType.User; - _depth = _isUserModule ? 0 : -1; + _moduleType = _module.ModuleType; _bufferVersion = -1; _analysisVersion = 0; @@ -101,12 +101,18 @@ public Task GetAnalysisAsync(CancellationToken cancellationTo public bool IsValidVersion(int version, out IPythonModule module, out PythonAst ast) { lock (_syncObj) { module = _module; - ast = module.GetAst(); - if (ast == null || module == null) { + ast = _ast; + + if (module == null) { return false; } - return _previousAnalysis is EmptyAnalysis || _isUserModule || _analysisVersion <= version; + if (ast == null) { + Debug.Assert(!(_previousAnalysis is LibraryAnalysis), $"Library module {module.Name} of type {module.ModuleType} has been analyzed already!"); + return false; + } + + return _previousAnalysis is EmptyAnalysis || _moduleType == ModuleType.User || _analysisVersion <= version; } } @@ -120,6 +126,11 @@ public void TrySetAnalysis(IDocumentAnalysis analysis, int version) { return; } + if (analysis is LibraryAnalysis) { + _ast = null; + _parserDependencies = null; + } + _analysisDependencies = null; UpdateAnalysisTcs(version); _previousAnalysis = analysis; @@ -209,20 +220,21 @@ public bool Invalidate(ImmutableArray analysisDependencies, int a } } - public bool Invalidate(IPythonModule module, int bufferVersion, int analysisVersion, out ImmutableArray dependencies) { + public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, int analysisVersion, out ImmutableArray dependencies) { dependencies = ImmutableArray.Empty; if (_bufferVersion >= bufferVersion) { return false; } - var dependenciesHashSet = FindDependencies(module, bufferVersion); + var dependenciesHashSet = FindDependencies(module, ast, bufferVersion); lock (_syncObj) { if (_analysisVersion >= analysisVersion && _bufferVersion >= bufferVersion) { return false; } + _ast = ast; _module = module; - _isUserModule = module.ModuleType == ModuleType.User; + _moduleType = module.ModuleType; _parserDependencies = dependenciesHashSet; Interlocked.Exchange(ref _bufferVersion, bufferVersion); @@ -234,13 +246,13 @@ public bool Invalidate(IPythonModule module, int bufferVersion, int analysisVers } } - private HashSet FindDependencies(IPythonModule module, int bufferVersion) { + private HashSet FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) { if (_bufferVersion > bufferVersion) { return new HashSet(); } var walker = new DependencyWalker(module); - module.GetAst().Walk(walker); + ast.Walk(walker); var dependencies = walker.Dependencies; dependencies.Remove(new AnalysisModuleKey(module)); return dependencies; @@ -285,8 +297,14 @@ public DependencyWalker(IPythonModule module) { } public override bool Walk(ImportStatement import) { + var forceAbsolute = import.ForceAbsolute; foreach (var moduleName in import.Names) { - HandleSearchResults(_pathResolver.FindImports(_module.FilePath, moduleName, import.ForceAbsolute)); + var importNames = ImmutableArray.Empty; + foreach (var nameExpression in moduleName.Names) { + importNames = importNames.Add(nameExpression.Name); + var imports = _pathResolver.GetImportsFromAbsoluteName(_module.FilePath, importNames, forceAbsolute); + HandleSearchResults(imports); + } } return false; diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index d2d9a58f1..a2347f281 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -19,14 +19,17 @@ using System.Runtime; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Caching; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Services; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { internal sealed class PythonAnalyzerSession { @@ -147,9 +150,9 @@ private async Task StartAsync() { if (!isCanceled) { _progress.ReportRemaining(remaining); if (isFinal) { - ActivityTracker.EndTracking(); - (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(ActivityTracker.ModuleCount, ActivityTracker.MillisecondsElapsed); - _log?.Log(TraceEventType.Verbose, $"Analysis complete: {ActivityTracker.ModuleCount} modules in { ActivityTracker.MillisecondsElapsed} ms."); + var (modulesCount, totalMilliseconds) = ActivityTracker.EndTracking(); + (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(modulesCount, totalMilliseconds); + _log?.Log(TraceEventType.Verbose, $"Analysis complete: {modulesCount} modules in {totalMilliseconds} ms."); } } } @@ -194,7 +197,7 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { if (isCanceled && !node.Value.NotAnalyzed) { remaining++; - node.Skip(); + node.MoveNext(); continue; } @@ -247,25 +250,23 @@ private void Analyze(IDependencyChainNode node, AsyncCountd } _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); - node.Skip(); return; } var startTime = stopWatch.Elapsed; - AnalyzeEntry(entry, module, _walker.Version, node.IsComplete); - node.Commit(); - ActivityTracker.OnModuleAnalysisComplete(node.Value.Module.FilePath); + AnalyzeEntry(node, entry, module, ast, _walker.Version); - LogCompleted(module, stopWatch, startTime); + LogCompleted(node, module, stopWatch, startTime); } catch (OperationCanceledException oce) { node.Value.TryCancel(oce, _walker.Version); - node.Skip(); LogCanceled(node.Value.Module); } catch (Exception exception) { node.Value.TrySetException(exception, _walker.Version); - node.Commit(); + node.MarkWalked(); LogException(node.Value.Module, exception); } finally { + node.MoveNext(); + bool isCanceled; lock (_syncObj) { isCanceled = _isCanceled; @@ -294,9 +295,9 @@ private void AnalyzeEntry() { var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero; - AnalyzeEntry(_entry, module, Version, true); + AnalyzeEntry(null, _entry, module, ast, Version); - LogCompleted(module, stopWatch, startTime); + LogCompleted(null, module, stopWatch, startTime); } catch (OperationCanceledException oce) { _entry.TryCancel(oce, Version); LogCanceled(_entry.Module); @@ -309,7 +310,7 @@ private void AnalyzeEntry() { } } - private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int version, bool isFinalPass) { + private void AnalyzeEntry(IDependencyChainNode node, PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version) { if (entry.PreviousAnalysis is LibraryAnalysis) { _log?.Log(TraceEventType.Verbose, $"Request to re-analyze finalized {module.Name}."); } @@ -318,8 +319,7 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int v var analyzable = module as IAnalyzable; analyzable?.NotifyAnalysisBegins(); - var ast = module.GetAst(); - var walker = new ModuleWalker(_services, module); + var walker = new ModuleWalker(_services, module, ast); ast.Walk(walker); _analyzerCancellationToken.ThrowIfCancellationRequested(); @@ -327,7 +327,18 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int v walker.Complete(); _analyzerCancellationToken.ThrowIfCancellationRequested(); - analyzable?.NotifyAnalysisComplete(version, walker, isFinalPass); + bool isCanceled; + lock (_syncObj) { + isCanceled = _isCanceled; + } + + if (!isCanceled) { + node?.MarkWalked(); + } + + var analysis = CreateAnalysis(node, (IDocument)module, ast, version, walker, isCanceled); + + analyzable?.NotifyAnalysisComplete(analysis); entry.TrySetAnalysis(module.Analysis, version); if (module.ModuleType == ModuleType.User) { @@ -336,9 +347,13 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int v } } - private void LogCompleted(IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { + private void LogCompleted(IDependencyChainNode node, IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { if (_log != null) { - _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."); + var completed = node != null && module.Analysis is LibraryAnalysis ? "completed for library" : "completed"; + var message = node != null + ? $"Analysis of {module.Name}({module.ModuleType}) on depth {node.VertexDepth} {completed} in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms." + : $"Out of order analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."; + _log.Log(TraceEventType.Verbose, message); } } @@ -354,6 +369,25 @@ private void LogException(IPythonModule module, Exception exception) { } } + private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker, bool isCanceled) { + var createLibraryAnalysis = !isCanceled && + node != null && + !node.HasMissingDependencies && + document.ModuleType == ModuleType.Library && + !document.IsOpen && + node.HasOnlyWalkedDependencies && + node.IsValidVersion; + + if (!createLibraryAnalysis) { + return new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); + } + + ast.Reduce(x => x is ImportStatement || x is FromImportStatement); + document.SetAst(ast); + var eval = new ExpressionEval(walker.Eval.Services, document, ast); + return new LibraryAnalysis(document, version, walker.GlobalScope, eval, walker.StarImportMemberNames); + } + private enum State { NotStarted = 0, Started = 1, diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs index e7cd7c3f0..58b06e866 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs @@ -50,9 +50,9 @@ private async Task LoadBuiltinTypesAsync(string root, IServiceManager sm, Cancel lock (_lock) { var builtinModule = _moduleResolution.CreateBuiltinsModule(); _builtinTypes[BuiltinTypeId.NoneType] - = new PythonType("NoneType", new Location(builtinModule, default), string.Empty, BuiltinTypeId.NoneType); + = new PythonType("NoneType", new Location(builtinModule), string.Empty, BuiltinTypeId.NoneType); _builtinTypes[BuiltinTypeId.Unknown] - = UnknownType = new PythonType("Unknown", new Location(builtinModule, default), string.Empty); + = UnknownType = new PythonType("Unknown", new Location(builtinModule), string.Empty); } await _moduleResolution.InitializeAsync(cancellationToken); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs index c26067c0e..915008d07 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs @@ -13,12 +13,15 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections; using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Symbols { @@ -50,22 +53,12 @@ public void EvaluateClass() { EvaluateInnerClasses(_classDef); _class = classInfo; - // Set bases to the class. - var bases = new List(); - foreach (var a in _classDef.Bases.Where(a => string.IsNullOrEmpty(a.Name))) { - // We cheat slightly and treat base classes as annotations. - var b = Eval.GetTypeFromAnnotation(a.Expression); - if (b != null) { - var t = b.GetPythonType(); - bases.Add(t); - t.AddReference(Eval.GetLocationOfName(a.Expression)); - } - } - _class.SetBases(bases); + var bases = ProcessBases(outerScope); + + _class.SetBases(bases); // Declare __class__ variable in the scope. Eval.DeclareVariable("__class__", _class, VariableSource.Declaration); - ProcessClassBody(); } } @@ -118,6 +111,47 @@ private void ProcessClassBody() { UpdateClassMembers(); } + private IEnumerable ProcessBases(Scope outerScope) { + var bases = new List(); + foreach (var a in _classDef.Bases.Where(a => string.IsNullOrEmpty(a.Name))) { + var expr = a.Expression; + + switch (expr) { + // class declared in the current module + case NameExpression nameExpression: + var name = Eval.GetInScope(nameExpression.Name, outerScope); + switch (name) { + case PythonClassType classType: + bases.Add(classType); + break; + case IPythonConstant constant: + ReportInvalidBase(constant.Value); + break; + default: + TryAddBase(bases, a); + break; + } + break; + default: + TryAddBase(bases, a); + break; + } + } + return bases; + } + + private void TryAddBase(List bases, Arg arg) { + // We cheat slightly and treat base classes as annotations. + var b = Eval.GetTypeFromAnnotation(arg.Expression); + if (b != null) { + var t = b.GetPythonType(); + bases.Add(t); + t.AddReference(Eval.GetLocationOfName(arg.Expression)); + } else { + ReportInvalidBase(arg.ToCodeString(Eval.Ast, CodeFormattingOptions.Traditional)); + } + } + private void EvaluateConstructors(ClassDefinition cd) { // Do not use foreach since walker list is dynamically modified and walkers are removed // after processing. Handle __init__ and __new__ first so class variables are initialized. @@ -152,6 +186,17 @@ private void UpdateClassMembers() { _class.AddMembers(members, false); } + private void ReportInvalidBase(object argVal) { + Eval.ReportDiagnostics(Eval.Module.Uri, + new DiagnosticsEntry( + Resources.InheritNonClass.FormatInvariant(argVal), + _classDef.NameExpression.GetLocation(Eval)?.Span ?? default, + Diagnostics.ErrorCodes.InheritNonClass, + Severity.Error, + DiagnosticSource.Analysis + )); + } + // Classes and functions are walked by their respective evaluators public override bool Walk(ClassDefinition node) => false; public override bool Walk(FunctionDefinition node) => false; diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 9732aea0e..342ffcfd2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -18,12 +18,15 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Analysis.Values.Collections; using Microsoft.Python.Core; +using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; +using ErrorCodes = Microsoft.Python.Analysis.Diagnostics.ErrorCodes; namespace Microsoft.Python.Analysis.Analyzer.Symbols { [DebuggerDisplay("{FunctionDefinition.Name}")] @@ -34,7 +37,6 @@ internal sealed class FunctionEvaluator : MemberEvaluator { public FunctionEvaluator(ExpressionEval eval, PythonFunctionOverload overload) : base(eval, overload.FunctionDefinition) { - _overload = overload; _function = overload.ClassMember ?? throw new NullReferenceException(nameof(overload.ClassMember)); _self = _function.DeclaringType as PythonClassType; @@ -77,7 +79,7 @@ private IPythonType TryDetermineReturnValue() { if (!annotationType.IsUnknown()) { // Annotations are typically types while actually functions return // instances unless specifically annotated to a type such as Type[T]. - var t = annotationType.CreateInstance(annotationType.Name, ArgumentSet.Empty); + var t = annotationType.CreateInstance(annotationType.Name, ArgumentSet.WithoutContext); // If instance could not be created, such as when return type is List[T] and // type of T is not yet known, just use the type. var instance = t.IsUnknown() ? annotationType : t; @@ -116,6 +118,18 @@ public override bool Walk(AssignmentStatement node) { public override bool Walk(ReturnStatement node) { var value = Eval.GetValueFromExpression(node.Expression); if (value != null) { + // although technically legal, __init__ in a constructor should not have a not-none return value + if (FunctionDefinition.Name.EqualsOrdinal("__init__") && _function.DeclaringType.MemberType == PythonMemberType.Class + && !value.IsOfType(BuiltinTypeId.NoneType)) { + + Eval.ReportDiagnostics(Module.Uri, new Diagnostics.DiagnosticsEntry( + Resources.ReturnInInit, + node.GetLocation(Eval).Span, + ErrorCodes.ReturnInInit, + Severity.Warning, + DiagnosticSource.Analysis)); + } + _overload.AddReturnValue(value); } return true; // We want to evaluate all code so all private variables in __new__ get defined diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index 1d2e76c6b..1644a1db2 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -31,10 +31,10 @@ internal sealed class DependencyResolver : IDependencyResolver _version; - public int ChangeValue(TKey key, TValue value, bool isRoot, params TKey[] incomingKeys) + public int ChangeValue(in TKey key, in TValue value, in bool isRoot, params TKey[] incomingKeys) => ChangeValue(key, value, isRoot, ImmutableArray.Create(incomingKeys)); - public int ChangeValue(TKey key, TValue value, bool isRoot, ImmutableArray incomingKeys) { + public int ChangeValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys) { lock (_syncObj) { if (!_keys.TryGetValue(key, out var index)) { index = _keys.Count; @@ -47,7 +47,7 @@ public int ChangeValue(TKey key, TValue value, bool isRoot, ImmutableArray } } - public int TryAddValue(TKey key, TValue value, bool isRoot, ImmutableArray incomingKeys) { + public int TryAddValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys) { lock (_syncObj) { if (!_keys.TryGetValue(key, out var index)) { index = _keys.Count; @@ -62,7 +62,7 @@ public int TryAddValue(TKey key, TValue value, bool isRoot, ImmutableArray } } - public int Remove(TKey key) { + public int Remove(in TKey key) { lock (_syncObj) { if (!_keys.TryGetValue(key, out var index)) { return _version; @@ -71,14 +71,14 @@ public int Remove(TKey key) { var version = Interlocked.Increment(ref _version); var vertex = _vertices[index]; - _vertices[index] = default; - if (vertex == null) { + if (vertex == default) { return version; } - + + _vertices[index] = default; foreach (var incomingIndex in vertex.Incoming) { var incoming = _vertices[incomingIndex]; - if (incoming != null && incoming.IsSealed) { + if (incoming != default && incoming.IsSealed) { _vertices[incomingIndex] = new DependencyVertex(incoming, version, false); } } @@ -89,7 +89,7 @@ public int Remove(TKey key) { foreach (var outgoingIndex in vertex.Outgoing) { var outgoing = _vertices[outgoingIndex]; - if (outgoing != null && !outgoing.IsNew) { + if (outgoing != default && !outgoing.IsNew) { _vertices[outgoingIndex] = new DependencyVertex(outgoing, version, true); } } @@ -100,7 +100,7 @@ public int Remove(TKey key) { public int RemoveKeys(params TKey[] keys) => RemoveKeys(ImmutableArray.Create(keys)); - public int RemoveKeys(ImmutableArray keys) { + public int RemoveKeys(in ImmutableArray keys) { lock (_syncObj) { foreach (var key in keys) { if (_keys.TryGetValue(key, out var index)) { @@ -138,7 +138,7 @@ public int RemoveKeys(ImmutableArray keys) { } } - private void Update(TKey key, TValue value, bool isRoot, ImmutableArray incomingKeys, int index) { + private void Update(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys, in int index) { var version = Interlocked.Increment(ref _version); var incoming = EnsureKeys(index, incomingKeys, version); @@ -147,7 +147,7 @@ private void Update(TKey key, TValue value, bool isRoot, ImmutableArray in _keys[key] = index; } - private ImmutableArray EnsureKeys(int index, ImmutableArray keys, int version) { + private ImmutableArray EnsureKeys(in int index, in ImmutableArray keys, in int version) { var incoming = ImmutableArray.Empty; foreach (var key in keys) { @@ -175,7 +175,7 @@ public IDependencyChainWalker CreateWalker() { } } - public bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyChainWalker walker) { + public bool TryCreateWalker(in int version, in int walkerDepthLimit, out IDependencyChainWalker walker) { ImmutableArray> vertices; lock (_syncObj) { @@ -210,8 +210,7 @@ public bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyCh } // Iterate original graph to get starting vertices - var startingVertices = walkingGraph.Where(v => v.IncomingCount == 0); - if (!TryFindMissingDependencies(vertices, walkingGraph, version)) { + if (!TryFindMissingDependencies(vertices, walkingGraph, version, out var missingKeys)) { walker = default; return false; } @@ -222,26 +221,12 @@ public bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyCh } var affectedValues = walkingGraph.Select(v => v.DependencyVertex.Value); - var missingKeys = ImmutableArray.Empty; - - lock (_syncObj) { - if (version != _version) { - walker = default; - return false; - } - - foreach (var (key, index) in _keys) { - if (vertices[index] == default/* && depths[index] <= walkerDepthLimit*/) { - missingKeys = missingKeys.Add(key); - } - } - } - - walker = new DependencyChainWalker(startingVertices, affectedValues, depths, missingKeys, totalNodesCount, version); - return true; + var startingVertices = walkingGraph.Where(v => !v.HasIncoming); + walker = new DependencyChainWalker(this, startingVertices, affectedValues, depths, missingKeys, totalNodesCount, version); + return version == _version; } - private bool TryBuildReverseGraph(in ImmutableArray> vertices, int version) { + private bool TryBuildReverseGraph(in ImmutableArray> vertices, in int version) { var reverseGraphIsBuilt = true; foreach (var vertex in vertices) { if (vertex != null && !vertex.IsSealed) { @@ -256,7 +241,7 @@ private bool TryBuildReverseGraph(in ImmutableArray[vertices.Count]; foreach (var vertex in vertices) { - if (vertex == null) { + if (vertex == default) { continue; } @@ -283,7 +268,7 @@ private bool TryBuildReverseGraph(in ImmutableArray> vertices, ImmutableArray depths, int version, out ImmutableArray> analysisGraph) { + private bool TryCreateWalkingGraph(in ImmutableArray> vertices, in ImmutableArray depths, in int version, out ImmutableArray> analysisGraph) { if (version != _version) { analysisGraph = default; return false; @@ -326,7 +311,7 @@ private bool TryCreateWalkingGraph(in ImmutableArray vertex, } } - private bool TryFindMissingDependencies(in ImmutableArray> vertices, in ImmutableArray> walkingGraph, int version) { + private bool TryFindMissingDependencies(in ImmutableArray> vertices, in ImmutableArray> walkingGraph, int version, out ImmutableArray missingKeys) { var haveMissingDependencies = new bool[vertices.Count]; var queue = new Queue>(); + var missingIndicesHashSet = new HashSet(); + // First, go through all the vertices and find those that have missing incoming edges foreach (var vertex in vertices) { if (vertex == default) { continue; @@ -554,11 +541,14 @@ private bool TryFindMissingDependencies(in ImmutableArray.Empty; + + // From them, go through the edges and mark all reachable vertices while (queue.Count > 0) { if (version != _version) { return false; @@ -566,6 +556,10 @@ private bool TryFindMissingDependencies(in ImmutableArray { + private readonly DependencyResolver _dependencyResolver; private readonly ImmutableArray> _startingVertices; private readonly ImmutableArray _depths; private readonly object _syncObj; @@ -604,14 +607,16 @@ public int Remaining { } } - public DependencyChainWalker( + public DependencyChainWalker(in DependencyResolver dependencyResolver, in ImmutableArray> startingVertices, in ImmutableArray affectedValues, in ImmutableArray depths, in ImmutableArray missingKeys, in int totalNodesCount, in int version) { + _syncObj = new object(); + _dependencyResolver = dependencyResolver; _startingVertices = startingVertices; _depths = depths; AffectedValues = affectedValues; @@ -638,18 +643,18 @@ public Task> GetNextAsync(CancellationToken cancell return ppc.ConsumeAsync(cancellationToken); } - public void MarkCompleted(WalkingVertex vertex, bool commitChanges) { + public void MoveNext(WalkingVertex vertex) { var verticesToProduce = new List>(); var isCompleted = false; lock (_syncObj) { _remaining--; foreach (var outgoing in vertex.Outgoing) { - if (outgoing.IncomingCount == 0) { + if (!outgoing.HasIncoming) { continue; } - outgoing.DecrementIncoming(); - if (outgoing.IncomingCount > 0) { + outgoing.DecrementIncoming(vertex.HasOnlyWalkedIncoming); + if (outgoing.HasIncoming) { continue; } @@ -661,10 +666,6 @@ public void MarkCompleted(WalkingVertex vertex, bool commitChanges } } - if (commitChanges && vertex.SecondPass == null) { - vertex.DependencyVertex.MarkWalked(); - } - if (isCompleted) { _ppc.Produce(null); } else { @@ -673,15 +674,24 @@ public void MarkCompleted(WalkingVertex vertex, bool commitChanges } } } + + public bool IsValidVersion { + get { + lock (_dependencyResolver._syncObj) { + return _dependencyResolver._version == Version; + } + } + } } private sealed class DependencyChainNode : IDependencyChainNode { private readonly WalkingVertex _vertex; private DependencyChainWalker _walker; public TValue Value => _vertex.DependencyVertex.Value; - public bool HasMissingDependencies => _vertex.HasMissingDependencies; public int VertexDepth { get; } - public bool IsComplete => _vertex.SecondPass == null && !HasMissingDependencies; + public bool HasMissingDependencies => _vertex.HasMissingDependencies; + public bool HasOnlyWalkedDependencies => _vertex.HasOnlyWalkedIncoming && _vertex.SecondPass == default; + public bool IsValidVersion => _walker.IsValidVersion; public DependencyChainNode(DependencyChainWalker walker, WalkingVertex vertex, int depth) { _walker = walker; @@ -689,8 +699,13 @@ public DependencyChainNode(DependencyChainWalker walker, WalkingVertex Interlocked.Exchange(ref _walker, null)?.MarkCompleted(_vertex, true); - public void Skip() => Interlocked.Exchange(ref _walker, null)?.MarkCompleted(_vertex, false); + public void MarkWalked() { + if (_vertex.SecondPass == null) { + _vertex.DependencyVertex.MarkWalked(); + } + } + + public void MoveNext() => Interlocked.Exchange(ref _walker, null)?.MoveNext(_vertex); } } } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs index 03bffead2..ddbf9160d 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyVertex.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics; using System.Threading; diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs index 299235cc6..a4f394d00 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs @@ -18,12 +18,19 @@ internal interface IDependencyChainNode { int VertexDepth { get; } /// - /// Shows if node has any direct or indirect dependencies that aren't added to the graph + /// Returns true if node has any direct or indirect dependencies that aren't added to the graph, otherwise false /// bool HasMissingDependencies { get; } + /// + /// Returns true if node has only direct and indirect dependencies that have been walked at least once, otherwise false + /// + bool HasOnlyWalkedDependencies { get; } + /// + /// Returns true if node version matches version of the walked graph + /// + bool IsValidVersion { get; } TValue Value { get; } - void Commit(); - void Skip(); - bool IsComplete { get; } + void MarkWalked(); + void MoveNext(); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs index a8328bfb9..02fdfbdfd 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyResolver.cs @@ -24,12 +24,12 @@ namespace Microsoft.Python.Analysis.Dependencies { /// concurrently. /// internal interface IDependencyResolver { - int TryAddValue(TKey key, TValue value, bool isRoot, ImmutableArray incomingKeys); - int ChangeValue(TKey key, TValue value, bool isRoot, ImmutableArray incomingKeys); - int Remove(TKey key); - int RemoveKeys(ImmutableArray keys); + int TryAddValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys); + int ChangeValue(in TKey key, in TValue value, in bool isRoot, in ImmutableArray incomingKeys); + int Remove(in TKey key); + int RemoveKeys(in ImmutableArray keys); IDependencyChainWalker CreateWalker(); - bool TryCreateWalker(int version, int walkerDepthLimit, out IDependencyChainWalker walker); + bool TryCreateWalker(in int version, in int walkerDepthLimit, out IDependencyChainWalker walker); } } diff --git a/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs b/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs index a2ddb90ca..d3debee4f 100644 --- a/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs +++ b/src/Analysis/Ast/Impl/Dependencies/WalkingVertex.cs @@ -21,16 +21,19 @@ namespace Microsoft.Python.Analysis.Dependencies { [DebuggerDisplay("{" + nameof(DebuggerDisplay) + ",nq}")] internal sealed class WalkingVertex { - public static Comparison> FirstPassIncomingComparison { get; } = (v1, v2) => v1.FirstPass.IncomingCount.CompareTo(v2.FirstPass.IncomingCount); + public static Comparison> FirstPassIncomingComparison { get; } = (v1, v2) => v1.FirstPass._incomingCount.CompareTo(v2.FirstPass._incomingCount); private readonly List> _outgoing; private bool _isSealed; + private int _incomingCount; + private int _walkedIncomingCount; public DependencyVertex DependencyVertex { get; } public IReadOnlyList> Outgoing => _outgoing; public int Index { get; set; } public int LoopNumber { get; set; } - public int IncomingCount { get; private set; } + public bool HasIncoming => _incomingCount != 0; + public bool HasOnlyWalkedIncoming => _walkedIncomingCount == 0; public bool HasMissingDependencies { get; private set; } public WalkingVertex FirstPass { get; } @@ -57,7 +60,8 @@ public void AddOutgoing(WalkingVertex outgoingVertex) { CheckNotSealed(); _outgoing.Add(outgoingVertex); - outgoingVertex.IncomingCount++; + outgoingVertex._incomingCount++; + outgoingVertex._walkedIncomingCount++; } public void AddOutgoing(HashSet> loop) { @@ -65,7 +69,8 @@ public void AddOutgoing(HashSet> loop) { _outgoing.AddRange(loop); foreach (var outgoingVertex in loop) { - outgoingVertex.IncomingCount++; + outgoingVertex._incomingCount++; + outgoingVertex._walkedIncomingCount++; } } @@ -74,7 +79,8 @@ public void RemoveOutgoingAt(int index) { var outgoingVertex = _outgoing[index]; _outgoing.RemoveAt(index); - outgoingVertex.IncomingCount--; + outgoingVertex._incomingCount--; + outgoingVertex._walkedIncomingCount--; } public WalkingVertex CreateSecondPassVertex() { @@ -86,9 +92,12 @@ public WalkingVertex CreateSecondPassVertex() { public void Seal() => _isSealed = true; - public void DecrementIncoming() { + public void DecrementIncoming(bool isWalkedIncoming) { CheckSealed(); - IncomingCount--; + _incomingCount--; + if (isWalkedIncoming) { + _walkedIncomingCount--; + } } private void CheckSealed() => Check.InvalidOperation(_isSealed); diff --git a/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs b/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs index 45c6a1407..6797c4044 100644 --- a/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs +++ b/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs @@ -25,5 +25,10 @@ public static class ErrorCodes { public const string UndefinedVariable = "undefined-variable"; public const string VariableNotDefinedGlobally= "variable-not-defined-globally"; public const string VariableNotDefinedNonLocal = "variable-not-defined-nonlocal"; + public const string UnsupportedOperandType = "unsupported-operand-type"; + public const string ReturnInInit = "return-in-init"; + public const string TypingNewTypeArguments = "typing-newtype-arguments"; + public const string TypingGenericArguments = "typing-generic-arguments"; + public const string InheritNonClass = "inherit-non-class"; } } diff --git a/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs b/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs index e03fdf3b6..dc1228ddb 100644 --- a/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs +++ b/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs @@ -51,11 +51,6 @@ public interface IDocument: IPythonModule, IDisposable { /// Task GetAnalysisAsync(int waitTime = 200, CancellationToken cancellationToken = default); - /// - /// Returns last known document AST. The AST may be out of date or null. - /// - PythonAst GetAnyAst(); - /// /// Returns last known document analysis. The analysis may be out of date. /// diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index 9fc6d389d..a88533762 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -103,7 +103,6 @@ public IDocument OpenDocument(Uri uri, string content, string filePath = null) { if (justOpened) { Opened?.Invoke(this, new DocumentEventArgs(document)); - _services.GetService().InvalidateAnalysis(document); } return document; @@ -248,8 +247,8 @@ private bool TryAddModulePath(ModuleCreationOptions mco) { private bool TryOpenDocument(DocumentEntry entry, string content) { if (!entry.Document.IsOpen) { - entry.Document.Reset(content); entry.Document.IsOpen = true; + entry.Document.Reset(content); entry.LockCount++; return true; } diff --git a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs index e5ccb7bfb..aa82e938a 100644 --- a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs @@ -13,18 +13,30 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis { public static class NodeExtensions { - public static LocationInfo GetLocation(this Node node, IPythonModule module) { + public static LocationInfo GetLocation(this Node node, IExpressionEvaluator eval) { if (node == null || node.StartIndex >= node.EndIndex) { return LocationInfo.Empty; } - var start = node.GetStart(module.GetAst()); - var end = node.GetEnd(module.GetAst()); + return GetLocation(node, eval.Ast, eval.Module); + } + public static LocationInfo GetLocation(this Node node, IDocumentAnalysis analysis) { + if (node == null || node.StartIndex >= node.EndIndex) { + return LocationInfo.Empty; + } + + return GetLocation(node, analysis.Ast, analysis.Document); + } + + private static LocationInfo GetLocation(Node node, PythonAst ast, IPythonModule module) { + var start = node.GetStart(ast); + var end = node.GetEnd(ast); return new LocationInfo(module.FilePath, module.Uri, start.Line, start.Column, end.Line, end.Column); } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs index 107c5b41d..81442bc96 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs @@ -33,6 +33,8 @@ public static void AddMemberReference(this IPythonType type, string name, IExpre eval.LookupNameInScopes(name, out _, out var v, LookupOptions.Local); v?.AddReference(location); } + } else if (type is IPythonModule module && module.GlobalScope != null && module.GlobalScope.Variables.TryGetVariable(name, out var variable)) { + variable.AddReference(location); } } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs index c33b4620f..b2553b65c 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -20,10 +20,16 @@ namespace Microsoft.Python.Analysis { public static class PythonModuleExtensions { internal static PythonAst GetAst(this IPythonModule module) - => ((IAstNodeContainer)module).GetAstNode(module); + => (PythonAst)((IAstNodeContainer)module).GetAstNode(module); + + internal static void SetAst(this IPythonModule module, PythonAst ast) { + var contained = (IAstNodeContainer)module; + contained.ClearContent(); + contained.AddAstNode(module, ast); + } internal static T GetAstNode(this IPythonModule module, object o) where T : Node - => ((IAstNodeContainer)module).GetAstNode(o); + => (T)((IAstNodeContainer)module).GetAstNode(o); internal static void AddAstNode(this IPythonModule module, object o, Node n) => ((IAstNodeContainer)module).AddAstNode(o, n); } diff --git a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs index 6ba17ce62..69a5d536a 100644 --- a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs @@ -27,6 +27,8 @@ public static int GetBodyStartIndex(this IScope scope) { return cd.HeaderIndex; case FunctionDefinition fd: return fd.HeaderIndex; + case null: + return 0; default: return scope.Node.StartIndex; } diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs index cc33b194e..5cdae2a1b 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs @@ -98,7 +98,7 @@ public override bool Walk(NameExpression node) { // y = x // x = 1 var variableDefinitionSpan = v.Definition.Span; - var nameUseLocation = node.GetLocation(analysis.Document); + var nameUseLocation = node.GetLocation(analysis); // Make sure we are in the same scope in order to avoid // def func(): diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index f792b565f..c86634ddf 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -121,7 +121,7 @@ private void SpecializeTypes() { _hiddenNames.Add("__builtin_module_names__"); - var location = new Location(this, default); + var location = new Location(this); if (_boolType != null) { Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Builtin, location); Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Builtin, location); diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs index 0ed36cb97..deeb65c63 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs @@ -24,18 +24,13 @@ internal interface IAstNodeContainer { /// Nodes are not available for library modules as AST is not retained /// in libraries in order to conserve memory. /// - T GetAstNode(object o) where T : Node; + Node GetAstNode(object o); /// /// Associated AST node with the object. /// void AddAstNode(object o, Node n); - /// - /// Removes all AST node associations. - /// - void ClearAst(); - void ClearContent(); } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.Analysis.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.Analysis.cs deleted file mode 100644 index 07deffadd..000000000 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.Analysis.cs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Diagnostics; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Caching; -using Microsoft.Python.Analysis.Diagnostics; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; - -namespace Microsoft.Python.Analysis.Modules { - /// - /// Primary base for all modules and user documents. Provides access - /// to AST and the module analysis. - /// - [DebuggerDisplay("{Name} : {ModuleType}")] - internal partial class PythonModule { - public void NotifyAnalysisBegins() { - lock (AnalysisLock) { - if (Analysis is LibraryAnalysis) { - var sw = Log != null ? Stopwatch.StartNew() : null; - lock (AnalysisLock) { - _astMap[this] = RecreateAst(); - } - sw?.Stop(); - Log?.Log(TraceEventType.Verbose, $"Reloaded AST of {Name} in {sw?.Elapsed.TotalMilliseconds} ms"); - } - - if (_updated) { - _updated = false; - // In all variables find those imported, then traverse imported modules - // and remove references to this module. If variable refers to a module, - // recurse into module but only process global scope. - - if (GlobalScope == null) { - return; - } - - // TODO: Figure out where the nulls below are coming from. - var importedVariables = ((IScope)GlobalScope) - .TraverseDepthFirst(c => c?.Children ?? Enumerable.Empty()) - .SelectMany(s => s?.Variables ?? VariableCollection.Empty) - .Where(v => v?.Source == VariableSource.Import); - - foreach (var v in importedVariables) { - v.RemoveReferences(this); - if (v.Value is IPythonModule module) { - RemoveReferencesInModule(module); - } - } - } - } - } - - public void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass) { - lock (AnalysisLock) { - if (version < Analysis.Version) { - return; - } - - Analysis = CreateAnalysis(version, walker, isFinalPass); - GlobalScope = Analysis.GlobalScope; - - // Derived classes can override OnAnalysisComplete if they want - // to perform additional actions on the completed analysis such - // as declare additional variables, etc. - OnAnalysisComplete(); - ContentState = State.Analyzed; - - if (ModuleType != ModuleType.User) { - _buffer.Reset(_buffer.Version, string.Empty); - } - } - - // Do not report issues with libraries or stubs - if (ModuleType == ModuleType.User) { - _diagnosticsService?.Replace(Uri, Analysis.Diagnostics, DiagnosticSource.Analysis); - } - - NewAnalysis?.Invoke(this, EventArgs.Empty); - } - - protected virtual void OnAnalysisComplete() { } - - public IDocumentAnalysis GetAnyAnalysis() => Analysis; - - public Task GetAnalysisAsync(int waitTime = 200, CancellationToken cancellationToken = default) - => Services.GetService().GetAnalysisAsync(this, waitTime, cancellationToken); - - private IDocumentAnalysis CreateAnalysis(int version, ModuleWalker walker, bool isFinalPass) { - var analysis = ModuleType == ModuleType.Library && isFinalPass - ? new LibraryAnalysis(this, version, walker.Eval.Services, walker.GlobalScope, walker.StarImportMemberNames) - : (IDocumentAnalysis)new DocumentAnalysis(this, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); - - if (analysis.Document.ModuleType != ModuleType.Stub) { - var dbs = Services.GetService(); - dbs?.StoreModuleAnalysisAsync(analysis).DoNotWait(); - } - - return analysis; - } - - private void RemoveReferencesInModule(IPythonModule module) { - if (module.GlobalScope?.Variables != null) { - foreach (var v in module.GlobalScope.Variables) { - v.RemoveReferences(this); - } - } - } - } -} diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.Ast.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.Ast.cs deleted file mode 100644 index 08b69fd28..000000000 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.Ast.cs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Diagnostics; -using Microsoft.Python.Core.Text; -using Microsoft.Python.Parsing; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.Analysis.Modules { - internal partial class PythonModule { - #region Parsing - /// - /// Returns document parse tree. - /// - public async Task GetAstAsync(CancellationToken cancellationToken = default) { - Task t = null; - while (true) { - lock (AnalysisLock) { - if (t == _parsingTask) { - break; - } - cancellationToken.ThrowIfCancellationRequested(); - t = _parsingTask; - } - try { - await (t ?? Task.CompletedTask); - break; - } catch (OperationCanceledException) { - // Parsing as canceled, try next task. - } - } - cancellationToken.ThrowIfCancellationRequested(); - return this.GetAst(); - } - - public PythonAst GetAnyAst() => GetAstNode(this); - - /// - /// Provides collection of parsing errors, if any. - /// - public IEnumerable GetParseErrors() => _parseErrors.ToArray(); - - private void Parse() { - _parseCts?.Cancel(); - _parseCts = new CancellationTokenSource(); - - _linkedParseCts?.Dispose(); - _linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_disposeToken.CancellationToken, _parseCts.Token); - - ContentState = State.Parsing; - _parsingTask = Task.Run(() => Parse(_linkedParseCts.Token), _linkedParseCts.Token); - } - - private void Parse(CancellationToken cancellationToken) { - CollectingErrorSink sink = null; - int version; - Parser parser; - - //Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name}"); - - lock (AnalysisLock) { - version = _buffer.Version; - var options = new ParserOptions { - StubFile = FilePath != null && Path.GetExtension(FilePath).Equals(".pyi", FileSystem.StringComparison) - }; - if (ModuleType == ModuleType.User) { - sink = new CollectingErrorSink(); - options.ErrorSink = sink; - } - parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, options); - } - - var ast = parser.ParseFile(Uri); - - //Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name}"); - - lock (AnalysisLock) { - cancellationToken.ThrowIfCancellationRequested(); - if (version != _buffer.Version) { - throw new OperationCanceledException(); - } - - // Stored nodes are no longer valid. - _astMap.Clear(); - _astMap[this] = ast; - - _parseErrors = sink?.Diagnostics ?? Array.Empty(); - - // Do not report issues with libraries or stubs - if (sink != null) { - _diagnosticsService?.Replace(Uri, _parseErrors, DiagnosticSource.Parser); - } - - ContentState = State.Parsed; - Analysis = new EmptyAnalysis(Services, this); - } - - NewAst?.Invoke(this, EventArgs.Empty); - - if (ContentState < State.Analyzing) { - ContentState = State.Analyzing; - - var analyzer = Services.GetService(); - analyzer.EnqueueDocumentForAnalysis(this, version); - } - - lock (AnalysisLock) { - _parsingTask = null; - } - } - - private class CollectingErrorSink : ErrorSink { - private readonly List _diagnostics = new List(); - - public IReadOnlyList Diagnostics => _diagnostics; - public override void Add(string message, SourceSpan span, int errorCode, Severity severity) - => _diagnostics.Add(new DiagnosticsEntry(message, span, $"parser-{errorCode}", severity, DiagnosticSource.Parser)); - } - #endregion - - #region IAstNodeContainer - public T GetAstNode(object o) where T : Node { - lock (AnalysisLock) { - return _astMap.TryGetValue(o, out var n) ? (T)n : null; - } - } - - public void AddAstNode(object o, Node n) { - lock (AnalysisLock) { - Debug.Assert(!_astMap.ContainsKey(o) || _astMap[o] == n); - _astMap[o] = n; - } - } - - public void ClearAst() { - lock (AnalysisLock) { - if (ModuleType != ModuleType.User) { - _astMap.Clear(); - } - } - } - public void ClearContent() { - lock (AnalysisLock) { - if (ModuleType != ModuleType.User) { - _buffer.Reset(_buffer.Version, string.Empty); - } - } - } - #endregion - - private PythonAst RecreateAst() { - lock (AnalysisLock) { - ContentState = State.None; - LoadContent(); - var parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, ParserOptions.Default); - var ast = parser.ParseFile(Uri); - ContentState = State.Parsed; - return ast; - } - } - } -} diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.Content.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.Content.cs deleted file mode 100644 index 911f604f8..000000000 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.Content.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Threading; -using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Core.IO; - -namespace Microsoft.Python.Analysis.Modules { - internal partial class PythonModule { - public void Update(IEnumerable changes) { - lock (AnalysisLock) { - _parseCts?.Cancel(); - _parseCts = new CancellationTokenSource(); - - _linkedParseCts?.Dispose(); - _linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_disposeToken.CancellationToken, _parseCts.Token); - - _buffer.Update(changes); - _updated = true; - - Parse(); - } - - Services.GetService().InvalidateAnalysis(this); - } - - public void Reset(string content) { - lock (AnalysisLock) { - if (content != Content) { - ContentState = State.None; - InitializeContent(content, _buffer.Version + 1); - } - } - - Services.GetService().InvalidateAnalysis(this); - } - - protected virtual string LoadContent() { - if (ContentState < State.Loading) { - ContentState = State.Loading; - try { - var code = FileSystem.ReadTextWithRetry(FilePath); - ContentState = State.Loaded; - return code; - } catch (IOException) { } catch (UnauthorizedAccessException) { } - } - return null; // Keep content as null so module can be loaded later. - } - - private void InitializeContent(string content, int version) { - lock (AnalysisLock) { - LoadContent(content, version); - - var startParse = ContentState < State.Parsing && (_parsingTask == null || version > 0); - if (startParse) { - Parse(); - } - } - } - - private void LoadContent(string content, int version) { - if (ContentState < State.Loading) { - try { - content = content ?? LoadContent(); - _buffer.Reset(version, content); - ContentState = State.Loaded; - } catch (IOException) { } catch (UnauthorizedAccessException) { } - } - } - } -} diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.Documentation.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.Documentation.cs deleted file mode 100644 index 121fd1320..000000000 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.Documentation.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.IO; -using System.Linq; -using System.Text; -using Microsoft.Python.Core; - -namespace Microsoft.Python.Analysis.Modules { - /// - /// Primary base for all modules and user documents. Provides access - /// to AST and the module analysis. - /// - internal partial class PythonModule { - private string TryGetDocFromModuleInitFile() { - if (string.IsNullOrEmpty(FilePath) || !FileSystem.FileExists(FilePath)) { - return string.Empty; - } - - try { - using (var sr = new StreamReader(FilePath)) { - string quote = null; - string line; - while (true) { - line = sr.ReadLine()?.Trim(); - if (line == null) { - break; - } - if (line.Length == 0 || line.StartsWithOrdinal("#")) { - continue; - } - if (line.StartsWithOrdinal("\"\"\"") || line.StartsWithOrdinal("r\"\"\"")) { - quote = "\"\"\""; - } else if (line.StartsWithOrdinal("'''") || line.StartsWithOrdinal("r'''")) { - quote = "'''"; - } - break; - } - - if (line != null && quote != null) { - // Check if it is a single-liner, but do distinguish from """ - // Also, handle quadruple+ quotes. - line = line.Trim(); - line = line.All(c => c == quote[0]) ? quote : line; - if (line.EndsWithOrdinal(quote) && line.IndexOf(quote, StringComparison.Ordinal) < line.LastIndexOf(quote, StringComparison.Ordinal)) { - return line.Substring(quote.Length, line.Length - 2 * quote.Length).Trim(); - } - var sb = new StringBuilder(); - while (true) { - line = sr.ReadLine(); - if (line == null || line.EndsWithOrdinal(quote)) { - break; - } - sb.AppendLine(line); - } - return sb.ToString(); - } - } - } catch (IOException) { } catch (UnauthorizedAccessException) { } - return string.Empty; - } - } -} diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 47db6d93e..8bac52f07 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -16,7 +16,9 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Linq; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer; @@ -30,6 +32,8 @@ using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Logging; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Modules { @@ -38,7 +42,7 @@ namespace Microsoft.Python.Analysis.Modules { /// to AST and the module analysis. /// [DebuggerDisplay("{Name} : {ModuleType}")] - internal partial class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable, IAstNodeContainer { + internal class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable, IAstNodeContainer { private enum State { None, Loading, @@ -51,6 +55,7 @@ private enum State { private readonly DocumentBuffer _buffer = new DocumentBuffer(); private readonly DisposeToken _disposeToken = DisposeToken.Create(); + private readonly object _syncObj = new object(); private IReadOnlyList _parseErrors = Array.Empty(); private readonly Dictionary _astMap = new Dictionary(); private readonly IDiagnosticsService _diagnosticsService; @@ -64,7 +69,6 @@ private enum State { protected ILogger Log { get; } protected IFileSystem FileSystem { get; } protected IServiceContainer Services { get; } - private object AnalysisLock { get; } = new object(); private State ContentState { get; set; } = State.None; protected PythonModule(string name, ModuleType moduleType, IServiceContainer services) : base(null) { @@ -75,7 +79,6 @@ protected PythonModule(string name, ModuleType moduleType, IServiceContainer ser Log = services.GetService(); Interpreter = services.GetService(); Analysis = new EmptyAnalysis(services, this); - GlobalScope = Analysis.GlobalScope; _diagnosticsService = services.GetService(); SetDeclaringModule(this); @@ -115,7 +118,6 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s #region IPythonType public string Name { get; } public string QualifiedName => Name; - public BuiltinTypeId TypeId => BuiltinTypeId.Module; public bool IsBuiltin => true; public bool IsAbstract => false; @@ -148,10 +150,10 @@ public virtual string Documentation { #endregion #region IMemberContainer - public virtual IMember GetMember(string name) => GlobalScope.Variables[name]?.Value; + public virtual IMember GetMember(string name) => Analysis.GlobalScope.Variables[name]?.Value; public virtual IEnumerable GetMemberNames() { // drop imported modules and typing. - return GlobalScope.Variables + return Analysis.GlobalScope.Variables .Where(v => { // Instances are always fine. if (v.Value is IPythonInstance) { @@ -172,9 +174,10 @@ public virtual IEnumerable GetMemberNames() { public override LocationInfo Definition => new LocationInfo(Uri.ToAbsolutePath(), Uri, 0, 0); #endregion - #region IPythonModule public virtual string FilePath { get; protected set; } public virtual Uri Uri { get; } + + #region IPythonModule public IDocumentAnalysis Analysis { get; private set; } public IPythonInterpreter Interpreter { get; } @@ -188,7 +191,7 @@ public virtual IEnumerable GetMemberNames() { /// /// Global cope of the module. /// - public virtual IGlobalScope GlobalScope { get; protected set; } + public IGlobalScope GlobalScope { get; protected set; } /// /// If module is a stub points to the primary module. @@ -196,7 +199,6 @@ public virtual IEnumerable GetMemberNames() { /// wants to see library code and not a stub. /// public IPythonModule PrimaryModule { get; private set; } - #endregion #region IDisposable @@ -232,11 +234,340 @@ protected virtual void Dispose(bool disposing) { /// /// Returns module content (code). /// - public string Content => _buffer.Text; + public string Content { + get { + lock (_syncObj) { + return _buffer.Text; + } + } + } + + #endregion + + #region Parsing + /// + /// Returns document parse tree. + /// + public async Task GetAstAsync(CancellationToken cancellationToken = default) { + Task t = null; + while (true) { + lock (_syncObj) { + if (t == _parsingTask) { + break; + } + cancellationToken.ThrowIfCancellationRequested(); + t = _parsingTask; + } + try { + await (t ?? Task.CompletedTask); + break; + } catch (OperationCanceledException) { + // Parsing as canceled, try next task. + } + } + cancellationToken.ThrowIfCancellationRequested(); + return this.GetAst(); + } + + /// + /// Provides collection of parsing errors, if any. + /// + public IEnumerable GetParseErrors() => _parseErrors.ToArray(); + + public void Update(IEnumerable changes) { + lock (_syncObj) { + _parseCts?.Cancel(); + _parseCts = new CancellationTokenSource(); + + _linkedParseCts?.Dispose(); + _linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_disposeToken.CancellationToken, _parseCts.Token); + + _buffer.Update(changes); + _updated = true; + + Parse(); + } + + Services.GetService().InvalidateAnalysis(this); + } + + public void Reset(string content) { + lock (_syncObj) { + if (content != Content) { + ContentState = State.None; + InitializeContent(content, _buffer.Version + 1); + } + } + + Services.GetService().InvalidateAnalysis(this); + } + + private void Parse() { + _parseCts?.Cancel(); + _parseCts = new CancellationTokenSource(); + + _linkedParseCts?.Dispose(); + _linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_disposeToken.CancellationToken, _parseCts.Token); + + ContentState = State.Parsing; + _parsingTask = Task.Run(() => Parse(_linkedParseCts.Token), _linkedParseCts.Token); + } + + private void Parse(CancellationToken cancellationToken) { + CollectingErrorSink sink = null; + int version; + Parser parser; + + //Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name}"); + + lock (_syncObj) { + version = _buffer.Version; + var options = new ParserOptions { + StubFile = FilePath != null && Path.GetExtension(FilePath).Equals(".pyi", FileSystem.StringComparison) + }; + if (ModuleType == ModuleType.User) { + sink = new CollectingErrorSink(); + options.ErrorSink = sink; + } + parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, options); + } + + var ast = parser.ParseFile(Uri); + + //Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name}"); + + lock (_syncObj) { + cancellationToken.ThrowIfCancellationRequested(); + if (version != _buffer.Version) { + throw new OperationCanceledException(); + } + + // Stored nodes are no longer valid. + _astMap.Clear(); + _astMap[this] = ast; + + _parseErrors = sink?.Diagnostics ?? Array.Empty(); + + // Do not report issues with libraries or stubs + if (sink != null) { + _diagnosticsService?.Replace(Uri, _parseErrors, DiagnosticSource.Parser); + } + + ContentState = State.Parsed; + Analysis = new EmptyAnalysis(Services, this); + } + + NewAst?.Invoke(this, EventArgs.Empty); + + if (ContentState < State.Analyzing) { + ContentState = State.Analyzing; + + var analyzer = Services.GetService(); + analyzer.EnqueueDocumentForAnalysis(this, ast, version); + } + + lock (_syncObj) { + _parsingTask = null; + } + } + + private class CollectingErrorSink : ErrorSink { + private readonly List _diagnostics = new List(); + + public IReadOnlyList Diagnostics => _diagnostics; + public override void Add(string message, SourceSpan span, int errorCode, Severity severity) + => _diagnostics.Add(new DiagnosticsEntry(message, span, $"parser-{errorCode}", severity, DiagnosticSource.Parser)); + } + #endregion + + #region IAnalyzable + + public void NotifyAnalysisBegins() { + lock (_syncObj) { + if (_updated) { + _updated = false; + // In all variables find those imported, then traverse imported modules + // and remove references to this module. If variable refers to a module, + // recurse into module but only process global scope. + + if (GlobalScope == null) { + return; + } + + // TODO: Figure out where the nulls below are coming from. + var importedVariables = ((IScope)GlobalScope) + .TraverseDepthFirst(c => c?.Children ?? Enumerable.Empty()) + .SelectMany(s => s?.Variables ?? VariableCollection.Empty) + .Where(v => v?.Source == VariableSource.Import); + + foreach (var v in importedVariables) { + v.RemoveReferences(this); + if (v.Value is IPythonModule module) { + RemoveReferencesInModule(module); + } + } + } + } + } + + public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { + lock (_syncObj) { + if (analysis.Version < Analysis.Version) { + return; + } + + Analysis = analysis; + GlobalScope = analysis.GlobalScope; + + // Derived classes can override OnAnalysisComplete if they want + // to perform additional actions on the completed analysis such + // as declare additional variables, etc. + OnAnalysisComplete(); + ContentState = State.Analyzed; + + if (ModuleType != ModuleType.User) { + _buffer.Reset(_buffer.Version, string.Empty); + } + } + + // Do not report issues with libraries or stubs + if (ModuleType == ModuleType.User) { + _diagnosticsService?.Replace(Uri, analysis.Diagnostics, DiagnosticSource.Analysis); + } + + NewAnalysis?.Invoke(this, EventArgs.Empty); + } + + protected virtual void OnAnalysisComplete() { } #endregion #region IEquatable public bool Equals(IPythonModule other) => Name.Equals(other?.Name) && FilePath.Equals(other?.FilePath); #endregion + + #region IAstNodeContainer + public Node GetAstNode(object o) { + lock (_syncObj) { + return _astMap.TryGetValue(o, out var n) ? n : null; + } + } + + public void AddAstNode(object o, Node n) { + lock (_syncObj) { + Debug.Assert(!_astMap.ContainsKey(o) || _astMap[o] == n); + _astMap[o] = n; + } + } + + public void ClearContent() { + lock (_syncObj) { + if (ModuleType != ModuleType.User) { + _buffer.Reset(_buffer.Version, string.Empty); + _astMap.Clear(); + } + } + } + #endregion + + #region Analysis + public IDocumentAnalysis GetAnyAnalysis() => Analysis; + + public Task GetAnalysisAsync(int waitTime = 200, CancellationToken cancellationToken = default) + => Services.GetService().GetAnalysisAsync(this, waitTime, cancellationToken); + + #endregion + + #region Content management + protected virtual string LoadContent() { + if (ContentState < State.Loading) { + ContentState = State.Loading; + try { + var code = FileSystem.ReadTextWithRetry(FilePath); + ContentState = State.Loaded; + return code; + } catch (IOException) { } catch (UnauthorizedAccessException) { } + } + return null; // Keep content as null so module can be loaded later. + } + + private void InitializeContent(string content, int version) { + lock (_syncObj) { + LoadContent(content, version); + + var startParse = ContentState < State.Parsing && (_parsingTask == null || version > 0); + if (startParse) { + Parse(); + } + } + } + + private void LoadContent(string content, int version) { + if (ContentState < State.Loading) { + try { + content = content ?? LoadContent(); + _buffer.Reset(version, content); + ContentState = State.Loaded; + } catch (IOException) { } catch (UnauthorizedAccessException) { } + } + } + #endregion + + #region Documentation + private string TryGetDocFromModuleInitFile() { + if (string.IsNullOrEmpty(FilePath) || !FileSystem.FileExists(FilePath)) { + return string.Empty; + } + + try { + using (var sr = new StreamReader(FilePath)) { + string quote = null; + string line; + while (true) { + line = sr.ReadLine()?.Trim(); + if (line == null) { + break; + } + if (line.Length == 0 || line.StartsWithOrdinal("#")) { + continue; + } + if (line.StartsWithOrdinal("\"\"\"") || line.StartsWithOrdinal("r\"\"\"")) { + quote = "\"\"\""; + } else if (line.StartsWithOrdinal("'''") || line.StartsWithOrdinal("r'''")) { + quote = "'''"; + } + break; + } + + if (line != null && quote != null) { + // Check if it is a single-liner, but do distinguish from """ + // Also, handle quadruple+ quotes. + line = line.Trim(); + line = line.All(c => c == quote[0]) ? quote : line; + if (line.EndsWithOrdinal(quote) && line.IndexOf(quote, StringComparison.Ordinal) < line.LastIndexOf(quote, StringComparison.Ordinal)) { + return line.Substring(quote.Length, line.Length - 2 * quote.Length).Trim(); + } + var sb = new StringBuilder(); + while (true) { + line = sr.ReadLine(); + if (line == null || line.EndsWithOrdinal(quote)) { + break; + } + sb.AppendLine(line); + } + return sb.ToString(); + } + } + } catch (IOException) { } catch (UnauthorizedAccessException) { } + return string.Empty; + } + #endregion + + private void RemoveReferencesInModule(IPythonModule module) { + if (module.GlobalScope?.Variables != null) { + foreach (var v in module.GlobalScope.Variables) { + v.RemoveReferences(this); + } + } + } } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index c9eb9c936..9da8870d8 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -37,7 +37,6 @@ namespace Microsoft.Python.Analysis.Modules.Resolution { internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement { private readonly ConcurrentDictionary _specialized = new ConcurrentDictionary(); private IRunningDocumentTable _rdt; - private IReadOnlyList _searchPaths; public MainModuleResolution(string root, IServiceContainer services) : base(root, services) { } @@ -62,29 +61,6 @@ internal async Task InitializeAsync(CancellationToken cancellationToken = defaul cancellationToken.ThrowIfCancellationRequested(); } - public async Task> GetSearchPathsAsync(CancellationToken cancellationToken = default) { - if (_searchPaths != null) { - return _searchPaths; - } - - _searchPaths = await GetInterpreterSearchPathsAsync(cancellationToken); - Debug.Assert(_searchPaths != null, "Should have search paths"); - _searchPaths = _searchPaths ?? Array.Empty(); - - _log?.Log(TraceEventType.Verbose, "Python search paths:"); - foreach (var s in _searchPaths) { - _log?.Log(TraceEventType.Verbose, $" {s}"); - } - - var configurationSearchPaths = Configuration.SearchPaths ?? Array.Empty(); - - _log?.Log(TraceEventType.Verbose, "Configuration search paths:"); - foreach (var s in configurationSearchPaths) { - _log?.Log(TraceEventType.Verbose, $" {s}"); - } - return _searchPaths; - } - protected override IPythonModule CreateModule(string name) { var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name); if (moduleImport == null) { @@ -140,11 +116,11 @@ protected override IPythonModule CreateModule(string name) { return GetRdt().AddModule(mco); } - private async Task> GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default) { + private async Task> GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default) { if (!_fs.FileExists(Configuration.InterpreterPath)) { _log?.Log(TraceEventType.Warning, "Interpreter does not exist:", Configuration.InterpreterPath); _ui?.ShowMessageAsync(Resources.InterpreterNotFound, TraceEventType.Error); - return Array.Empty(); + return Array.Empty(); } _log?.Log(TraceEventType.Information, "GetCurrentSearchPaths", Configuration.InterpreterPath); @@ -153,11 +129,11 @@ private async Task> GetInterpreterSearchPathsAsync(Cancell var ps = _services.GetService(); var paths = await PythonLibraryPath.GetSearchPathsAsync(Configuration, fs, ps, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); - return paths.MaybeEnumerate().Select(p => p.Path).ToArray(); + return paths.ToArray(); } catch (InvalidOperationException ex) { _log?.Log(TraceEventType.Warning, "Exception getting search paths", ex); _ui?.ShowMessageAsync(Resources.ExceptionGettingSearchPaths, TraceEventType.Error); - return Array.Empty(); + return Array.Empty(); } } @@ -216,16 +192,13 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) { var addedRoots = new HashSet(); addedRoots.UnionWith(PathResolver.SetRoot(Root)); - InterpreterPaths = await GetSearchPathsAsync(cancellationToken); + var ps = _services.GetService(); - IEnumerable userSearchPaths = Configuration.SearchPaths; - InterpreterPaths = InterpreterPaths.Except(userSearchPaths, StringExtensions.PathsStringComparer).Where(p => !p.PathEquals(Root)); + var paths = await GetInterpreterSearchPathsAsync(cancellationToken); + var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, _fs, paths, Configuration.SearchPaths); - if (Root != null) { - var underRoot = userSearchPaths.ToLookup(p => _fs.IsPathUnderRoot(Root, p)); - userSearchPaths = underRoot[true]; - InterpreterPaths = underRoot[false].Concat(InterpreterPaths); - } + InterpreterPaths = interpreterPaths.Select(p => p.Path); + var userSearchPaths = userPaths.Select(p => p.Path); _log?.Log(TraceEventType.Information, "Interpreter search paths:"); foreach (var s in InterpreterPaths) { diff --git a/src/Analysis/Ast/Impl/Resources.Designer.cs b/src/Analysis/Ast/Impl/Resources.Designer.cs index 785cbf9a8..1b1a34bad 100644 --- a/src/Analysis/Ast/Impl/Resources.Designer.cs +++ b/src/Analysis/Ast/Impl/Resources.Designer.cs @@ -222,6 +222,33 @@ internal static string ExceptionGettingSearchPaths { } } + /// + /// Looks up a localized string similar to Arguments to Generic must all be type parameters.. + /// + internal static string GenericNotAllTypeParameters { + get { + return ResourceManager.GetString("GenericNotAllTypeParameters", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Arguments to Generic must all be unique.. + /// + internal static string GenericNotAllUnique { + get { + return ResourceManager.GetString("GenericNotAllUnique", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Inheriting '{0}', which is not a class.. + /// + internal static string InheritNonClass { + get { + return ResourceManager.GetString("InheritNonClass", resourceCulture); + } + } + /// /// Looks up a localized string similar to Interpreter does not exist; analysis will not be available.. /// @@ -231,6 +258,15 @@ internal static string InterpreterNotFound { } } + /// + /// Looks up a localized string similar to The first argument to NewType must be a string, but it is of type '{0}'.. + /// + internal static string NewTypeFirstArgNotString { + get { + return ResourceManager.GetString("NewTypeFirstArgNotString", resourceCulture); + } + } + /// /// Looks up a localized string similar to property of type {0}. /// @@ -249,6 +285,15 @@ internal static string PropertyOfUnknownType { } } + /// + /// Looks up a localized string similar to Explicit return in __init__ . + /// + internal static string ReturnInInit { + get { + return ResourceManager.GetString("ReturnInInit", resourceCulture); + } + } + /// /// Looks up a localized string similar to Unable to determine analysis cache path. Using default '{0}'.. /// @@ -275,5 +320,14 @@ internal static string UndefinedVariable { return ResourceManager.GetString("UndefinedVariable", resourceCulture); } } + + /// + /// Looks up a localized string similar to Unsupported operand types for '{0}': '{1}' and '{2}'. + /// + internal static string UnsupporedOperandType { + get { + return ResourceManager.GetString("UnsupporedOperandType", resourceCulture); + } + } } } diff --git a/src/Analysis/Ast/Impl/Resources.resx b/src/Analysis/Ast/Impl/Resources.resx index 95cd89002..9fe003e70 100644 --- a/src/Analysis/Ast/Impl/Resources.resx +++ b/src/Analysis/Ast/Impl/Resources.resx @@ -189,4 +189,22 @@ Unable to determine analysis cache path. Exception: {0}. Using default '{1}'. + + The first argument to NewType must be a string, but it is of type '{0}'. + + + Unsupported operand types for '{0}': '{1}' and '{2}' + + + Explicit return in __init__ + + + Arguments to Generic must all be type parameters. + + + Arguments to Generic must all be unique. + + + Inheriting '{0}', which is not a class. + \ No newline at end of file diff --git a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs index 54c905ed3..4ce6ee809 100644 --- a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs +++ b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs @@ -67,7 +67,7 @@ public static IMember Next(IPythonModule module, IPythonFunctionOverload overloa } public static IMember __iter__(IPythonInterpreter interpreter, BuiltinTypeId contentTypeId) { - var location = new Location(interpreter.ModuleResolution.BuiltinsModule, default); + var location = new Location(interpreter.ModuleResolution.BuiltinsModule); var fn = new PythonFunctionType(@"__iter__", location, null, string.Empty); var o = new PythonFunctionOverload(fn.Name, location); o.AddReturnValue(PythonTypeIterator.FromTypeId(interpreter, contentTypeId)); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs index 8e280f25b..9977389eb 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs @@ -15,6 +15,7 @@ using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Specializations.Typing.Types; using Microsoft.Python.Analysis.Types; @@ -43,7 +44,7 @@ public static IPythonModule Create(IServiceContainer services) { #endregion private void SpecializeMembers() { - var location = new Location(this, default); + var location = new Location(this); // TypeVar var fn = PythonFunctionType.Specialize("TypeVar", this, GetMemberDocumentation("TypeVar")); @@ -61,7 +62,7 @@ private void SpecializeMembers() { o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, args) => CreateTypeAlias(args.Values())); + o.SetReturnValueProvider((interpreter, overload, args) => CreateTypeAlias(args)); fn.AddOverload(o); _members["NewType"] = fn; @@ -81,41 +82,41 @@ private void SpecializeMembers() { _members["Iterable"] = new GenericType("Iterable", typeArgs => CreateListType("Iterable", BuiltinTypeId.List, typeArgs, false), this); _members["Sequence"] = new GenericType("Sequence", typeArgs => CreateListType("Sequence", BuiltinTypeId.List, typeArgs, false), this); - _members["MutableSequence"] = new GenericType("MutableSequence", + _members["MutableSequence"] = new GenericType("MutableSequence", typeArgs => CreateListType("MutableSequence", BuiltinTypeId.List, typeArgs, true), this); - _members["List"] = new GenericType("List", + _members["List"] = new GenericType("List", typeArgs => CreateListType("List", BuiltinTypeId.List, typeArgs, true), this); - _members["MappingView"] = new GenericType("MappingView", + _members["MappingView"] = new GenericType("MappingView", typeArgs => CreateDictionary("MappingView", typeArgs, false), this); _members["KeysView"] = new GenericType("KeysView", CreateKeysViewType, this); _members["ValuesView"] = new GenericType("ValuesView", CreateValuesViewType, this); _members["ItemsView"] = new GenericType("ItemsView", CreateItemsViewType, this); - _members["Set"] = new GenericType("Set", + _members["Set"] = new GenericType("Set", typeArgs => CreateListType("Set", BuiltinTypeId.Set, typeArgs, true), this); - _members["MutableSet"] = new GenericType("MutableSet", + _members["MutableSet"] = new GenericType("MutableSet", typeArgs => CreateListType("MutableSet", BuiltinTypeId.Set, typeArgs, true), this); - _members["FrozenSet"] = new GenericType("FrozenSet", + _members["FrozenSet"] = new GenericType("FrozenSet", typeArgs => CreateListType("FrozenSet", BuiltinTypeId.Set, typeArgs, false), this); _members["Tuple"] = new GenericType("Tuple", CreateTupleType, this); - _members["Mapping"] = new GenericType("Mapping", + _members["Mapping"] = new GenericType("Mapping", typeArgs => CreateDictionary("Mapping", typeArgs, false), this); - _members["MutableMapping"] = new GenericType("MutableMapping", + _members["MutableMapping"] = new GenericType("MutableMapping", typeArgs => CreateDictionary("MutableMapping", typeArgs, true), this); - _members["Dict"] = new GenericType("Dict", + _members["Dict"] = new GenericType("Dict", typeArgs => CreateDictionary("Dict", typeArgs, true), this); - _members["OrderedDict"] = new GenericType("OrderedDict", + _members["OrderedDict"] = new GenericType("OrderedDict", typeArgs => CreateDictionary("OrderedDict", typeArgs, true), this); - _members["DefaultDict"] = new GenericType("DefaultDict", + _members["DefaultDict"] = new GenericType("DefaultDict", typeArgs => CreateDictionary("DefaultDict", typeArgs, true), this); _members["Union"] = new GenericType("Union", CreateUnion, this); - _members["Counter"] = Specialized.Function("Counter", this, GetMemberDocumentation("Counter"), + _members["Counter"] = Specialized.Function("Counter", this, GetMemberDocumentation("Counter"), new PythonInstance(Interpreter.GetBuiltinType(BuiltinTypeId.Int))); _members["SupportsInt"] = Interpreter.GetBuiltinType(BuiltinTypeId.Int); @@ -132,16 +133,7 @@ private void SpecializeMembers() { _members["Any"] = new AnyType(this); - // AnyStr - var str = Interpreter.GetBuiltinType(BuiltinTypeId.Str); - var bytes = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); - var unicode = Interpreter.GetBuiltinType(BuiltinTypeId.Unicode); - var anyStrName = new PythonConstant("AnyStr", str); - - var anyStrArgs = Interpreter.LanguageVersion.Is3x() - ? new IMember[] { anyStrName, str, bytes } - : new IMember[] { anyStrName, str, unicode }; - _members["AnyStr"] = GenericTypeParameter.FromTypeVar(new ArgumentSet(anyStrArgs), this); + _members["AnyStr"] = CreateAnyStr(); _members["Optional"] = new GenericType("Optional", CreateOptional, this); _members["Type"] = new GenericType("Type", CreateType, this); @@ -226,13 +218,25 @@ private IPythonType CreateItemsViewType(IReadOnlyList typeArgs) { return Interpreter.UnknownType; } - private IPythonType CreateTypeAlias(IReadOnlyList typeArgs) { + private IPythonType CreateTypeAlias(IArgumentSet args) { + var typeArgs = args.Values(); if (typeArgs.Count == 2) { var typeName = (typeArgs[0] as IPythonConstant)?.Value as string; if (!string.IsNullOrEmpty(typeName)) { return new TypeAlias(typeName, typeArgs[1].GetPythonType() ?? Interpreter.UnknownType); } - // TODO: report incorrect first argument to NewVar + + var firstArgType = (typeArgs[0] as PythonInstance)?.Type.Name; + var eval = args.Eval; + var expression = args.Expression; + + eval.ReportDiagnostics( + eval.Module?.Uri, + new DiagnosticsEntry(Resources.NewTypeFirstArgNotString.FormatInvariant(firstArgType), + expression?.GetLocation(eval)?.Span ?? default, + Diagnostics.ErrorCodes.TypingNewTypeArguments, + Severity.Error, DiagnosticSource.Analysis) + ); } // TODO: report wrong number of arguments return Interpreter.UnknownType; @@ -310,6 +314,21 @@ private IPythonType CreateType(IReadOnlyList typeArgs) { return Interpreter.UnknownType; } + private IPythonType CreateAnyStr() { + var str = Interpreter.GetBuiltinType(BuiltinTypeId.Str); + var bytes = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); + var unicode = Interpreter.GetBuiltinType(BuiltinTypeId.Unicode); + var name = "AnyStr"; + + var constraints = Interpreter.LanguageVersion.Is3x() + ? new IPythonType[] { str, bytes } + : new IPythonType[] { str, unicode }; + var docArgs = new[] { $"'{name}'" }.Concat(constraints.Select(c => c.Name)); + var documentation = CodeFormatter.FormatSequence("TypeVar", '(', docArgs); + + return new GenericTypeParameter(name, this, constraints, documentation, default); + } + private IPythonType CreateGenericClassParameter(IReadOnlyList typeArgs) { // Handle Generic[_T1, _T2, ...]. _T1, et al are IGenericTypeParameter from TypeVar. // Hold the parameter until concrete type is provided at the time of the class instantiation. @@ -325,7 +344,7 @@ private IPythonType CreateGenericClassParameter(IReadOnlyList typeA return Interpreter.UnknownType; } - private IPythonType ToGenericTemplate(string typeName, IGenericTypeDefinition[] typeArgs, BuiltinTypeId typeId) + private IPythonType ToGenericTemplate(string typeName, IGenericTypeDefinition[] typeArgs, BuiltinTypeId typeId) => _members[typeName] is GenericType gt ? new GenericType(CodeFormatter.FormatSequence(typeName, '[', typeArgs), gt.SpecificTypeConstructor, this, typeId, typeArgs) : Interpreter.UnknownType; diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingIterator.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingIterator.cs index 16d172527..124c7c862 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingIterator.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingIterator.cs @@ -39,7 +39,7 @@ public override IMember Next { } else if (_index < _iteratorType.ItemTypes.Count) { itemType = _iteratorType.ItemTypes[_index++]; } - return itemType?.CreateInstance(itemType.Name, ArgumentSet.Empty) ?? UnknownType; + return itemType?.CreateInstance(itemType.Name, ArgumentSet.WithoutContext) ?? UnknownType; } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingTuple.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingTuple.cs index 93c9b52a2..52960406d 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingTuple.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Values/TypingTuple.cs @@ -33,6 +33,6 @@ public override IPythonIterator GetIterator() { } public override IMember Index(object index) - => _collectionType.Index(this, index).GetPythonType().CreateInstance(null, ArgumentSet.Empty); + => _collectionType.Index(this, index).GetPythonType().CreateInstance(null, ArgumentSet.WithoutContext); } } diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index e0acc25a7..5854abeda 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -17,8 +17,8 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -37,7 +37,7 @@ internal sealed class ArgumentSet : IArgumentSet { private readonly DictArg _dictArgument; private bool _evaluated; - public static IArgumentSet Empty = new ArgumentSet(); + public static IArgumentSet WithoutContext = new ArgumentSet(); /// Module that declares the function public IPythonModule DeclaringModule { get; } @@ -48,22 +48,39 @@ internal sealed class ArgumentSet : IArgumentSet { public int OverloadIndex { get; } public IExpressionEvaluator Eval { get; } + public Expression Expression { get; } private ArgumentSet() { } - public ArgumentSet(IReadOnlyList typeArgs) { - _arguments = typeArgs.Select(t => new Argument(t)).ToList(); - _evaluated = true; + /// + /// Creates an empty argument set with some context in how the argument set was used. + /// + /// Expression associated with argument set. + /// Evaluator for the expression involving the argument set. + /// + public static ArgumentSet Empty(Expression expr, IExpressionEvaluator eval) { + return new ArgumentSet(new List(), expr, eval); } + + /// + /// Creates a set of arguments for a call + /// + /// Use in the cases a corresponding function is unknown, but it is still convenient to have the context + /// of the expression which the arguments are needed for and the evaluator that is analyzing + /// that expression. + /// + /// + /// Arguments for the call. + /// Expression for the call. + /// Evaluator of the current analysis. + public ArgumentSet(IReadOnlyList args, Expression expr, IExpressionEvaluator eval) { + _arguments = args.Select(t => new Argument(t)).ToList(); + Expression = expr; + Eval = eval; - public ArgumentSet(IReadOnlyList memberArgs) { - _arguments = memberArgs.Select(t => new Argument(t)).ToList(); _evaluated = true; } - public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, ExpressionEval eval) : - this(fn, overloadIndex, instance, callExpr, eval.Module, eval) { } - /// /// Creates set of arguments for a function call based on the call expression /// and the function signature. The result contains expressions @@ -74,12 +91,12 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in /// Function overload to call. /// Type instance the function is bound to. For derived classes it is different from the declared type. /// Call expression that invokes the function. - /// Module that contains the call expression. /// Evaluator that can calculate values of arguments from their respective expressions. - public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, IPythonModule module, IExpressionEvaluator eval) { + public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, IExpressionEvaluator eval) { Eval = eval; OverloadIndex = overloadIndex; DeclaringModule = fn.DeclaringModule; + Expression = callExpr; if (callExpr == null) { // Typically invoked by specialization code without call expression in the code. @@ -107,7 +124,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in return; } - var callLocation = callExpr.GetLocation(module); + var callLocation = callExpr.GetLocation(eval); // https://www.python.org/dev/peps/pep-3102/#id5 // For each formal parameter, there is a slot which will be used to contain @@ -158,7 +175,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in if (formalParamIndex >= overload.Parameters.Count) { // We ran out of formal parameters and yet haven't seen // any sequence or dictionary ones. This looks like an error. - _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(eval).Span, ErrorCodes.TooManyFunctionArguments, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -167,7 +184,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in if (formalParam.Kind == ParameterKind.List) { if (string.IsNullOrEmpty(formalParam.Name)) { // If the next unfilled slot is a vararg slot, and it does not have a name, then it is an error. - _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(eval).Span, ErrorCodes.TooManyPositionalArgumentsBeforeStar, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -175,7 +192,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in // If the next unfilled slot is a vararg slot then all remaining // non-keyword arguments are placed into the vararg slot. if (_listArgument == null) { - _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(eval).Span, ErrorCodes.TooManyFunctionArguments, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -195,7 +212,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in if (formalParam.Kind == ParameterKind.Dictionary) { // Next slot is a dictionary slot, but we have positional arguments still. - _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(eval).Span, ErrorCodes.TooManyPositionalArgumentsBeforeStar, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -209,7 +226,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in var arg = callExpr.Args[callParamIndex]; if (string.IsNullOrEmpty(arg.Name)) { - _errors.Add(new DiagnosticsEntry(Resources.Analysis_PositionalArgumentAfterKeyword, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_PositionalArgumentAfterKeyword, arg.GetLocation(eval).Span, ErrorCodes.PositionalArgumentAfterKeyword, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -221,13 +238,13 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in // to the dictionary using the keyword name as the dictionary key, // unless there is already an entry with that key, in which case it is an error. if (_dictArgument == null) { - _errors.Add(new DiagnosticsEntry(Resources.Analysis_UnknownParameterName, arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_UnknownParameterName, arg.GetLocation(eval).Span, ErrorCodes.UnknownParameterName, Severity.Warning, DiagnosticSource.Analysis)); return; } if (_dictArgument.Arguments.ContainsKey(arg.Name)) { - _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterAlreadySpecified.FormatUI(arg.Name), arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterAlreadySpecified.FormatUI(arg.Name), arg.GetLocation(eval).Span, ErrorCodes.ParameterAlreadySpecified, Severity.Warning, DiagnosticSource.Analysis)); return; } @@ -238,7 +255,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in if (nvp.ValueExpression != null || nvp.Value != null) { // Slot is already filled. - _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterAlreadySpecified.FormatUI(arg.Name), arg.GetLocation(module).Span, + _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterAlreadySpecified.FormatUI(arg.Name), arg.GetLocation(eval).Span, ErrorCodes.ParameterAlreadySpecified, Severity.Warning, DiagnosticSource.Analysis)); return; } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs index bc1caa61b..4387b9efa 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs @@ -138,8 +138,13 @@ public interface IArgumentSet { int OverloadIndex { get; } /// - /// Evaluator associated with the set. + /// Evaluator associated with the argument set. /// IExpressionEvaluator Eval { get; } + + /// + /// Expression associated with the argument set + /// + Expression Expression { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs index d752e3f4f..36899d865 100644 --- a/src/Analysis/Ast/Impl/Types/LocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -23,7 +23,7 @@ namespace Microsoft.Python.Analysis.Types { internal abstract class LocatedMember : ILocatedMember { private HashSet _references; - protected LocatedMember(IPythonModule module) : this(new Location(module, default)) { } + protected LocatedMember(IPythonModule module) : this(new Location(module)) { } protected LocatedMember(Location location) { Location = location; diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs index fd2275707..2485d307d 100644 --- a/src/Analysis/Ast/Impl/Types/Location.cs +++ b/src/Analysis/Ast/Impl/Types/Location.cs @@ -17,7 +17,7 @@ using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { - public struct Location { + public readonly struct Location { public Location(IPythonModule module) : this(module, default) { } public Location(IPythonModule module, IndexSpan indexSpan) { diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index b3e2de521..c8d72c3cd 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -17,6 +17,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types.Collections; @@ -191,6 +193,7 @@ internal void SetBases(IEnumerable bases) { } bases = bases != null ? bases.Where(b => !b.GetPythonType().IsUnknown()).ToArray() : Array.Empty(); + // For Python 3+ attach object as a base class by default except for the object class itself. if (DeclaringModule.Interpreter.LanguageVersion.Is3x() && DeclaringModule.ModuleType != ModuleType.Builtins) { var objectType = DeclaringModule.Interpreter.GetBuiltinType(BuiltinTypeId.Object); @@ -403,7 +406,7 @@ public IPythonType CreateSpecificType(IArgumentSet args) { .Where(p => !p.IsUnknown()) .ToArray(); if (st.Length > 0) { - var type = gt.CreateSpecificType(new ArgumentSet(st)); + var type = gt.CreateSpecificType(new ArgumentSet(st, args.Expression, args.Eval)); if (!type.IsUnknown()) { bases.Add(type); } diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 0e5f90a51..77187a780 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -167,7 +167,7 @@ private IMember CreateSpecificReturnFromClassType(IPythonClassType selfClassType } if (typeArgs != null) { - var specificReturnValue = returnClassType.CreateSpecificType(new ArgumentSet(typeArgs)); + var specificReturnValue = returnClassType.CreateSpecificType(new ArgumentSet(typeArgs, args?.Expression, args?.Eval)); return new PythonInstance(specificReturnValue); } diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index 865fdac58..a0f6079e2 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -33,7 +33,7 @@ internal sealed class PythonFunctionType : PythonType, IPythonFunctionType { /// Creates function for specializations /// public static PythonFunctionType Specialize(string name, IPythonModule declaringModule, string documentation) - => new PythonFunctionType(name, new Location(declaringModule, default), documentation, true); + => new PythonFunctionType(name, new Location(declaringModule), documentation, true); private PythonFunctionType(string name, Location location, string documentation, bool isSpecialized = false) : base(name, location, documentation ?? string.Empty, BuiltinTypeId.Function) { diff --git a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs index 0eba0d916..fdf4b7a66 100644 --- a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs @@ -53,7 +53,7 @@ public string Description { public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => _getter.Call(args, instance?.GetPythonType() ?? DeclaringType); - public IMember ReturnType => _getter?.Call(ArgumentSet.Empty, DeclaringType); + public IMember ReturnType => _getter?.Call(ArgumentSet.WithoutContext, DeclaringType); #endregion internal void AddOverload(IPythonFunctionOverload overload) => _getter = _getter ?? overload; diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs index 95cc1a0c9..4fe0697e9 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonDictionary.cs @@ -62,7 +62,7 @@ public IReadOnlyList Items _contents.TryGetValue(key, out var value) ? value : UnknownType; public override IPythonIterator GetIterator() => - Call(@"iterkeys", ArgumentSet.Empty) as IPythonIterator ?? new EmptyIterator(Type.DeclaringModule.Interpreter.UnknownType); + Call(@"iterkeys", ArgumentSet.WithoutContext) as IPythonIterator ?? new EmptyIterator(Type.DeclaringModule.Interpreter.UnknownType); public override IMember Index(object key) => key is IMember m ? this[m] : UnknownType; diff --git a/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs b/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs index 53ec70cb4..ae3d16d7a 100644 --- a/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs +++ b/src/Analysis/Ast/Impl/Values/Collections/PythonInstanceIterator.cs @@ -29,7 +29,7 @@ public PythonInstanceIterator(IMember instance, IPythonInterpreter interpreter) __next__ = instance.GetPythonType().GetMember(@"__next__") as IPythonFunctionType; } - public IMember Next => __next__?.Call(null, @"__next__", ArgumentSet.Empty) ?? UnknownType; + public IMember Next => __next__?.Call(null, @"__next__", ArgumentSet.WithoutContext) ?? UnknownType; public override IMember Call(string memberName, IArgumentSet args) { // Specializations diff --git a/src/Analysis/Ast/Impl/Values/GlobalScope.cs b/src/Analysis/Ast/Impl/Values/GlobalScope.cs index 699809d20..44f35bfd1 100644 --- a/src/Analysis/Ast/Impl/Values/GlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/GlobalScope.cs @@ -19,18 +19,21 @@ namespace Microsoft.Python.Analysis.Values { internal sealed class GlobalScope: Scope, IGlobalScope { - public GlobalScope(IPythonModule module): base(null, null, module) { + private readonly PythonAst _ast; + + public GlobalScope(IPythonModule module, PythonAst ast): base(null, null, module) { + _ast = ast; DeclareBuiltinVariables(); } - public override ScopeStatement Node => Module.GetAst(); + public override ScopeStatement Node => _ast; private void DeclareBuiltinVariables() { if (Module.ModuleType != ModuleType.User) { return; } - var location = new Location(Module, default); + var location = new Location(Module); var boolType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Bool); var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); diff --git a/src/Analysis/Ast/Impl/Values/PythonInstance.cs b/src/Analysis/Ast/Impl/Values/PythonInstance.cs index 2aed9f4cb..0da044063 100644 --- a/src/Analysis/Ast/Impl/Values/PythonInstance.cs +++ b/src/Analysis/Ast/Impl/Values/PythonInstance.cs @@ -56,7 +56,7 @@ public virtual IMember Call(string memberName, IArgumentSet args) { public virtual IPythonIterator GetIterator() { var iteratorFunc = Type.GetMember(@"__iter__") as IPythonFunctionType; var o = iteratorFunc?.Overloads.FirstOrDefault(); - var instance = o?.Call(ArgumentSet.Empty, Type); + var instance = o?.Call(ArgumentSet.WithoutContext, Type); if (instance != null) { return new PythonInstanceIterator(instance, Type.DeclaringModule.Interpreter); } diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index d517c4457..0ffeba6c5 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -42,7 +42,7 @@ public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) { #region IScope public string Name => Node?.Name ?? ""; - public virtual ScopeStatement Node => Module.GetAstNode(this); + public virtual ScopeStatement Node => Module.GetAstNode(this) ?? Module.GetAst(); public IScope OuterScope { get; } public IPythonModule Module { get; } @@ -96,7 +96,7 @@ private void DeclareBuiltinVariables() { return; } - var location = new Location(Module, default); + var location = new Location(Module); var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var objType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); @@ -126,9 +126,9 @@ public EmptyGlobalScope(IPythonModule module) { } public IPythonModule Module { get; } public string Name => string.Empty; - public ScopeStatement Node => null; + public ScopeStatement Node => Module.Analysis.Ast; public IScope OuterScope => null; - public IGlobalScope GlobalScope { get; protected set; } + public IGlobalScope GlobalScope { get; } public IReadOnlyList Children => Array.Empty(); public IEnumerable EnumerateTowardsGlobal => Enumerable.Repeat(this, 1); public IEnumerable EnumerateFromGlobal => Enumerable.Repeat(this, 1); diff --git a/src/Analysis/Ast/Impl/get_search_paths.py b/src/Analysis/Ast/Impl/get_search_paths.py index 290662f48..53cec1cf0 100644 --- a/src/Analysis/Ast/Impl/get_search_paths.py +++ b/src/Analysis/Ast/Impl/get_search_paths.py @@ -45,6 +45,11 @@ def clean(path): BEFORE_SITE = set(clean(p) for p in BEFORE_SITE) AFTER_SITE = set(clean(p) for p in AFTER_SITE) +try: + SITE_PKGS = set(clean(p) for p in site.getsitepackages()) +except AttributeError: + SITE_PKGS = set() + for prefix in [ sys.prefix, sys.exec_prefix, @@ -69,4 +74,7 @@ def clean(path): if p in BEFORE_SITE: print("%s|stdlib|" % p) elif p in AFTER_SITE: - print("%s||" % p) + if p in SITE_PKGS: + print("%s|site|" % p) + else: + print("%s|pth|" % p) diff --git a/src/Analysis/Ast/Test/ArgumentSetTests.cs b/src/Analysis/Ast/Test/ArgumentSetTests.cs index 2f4bee636..e0a3eb541 100644 --- a/src/Analysis/Ast/Test/ArgumentSetTests.cs +++ b/src/Analysis/Ast/Test/ArgumentSetTests.cs @@ -370,14 +370,14 @@ private async Task GetArgSetAsync(string code, string funcName = "f var analysis = await GetAnalysisAsync(code); var f = analysis.Should().HaveFunction(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f, 0, null, call, analysis.Document, analysis.ExpressionEvaluator); + return new ArgumentSet(f, 0, null, call, analysis.ExpressionEvaluator); } private async Task GetUnboundArgSetAsync(string code, string funcName = "f") { var analysis = await GetAnalysisAsync(code); var f = analysis.Should().HaveVariable(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f.Value.GetPythonType(), 0, null, call, analysis.Document, analysis.ExpressionEvaluator); + return new ArgumentSet(f.Value.GetPythonType(), 0, null, call, analysis.ExpressionEvaluator); } private async Task GetClassArgSetAsync(string code, string className = "A", string funcName = "f") { @@ -385,7 +385,7 @@ private async Task GetClassArgSetAsync(string code, string classNam var cls = analysis.Should().HaveClass(className).Which; var f = cls.Should().HaveMethod(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f, 0, new PythonInstance(cls), call, analysis.Document, analysis.ExpressionEvaluator); + return new ArgumentSet(f, 0, new PythonInstance(cls), call, analysis.ExpressionEvaluator); } private CallExpression GetCall(PythonAst ast) { diff --git a/src/Analysis/Ast/Test/ClassesTests.cs b/src/Analysis/Ast/Test/ClassesTests.cs index 57400a6a1..19e6639d2 100644 --- a/src/Analysis/Ast/Test/ClassesTests.cs +++ b/src/Analysis/Ast/Test/ClassesTests.cs @@ -89,7 +89,7 @@ public async Task Mro() { var o = interpreter.GetBuiltinType(BuiltinTypeId.Object); var m = new SentinelModule("test", s); - var location = new Location(m, default); + var location = new Location(m); var O = new PythonClassType("O", location); var A = new PythonClassType("A", location); var B = new PythonClassType("B", location); diff --git a/src/Analysis/Ast/Test/DependencyResolverTests.cs b/src/Analysis/Ast/Test/DependencyResolverTests.cs index a9c476b66..8b5adae65 100644 --- a/src/Analysis/Ast/Test/DependencyResolverTests.cs +++ b/src/Analysis/Ast/Test/DependencyResolverTests.cs @@ -44,8 +44,8 @@ public void TestInitialize() [DataRow("A:C|C:B|B:A|D:AF|F:CE|E:BD", "ABCABCDEFDEF", "F")] [DataRow("A:BC|B:AC|C:BA|D:BC", "ACBACBD", "D")] [DataRow("A|B|C|D:AB|E:BC", "[ABC][DE]", "D|E")] - [DataRow("A:CE|B:A|C:B|D:BC|E|F:C", "[CE]ABCAB[DF]", "F")] - [DataRow("A:D|B:E|C:F|D:E|E:F|F:D", "DFEDFE[ABC]", "A")] + [DataRow("A:CE|B:A|C:B|D:BC|E|F:C", "[CE]ABCAB[DF]", "D|F")] + [DataRow("A:D|B:E|C:F|D:E|E:F|F:D", "DFEDFE[ABC]", "A|B|C")] // ReSharper restore StringLiteralTypo [DataTestMethod] public void ChangeValue(string input, string output, string root) { @@ -71,7 +71,8 @@ public void ChangeValue(string input, string output, string root) { foreach (var task in tasks) { result.Append(task.Result.Value[0]); - task.Result.Commit(); + task.Result.MarkWalked(); + task.Result.MoveNext(); } if (tasks.Count > 1) { @@ -87,7 +88,7 @@ public void ChangeValue(string input, string output, string root) { } [TestMethod] - public async Task ChangeValue_RepeatedChange() { + public async Task ChangeValue_ChangeToIdentical() { var resolver = new DependencyResolver(); resolver.ChangeValue("A", "A:B", true, "B"); resolver.ChangeValue("B", "B:C", false, "C"); @@ -98,7 +99,9 @@ public async Task ChangeValue_RepeatedChange() { while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); } result.ToString().Should().Be("CBA"); @@ -110,14 +113,16 @@ public async Task ChangeValue_RepeatedChange() { while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); } result.ToString().Should().Be("BA"); } [TestMethod] - public async Task ChangeValue_RepeatedChange2() { + public async Task ChangeValue_TwoChanges() { var resolver = new DependencyResolver(); resolver.ChangeValue("A", "A:B", true, "B"); resolver.ChangeValue("B", "B", true); @@ -129,7 +134,9 @@ public async Task ChangeValue_RepeatedChange2() { while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); } result.ToString().Should().Be("BDAC"); @@ -142,7 +149,9 @@ public async Task ChangeValue_RepeatedChange2() { while (walker.Remaining > 0) { var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); } result.ToString().Should().Be("DCBA"); @@ -159,12 +168,16 @@ public async Task ChangeValue_MissingKeys() { var result = new StringBuilder(); var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + walker.MissingKeys.Should().Equal("D"); result.ToString().Should().Be("BC"); @@ -189,10 +202,13 @@ public async Task ChangeValue_Add() { var node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BD"); node.HasMissingDependencies.Should().BeTrue(); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); walker.Remaining.Should().Be(0); - + + // Add B resolver.ChangeValue("B", "B", false); walker = resolver.CreateWalker(); walker.MissingKeys.Should().Equal("D"); @@ -200,15 +216,20 @@ public async Task ChangeValue_Add() { node = await walker.GetNextAsync(default); node.Value.Should().Be("B"); node.HasMissingDependencies.Should().BeFalse(); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BD"); node.HasMissingDependencies.Should().BeTrue(); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + walker.Remaining.Should().Be(0); + // Add D resolver.ChangeValue("D", "D:C", false, "C"); walker = resolver.CreateWalker(); walker.MissingKeys.Should().BeEmpty(); @@ -216,18 +237,104 @@ public async Task ChangeValue_Add() { node = await walker.GetNextAsync(default); node.Value.Should().Be("C"); node.HasMissingDependencies.Should().BeFalse(); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); node = await walker.GetNextAsync(default); node.Value.Should().Be("D:C"); node.HasMissingDependencies.Should().BeFalse(); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BD"); node.HasMissingDependencies.Should().BeFalse(); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + } + + [TestMethod] + public async Task ChangeValue_Add_ParallelWalkers() { + var resolver = new DependencyResolver(); + resolver.ChangeValue("A", "A:BD", true, "B", "D"); + resolver.ChangeValue("B", "B:C", false, "C"); + resolver.ChangeValue("C", "C", false); + + var walker = resolver.CreateWalker(); + walker.MissingKeys.Should().Equal("D"); + + var node = await walker.GetNextAsync(default); + node.Value.Should().Be("C"); + node.HasMissingDependencies.Should().BeFalse(); + node.IsValidVersion.Should().BeTrue(); + + // Add D + resolver.ChangeValue("D", "D:C", false, "C"); + var newWalker = resolver.CreateWalker(); + newWalker.MissingKeys.Should().BeEmpty(); + + // MarkWalked node from old walker + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:BD"); + node.HasMissingDependencies.Should().BeTrue(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + + // Walk new walker + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("D:C"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await newWalker.GetNextAsync(default); + node.Value.Should().Be("A:BD"); + node.HasMissingDependencies.Should().BeFalse(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.IsValidVersion.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + walker.Remaining.Should().Be(0); } @@ -242,15 +349,21 @@ public async Task ChangeValue_Remove() { walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); node.Value.Should().Be("C"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); node = await walker.GetNextAsync(default); node.Value.Should().Be("B:C"); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BC"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); resolver.Remove("B"); walker = resolver.CreateWalker(); @@ -258,7 +371,106 @@ public async Task ChangeValue_Remove() { node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BC"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + } + + [TestMethod] + public async Task ChangeValue_ChangeChangeRemove() { + var resolver = new DependencyResolver(); + resolver.ChangeValue("A", "A:B", true, "B"); + resolver.ChangeValue("B", "B:C", true, "C"); + resolver.ChangeValue("C", "C:AD", true, "A", "D"); + + var walker = resolver.CreateWalker(); + walker.MissingKeys.Should().Equal("D"); + walker.AffectedValues.Should().Equal("A:B", "B:C", "C:AD"); + walker.Remaining.Should().Be(6); + + //resolver.ChangeValue("D", "D:B", true, "B"); + resolver.ChangeValue("A", "A", true); + resolver.ChangeValue("B", "B", true); + resolver.Remove("B"); + + walker = resolver.CreateWalker(); + walker.MissingKeys.Should().Equal("D"); + walker.Remaining.Should().Be(2); + + var node = await walker.GetNextAsync(default); + node.Value.Should().Be("A"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("C:AD"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + } + + [TestMethod] + public async Task ChangeValue_RemoveFromLoop() { + var resolver = new DependencyResolver(); + resolver.ChangeValue("A", "A:B", true, "B"); + resolver.ChangeValue("B", "B:C", false, "C"); + resolver.ChangeValue("C", "C:A", false, "A"); + + var walker = resolver.CreateWalker(); + walker.MissingKeys.Should().BeEmpty(); + + var node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:B"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("C:A"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasOnlyWalkedDependencies.Should().BeFalse(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:B"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("C:A"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("B:C"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + + walker.Remaining.Should().Be(0); + + resolver.Remove("B"); + walker = resolver.CreateWalker(); + walker.MissingKeys.Should().Equal("B"); + + node = await walker.GetNextAsync(default); + node.Value.Should().Be("A:B"); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); walker.Remaining.Should().Be(0); } @@ -275,19 +487,27 @@ public async Task ChangeValue_RemoveKeys() { walker.MissingKeys.Should().BeEmpty(); var node = await walker.GetNextAsync(default); node.Value.Should().Be("D"); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("C:D"); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("B:C"); - node.Commit(); - + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); + node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BC"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); resolver.RemoveKeys("B", "D"); walker = resolver.CreateWalker(); @@ -295,11 +515,15 @@ public async Task ChangeValue_RemoveKeys() { node = await walker.GetNextAsync(default); node.Value.Should().Be("C:D"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); node = await walker.GetNextAsync(default); node.Value.Should().Be("A:BC"); - node.Commit(); + node.HasOnlyWalkedDependencies.Should().BeTrue(); + node.MarkWalked(); + node.MoveNext(); walker.Remaining.Should().Be(0); } @@ -316,11 +540,11 @@ public async Task ChangeValue_Skip() { var result = new StringBuilder(); var node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Skip(); + node.MoveNext(); node = await walker.GetNextAsync(default); result.Append(node.Value[0]); - node.Skip(); + node.MoveNext(); result.ToString().Should().Be("BD"); diff --git a/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionOverloadAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionOverloadAssertions.cs index 2744f69b3..6ab5b0f8e 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionOverloadAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/PythonFunctionOverloadAssertions.cs @@ -31,7 +31,7 @@ public PythonFunctionOverloadAssertions(IPythonFunctionOverload pythonFunctionOv protected override string Identifier => nameof(IPythonFunctionOverload); public AndWhichConstraint HaveReturnType(string because = "", params object[] reasonArgs) { - var returnType = Subject.Call(ArgumentSet.Empty, null); + var returnType = Subject.Call(ArgumentSet.WithoutContext, null); Execute.Assertion.ForCondition(returnType != null) .BecauseOf(because, reasonArgs) .FailWith($"Expected {Subject.Name} overload to have a return type{{reason}}, but it has none."); @@ -40,7 +40,7 @@ public AndWhichConstraint HaveRet } public AndWhichConstraint HaveReturnType(BuiltinTypeId typeid, string because = "", params object[] reasonArgs) { - Subject.Call(ArgumentSet.Empty, null).GetPythonType().TypeId.Should().Be(typeid); + Subject.Call(ArgumentSet.WithoutContext, null).GetPythonType().TypeId.Should().Be(typeid); return new AndWhichConstraint(this, Subject); } @@ -106,7 +106,7 @@ public AndConstraint HaveNoParameters(string b => HaveParameters(Enumerable.Empty(), because, reasonArgs); public AndConstraint HaveReturnType(string type, string because = "", params object[] reasonArgs) { - var returnType = Subject.Call(ArgumentSet.Empty, null).GetPythonType(); + var returnType = Subject.Call(ArgumentSet.WithoutContext, null).GetPythonType(); Execute.Assertion.ForCondition(string.Equals(returnType.Name, type, StringComparison.Ordinal)) .BecauseOf(because, reasonArgs) .FailWith($"Expected {Subject.Name} to have return type [{type}]{{reason}}, but it has [{returnType}]."); diff --git a/src/Analysis/Ast/Test/InheritNonClassTests.cs b/src/Analysis/Ast/Test/InheritNonClassTests.cs new file mode 100644 index 000000000..8368f7f66 --- /dev/null +++ b/src/Analysis/Ast/Test/InheritNonClassTests.cs @@ -0,0 +1,173 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class InheritNonClassTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task InheritFromRenamedBuiltin() { + const string code = @" +tmp = str + +class C(tmp): + def method(self): + return 'test' +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + + [TestMethod, Priority(0)] + public async Task InheritFromBuiltin() { + const string code = @" +class C(str): + def method(self): + return 'test' +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + + [TestMethod, Priority(0)] + public async Task InheritFromUserClass() { + const string code = @" +class D: + def hello(self): + pass + +class C(D): + def method(self): + return 'test' +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + + [TestMethod, Priority(0)] + public async Task InheritFromConstant() { + const string code = @" +class C(5): + def method(self): + return 'test' +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.SourceSpan.Should().Be(2, 7, 2, 8); + diagnostic.Message.Should().Be(Resources.InheritNonClass.FormatInvariant("5")); + diagnostic.ErrorCode.Should().Be(ErrorCodes.InheritNonClass); + } + + + [TestMethod, Priority(0)] + public async Task InheritFromConstantVar() { + const string code = @" +x = 'str' + +class C(x): + def method(self): + return 'test' + +x = 5 + +class D(x): + def method(self): + return 'test' +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(2); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.SourceSpan.Should().Be(4, 7, 4, 8); + diagnostic.Message.Should().Be(Resources.InheritNonClass.FormatInvariant("str")); + diagnostic.ErrorCode.Should().Be(ErrorCodes.InheritNonClass); + + diagnostic = analysis.Diagnostics.ElementAt(1); + diagnostic.SourceSpan.Should().Be(10, 7, 10, 8); + diagnostic.Message.Should().Be(Resources.InheritNonClass.FormatInvariant("5")); + diagnostic.ErrorCode.Should().Be(ErrorCodes.InheritNonClass); + } + + [Ignore] + [TestMethod, Priority(0)] + public async Task InheritFromBinaryOp() { + const string code = @" +x = 5 + +class C(x + 2): + def method(self): + return 'test' +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.SourceSpan.Should().Be(4, 7, 4, 8); + diagnostic.Message.Should().Be(Resources.InheritNonClass.FormatInvariant("x + 2")); + diagnostic.ErrorCode.Should().Be(ErrorCodes.InheritNonClass); + } + + + [TestMethod, Priority(0)] + public async Task InheritFromOtherModule() { + const string code = @" +import typing + +class C(typing.TypeVar): + def method(self): + return 'test' +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task InheritFromRenamedOtherModule() { + const string code = @" +import typing + +tmp = typing.TypeVar +class C(tmp): + def method(self): + return 'test' +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + } +} diff --git a/src/Analysis/Ast/Test/LintGenericTests.cs b/src/Analysis/Ast/Test/LintGenericTests.cs new file mode 100644 index 000000000..e0ce8b18a --- /dev/null +++ b/src/Analysis/Ast/Test/LintGenericTests.cs @@ -0,0 +1,105 @@ +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + + +namespace Microsoft.Python.Analysis.Tests { + + [TestClass] + public class LintGenericTests : AnalysisTestBase { + + public const string GenericSetup = @" +from typing import Generic, TypeVar +T = TypeVar('T', int, str) +T1 = TypeVar('T1', int, str) + +_X = TypeVar('_X', str, int) +_T = _X +"; + + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [DataRow("x = Generic[T, str]")] + [DataRow("x = Generic[T, T1, int]")] + [DataRow("x = Generic[T, str, int, T1]")] + [DataRow("x = Generic[str, int]")] + [DataRow("x = Generic[str]")] + [DataTestMethod, Priority(0)] + public async Task GenericNotAllTypeParameters(string decl) { + string code = GenericSetup + decl; + + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + var start = decl.IndexOf("Generic") + 1; + // adding 1 because SourceSpan.End is exclusive and another 1 because SourceSpan is 1-indexed + var end = decl.IndexOf("]", start) + 2; + + diagnostic.SourceSpan.Should().Be(8, start, 8, end); + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.TypingGenericArguments); + diagnostic.Message.Should().Be(Resources.GenericNotAllTypeParameters); + } + + [DataRow("x = Generic[_T, _X]")] + [DataRow("x = Generic[_T, T, T1, _X]")] + [DataRow("x = Generic[_T,_T, T]")] + [DataRow("x = Generic[T,T]")] + [DataTestMethod, Priority(0)] + public async Task GenericDuplicateArguments(string decl) { + string code = GenericSetup + decl; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + var start = decl.IndexOf("Generic") + 1; + // adding 1 because SourceSpan.End is exclusive and another 1 because SourceSpan is 1-indexed + var end = decl.IndexOf("]", start) + 2; + diagnostic.SourceSpan.Should().Be(8, start, 8, end); + + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.TypingGenericArguments); + diagnostic.Message.Should().Be(Resources.GenericNotAllUnique); + } + + [DataRow("x = Generic[_X, T]")] + [DataRow("x = Generic[T1, T]")] + [DataRow("x = Generic[T]")] + [DataRow("x = Generic[T,T1, _X]")] + [DataTestMethod, Priority(0)] + public async Task GenericArgumentsNoDiagnosticOnValid(string decl) { + string code = GenericSetup + decl; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task GenericNoArgumentsNoDiagnostic() { + string code = GenericSetup + @" +x = Generic[] +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + + [TestMethod, Priority(0)] + public async Task GenericArgumentSpaceNoDiagnostic() { + string code = GenericSetup + @" +x = Generic[ ] +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + } +} diff --git a/src/Analysis/Ast/Test/LintNewTypeTests.cs b/src/Analysis/Ast/Test/LintNewTypeTests.cs new file mode 100644 index 000000000..aacddbf87 --- /dev/null +++ b/src/Analysis/Ast/Test/LintNewTypeTests.cs @@ -0,0 +1,130 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class LintNewTypeTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task NewTypeIntFirstArg() { + const string code = @" +from typing import NewType + +T = NewType(5, int) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.SourceSpan.Should().Be(4, 5, 4, 20); + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.TypingNewTypeArguments); + diagnostic.Message.Should().Be(Resources.NewTypeFirstArgNotString.FormatInvariant("int")); + } + + [DataRow("float", "float")] + [DataRow("int", "int")] + [DataRow("complex", "str")] + [DataTestMethod, Priority(0)] + public async Task DifferentTypesFirstArg(string nameType, string type) { + string code = $@" +from typing import NewType + +T = NewType({nameType}(10), {type}) + +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.TypingNewTypeArguments); + diagnostic.Message.Should().Be(Resources.NewTypeFirstArgNotString.FormatInvariant(nameType)); + } + + [TestMethod, Priority(0)] + public async Task ObjectFirstArg() { + string code = $@" +from typing import NewType + +class X: + def hello(): + pass + +h = X() + +T = NewType(h, int) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.SourceSpan.Should().Be(10, 5, 10, 20); + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.TypingNewTypeArguments); + diagnostic.Message.Should().Be(Resources.NewTypeFirstArgNotString.FormatInvariant("X")); + } + + [TestMethod, Priority(0)] + public async Task GenericFirstArg() { + string code = $@" +from typing import NewType, Generic, TypeVar + +T = TypeVar('T', str, int) + +class X(Generic[T]): + def __init__(self, p: T): + self.x = p + +h = X(5) +T = NewType(h, int) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.SourceSpan.Should().Be(11, 5, 11, 20); + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.TypingNewTypeArguments); + diagnostic.Message.Should().Be(Resources.NewTypeFirstArgNotString.FormatInvariant("X[int]")); + } + + [DataRow("test", "float")] + [DataRow("testing", "int")] + [DataTestMethod, Priority(0)] + public async Task NoDiagnosticOnStringFirstArg(string name, string type) { + string code = $@" +from typing import NewType + +T = NewType('{name}', {type}) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(0); + } + } +} diff --git a/src/Analysis/Ast/Test/LintOperatorTests.cs b/src/Analysis/Ast/Test/LintOperatorTests.cs new file mode 100644 index 000000000..0578dbdbd --- /dev/null +++ b/src/Analysis/Ast/Test/LintOperatorTests.cs @@ -0,0 +1,86 @@ +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class LintOperatorTests : AnalysisTestBase { + + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task IncompatibleTypesBinaryOpBasic() { + var code = $@" +a = 5 + 'str' +"; + + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.SourceSpan.Should().Be(2, 5, 2, 14); + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.UnsupportedOperandType); + diagnostic.Message.Should().Be(Resources.UnsupporedOperandType.FormatInvariant("+", "int", "str")); + } + + [DataRow("str", "int", "+")] + [DataRow("str", "int", "-")] + [DataRow("str", "int", "/")] + [DataRow("str", "float", "+")] + [DataRow("str", "float", "-")] + [DataRow("str", "float", "*")] + [DataTestMethod, Priority(0)] + public async Task IncompatibleTypesBinaryOp(string leftType, string rightType, string op) { + var code = $@" +x = 1 +y = 2 + +z = {leftType}(x) {op} {rightType}(y) +"; + + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + + + string line = $"z = {leftType}(x) {op} {rightType}(y)"; + // source span is 1 indexed + diagnostic.SourceSpan.Should().Be(5, line.IndexOf(leftType) + 1, 5, line.IndexOf("(y)") + 4); + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.UnsupportedOperandType); + diagnostic.Message.Should().Be(Resources.UnsupporedOperandType.FormatInvariant(op, leftType, rightType)); + } + + [DataRow("str", "str", "+")] + [DataRow("int", "int", "-")] + [DataRow("bool", "int", "/")] + [DataRow("float", "int", "+")] + [DataRow("complex", "float", "-")] + [DataRow("str", "int", "*")] + [DataTestMethod, Priority(0)] + public async Task CompatibleTypesBinaryOp(string leftType, string rightType, string op) { + var code = $@" +x = 1 +y = 2 + +z = {leftType}(x) {op} {rightType}(y) +"; + + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Diagnostics.Should().BeEmpty(); + } + } + +} diff --git a/src/Analysis/Ast/Test/LintReturnInInitTests.cs b/src/Analysis/Ast/Test/LintReturnInInitTests.cs new file mode 100644 index 000000000..4ae7626a2 --- /dev/null +++ b/src/Analysis/Ast/Test/LintReturnInInitTests.cs @@ -0,0 +1,127 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class LintReturnInInitTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task ReturnInInit() { + const string code = @" +class Rectangle: + def __init__(self, width, height): + self.width = width + self.height = height + self.area = width * height + return self.area + +r = Rectangle(10, 10) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.SourceSpan.Should().Be(7, 9, 7, 25); + diagnostic.Severity.Should().Be(Severity.Warning); + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.ReturnInInit); + diagnostic.Message.Should().Be(Resources.ReturnInInit); + } + + [TestMethod, Priority(0)] + public async Task ReturnInitBasic() { + const string code = @" +class Rectangle: + def __init__(self, width, height): + return 2 + +r = Rectangle(10, 10) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.SourceSpan.Should().Be(4, 9, 4, 17); + diagnostic.Severity.Should().Be(Severity.Warning); + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.ReturnInInit); + diagnostic.Message.Should().Be(Resources.ReturnInInit); + } + + [TestMethod, Priority(0)] + public async Task ReturnInInitConditional() { + const string code = @" +class A: + def __init__(self, x): + self.x = x + if x > 0: + return 10 + +a = A(1) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().HaveCount(1); + + var diagnostic = analysis.Diagnostics.ElementAt(0); + diagnostic.SourceSpan.Should().Be(6, 13, 6, 22); + diagnostic.Severity.Should().Be(Severity.Warning); + diagnostic.ErrorCode.Should().Be(Diagnostics.ErrorCodes.ReturnInInit); + diagnostic.Message.Should().Be(Resources.ReturnInInit); + } + + [TestMethod, Priority(0)] + public async Task ReturnNoneInInit() { + const string code = @" +class A: + def __init__(self, x): + self.x = x + self.x += 1 + return None + +a = A(1) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task EmptyReturnInInit() { + const string code = @" +class A: + def __init__(self, x): + self.x = x + return +a = A(1) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + } +} diff --git a/src/Analysis/Ast/Test/PathClassificationTests.cs b/src/Analysis/Ast/Test/PathClassificationTests.cs new file mode 100644 index 000000000..429df72b7 --- /dev/null +++ b/src/Analysis/Ast/Test/PathClassificationTests.cs @@ -0,0 +1,218 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.IO; +using FluentAssertions; +using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Core.OS; +using Microsoft.Python.Tests.Utilities.FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class PathClassificationTests { + private readonly FileSystem _fs = new FileSystem(new OSPlatform()); + + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod] + public void Plain() { + var appPath = TestData.GetTestSpecificPath("app.py"); + var root = Path.GetDirectoryName(appPath); + + var venv = Path.Combine(root, "venv"); + var venvLib = Path.Combine(venv, "Lib"); + var venvSitePackages = Path.Combine(venvLib, "site-packages"); + + var fromInterpreter = new[] { + new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), + }; + + var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, Array.Empty()); + + interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] { + new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), + }); + + userPaths.Should().BeEmpty(); + } + + [TestMethod] + public void WithSrcDir() { + var appPath = TestData.GetTestSpecificPath("app.py"); + var root = Path.GetDirectoryName(appPath); + + var venv = Path.Combine(root, "venv"); + var venvLib = Path.Combine(venv, "Lib"); + var venvSitePackages = Path.Combine(venvLib, "site-packages"); + + var src = Path.Combine(root, "src"); + + var fromInterpreter = new[] { + new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), + }; + + var fromUser = new[] { + "./src", + }; + + var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, fromUser); + + interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] { + new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), + }); + + userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { + new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), + }); + } + + [TestMethod] + public void NormalizeUser() { + var appPath = TestData.GetTestSpecificPath("app.py"); + var root = Path.GetDirectoryName(appPath); + + var src = Path.Combine(root, "src"); + + var fromUser = new[] { + "./src/", + }; + + var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, Array.Empty(), fromUser); + + interpreterPaths.Should().BeEmpty(); + + userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { + new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), + }); + } + + [TestMethod] + public void NestedUser() { + var appPath = TestData.GetTestSpecificPath("app.py"); + var root = Path.GetDirectoryName(appPath); + + var src = Path.Combine(root, "src"); + var srcSomething = Path.Combine(src, "something"); + var srcFoo = Path.Combine(src, "foo"); + var srcFooBar = Path.Combine(srcFoo, "bar"); + + var fromUser = new[] { + "./src", + "./src/something", + "./src/foo/bar", + "./src/foo", + }; + + var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, Array.Empty(), fromUser); + + interpreterPaths.Should().BeEmpty(); + + userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { + new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), + new PythonLibraryPath(srcSomething, PythonLibraryPathType.Unspecified), + new PythonLibraryPath(srcFooBar, PythonLibraryPathType.Unspecified), + new PythonLibraryPath(srcFoo, PythonLibraryPathType.Unspecified), + }); + } + + [TestMethod] + public void NestedUserOrdering() { + var appPath = TestData.GetTestSpecificPath("app.py"); + var root = Path.GetDirectoryName(appPath); + + var src = Path.Combine(root, "src"); + var srcSomething = Path.Combine(src, "something"); + var srcFoo = Path.Combine(src, "foo"); + var srcFooBar = Path.Combine(srcFoo, "bar"); + + var fromUser = new[] { + "./src/foo", + "./src/foo/bar", + "./src", + "./src/something", + }; + + var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, Array.Empty(), fromUser); + + interpreterPaths.Should().BeEmpty(); + + userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { + new PythonLibraryPath(srcFoo, PythonLibraryPathType.Unspecified), + new PythonLibraryPath(srcFooBar, PythonLibraryPathType.Unspecified), + new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), + new PythonLibraryPath(srcSomething, PythonLibraryPathType.Unspecified), + }); + } + + [TestMethod] + public void InsideStdLib() { + var appPath = TestData.GetTestSpecificPath("app.py"); + var root = Path.GetDirectoryName(appPath); + + var venv = Path.Combine(root, "venv"); + var venvLib = Path.Combine(venv, "Lib"); + var venvSitePackages = Path.Combine(venvLib, "site-packages"); + var inside = Path.Combine(venvSitePackages, "inside"); + var what = Path.Combine(venvSitePackages, "what"); + + var src = Path.Combine(root, "src"); + + var fromInterpreter = new[] { + new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), + new PythonLibraryPath(inside, PythonLibraryPathType.Pth), + }; + + var fromUser = new[] { + "./src", + "./venv/Lib/site-packages/what", + }; + + var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, fromUser); + + interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] { + new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib), + new PythonLibraryPath(venv, PythonLibraryPathType.StdLib), + new PythonLibraryPath(what, PythonLibraryPathType.Unspecified), + new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site), + new PythonLibraryPath(inside, PythonLibraryPathType.Pth), + }); + + userPaths.Should().BeEquivalentToWithStrictOrdering(new[] { + new PythonLibraryPath(src, PythonLibraryPathType.Unspecified), + }); + } + } +} diff --git a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs index 85a3274c7..132d06ff1 100644 --- a/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs +++ b/src/Analysis/Core/Impl/DependencyResolution/AstUtilities.cs @@ -18,9 +18,6 @@ namespace Microsoft.Python.Analysis.Core.DependencyResolution { public static class AstUtilities { - public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, ModuleName importName, bool forceAbsolute) - => pathResolver.GetImportsFromAbsoluteName(modulePath, importName.Names.Select(n => n.Name), forceAbsolute); - public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, FromImportStatement fromImportStatement) { var rootNames = fromImportStatement.Root.Names.Select(n => n.Name); return fromImportStatement.Root is RelativeModuleName relativeName diff --git a/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs b/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs index d95e35cf1..2c4106e13 100644 --- a/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs +++ b/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs @@ -18,7 +18,6 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Core; @@ -27,75 +26,103 @@ using IOPath = System.IO.Path; namespace Microsoft.Python.Analysis.Core.Interpreter { - public sealed class PythonLibraryPath { - private readonly string _modulePrefix; - - private static readonly Regex ParseRegex = new Regex( - @"(?[^|]+)\|(?stdlib)?\|(?[^|]+)?" - ); + public enum PythonLibraryPathType { + Unspecified, + StdLib, + Site, + Pth, + } - public PythonLibraryPath(string path, LibraryType libraryType, string modulePrefix) { - Path = path; - LibraryType = libraryType; - _modulePrefix = modulePrefix; + public sealed class PythonLibraryPath : IEquatable { + public PythonLibraryPath(string path, PythonLibraryPathType type = PythonLibraryPathType.Unspecified, string modulePrefix = null) { + Path = PathUtils.NormalizePathAndTrim(path); + Type = type; + ModulePrefix = modulePrefix ?? string.Empty; } + public PythonLibraryPath(string path, bool isStandardLibrary, string modulePrefix) : + this(path, isStandardLibrary ? PythonLibraryPathType.StdLib : PythonLibraryPathType.Unspecified, modulePrefix) { } + public string Path { get; } - public LibraryType LibraryType { get; } - public string ModulePrefix => _modulePrefix ?? string.Empty; + public PythonLibraryPathType Type { get; } + + public string ModulePrefix { get; } = string.Empty; + + public bool IsStandardLibrary => Type == PythonLibraryPathType.StdLib; + + public override string ToString() { + var type = string.Empty; + + switch (Type) { + case PythonLibraryPathType.StdLib: + type = "stdlib"; + break; + case PythonLibraryPathType.Site: + type = "site"; + break; + case PythonLibraryPathType.Pth: + type = "pth"; + break; + } - public override string ToString() - => "{0}|{1}|{2}".FormatInvariant(Path, LibraryType == LibraryType.Standard ? "stdlib" : "", _modulePrefix ?? string.Empty); + return "{0}|{1}|{2}".FormatInvariant(Path, type, ModulePrefix); + } - public static PythonLibraryPath FromLibraryPath(string s, IFileSystem fs, string standardLibraryPath) { + public static PythonLibraryPath Parse(string s) { if (string.IsNullOrEmpty(s)) { - throw new ArgumentNullException(nameof(s)); + throw new ArgumentNullException("source"); } - - var m = ParseRegex.Match(s); - if (!m.Success || !m.Groups["path"].Success) { + + var parts = s.Split(new[] { '|' }, 3); + if (parts.Length < 3) { throw new FormatException(); } - var libraryType = LibraryType.Other; - var sitePackagesPath = GetSitePackagesPath(standardLibraryPath); - var path = m.Groups["path"].Value; - if (m.Groups["stdlib"].Success) { - libraryType = LibraryType.Standard; - } else if(fs.IsPathUnderRoot(sitePackagesPath, path)) { - libraryType = LibraryType.SitePackages; + var path = parts[0]; + var ty = parts[1]; + var prefix = parts[2]; + + PythonLibraryPathType type = PythonLibraryPathType.Unspecified; + + switch (ty) { + case "stdlib": + type = PythonLibraryPathType.StdLib; + break; + case "site": + type = PythonLibraryPathType.Site; + break; + case "pth": + type = PythonLibraryPathType.Pth; + break; } - return new PythonLibraryPath( - m.Groups["path"].Value, - libraryType, - m.Groups["prefix"].Success ? m.Groups["prefix"].Value : null - ); + return new PythonLibraryPath(path, type, prefix); } /// /// Gets the default set of search paths based on the path to the root /// of the standard library. /// - /// Root of the standard library. + /// Root of the standard library. /// A list of search paths for the interpreter. - public static List GetDefaultSearchPaths(string standardLibraryPath) { + /// New in 2.2, moved in 3.3 + public static List GetDefaultSearchPaths(string library) { var result = new List(); - if (!Directory.Exists(standardLibraryPath)) { + if (!Directory.Exists(library)) { return result; } - result.Add(new PythonLibraryPath(standardLibraryPath, LibraryType.Standard, null)); + result.Add(new PythonLibraryPath(library, PythonLibraryPathType.StdLib)); - var sitePackages = GetSitePackagesPath(standardLibraryPath); + var sitePackages = IOPath.Combine(library, "site-packages"); if (!Directory.Exists(sitePackages)) { return result; } - result.Add(new PythonLibraryPath(sitePackages, LibraryType.SitePackages, null)); + result.Add(new PythonLibraryPath(sitePackages)); result.AddRange(ModulePath.ExpandPathFiles(sitePackages) - .Select(p => new PythonLibraryPath(p, LibraryType.SitePackages, null)) + .Select(p => new PythonLibraryPath(p)) ); return result; @@ -107,7 +134,7 @@ public static List GetDefaultSearchPaths(string standardLibra public static async Task> GetSearchPathsAsync(InterpreterConfiguration config, IFileSystem fs, IProcessServices ps, CancellationToken cancellationToken = default) { for (int retries = 5; retries > 0; --retries) { try { - return await GetSearchPathsFromInterpreterAsync(config, fs, ps, cancellationToken); + return await GetSearchPathsFromInterpreterAsync(config.InterpreterPath, fs, ps, cancellationToken); } catch (InvalidOperationException) { // Failed to get paths break; @@ -133,18 +160,18 @@ public static string GetStandardLibraryPath(InterpreterConfiguration config) { public static string GetSitePackagesPath(InterpreterConfiguration config) => GetSitePackagesPath(GetStandardLibraryPath(config)); - public static string GetSitePackagesPath(string standardLibraryPath) + public static string GetSitePackagesPath(string standardLibraryPath) => !string.IsNullOrEmpty(standardLibraryPath) ? IOPath.Combine(standardLibraryPath, "site-packages") : string.Empty; /// /// Gets the set of search paths by running the interpreter. /// - /// Interpreter configuration. + /// Path to the interpreter. /// File system services. /// Process services. /// Cancellation token. /// A list of search paths for the interpreter. - public static async Task> GetSearchPathsFromInterpreterAsync(InterpreterConfiguration config, IFileSystem fs, IProcessServices ps, CancellationToken cancellationToken = default) { + public static async Task> GetSearchPathsFromInterpreterAsync(string interpreter, IFileSystem fs, IProcessServices ps, CancellationToken cancellationToken = default) { // sys.path will include the working directory, so we make an empty // path that we can filter out later var tempWorkingDir = IOPath.Combine(IOPath.GetTempPath(), IOPath.GetRandomFileName()); @@ -156,7 +183,7 @@ public static async Task> GetSearchPathsFromInterpreterA File.Copy(srcGetSearchPaths, getSearchPaths); var startInfo = new ProcessStartInfo( - config.InterpreterPath, + interpreter, new[] { "-S", "-E", getSearchPaths }.AsQuotedArguments() ) { WorkingDirectory = tempWorkingDir, @@ -169,13 +196,12 @@ public static async Task> GetSearchPathsFromInterpreterA try { var output = await ps.ExecuteAndCaptureOutputAsync(startInfo, cancellationToken); - var standardLibraryPath = GetSitePackagesPath(config); - return output.Split(new [] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries).Select(s => { + return output.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries).Select(s => { if (s.PathStartsWith(tempWorkingDir)) { return null; } try { - return FromLibraryPath(s, fs, standardLibraryPath); + return Parse(s); } catch (ArgumentException) { Debug.Fail("Invalid search path: " + (s ?? "")); return null; @@ -188,5 +214,88 @@ public static async Task> GetSearchPathsFromInterpreterA fs.DeleteDirectory(tempWorkingDir, true); } } + + public static (IReadOnlyList interpreterPaths, IReadOnlyList userPaths) ClassifyPaths( + string root, + IFileSystem fs, + IEnumerable fromInterpreter, + IEnumerable fromUser + ) { +#if DEBUG + Debug.Assert(root == null || root.PathEquals(PathUtils.NormalizePathAndTrim(root))); + Debug.Assert(!fromInterpreter.Any(p => !p.Path.PathEquals(PathUtils.NormalizePathAndTrim(p.Path)))); +#endif + + // Clean up user configured paths. + // 1) Normalize paths. + // 2) If a path isn't rooted, then root it relative to the workspace root. If there is no root, just continue. + // 3) Trim off any ending separators for consistency. + // 4) Remove any empty paths, FS root paths (bad idea), or paths equal to the root. + // 5) Deduplicate, preserving the order specified by the user. + var fromUserList = fromUser + .Select(PathUtils.NormalizePath) + .Select(p => root == null || IOPath.IsPathRooted(p) ? p : IOPath.GetFullPath(IOPath.Combine(root, p))) // TODO: Replace with GetFullPath(p, root) when .NET Standard 2.1 is out. + .Select(PathUtils.TrimEndSeparator) + .Where(p => !string.IsNullOrWhiteSpace(p) && p != "/" && !p.PathEquals(root)) + .Distinct(PathEqualityComparer.Instance) + .ToList(); + + // Remove any interpreter paths specified in the user config so they can be reclassified. + // The user list is usually small; List.Contains should not be too slow. + fromInterpreter.Where(p => !fromUserList.Contains(p.Path, PathEqualityComparer.Instance)) + .Split(p => p.Type == PythonLibraryPathType.StdLib, out var stdlib, out var withoutStdlib); + + // Pull out stdlib paths, and make them always be interpreter paths. + var interpreterPaths = new List(stdlib); + + var userPaths = new List(); + + var allPaths = fromUserList.Select(p => new PythonLibraryPath(p)) + .Concat(withoutStdlib.Where(p => !p.Path.PathEquals(root))); + + foreach (var p in allPaths) { + // If path is within a stdlib path, then treat it as interpreter. + if (stdlib.Any(s => fs.IsPathUnderRoot(s.Path, p.Path))) { + interpreterPaths.Add(p); + continue; + } + + // If path is outside the workspace, then treat it as interpreter. + if (root == null || !fs.IsPathUnderRoot(root, p.Path)) { + interpreterPaths.Add(p); + continue; + } + + userPaths.Add(p); + } + + return (interpreterPaths, userPaths.ToList()); + } + + public override bool Equals(object obj) => obj is PythonLibraryPath other && Equals(other); + + public override int GetHashCode() { + // TODO: Replace with HashCode.Combine when .NET Standard 2.1 is out. + unchecked { + var hashCode = Path.GetHashCode(); + hashCode = (hashCode * 397) ^ Type.GetHashCode(); + hashCode = (hashCode * 397) ^ ModulePrefix.GetHashCode(); + return hashCode; + } + } + + public bool Equals(PythonLibraryPath other) { + if (other is null) { + return false; + } + + return Path.PathEquals(other.Path) + && Type == other.Type + && ModulePrefix == other.ModulePrefix; + } + + public static bool operator ==(PythonLibraryPath left, PythonLibraryPath right) => left?.Equals(right) ?? right is null; + + public static bool operator !=(PythonLibraryPath left, PythonLibraryPath right) => !(left?.Equals(right) ?? right is null); } } diff --git a/src/Caching/Impl/Factories/ModuleFactory.cs b/src/Caching/Impl/Factories/ModuleFactory.cs index ad0fe4d6c..a2ba630c3 100644 --- a/src/Caching/Impl/Factories/ModuleFactory.cs +++ b/src/Caching/Impl/Factories/ModuleFactory.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using System.Collections; using System.Collections.Generic; using System.Diagnostics; using System.Linq; diff --git a/src/Core/Impl/Extensions/StringExtensions.cs b/src/Core/Impl/Extensions/StringExtensions.cs index 378ffc3b8..a023b3363 100644 --- a/src/Core/Impl/Extensions/StringExtensions.cs +++ b/src/Core/Impl/Extensions/StringExtensions.cs @@ -204,6 +204,9 @@ public static bool EqualsOrdinal(this string s, string other) public static bool PathEquals(this string s, string other) => string.Equals(s, other, PathsStringComparison); + public static int PathCompare(this string s, string other) + => string.Compare(s, other, PathsStringComparison); + public static bool EqualsOrdinal(this string s, int index, string other, int otherIndex, int length, bool ignoreCase = false) => string.Compare(s, index, other, otherIndex, length, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal) == 0; public static bool ContainsOrdinal(this string s, string value, bool ignoreCase = false) diff --git a/src/Core/Impl/IO/PathUtils.cs b/src/Core/Impl/IO/PathUtils.cs index a9513ee59..a5140c1a0 100644 --- a/src/Core/Impl/IO/PathUtils.cs +++ b/src/Core/Impl/IO/PathUtils.cs @@ -493,5 +493,7 @@ public static string NormalizePath(string path) { ); return isDir ? EnsureEndSeparator(newPath) : newPath; } + + public static string NormalizePathAndTrim(string path) => TrimEndSeparator(NormalizePath(path)); } } diff --git a/src/LanguageServer/Impl/Completion/CompletionItemSource.cs b/src/LanguageServer/Impl/Completion/CompletionItemSource.cs index f202be5c4..60e43e7a2 100644 --- a/src/LanguageServer/Impl/Completion/CompletionItemSource.cs +++ b/src/LanguageServer/Impl/Completion/CompletionItemSource.cs @@ -28,13 +28,14 @@ internal class CompletionItemSource { public static readonly CompletionItem Star = CreateCompletionItem("*", CompletionItemKind.Keyword); private readonly IDocumentationSource _docSource; - private readonly ServerSettings.PythonCompletionOptions _options; public CompletionItemSource(IDocumentationSource docSource, ServerSettings.PythonCompletionOptions options) { _docSource = docSource; - _options = options; + Options = options; } + public ServerSettings.PythonCompletionOptions Options { get; set; } + public CompletionItem CreateCompletionItem(string text, IMember member, IPythonType self = null, string label = null) => CreateCompletionItem(text, ToCompletionItemKind(member?.MemberType ?? PythonMemberType.Class), member, self, label); @@ -42,7 +43,8 @@ public CompletionItemEx CreateCompletionItem(string text, CompletionItemKind kin var t = member?.GetPythonType(); var docFormat = _docSource.DocumentationFormat; - if (_options.addBrackets && (kind == CompletionItemKind.Constructor || kind == CompletionItemKind.Function || kind == CompletionItemKind.Method)) { + if (Options.addBrackets && (kind == CompletionItemKind.Constructor || kind == CompletionItemKind.Function || kind == CompletionItemKind.Method)) { + label = text; text += "($0)"; docFormat = InsertTextFormat.Snippet; } diff --git a/src/LanguageServer/Impl/Completion/CompletionSource.cs b/src/LanguageServer/Impl/Completion/CompletionSource.cs index 7f635c5c3..7b9d51cf4 100644 --- a/src/LanguageServer/Impl/Completion/CompletionSource.cs +++ b/src/LanguageServer/Impl/Completion/CompletionSource.cs @@ -29,6 +29,11 @@ public CompletionSource(IDocumentationSource docSource, ServerSettings.PythonCom _itemSource = new CompletionItemSource(docSource, completionSettings); } + public ServerSettings.PythonCompletionOptions Options { + get => _itemSource.Options; + set => _itemSource.Options = value; + } + public CompletionResult GetCompletions(IDocumentAnalysis analysis, SourceLocation location) { if(analysis.Document.ModuleType != ModuleType.User) { return CompletionResult.Empty; diff --git a/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs b/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs index d95d8366d..a199acae7 100644 --- a/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs @@ -76,6 +76,7 @@ private static IEnumerable GetClassItems(IPythonClassType cls, E if (m is IVariable v && v.Source != VariableSource.Declaration) { continue; } + // If this is class member completion, unmangle private member names. var unmangledName = cls.UnmangleMemberName(t); if (!string.IsNullOrEmpty(unmangledName)) { diff --git a/src/LanguageServer/Impl/Implementation/Server.Symbols.cs b/src/LanguageServer/Impl/Implementation/Server.Symbols.cs index 09a000228..f9318f06e 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Symbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Symbols.cs @@ -36,7 +36,7 @@ public async Task HierarchicalDocumentSymbol(DocumentSymbolPar var path = @params.textDocument.uri.AbsolutePath; var symbols = await _indexManager.HierarchicalDocumentSymbolsAsync(path, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); - return symbols.Select(hSym => MakeDocumentSymbol(hSym)).ToArray(); + return symbols.Select(MakeDocumentSymbol).ToArray(); } private static SymbolInformation MakeSymbolInfo(FlatSymbol s) { diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index ea00a82c6..3f655f73a 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -116,8 +116,7 @@ public async Task InitializeAsync(InitializeParams @params, Ca _rootDir = @params.rootUri != null ? @params.rootUri.ToAbsolutePath() : @params.rootPath; if (_rootDir != null) { - _rootDir = PathUtils.NormalizePath(_rootDir); - _rootDir = PathUtils.TrimEndSeparator(_rootDir); + _rootDir = PathUtils.NormalizePathAndTrim(_rootDir); } Version.TryParse(@params.initializationOptions.interpreter.properties?.Version, out var version); @@ -126,19 +125,10 @@ public async Task InitializeAsync(InitializeParams @params, Ca interpreterPath: @params.initializationOptions.interpreter.properties?.InterpreterPath, version: version ) { - // 1) Split on ';' to support older VS Code extension versions which send paths as a single entry separated by ';'. TODO: Eventually remove. - // 2) Normalize paths. - // 3) If a path isn't rooted, then root it relative to the workspace root. If _rootDir is null, then accept the path as-is. - // 4) Trim off any ending separator for a consistent style. - // 5) Filter out any entries which are the same as the workspace root; they are redundant. Also ignore "/" to work around the extension (for now). - // 6) Remove duplicates. + // Split on ';' to support older VS Code extension versions which send paths as a single entry separated by ';'. TODO: Eventually remove. + // Note that the actual classification of these paths as user/library is done later in MainModuleResolution.ReloadAsync. SearchPaths = @params.initializationOptions.searchPaths .Select(p => p.Split(';', StringSplitOptions.RemoveEmptyEntries)).SelectMany() - .Select(PathUtils.NormalizePath) - .Select(p => _rootDir == null || Path.IsPathRooted(p) ? p : Path.GetFullPath(p, _rootDir)) - .Select(PathUtils.TrimEndSeparator) - .Where(p => !string.IsNullOrWhiteSpace(p) && p != "/" && !p.PathEquals(_rootDir)) - .Distinct(PathEqualityComparer.Instance) .ToList(), TypeshedPath = @params.initializationOptions.typeStubSearchPaths.FirstOrDefault() }; @@ -214,6 +204,7 @@ private bool HandleConfigurationChanges(ServerSettings newSettings) { Settings = newSettings; _symbolHierarchyMaxSymbols = Settings.analysis.symbolsHierarchyMaxSymbols; + _completionSource.Options = Settings.completion; if (oldSettings == null) { return true; diff --git a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs index 277267d09..2096e26fc 100644 --- a/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs +++ b/src/LanguageServer/Impl/Indexing/MostRecentDocumentSymbols.cs @@ -119,8 +119,7 @@ public void Dispose() { } } - private async Task> IndexAsync(IDocument doc, - CancellationToken indexCt) { + private async Task> IndexAsync(IDocument doc, CancellationToken indexCt) { var ast = await doc.GetAstAsync(indexCt); indexCt.ThrowIfCancellationRequested(); var walker = new SymbolIndexWalker(ast); diff --git a/src/LanguageServer/Impl/LanguageServer.cs b/src/LanguageServer/Impl/LanguageServer.cs index cb172f7c1..f61e951ce 100644 --- a/src/LanguageServer/Impl/LanguageServer.cs +++ b/src/LanguageServer/Impl/LanguageServer.cs @@ -432,7 +432,7 @@ private async Task ConsumerLoop() { return item.Task; } - private struct QueueItem { + private readonly struct QueueItem { private readonly TaskCompletionSource _tcs; public Task Task => _tcs.Task; public bool IsAwaitable { get; } diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs index 93eb1e189..8b0416bb2 100644 --- a/src/LanguageServer/Test/CompletionTests.cs +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -1194,5 +1194,25 @@ def test(x: Foo = func()): var comps = cs.GetCompletions(analysis, new SourceLocation(13, 7)); comps.Should().HaveLabels("name", "z"); } + + [TestMethod, Priority(0)] + public async Task AddBrackets() { + const string code = @"prin"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + + ServerSettings.completion.addBrackets = true; + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var comps = cs.GetCompletions(analysis, new SourceLocation(1, 5)); + var print = comps.Completions.FirstOrDefault(x => x.label == "print"); + print.Should().NotBeNull(); + print.insertText.Should().Be("print($0)"); + + cs.Options.addBrackets = false; + comps = cs.GetCompletions(analysis, new SourceLocation(1, 5)); + print = comps.Completions.FirstOrDefault(x => x.label == "print"); + print.Should().NotBeNull(); + print.insertText.Should().Be("print"); + } } } diff --git a/src/LanguageServer/Test/ImportsTests.cs b/src/LanguageServer/Test/ImportsTests.cs index bc9d10265..1df363597 100644 --- a/src/LanguageServer/Test/ImportsTests.cs +++ b/src/LanguageServer/Test/ImportsTests.cs @@ -229,8 +229,8 @@ public async Task UncSearchPaths() { [TestMethod, Priority(0)] public async Task UserSearchPathsInsideWorkspace() { - var folder1 = TestData.GetTestSpecificPath("folder1"); - var folder2 = TestData.GetTestSpecificPath("folder2"); + var folder2 = TestData.GetTestSpecificPath("src"); + var folder1 = TestData.GetTestSpecificPath("src", "virtualenv"); var packageInFolder1 = Path.Combine(folder1, "package"); var packageInFolder2 = Path.Combine(folder2, "package"); var module1Path = Path.Combine(packageInFolder1, "module1.py"); diff --git a/src/LanguageServer/Test/ReferencesTests.cs b/src/LanguageServer/Test/ReferencesTests.cs index cd53d92f4..405808b5c 100644 --- a/src/LanguageServer/Test/ReferencesTests.cs +++ b/src/LanguageServer/Test/ReferencesTests.cs @@ -61,49 +61,79 @@ def func(x): } [TestMethod, Priority(0)] - public async Task TwoOpenFiles() { + public async Task FindAllReferences_MultipleOpenFiles() { const string code1 = @" x = 1 def func(x): return x +class cls: + f = 3 + +c = cls() y = func(x) x = 2 "; - var code2 = $@" -from module1 import x -y = x -"; + var code2 = @" +from module1 import x, c +y = x, +f = c.f"; + + var code3 = @" +from package import module2 as m +from package.module2 import x +a = m.x +b = m.y +c = x"; + var uri1 = await TestData.CreateTestSpecificFileAsync("module1.py", code1); - var uri2 = await TestData.CreateTestSpecificFileAsync("module2.py", code2); + var uri2 = await TestData.CreateTestSpecificFileAsync(Path.Combine("package", "module2.py"), code2); + var uri3 = await TestData.CreateTestSpecificFileAsync("module3.py", code3); - await CreateServicesAsync(PythonVersions.LatestAvailable3X, uri1.AbsolutePath); + await CreateServicesAsync(PythonVersions.LatestAvailable3X); var rdt = Services.GetService(); - rdt.OpenDocument(uri1, code1); - rdt.OpenDocument(uri2, code2); + var doc1 = rdt.OpenDocument(uri1, code1); + var doc2 = rdt.OpenDocument(uri2, code2); + var doc3 = rdt.OpenDocument(uri3, code3); - var doc1 = rdt.GetDocument(uri1); - var analysis = await GetDocumentAnalysisAsync(doc1); + await doc1.GetAnalysisAsync(); + await doc2.GetAnalysisAsync(); + await doc3.GetAnalysisAsync(); var rs = new ReferenceSource(Services); - var refs = await rs.FindAllReferencesAsync(analysis.Document.Uri, new SourceLocation(7, 10), ReferenceSearchOptions.All); + var refs = await rs.FindAllReferencesAsync(uri1, new SourceLocation(11, 10), ReferenceSearchOptions.All); - refs.Should().HaveCount(5); + refs.Should().HaveCount(8); refs[0].range.Should().Be(1, 0, 1, 1); refs[0].uri.Should().Be(uri1); - refs[1].range.Should().Be(6, 9, 6, 10); + refs[1].range.Should().Be(10, 9, 10, 10); refs[1].uri.Should().Be(uri1); - refs[2].range.Should().Be(7, 0, 7, 1); + refs[2].range.Should().Be(11, 0, 11, 1); refs[2].uri.Should().Be(uri1); - refs[3].range.Should().Be(1, 20, 1, 21); - refs[3].uri.Should().Be(uri2); - refs[4].range.Should().Be(2, 4, 2, 5); - refs[4].uri.Should().Be(uri2); + refs[3].range.Should().Be(2, 28, 2, 29); + refs[3].uri.Should().Be(uri3); + refs[4].range.Should().Be(3, 6, 3, 7); + refs[4].uri.Should().Be(uri3); + refs[5].range.Should().Be(5, 4, 5, 5); + refs[5].uri.Should().Be(uri3); + + refs[6].range.Should().Be(1, 20, 1, 21); + refs[6].uri.Should().Be(uri2); + refs[7].range.Should().Be(2, 4, 2, 5); + refs[7].uri.Should().Be(uri2); + + refs = await rs.FindAllReferencesAsync(uri1, new SourceLocation(8, 5), ReferenceSearchOptions.All); + refs.Should().HaveCount(2); + + refs[0].range.Should().Be(7, 4, 7, 5); + refs[0].uri.Should().Be(uri1); + refs[1].range.Should().Be(3, 6, 3, 7); + refs[1].uri.Should().Be(uri2); } [TestMethod, Priority(0)] diff --git a/src/LanguageServer/Test/RenameTests.cs b/src/LanguageServer/Test/RenameTests.cs index a69d5755c..6b6bcb2f5 100644 --- a/src/LanguageServer/Test/RenameTests.cs +++ b/src/LanguageServer/Test/RenameTests.cs @@ -149,6 +149,62 @@ from module import x wse.changes[uri3][1].range.Should().Be(2, 4, 2, 5); } + [TestMethod, Priority(0)] + public async Task Rename_ImportInitPy() { + const string packageInitCode = @"x = 1"; + const string moduleCode = @"import package +y = package.x"; + var initPyUri = TestData.GetTestSpecificUri("package", "__init__.py"); + var moduleUri = TestData.GetTestSpecificUri("module.py"); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var initPy = rdt.OpenDocument(initPyUri, packageInitCode); + var module = rdt.OpenDocument(moduleUri, moduleCode); + + await initPy.GetAnalysisAsync(); + await module.GetAnalysisAsync(); + + var rs = new RenameSource(Services); + var wse = await rs.RenameAsync(initPyUri, new SourceLocation(1, 1), "z"); + + wse.changes.Should().HaveCount(2); + + wse.changes[initPyUri].Should().HaveCount(1); + wse.changes[initPyUri][0].range.Should().Be(0, 0, 0, 1); + + wse.changes[moduleUri].Should().HaveCount(1); + wse.changes[moduleUri][0].range.Should().Be(1, 12, 1, 13); + } + + [TestMethod, Priority(0)] + public async Task Rename_ImportSubmodule() { + const string packageInitCode = @"x = 1"; + const string moduleCode = @"import package.submodule +y = package.x"; + var initPyUri = TestData.GetTestSpecificUri("package", "__init__.py"); + var moduleUri = TestData.GetTestSpecificUri("module.py"); + + await CreateServicesAsync(PythonVersions.LatestAvailable3X); + var rdt = Services.GetService(); + var initPy = rdt.OpenDocument(initPyUri, packageInitCode); + var module = rdt.OpenDocument(moduleUri, moduleCode); + + await initPy.GetAnalysisAsync(); + await module.GetAnalysisAsync(); + + var rs = new RenameSource(Services); + var wse = await rs.RenameAsync(initPyUri, new SourceLocation(1, 1), "z"); + + wse.changes.Should().HaveCount(2); + + wse.changes[initPyUri].Should().HaveCount(1); + wse.changes[initPyUri][0].range.Should().Be(0, 0, 0, 1); + + wse.changes[moduleUri].Should().HaveCount(1); + wse.changes[moduleUri][0].range.Should().Be(1, 12, 1, 13); + } + [TestMethod, Priority(0)] public async Task NoRenameInCompiled() { const string code = "from sys import path";