From 500405061f4a483e2ff8319f6c22ee84a71cb2a9 Mon Sep 17 00:00:00 2001 From: "ASHER-HOME\\Alexander Sher" Date: Thu, 29 Aug 2019 17:33:49 -0500 Subject: [PATCH 1/2] Fix #1156: Unresolved import for builtin packages after reload --- .../Ast/Impl/Analyzer/PythonInterpreter.cs | 78 +++++----- .../Modules/Definitions/IModuleManagement.cs | 10 ++ .../Modules/Definitions/IModuleResolution.cs | 10 -- .../Resolution/MainModuleResolution.cs | 140 +++++++++--------- .../Resolution/ModuleResolutionBase.cs | 43 ++---- .../Modules/Resolution/TypeshedResolution.cs | 25 ++-- .../Ast/Impl/Types/PythonClassType.cs | 2 - .../Ast/Impl/Types/PythonFunctionType.cs | 1 - src/Analysis/Ast/Impl/Types/PythonType.cs | 9 +- .../Impl/Interpreter/PythonLibraryPath.cs | 27 ++-- .../Test/GoToDefinitionTests.cs | 2 +- 11 files changed, 166 insertions(+), 181 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs index 001437e7b..a1a957df7 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs @@ -33,6 +33,7 @@ namespace Microsoft.Python.Analysis.Analyzer { public sealed class PythonInterpreter : IPythonInterpreter { private MainModuleResolution _moduleResolution; private TypeshedResolution _stubResolution; + private IPythonType _unknownType; private readonly object _lock = new object(); private readonly Dictionary _builtinTypes = new Dictionary(); @@ -42,30 +43,20 @@ private PythonInterpreter(InterpreterConfiguration configuration) { LanguageVersion = Configuration.Version.ToLanguageVersion(); } - private async Task LoadBuiltinTypesAsync(string root, IServiceManager sm, CancellationToken cancellationToken = default) { + private async Task InitializeAsync(string root, IServiceManager sm, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); sm.AddService(this); _moduleResolution = new MainModuleResolution(root, sm); - _stubResolution = new TypeshedResolution(sm); - - await _moduleResolution.InitializeAsync(cancellationToken); - await _stubResolution.InitializeAsync(cancellationToken); - - lock (_lock) { - var builtinModule = _moduleResolution.CreateBuiltinsModule(); - _builtinTypes[BuiltinTypeId.NoneType] - = new PythonType("NoneType", new Location(builtinModule), string.Empty, BuiltinTypeId.NoneType); - _builtinTypes[BuiltinTypeId.Unknown] - = UnknownType = new PythonType("Unknown", new Location(builtinModule), string.Empty); - } - - await _moduleResolution.LoadBuiltinTypesAsync(cancellationToken); + _stubResolution = new TypeshedResolution(Configuration.TypeshedPath, sm); + + await _stubResolution.ReloadAsync(cancellationToken); + await _moduleResolution.ReloadAsync(cancellationToken); } public static async Task CreateAsync(InterpreterConfiguration configuration, string root, IServiceManager sm, CancellationToken cancellationToken = default) { var pi = new PythonInterpreter(configuration); - await pi.LoadBuiltinTypesAsync(root, sm, cancellationToken); + await pi.InitializeAsync(root, sm, cancellationToken); // Specialize typing TypingModule.Create(sm); @@ -92,6 +83,24 @@ public static async Task CreateAsync(InterpreterConfiguratio /// public IModuleResolution TypeshedResolution => _stubResolution; + /// + /// Unknown type. + /// + public IPythonType UnknownType { + get { + lock (_lock) { + var type = _unknownType; + if (type != null) { + return type; + } + + _unknownType = new PythonType("Unknown", new Location(_moduleResolution.BuiltinsModule), string.Empty); + _builtinTypes[BuiltinTypeId.Unknown] = _unknownType; + return _unknownType; + } + } + } + /// /// Gets a well known built-in type such as int, list, dict, etc... /// @@ -106,31 +115,34 @@ public IPythonType GetBuiltinType(BuiltinTypeId id) { } lock (_lock) { - if (_builtinTypes.TryGetValue(id, out var res) && res != null) { - return res; + if (_builtinTypes.TryGetValue(id, out var type) && type != null) { + return type; } - var bm = ModuleResolution.BuiltinsModule; - var typeName = id.GetTypeName(LanguageVersion); - if (typeName != null) { - res = bm.GetMember(typeName) as IPythonType; + if (id == BuiltinTypeId.Unknown) { + return UnknownType; } - if (res == null) { - res = bm.GetAnyMember("__{0}__".FormatInvariant(id)) as IPythonType; - if (res == null) { - return _builtinTypes[BuiltinTypeId.Unknown]; + if (id == BuiltinTypeId.NoneType) { + type = new PythonType("NoneType", new Location(_moduleResolution.BuiltinsModule), string.Empty, BuiltinTypeId.NoneType); + } else { + var bm = _moduleResolution.BuiltinsModule; + var typeName = id.GetTypeName(LanguageVersion); + if (typeName != null) { + type = _moduleResolution.BuiltinsModule.GetMember(typeName) as IPythonType; + } + + if (type == null) { + type = bm.GetAnyMember("__{0}__".FormatInvariant(id)) as IPythonType; + if (type == null) { + return UnknownType; + } } } - _builtinTypes[id] = res; - return res; + _builtinTypes[id] = type; + return type; } } - - /// - /// Unknown type. - /// - public IPythonType UnknownType { get; private set; } } } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs index 28791fc14..567ac5002 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs @@ -26,6 +26,16 @@ namespace Microsoft.Python.Analysis.Modules { /// Represents module resolution and search subsystem. /// public interface IModuleManagement: IModuleResolution { + /// + /// Builtins module name. + /// + string BuiltinModuleName { get; } + + /// + /// Builtins module. + /// + IBuiltinsPythonModule BuiltinsModule { get; } + /// /// Locates module by path. /// diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs index ef66855c8..4b8ce7744 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs @@ -23,16 +23,6 @@ namespace Microsoft.Python.Analysis.Modules { /// Represents basic module resolution and search subsystem. /// public interface IModuleResolution { - /// - /// Builtins module name. - /// - string BuiltinModuleName { get; } - - /// - /// Builtins module. - /// - IBuiltinsPythonModule BuiltinsModule { get; } - /// /// Path resolver providing file resolution in module imports. /// diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 86364d9dc..8f89803c8 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -33,41 +33,30 @@ using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO; using Microsoft.Python.Core.OS; +using Microsoft.Python.Core.Services; namespace Microsoft.Python.Analysis.Modules.Resolution { internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement { private readonly ConcurrentDictionary _specialized = new ConcurrentDictionary(); + private readonly IUIService _ui; private IRunningDocumentTable _rdt; private ImmutableArray _userPaths = ImmutableArray.Empty; public MainModuleResolution(string root, IServiceContainer services) - : base(root, services) { } + : base(root, services) { - internal IBuiltinsPythonModule CreateBuiltinsModule() { - if (BuiltinsModule == null) { - // Initialize built-in - var moduleName = BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion); - - StubCache = _services.GetService(); - var modulePath = StubCache.GetCacheFilePath(_interpreter.Configuration.InterpreterPath); - - var b = new BuiltinsPythonModule(moduleName, modulePath, _services); - BuiltinsModule = b; - Modules[BuiltinModuleName] = new ModuleRef(b); - } - return BuiltinsModule; + _ui = services.GetService(); } + + public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(Interpreter.LanguageVersion); - internal async Task InitializeAsync(CancellationToken cancellationToken = default) { - await ReloadAsync(cancellationToken); - cancellationToken.ThrowIfCancellationRequested(); - } + public IBuiltinsPythonModule BuiltinsModule { get; private set; } protected override IPythonModule CreateModule(string name) { var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name); if (moduleImport == null) { - _log?.Log(TraceEventType.Verbose, "Import not found: ", name); + Log?.Log(TraceEventType.Verbose, "Import not found: ", name); return null; } @@ -83,7 +72,7 @@ protected override IPythonModule CreateModule(string name) { // First check stub next to the module. if (!TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) { // If nothing found, try Typeshed. - stub = _interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); + stub = Interpreter.TypeshedResolution.GetOrLoadModule(moduleImport.IsBuiltin ? name : moduleImport.FullName); } // If stub is created and its path equals to module, return that stub as module @@ -92,16 +81,16 @@ protected override IPythonModule CreateModule(string name) { } if (moduleImport.IsBuiltin) { - _log?.Log(TraceEventType.Verbose, "Create built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); - return new CompiledBuiltinPythonModule(name, stub, _services); + Log?.Log(TraceEventType.Verbose, "Create built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); + return new CompiledBuiltinPythonModule(name, stub, Services); } if (moduleImport.IsCompiled) { - _log?.Log(TraceEventType.Verbose, "Create compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath); - return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, _services); + Log?.Log(TraceEventType.Verbose, "Create compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath); + return new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, Services); } - _log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); + Log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); // Module inside workspace == user code. var mco = new ModuleCreationOptions { @@ -114,24 +103,22 @@ protected override IPythonModule CreateModule(string name) { return GetRdt().AddModule(mco); } - private async Task> GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default) { - if (!_fs.FileExists(Configuration.InterpreterPath)) { - _log?.Log(TraceEventType.Warning, "Interpreter does not exist:", Configuration.InterpreterPath); + private async Task> GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default) { + if (!FileSystem.FileExists(Configuration.InterpreterPath)) { + Log?.Log(TraceEventType.Warning, "Interpreter does not exist:", Configuration.InterpreterPath); _ui?.ShowMessageAsync(Resources.InterpreterNotFound, TraceEventType.Error); - return Array.Empty(); + return ImmutableArray.Empty; } - _log?.Log(TraceEventType.Information, "GetCurrentSearchPaths", Configuration.InterpreterPath); + Log?.Log(TraceEventType.Information, "GetCurrentSearchPaths", Configuration.InterpreterPath); try { - var fs = _services.GetService(); - var ps = _services.GetService(); - var paths = await PythonLibraryPath.GetSearchPathsAsync(Configuration, fs, ps, cancellationToken); - cancellationToken.ThrowIfCancellationRequested(); - return paths.ToArray(); + var fs = Services.GetService(); + var ps = Services.GetService(); + return await PythonLibraryPath.GetSearchPathsAsync(Configuration, fs, ps, cancellationToken); } catch (InvalidOperationException ex) { - _log?.Log(TraceEventType.Warning, "Exception getting search paths", ex); + Log?.Log(TraceEventType.Warning, "Exception getting search paths", ex); _ui?.ShowMessageAsync(Resources.ExceptionGettingSearchPaths, TraceEventType.Error); - return Array.Empty(); + return ImmutableArray.Empty; } } @@ -157,8 +144,8 @@ public IPythonModule SpecializeModule(string name, Func s public IPythonModule GetSpecializedModule(string name) => _specialized.TryGetValue(name, out var module) ? module : null; - internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken = default) { - var analyzer = _services.GetService(); + internal async Task AddBuiltinTypesToPathResolverAsync(CancellationToken cancellationToken = default) { + var analyzer = Services.GetService(); await analyzer.GetAnalysisAsync(BuiltinsModule, -1, cancellationToken); Check.InvalidOperation(!(BuiltinsModule.Analysis is EmptyAnalysis), "After await"); @@ -172,26 +159,6 @@ internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken = } } - internal async Task ReloadSearchPaths(CancellationToken cancellationToken = default) { - var ps = _services.GetService(); - - var paths = await GetInterpreterSearchPathsAsync(cancellationToken); - var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, _fs, paths, Configuration.SearchPaths); - - InterpreterPaths = interpreterPaths.Select(p => p.Path); - _userPaths = userPaths.Select(p => p.Path); - - _log?.Log(TraceEventType.Information, "Interpreter search paths:"); - foreach (var s in InterpreterPaths) { - _log?.Log(TraceEventType.Information, $" {s}"); - } - - _log?.Log(TraceEventType.Information, "User search paths:"); - foreach (var s in _userPaths) { - _log?.Log(TraceEventType.Information, $" {s}"); - } - } - public async Task ReloadAsync(CancellationToken cancellationToken = default) { foreach (var uri in Modules .Where(m => m.Value.Value?.Name != BuiltinModuleName) @@ -201,19 +168,52 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) { } // Preserve builtins, they don't need to be reloaded since interpreter does not change. - if (Modules.TryGetValue(BuiltinModuleName, out var builtins)) { - Modules.Clear(); - Modules[BuiltinModuleName] = builtins; - } + var builtinsIsCreated = Modules.TryGetValue(BuiltinModuleName, out var builtinsRef); + Modules.Clear(); await ReloadSearchPaths(cancellationToken); - PathResolver = new PathResolver(_interpreter.LanguageVersion, Root, InterpreterPaths, _userPaths); + PathResolver = new PathResolver(Interpreter.LanguageVersion, Root, InterpreterPaths, _userPaths); var addedRoots = new HashSet { Root }; addedRoots.UnionWith(InterpreterPaths); addedRoots.UnionWith(_userPaths); ReloadModulePaths(addedRoots); + + if (!builtinsIsCreated) { + var builtinsModule = CreateBuiltinsModule(Services, Interpreter, StubCache); + BuiltinsModule = builtinsModule; + builtinsRef = new ModuleRef(builtinsModule); + } + + Modules[BuiltinModuleName] = builtinsRef; + await AddBuiltinTypesToPathResolverAsync(cancellationToken); + } + + private static IBuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) { + var moduleName = BuiltinTypeId.Unknown.GetModuleName(interpreter.LanguageVersion); + var modulePath = stubCache.GetCacheFilePath(interpreter.Configuration.InterpreterPath); + return new BuiltinsPythonModule(moduleName, modulePath, services); + } + + private async Task ReloadSearchPaths(CancellationToken cancellationToken = default) { + var paths = await GetInterpreterSearchPathsAsync(cancellationToken); + var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, FileSystem, paths, Configuration.SearchPaths); + + InterpreterPaths = interpreterPaths.Select(p => p.Path); + _userPaths = userPaths.Select(p => p.Path); + + if (Log != null) { + Log.Log(TraceEventType.Information, "Interpreter search paths:"); + foreach (var s in InterpreterPaths) { + Log.Log(TraceEventType.Information, $" {s}"); + } + + Log.Log(TraceEventType.Information, "User search paths:"); + foreach (var s in _userPaths) { + Log.Log(TraceEventType.Information, $" {s}"); + } + } } public bool TryAddModulePath(in string path, in long fileSize, in bool allowNonRooted, out string fullModuleName) @@ -221,25 +221,25 @@ public bool TryAddModulePath(in string path, in long fileSize, in bool allowNonR // For tests internal void AddUnimportableModule(string moduleName) - => Modules[moduleName] = new ModuleRef(new SentinelModule(moduleName, _services)); + => Modules[moduleName] = new ModuleRef(new SentinelModule(moduleName, Services)); private bool TryCreateModuleStub(string name, string modulePath, out IPythonModule module) { // First check stub next to the module. if (!string.IsNullOrEmpty(modulePath)) { var pyiPath = Path.ChangeExtension(modulePath, "pyi"); - if (_fs.FileExists(pyiPath)) { - module = new StubPythonModule(name, pyiPath, false, _services); + if (FileSystem.FileExists(pyiPath)) { + module = new StubPythonModule(name, pyiPath, false, Services); return true; } } // Try location of stubs that are in a separate folder next to the package. - var stubPath = CurrentPathResolver.GetPossibleModuleStubPaths(name).FirstOrDefault(p => _fs.FileExists(p)); - module = !string.IsNullOrEmpty(stubPath) ? new StubPythonModule(name, stubPath, false, _services) : null; + var stubPath = CurrentPathResolver.GetPossibleModuleStubPaths(name).FirstOrDefault(p => FileSystem.FileExists(p)); + module = !string.IsNullOrEmpty(stubPath) ? new StubPythonModule(name, stubPath, false, Services) : null; return module != null; } private IRunningDocumentTable GetRdt() - => _rdt ?? (_rdt = _services.GetService()); + => _rdt ?? (_rdt = Services.GetService()); } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index 110340cf6..f486909ee 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -31,58 +31,47 @@ namespace Microsoft.Python.Analysis.Modules.Resolution { internal abstract class ModuleResolutionBase { - protected readonly IServiceContainer _services; - protected readonly IPythonInterpreter _interpreter; - protected readonly IFileSystem _fs; - protected readonly ILogger _log; - protected readonly IUIService _ui; - protected readonly bool _requireInitPy; + protected IServiceContainer Services { get; } + protected IFileSystem FileSystem { get; } + protected IPythonInterpreter Interpreter { get; } + protected ILogger Log { get; } protected ConcurrentDictionary Modules { get; } = new ConcurrentDictionary(); protected PathResolver PathResolver { get; set; } - protected InterpreterConfiguration Configuration => _interpreter.Configuration; + protected InterpreterConfiguration Configuration => Interpreter.Configuration; + + public string Root { get; } + public IStubCache StubCache { get; } protected ModuleResolutionBase(string root, IServiceContainer services) { Root = root; + Services = services; + FileSystem = services.GetService(); - _services = services; - _interpreter = services.GetService(); - _fs = services.GetService(); - _log = services.GetService(); - _ui = services.GetService(); - - _requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_interpreter.Configuration.Version); + Interpreter = services.GetService(); + StubCache = services.GetService(); + Log = services.GetService(); } - public string Root { get; protected set; } public ImmutableArray InterpreterPaths { get; protected set; } = ImmutableArray.Empty; - public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion); - /// /// Path resolver providing file resolution in module imports. /// public PathResolverSnapshot CurrentPathResolver => PathResolver.CurrentSnapshot; - /// - /// Builtins module. - /// - public IBuiltinsPythonModule BuiltinsModule { get; protected set; } - protected abstract IPythonModule CreateModule(string name); - public IStubCache StubCache { get; protected set; } - public IPythonModule GetImportedModule(string name) - => Modules.TryGetValue(name, out var moduleRef) ? moduleRef.Value : _interpreter.ModuleResolution.GetSpecializedModule(name); + => Modules.TryGetValue(name, out var moduleRef) ? moduleRef.Value : Interpreter.ModuleResolution.GetSpecializedModule(name); public IPythonModule GetOrLoadModule(string name) { if (Modules.TryGetValue(name, out var moduleRef)) { return moduleRef.GetOrCreate(name, this); } - var module = _interpreter.ModuleResolution.GetSpecializedModule(name); + var module = Interpreter.ModuleResolution.GetSpecializedModule(name); if (module != null) { return module; } @@ -105,7 +94,7 @@ public ModulePath FindModule(string filePath) { } protected void ReloadModulePaths(in IEnumerable rootPaths) { - foreach (var moduleFile in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(_fs, p))) { + foreach (var moduleFile in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(FileSystem, p))) { PathResolver.TryAddModulePath(moduleFile.FullName, moduleFile.Length, false, out _); } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs index bcb56b6e5..64762559e 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs @@ -32,35 +32,28 @@ namespace Microsoft.Python.Analysis.Modules.Resolution { internal sealed class TypeshedResolution : ModuleResolutionBase, IModuleResolution { private readonly ImmutableArray _typeStubPaths; - public TypeshedResolution(IServiceContainer services) : base(null, services) { - BuiltinsModule = _interpreter.ModuleResolution.BuiltinsModule; - Modules[BuiltinModuleName] = new ModuleRef(BuiltinsModule); - - var stubs = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Stubs"); - Root = _interpreter.Configuration?.TypeshedPath; + public TypeshedResolution(string root, IServiceContainer services) : base(root, services) { // TODO: merge with user-provided stub paths + var stubs = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "Stubs"); _typeStubPaths = GetTypeShedPaths(Root) .Concat(GetTypeShedPaths(stubs)) .Where(services.GetService().DirectoryExists) .ToImmutableArray(); - _log?.Log(TraceEventType.Verbose, @"Typeshed paths:"); + Log?.Log(TraceEventType.Verbose, @"Typeshed paths:"); foreach (var p in _typeStubPaths) { - _log?.Log(TraceEventType.Verbose, $" {p}"); + Log?.Log(TraceEventType.Verbose, $" {p}"); } } - internal Task InitializeAsync(CancellationToken cancellationToken = default) - => ReloadAsync(cancellationToken); - protected override IPythonModule CreateModule(string name) { var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name); if (moduleImport != default) { if (moduleImport.IsCompiled) { - _log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", moduleImport.FullName, moduleImport.ModulePath); + Log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", moduleImport.FullName, moduleImport.ModulePath); return null; } - return new StubPythonModule(moduleImport.FullName, moduleImport.ModulePath, true, _services); + return new StubPythonModule(moduleImport.FullName, moduleImport.ModulePath, true, Services); } var i = name.IndexOf('.'); @@ -69,13 +62,13 @@ protected override IPythonModule CreateModule(string name) { return null; } - var stubPath = CurrentPathResolver.GetPossibleModuleStubPaths(name).FirstOrDefault(p => _fs.FileExists(p)); - return stubPath != null ? new StubPythonModule(name, stubPath, true, _services) : null; + var stubPath = CurrentPathResolver.GetPossibleModuleStubPaths(name).FirstOrDefault(p => FileSystem.FileExists(p)); + return stubPath != null ? new StubPythonModule(name, stubPath, true, Services) : null; } public Task ReloadAsync(CancellationToken cancellationToken = default) { Modules.Clear(); - PathResolver = new PathResolver(_interpreter.LanguageVersion, Root, _typeStubPaths, ImmutableArray.Empty); + PathResolver = new PathResolver(Interpreter.LanguageVersion, Root, _typeStubPaths, ImmutableArray.Empty); ReloadModulePaths(_typeStubPaths.Prepend(Root)); cancellationToken.ThrowIfCancellationRequested(); return Task.CompletedTask; diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index 5abd7d7cf..1a79a3b1e 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -42,7 +42,6 @@ internal partial class PythonClassType : PythonType, IPythonClassType, IGenericT // For tests internal PythonClassType(string name, Location location) : base(name, location, string.Empty, BuiltinTypeId.Type) { - Check.ArgumentNotNull(nameof(location), location.Module); } public PythonClassType( @@ -50,7 +49,6 @@ public PythonClassType( Location location, BuiltinTypeId builtinTypeId = BuiltinTypeId.Type ) : base(classDefinition.Name, location, classDefinition.GetDocumentation(), builtinTypeId) { - Check.ArgumentNotNull(nameof(location), location.Module); location.Module.AddAstNode(this, classDefinition); } diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index 63208bc8b..c0c9a0e43 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -37,7 +37,6 @@ public static PythonFunctionType Specialize(string name, IPythonModule declaring private PythonFunctionType(string name, Location location, string documentation, bool isSpecialized = false) : base(name, location, documentation ?? string.Empty, BuiltinTypeId.Function) { - Check.ArgumentNotNull(nameof(location), location.Module); _isSpecialized = isSpecialized; } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index a0ea79878..7d5f0fb67 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -34,18 +34,13 @@ internal class PythonType : LocatedMember, IPythonType {//, IEquatable WritableMembers => _members ?? (_members = new Dictionary()); - public PythonType( - string name, - Location location, - string documentation, - BuiltinTypeId typeId = BuiltinTypeId.Unknown - ) : this(name, location, typeId) { + public PythonType(string name, Location location, string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown) + : this(name, location, typeId) { BaseName = name ?? throw new ArgumentNullException(nameof(name)); Documentation = documentation; } private PythonType(string name, Location location, BuiltinTypeId typeId) : base(location) { - Check.ArgumentNotNull(nameof(location), location.Module); BaseName = name ?? throw new ArgumentNullException(nameof(name)); _typeId = typeId; } diff --git a/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs b/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs index 7f04d8f3a..48a9f79c2 100644 --- a/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs +++ b/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs @@ -109,31 +109,30 @@ public static PythonLibraryPath Parse(string s) { /// Root of the standard library. /// A list of search paths for the interpreter. /// New in 2.2, moved in 3.3 - private static List GetDefaultSearchPaths(IFileSystem fs, string library) { - var result = new List(); + private static ImmutableArray GetDefaultSearchPaths(IFileSystem fs, string library) { + var result = ImmutableArray.Empty; if (!Directory.Exists(library)) { return result; } - result.Add(new PythonLibraryPath(library, PythonLibraryPathType.StdLib)); + result = result.Add(new PythonLibraryPath(library, PythonLibraryPathType.StdLib)); var sitePackages = IOPath.Combine(library, "site-packages"); if (!Directory.Exists(sitePackages)) { return result; } - result.Add(new PythonLibraryPath(sitePackages)); - result.AddRange(ModulePath.ExpandPathFiles(fs, sitePackages) - .Select(p => new PythonLibraryPath(p)) - ); - + result = result.Add(new PythonLibraryPath(sitePackages)); + foreach (var pathFile in ModulePath.ExpandPathFiles(fs, sitePackages)) { + result = result.Add(new PythonLibraryPath(pathFile)); + } return result; } /// /// Gets the set of search paths for the specified factory. /// - public static async Task> GetSearchPathsAsync(InterpreterConfiguration config, IFileSystem fs, IProcessServices ps, CancellationToken cancellationToken = default) { + public static async Task> GetSearchPathsAsync(InterpreterConfiguration config, IFileSystem fs, IProcessServices ps, CancellationToken cancellationToken = default) { for (int retries = 5; retries > 0; --retries) { try { return await GetSearchPathsFromInterpreterAsync(config.InterpreterPath, fs, ps, cancellationToken); @@ -142,7 +141,7 @@ public static async Task> GetSearchPathsAsync(Interpret break; } catch (Exception e) when (e is IOException || e is UnauthorizedAccessException) { // Failed to get paths due to IO exception - sleep and then loop - Thread.Sleep(50); + await Task.Delay(50, cancellationToken); } } @@ -152,7 +151,7 @@ public static async Task> GetSearchPathsAsync(Interpret return GetDefaultSearchPaths(fs, standardLibraryPath); } - return Array.Empty(); + return ImmutableArray.Empty; } /// @@ -163,13 +162,13 @@ public static async Task> GetSearchPathsAsync(Interpret /// Process services. /// Cancellation token. /// A list of search paths for the interpreter. - public static async Task> GetSearchPathsFromInterpreterAsync(string interpreter, IFileSystem fs, IProcessServices ps, CancellationToken cancellationToken = default) { + public static async Task> GetSearchPathsFromInterpreterAsync(string interpreter, IFileSystem fs, IProcessServices ps, CancellationToken cancellationToken = default) { // sys.path will include the working directory, so we make an empty // path that we can filter out later var tempWorkingDir = IOPath.Combine(IOPath.GetTempPath(), IOPath.GetRandomFileName()); fs.CreateDirectory(tempWorkingDir); if (!InstallPath.TryGetFile("get_search_paths.py", out var srcGetSearchPaths)) { - return new List(); + return ImmutableArray.Empty; } var getSearchPaths = IOPath.Combine(tempWorkingDir, PathUtils.GetFileName(srcGetSearchPaths)); File.Copy(srcGetSearchPaths, getSearchPaths); @@ -204,7 +203,7 @@ public static async Task> GetSearchPathsFromInterpreterA Debug.Fail("Invalid format for search path: " + s); return null; } - }).Where(p => p != null).ToList(); + }).Where(p => p != null).ToImmutableArray(); } finally { fs.DeleteDirectory(tempWorkingDir, true); } diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs index 35ddf0619..e5369c56c 100644 --- a/src/LanguageServer/Test/GoToDefinitionTests.cs +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -381,8 +381,8 @@ def X(): ... reference = ds.FindDefinition(analysis, new SourceLocation(1, 16), out _); reference.Should().NotBeNull(); - reference.range.Should().Be(2, 4, 2, 5); reference.uri.AbsolutePath.Should().Contain("b.py"); + reference.range.Should().Be(2, 4, 2, 5); } From 71a955376d4a1be64f7f807c9cc919085d09d97d Mon Sep 17 00:00:00 2001 From: "ASHER-HOME\\Alexander Sher" Date: Fri, 30 Aug 2019 10:55:49 -0500 Subject: [PATCH 2/2] Address CR comments --- src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs index a1a957df7..2ee064d35 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs @@ -115,14 +115,14 @@ public IPythonType GetBuiltinType(BuiltinTypeId id) { } lock (_lock) { - if (_builtinTypes.TryGetValue(id, out var type) && type != null) { - return type; - } - if (id == BuiltinTypeId.Unknown) { return UnknownType; } + if (_builtinTypes.TryGetValue(id, out var type) && type != null) { + return type; + } + if (id == BuiltinTypeId.NoneType) { type = new PythonType("NoneType", new Location(_moduleResolution.BuiltinsModule), string.Empty, BuiltinTypeId.NoneType); } else {