From 521eb47dfa68099e87cd06183d8c06acb5cc122f Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 30 Jul 2019 15:07:51 -0700 Subject: [PATCH 01/14] First attempt at ensuring moduled dropped during reload are GC'd --- .../Ast/Impl/Analyzer/ModuleWalker.cs | 117 +++++++++++++----- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 2 +- .../Impl/Analyzer/PythonAnalyzerSession.cs | 14 ++- .../Impl/Documents/RunningDocumentTable.cs | 8 ++ .../Ast/Impl/Modules/BuiltinsPythonModule.cs | 2 + src/Analysis/Ast/Impl/Modules/PythonModule.cs | 28 +++-- .../Impl/Implementation/Server.cs | 16 ++- src/LanguageServer/Impl/Program.cs | 2 +- 8 files changed, 136 insertions(+), 53 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index dd8a30c06..12fa66b5d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -228,6 +228,8 @@ private void MergeStub() { return; } + var builtins = Module.Interpreter.ModuleResolution.BuiltinsModule; + // Note that scrape can pick up more functions than the stub contains // Or the stub can have definitions that scraping had missed. Therefore // merge is the combination of the two with the documentation coming @@ -240,53 +242,102 @@ private void MergeStub() { var sourceVar = Eval.GlobalScope.Variables[v.Name]; var sourceType = sourceVar?.Value.GetPythonType(); - + // If stub says 'Any' but we have better type, keep the current type. if (!IsStubBetterType(sourceType, stubType)) { - continue;; + continue; } - // If types are the classes, merge members. - // Otherwise, replace type from one from the stub. - if (sourceType is PythonClassType cls && Module.Equals(cls.DeclaringModule)) { - // If class exists and belongs to this module, add or replace - // its members with ones from the stub, preserving documentation. - // Don't augment types that do not come from this module. - foreach (var name in stubType.GetMemberNames()) { - var stubMember = stubType.GetMember(name); - var member = cls.GetMember(name); - - var memberType = member?.GetPythonType(); - var stubMemberType = stubMember.GetPythonType(); - if (!IsStubBetterType(memberType, stubMemberType)) { - continue; + // If type does not exist in module, but exists in stub, declare it unless it is an import. + // If types are the classes, merge members. Otherwise, replace type from one from the stub. + switch (sourceType) { + case null: + if (v.Source == VariableSource.Declaration) { + Eval.DeclareVariable(v.Name, v.Value, v.Source); } - // Get documentation from the current type, if any, since stubs - // typically do not contain documentation while scraped code does. - memberType?.TransferDocumentationAndLocation(stubMemberType); - cls.AddMember(name, stubMember, overwrite: true); - } - } else { - // Re-declare variable with the data from the stub unless member is a module. - // Modules members that are modules should remain as they are, i.e. os.path - // should remain library with its own stub attached. - if (!(stubType is IPythonModule)) { - sourceType.TransferDocumentationAndLocation(stubType); - // TODO: choose best type between the scrape and the stub. Stub probably should always win. - var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? VariableSource.Declaration; - Eval.DeclareVariable(v.Name, v.Value, source); - } + break; + + case PythonClassType cls when Module.Equals(cls.DeclaringModule): + // If class exists and belongs to this module, add or replace + // its members with ones from the stub, preserving documentation. + // Don't augment types that do not come from this module. + // Do not replace __class__ since it has to match class type and we are not + // replacing class type, we are only merging members. + foreach (var name in stubType.GetMemberNames().Except(new[] { "__class__", "__base__", "__bases__", "__mro__", "mro" })) { + var stubMember = stubType.GetMember(name); + var member = cls.GetMember(name); + + var memberType = member?.GetPythonType(); + var stubMemberType = stubMember.GetPythonType(); + + if (builtins.Equals(memberType?.DeclaringModule) || builtins.Equals(stubMemberType?.DeclaringModule)) { + continue; // Leave builtins alone. + } + + if (!IsStubBetterType(memberType, stubMemberType)) { + continue; + } + + // Get documentation from the current type, if any, since stubs + // typically do not contain documentation while scraped code does. + TransferDocumentationAndLocation(memberType, stubMemberType); + cls.AddMember(name, stubMember, overwrite: true); + } + break; + + + default: + // Re-declare variable with the data from the stub unless member is a module. + // Modules members that are modules should remain as they are, i.e. os.path + // should remain library with its own stub attached. + var stubModule = stubType.DeclaringModule; + if (!(stubType is IPythonModule) && !builtins.Equals(stubModule)) { + TransferDocumentationAndLocation(sourceType, stubType); + // TODO: choose best type between the scrape and the stub. Stub probably should always win. + var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? v.Source; + Eval.DeclareVariable(v.Name, v.Value, source); + } + break; } } + + + var o = Eval.Interpreter.GetBuiltinType(BuiltinTypeId.Object); + Debug.Assert(o.DeclaringModule.ModuleType == ModuleType.Builtins); } private static bool IsStubBetterType(IPythonType sourceType, IPythonType stubType) { - // If stub says 'Any' but we have better type, keep the current type. if (stubType.IsUnknown()) { + // Do not use worse types than what is in the module. return false; } - return sourceType.IsUnknown() || !(stubType.DeclaringModule is TypingModule) || stubType.Name != "Any"; + if (sourceType.IsUnknown()) { + return true; // Anything is better than unknowns. + } + if (sourceType.MemberType != stubType.MemberType) { + // Types should match, we are not replacing unrelated types. + return false; + } + // If stub says 'Any' but we have better type, keep the current type. + return !(stubType.DeclaringModule is TypingModule) || stubType.Name != "Any"; + } + + private static void TransferDocumentationAndLocation(IPythonType s, IPythonType d) { + if (s.IsUnknown()) { + return; // Do not transfer location of unknowns + } + // Documentation and location are always get transferred from module type + // to the stub type and never the other way around. This makes sure that + // we show documentation from the original module and goto definition + // navigates to the module source and not to the stub. + if (s != d && s is PythonType src && d is PythonType dst) { + var documentation = src.Documentation; + if (!string.IsNullOrEmpty(documentation)) { + dst.SetDocumentation(documentation); + } + dst.Location = src.Location; + } } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index c7b35901b..a2816b872 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -198,7 +198,7 @@ public IReadOnlyList LintModule(IPythonModule module) { public void ResetAnalyzer() { lock (_syncObj) { - _analysisEntries.Split(kvp => kvp.Key.IsTypeshed || kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out var entriesToRemove); + _analysisEntries.Split(kvp => kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out var entriesToRemove); _analysisEntries.Clear(); foreach (var (key, entry) in entriesToPreserve) { _analysisEntries.Add(key, entry); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 97f5c9bfa..e6d298e87 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -368,10 +368,22 @@ private void LogException(IPythonModule module, Exception exception) { } private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker, bool isCanceled) { + var mtIsDroppable = false; + + switch (document.ModuleType) { + case ModuleType.Library: + case ModuleType.Stub: + case ModuleType.Compiled: + //case ModuleType.CompiledBuiltin: + //case ModuleType.Builtins: + mtIsDroppable = true; + break; + } + var createLibraryAnalysis = !isCanceled && node != null && !node.HasMissingDependencies && - document.ModuleType == ModuleType.Library && + mtIsDroppable && !document.IsOpen && node.HasOnlyWalkedDependencies && node.IsValidVersion; diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index cb99f182d..e93991f11 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -200,12 +200,17 @@ public void ReloadAll() { foreach (var (uri, entry) in closed) { _documentsByUri.Remove(uri); entry.Document.Dispose(); + (entry.Document.PrimaryModule as IDisposable)?.Dispose(); + (entry.Document.Stub as IDisposable)?.Dispose(); } } + var analyzer = _services.GetService(); + foreach (var (_, entry) in closed) { Closed?.Invoke(this, new DocumentEventArgs(entry.Document)); Removed?.Invoke(this, new DocumentEventArgs(entry.Document)); + analyzer.RemoveAnalysis(entry.Document); // Dispose does this??? } foreach (var (_, entry) in opened) { @@ -213,6 +218,9 @@ public void ReloadAll() { } } + // For debugging. + private IPythonAnalyzer Analyzer => _services.GetService(); + public void Dispose() { lock (_lock) { foreach (var d in _documentsByUri.Values.OfType()) { diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index 095df7033..efb938cc4 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -54,6 +54,8 @@ protected override void OnAnalysisComplete() { } base.OnAnalysisComplete(); + + ClearContent(); } private void SpecializeTypes() { diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index aeeaa41af..cf06a1212 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -394,18 +394,22 @@ public void NotifyAnalysisBegins() { return; } - // TODO: Figure out where the nulls below are coming from. - var importedVariables = ((IScope)GlobalScope) - .TraverseDepthFirst(c => c?.Children ?? Enumerable.Empty()) - .SelectMany(s => s?.Variables ?? VariableCollection.Empty) - .Where(v => v?.Source == VariableSource.Import); - - foreach (var v in importedVariables) { - v.RemoveReferences(this); - if (v.Value is IPythonModule module) { - RemoveReferencesInModule(module); - } - } + RemoveAllReferences(); + } + } + } + + public void RemoveAllReferences() { + // TODO: Figure out where the nulls below are coming from. + var importedVariables = ((IScope)GlobalScope) + .TraverseDepthFirst(c => c?.Children ?? Enumerable.Empty()) + .SelectMany(s => s?.Variables ?? VariableCollection.Empty) + .Where(v => v?.Source == VariableSource.Import); + + foreach (var v in importedVariables) { + v.RemoveReferences(this); + if (v.Value is IPythonModule module) { + RemoveReferencesInModule(module); } } } diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 575bb9f8b..e26d6f9e3 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -18,6 +18,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Runtime; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis; @@ -269,10 +270,10 @@ private void ResetPathWatcher() { public void NotifyPackagesChanged(CancellationToken cancellationToken = default) { var interpreter = _services.GetService(); _log?.Log(TraceEventType.Information, Resources.ReloadingModules); - // No need to reload typeshed resolution since it is a static storage. - // User does can add stubs while application is running, but it is - // by design at this time that the app should be restarted. - interpreter.ModuleResolution.ReloadAsync(cancellationToken).ContinueWith(t => { + + interpreter.TypeshedResolution.ReloadAsync().ContinueWith(async t => { + await interpreter.ModuleResolution.ReloadAsync(); + _log?.Log(TraceEventType.Information, Resources.Done); _log?.Log(TraceEventType.Information, Resources.AnalysisRestarted); @@ -281,7 +282,12 @@ public void NotifyPackagesChanged(CancellationToken cancellationToken = default) if (_watchSearchPaths) { ResetPathWatcher(); } - }, cancellationToken).DoNotWait(); + + await Task.Delay(10000); + + GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; + GC.Collect(); + }).DoNotWait(); } diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 024f9af18..24adfa924 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -// #define WAIT_FOR_DEBUGGER +#define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; From 74dec25c5a1782237ba319a7c2da6649bde01ade Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 31 Jul 2019 09:23:14 -0700 Subject: [PATCH 02/14] Make sure AST is set after content is cleared --- src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs | 2 +- src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index efb938cc4..d120d965b 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -55,7 +55,7 @@ protected override void OnAnalysisComplete() { base.OnAnalysisComplete(); - ClearContent(); + this.SetAst(Analysis.Ast); } private void SpecializeTypes() { diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs index b017e0708..64ab06683 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleType.cs @@ -13,10 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; - namespace Microsoft.Python.Analysis.Modules { - [Flags] public enum ModuleType { /// /// Module is user file in the workspace. From 6abeb90b0c35f4b3230451210513a796ac5394f5 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 31 Jul 2019 13:03:46 -0700 Subject: [PATCH 03/14] More tests passing --- .../Ast/Impl/Analyzer/ModuleWalker.cs | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 12fa66b5d..ecd217c61 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -255,7 +255,6 @@ private void MergeStub() { if (v.Source == VariableSource.Declaration) { Eval.DeclareVariable(v.Name, v.Value, v.Source); } - break; case PythonClassType cls when Module.Equals(cls.DeclaringModule): @@ -286,6 +285,13 @@ private void MergeStub() { } break; + case IPythonClassType _: + // We do not re-declare classes, we only transfer members, see above. + break; + + case IPythonModule _: + // We do not re-declare modules. + break; default: // Re-declare variable with the data from the stub unless member is a module. @@ -301,10 +307,6 @@ private void MergeStub() { break; } } - - - var o = Eval.Interpreter.GetBuiltinType(BuiltinTypeId.Object); - Debug.Assert(o.DeclaringModule.ModuleType == ModuleType.Builtins); } private static bool IsStubBetterType(IPythonType sourceType, IPythonType stubType) { @@ -315,10 +317,16 @@ private static bool IsStubBetterType(IPythonType sourceType, IPythonType stubTyp if (sourceType.IsUnknown()) { return true; // Anything is better than unknowns. } - if (sourceType.MemberType != stubType.MemberType) { - // Types should match, we are not replacing unrelated types. - return false; - } + + // Types should match, we are not replacing unrelated types + // except when it is a method/function replacement. + //var compatibleReplacement = + // sourceType.MemberType == stubType.MemberType || + // (sourceType.MemberType == PythonMemberType.Function && stubType.MemberType == PythonMemberType.Method) || + // (sourceType.MemberType == PythonMemberType.Method && stubType.MemberType == PythonMemberType.Function); + //if (!compatibleReplacement) { + // return false; + //} // If stub says 'Any' but we have better type, keep the current type. return !(stubType.DeclaringModule is TypingModule) || stubType.Name != "Any"; } From ddfb30538f86ae023d5ca0e4f7246f1a4253d51e Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 31 Jul 2019 16:05:58 -0700 Subject: [PATCH 04/14] ModuleWalker updates --- src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index ecd217c61..253d12510 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -252,6 +252,7 @@ private void MergeStub() { // If types are the classes, merge members. Otherwise, replace type from one from the stub. switch (sourceType) { case null: + // Nothing in sources, but there is type in the stub. Declare it. if (v.Source == VariableSource.Declaration) { Eval.DeclareVariable(v.Name, v.Value, v.Source); } @@ -285,10 +286,6 @@ private void MergeStub() { } break; - case IPythonClassType _: - // We do not re-declare classes, we only transfer members, see above. - break; - case IPythonModule _: // We do not re-declare modules. break; @@ -317,16 +314,6 @@ private static bool IsStubBetterType(IPythonType sourceType, IPythonType stubTyp if (sourceType.IsUnknown()) { return true; // Anything is better than unknowns. } - - // Types should match, we are not replacing unrelated types - // except when it is a method/function replacement. - //var compatibleReplacement = - // sourceType.MemberType == stubType.MemberType || - // (sourceType.MemberType == PythonMemberType.Function && stubType.MemberType == PythonMemberType.Method) || - // (sourceType.MemberType == PythonMemberType.Method && stubType.MemberType == PythonMemberType.Function); - //if (!compatibleReplacement) { - // return false; - //} // If stub says 'Any' but we have better type, keep the current type. return !(stubType.DeclaringModule is TypingModule) || stubType.Name != "Any"; } From bfc49557a4becac35b8e1a2d4578e69fe2708841 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 1 Aug 2019 16:33:02 -0700 Subject: [PATCH 05/14] Also don't merge against specialized modules, do builtin _astMap reset before reanalyzing instead of after any analysis --- src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs | 4 ++++ src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs | 2 ++ .../Ast/Impl/Types/Definitions/IBuiltinPythonModule.cs | 2 ++ src/LanguageServer/Impl/Implementation/Server.cs | 2 ++ 4 files changed, 10 insertions(+) diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 253d12510..778bfb055 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -275,6 +275,10 @@ private void MergeStub() { continue; // Leave builtins alone. } + if (memberType?.DeclaringModule is SpecializedModule || stubMemberType?.DeclaringModule is SpecializedModule) { + continue; // Leave specialized modules like typing alone. + } + if (!IsStubBetterType(memberType, stubMemberType)) { continue; } diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index d120d965b..2cc924d89 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -54,7 +54,9 @@ protected override void OnAnalysisComplete() { } base.OnAnalysisComplete(); + } + public void ResetAst() { this.SetAst(Analysis.Ast); } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IBuiltinPythonModule.cs b/src/Analysis/Ast/Impl/Types/Definitions/IBuiltinPythonModule.cs index 28f4d1177..dc568bb44 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IBuiltinPythonModule.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IBuiltinPythonModule.cs @@ -33,5 +33,7 @@ namespace Microsoft.Python.Analysis.Types { /// public interface IBuiltinsPythonModule : IPythonModule { IMember GetAnyMember(string name); + + void ResetAst(); } } diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index e26d6f9e3..0dce02694 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -271,6 +271,8 @@ public void NotifyPackagesChanged(CancellationToken cancellationToken = default) var interpreter = _services.GetService(); _log?.Log(TraceEventType.Information, Resources.ReloadingModules); + interpreter.ModuleResolution.BuiltinsModule.ResetAst(); + interpreter.TypeshedResolution.ReloadAsync().ContinueWith(async t => { await interpreter.ModuleResolution.ReloadAsync(); From 2425d1e28b46ef6c599252c23bd36df167d1a81a Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Thu, 1 Aug 2019 16:40:23 -0700 Subject: [PATCH 06/14] Don't call dispose directly, it's done elsewhere and has no effect --- src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs | 8 -------- src/LanguageServer/Impl/Program.cs | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index e93991f11..cb99f182d 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -200,17 +200,12 @@ public void ReloadAll() { foreach (var (uri, entry) in closed) { _documentsByUri.Remove(uri); entry.Document.Dispose(); - (entry.Document.PrimaryModule as IDisposable)?.Dispose(); - (entry.Document.Stub as IDisposable)?.Dispose(); } } - var analyzer = _services.GetService(); - foreach (var (_, entry) in closed) { Closed?.Invoke(this, new DocumentEventArgs(entry.Document)); Removed?.Invoke(this, new DocumentEventArgs(entry.Document)); - analyzer.RemoveAnalysis(entry.Document); // Dispose does this??? } foreach (var (_, entry) in opened) { @@ -218,9 +213,6 @@ public void ReloadAll() { } } - // For debugging. - private IPythonAnalyzer Analyzer => _services.GetService(); - public void Dispose() { lock (_lock) { foreach (var d in _documentsByUri.Values.OfType()) { diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 24adfa924..024f9af18 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -#define WAIT_FOR_DEBUGGER +// #define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; From e843a6032e9e1a5b2bd11bfae9c1008ea1905d2f Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 2 Aug 2019 15:48:52 -0700 Subject: [PATCH 07/14] Force GC via normal session instead of waiting for a hardcoded time --- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 18 ++++++++++++++++-- .../Ast/Impl/Analyzer/PythonAnalyzerSession.cs | 14 ++++++++------ src/Analysis/Ast/Impl/Types/LocatedMember.cs | 4 ++++ .../Impl/Implementation/Server.cs | 10 +++------- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index a2816b872..f8ed9a66e 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -49,6 +49,7 @@ public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable { private int _version; private PythonAnalyzerSession _currentSession; private PythonAnalyzerSession _nextSession; + private bool _forceGCOnNextSession; public PythonAnalyzer(IServiceManager services, string cacheFolderPath = null) { _services = services; @@ -218,6 +219,12 @@ public IReadOnlyList LoadedModules { public event EventHandler AnalysisComplete; + public void GCNextSession() { + lock (_syncObj) { + _forceGCOnNextSession = true; + } + } + internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) => AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); @@ -301,8 +308,15 @@ private void StartNextSession(Task task) { session.Start(false); } - private PythonAnalyzerSession CreateSession(in IDependencyChainWalker walker, in PythonAnalyzerEntry entry) - => new PythonAnalyzerSession(_services, _progress, _analysisCompleteEvent, _startNextSession, _disposeToken.CancellationToken, walker, _version, entry); + private PythonAnalyzerSession CreateSession(in IDependencyChainWalker walker, in PythonAnalyzerEntry entry) { + bool forceGC; + lock (_syncObj) { + forceGC = _forceGCOnNextSession; + _forceGCOnNextSession = false; + } + + return new PythonAnalyzerSession(_services, _progress, _analysisCompleteEvent, _startNextSession, _disposeToken.CancellationToken, walker, _version, entry, forceGC: forceGC); + } private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray missingKeys) { if (missingKeys.Count == 0) { diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index e6d298e87..8e3eec750 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -45,7 +45,7 @@ internal sealed class PythonAnalyzerSession { private readonly IProgressReporter _progress; private readonly IPythonAnalyzer _analyzer; private readonly ILogger _log; - private readonly ITelemetryService _telemetry; + private readonly bool _forceGC; private State _state; private bool _isCanceled; @@ -69,7 +69,8 @@ public PythonAnalyzerSession(IServiceManager services, CancellationToken analyzerCancellationToken, IDependencyChainWalker walker, int version, - PythonAnalyzerEntry entry) { + PythonAnalyzerEntry entry, + bool forceGC = false) { _services = services; _analysisCompleteEvent = analysisCompleteEvent; @@ -80,11 +81,11 @@ public PythonAnalyzerSession(IServiceManager services, _walker = walker; _entry = entry; _state = State.NotStarted; + _forceGC = forceGC; _diagnosticsService = _services.GetService(); _analyzer = _services.GetService(); _log = _services.GetService(); - _telemetry = _services.GetService(); _progress = progress; } @@ -158,11 +159,12 @@ private async Task StartAsync() { var elapsed = stopWatch.Elapsed.TotalMilliseconds; LogResults(_log, elapsed, originalRemaining, remaining, Version); - ForceGCIfNeeded(originalRemaining, remaining); + ForceGCIfNeeded(_log, originalRemaining, remaining, _forceGC); } - private static void ForceGCIfNeeded(int originalRemaining, int remaining) { - if (originalRemaining - remaining > 1000) { + private static void ForceGCIfNeeded(ILogger logger, int originalRemaining, int remaining, bool force) { + if (force || originalRemaining - remaining > 1000) { + logger?.Log(TraceEventType.Verbose, "Forcing full garbage collection and heap compaction."); GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; GC.Collect(); } diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs index 36899d865..09cd113b6 100644 --- a/src/Analysis/Ast/Impl/Types/LocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -53,6 +53,10 @@ public virtual IReadOnlyList References { } public virtual void AddReference(Location location) { + if (DeclaringModule == null || DeclaringModule.ModuleType == ModuleType.Builtins) { + return; + } + lock (this) { // Don't add references to library code. if (location.Module?.ModuleType == ModuleType.User && !location.Equals(Location)) { diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 0dce02694..46f2511ac 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -276,19 +276,15 @@ public void NotifyPackagesChanged(CancellationToken cancellationToken = default) interpreter.TypeshedResolution.ReloadAsync().ContinueWith(async t => { await interpreter.ModuleResolution.ReloadAsync(); - _log?.Log(TraceEventType.Information, Resources.Done); - _log?.Log(TraceEventType.Information, Resources.AnalysisRestarted); + _services.GetService().GCNextSession(); RestartAnalysis(); if (_watchSearchPaths) { ResetPathWatcher(); } - - await Task.Delay(10000); - - GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; - GC.Collect(); + _log?.Log(TraceEventType.Information, Resources.Done); + _log?.Log(TraceEventType.Information, Resources.AnalysisRestarted); }).DoNotWait(); } From 30bd0019709528da8062bcd844cfdb340a951a2f Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 2 Aug 2019 15:53:57 -0700 Subject: [PATCH 08/14] Un-refactor function since it's not being called externally anymore --- .../Impl/Analyzer/PythonAnalyzerSession.cs | 4 +-- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 28 ++++++++----------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 8e3eec750..2761915bf 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -372,12 +372,12 @@ private void LogException(IPythonModule module, Exception exception) { private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker, bool isCanceled) { var mtIsDroppable = false; + // Don't try to drop builtins; it causes issues elsewhere. + // We probably want the builtin module's AST and other info for evaluation. switch (document.ModuleType) { case ModuleType.Library: case ModuleType.Stub: case ModuleType.Compiled: - //case ModuleType.CompiledBuiltin: - //case ModuleType.Builtins: mtIsDroppable = true; break; } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index cf06a1212..aeeaa41af 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -394,22 +394,18 @@ public void NotifyAnalysisBegins() { return; } - RemoveAllReferences(); - } - } - } - - public void RemoveAllReferences() { - // TODO: Figure out where the nulls below are coming from. - var importedVariables = ((IScope)GlobalScope) - .TraverseDepthFirst(c => c?.Children ?? Enumerable.Empty()) - .SelectMany(s => s?.Variables ?? VariableCollection.Empty) - .Where(v => v?.Source == VariableSource.Import); - - foreach (var v in importedVariables) { - v.RemoveReferences(this); - if (v.Value is IPythonModule module) { - RemoveReferencesInModule(module); + // TODO: Figure out where the nulls below are coming from. + var importedVariables = ((IScope)GlobalScope) + .TraverseDepthFirst(c => c?.Children ?? Enumerable.Empty()) + .SelectMany(s => s?.Variables ?? VariableCollection.Empty) + .Where(v => v?.Source == VariableSource.Import); + + foreach (var v in importedVariables) { + v.RemoveReferences(this); + if (v.Value is IPythonModule module) { + RemoveReferencesInModule(module); + } + } } } } From 0336dfbfc7169dcbf7f85c381ea9937ffef2e2f3 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 2 Aug 2019 16:00:23 -0700 Subject: [PATCH 09/14] Formatting/usings --- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs | 9 ++++----- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs | 2 +- src/LanguageServer/Impl/Implementation/Server.cs | 4 +--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index f8ed9a66e..84cd2da32 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -17,7 +17,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Caching; @@ -225,7 +224,7 @@ public void GCNextSession() { } } - internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) + internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) => AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry entry, in ImmutableArray dependencies) { @@ -248,7 +247,7 @@ private void AnalyzeDocument(in AnalysisModuleKey key, in PythonAnalyzerEntry en session.Start(true); } } - + private bool TryCreateSession(in int graphVersion, in PythonAnalyzerEntry entry, out PythonAnalyzerSession session) { var analyzeUserModuleOutOfOrder = false; lock (_syncObj) { @@ -283,7 +282,7 @@ private bool TryCreateSession(in int graphVersion, in PythonAnalyzerEntry entry, session = null; return false; } - + if (_currentSession.IsCompleted) { _currentSession = session = CreateSession(walker, null); return true; @@ -322,7 +321,7 @@ private void LoadMissingDocuments(IPythonInterpreter interpreter, ImmutableArray if (missingKeys.Count == 0) { return; } - + var foundKeys = ImmutableArray.Empty; foreach (var missingKey in missingKeys) { lock (_syncObj) { diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 2761915bf..bc35dd2e7 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -231,7 +231,7 @@ private async Task AnalyzeAffectedEntriesAsync(Stopwatch stopWatch) { } - private bool IsAnalyzedLibraryInLoop(IDependencyChainNode node) + private bool IsAnalyzedLibraryInLoop(IDependencyChainNode node) => !node.HasMissingDependencies && node.Value.IsAnalyzedLibrary(_walker.Version) && node.IsWalkedWithDependencies && node.IsValidVersion; private Task StartAnalysis(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 46f2511ac..7fad0f794 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -15,10 +15,8 @@ using System; using System.Diagnostics; -using System.IO; using System.Linq; using System.Reflection; -using System.Runtime; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis; @@ -279,7 +277,7 @@ public void NotifyPackagesChanged(CancellationToken cancellationToken = default) _services.GetService().GCNextSession(); RestartAnalysis(); - + if (_watchSearchPaths) { ResetPathWatcher(); } From 4c516036bc16b4ebb94e8cda9497d9070610cc6c Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 2 Aug 2019 16:25:56 -0700 Subject: [PATCH 10/14] PR feedback --- src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs | 2 +- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 778bfb055..e38f983e5 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -271,7 +271,7 @@ private void MergeStub() { var memberType = member?.GetPythonType(); var stubMemberType = stubMember.GetPythonType(); - if (builtins.Equals(memberType?.DeclaringModule) || builtins.Equals(stubMemberType?.DeclaringModule)) { + if (sourceType.IsBuiltin || stubType.IsBuiltin) { continue; // Leave builtins alone. } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index bc35dd2e7..e35f4e76f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -370,7 +370,7 @@ private void LogException(IPythonModule module, Exception exception) { } private IDocumentAnalysis CreateAnalysis(IDependencyChainNode node, IDocument document, PythonAst ast, int version, ModuleWalker walker, bool isCanceled) { - var mtIsDroppable = false; + var canHaveLibraryAnalysis = false; // Don't try to drop builtins; it causes issues elsewhere. // We probably want the builtin module's AST and other info for evaluation. @@ -378,14 +378,14 @@ private IDocumentAnalysis CreateAnalysis(IDependencyChainNode Date: Mon, 5 Aug 2019 13:00:08 -0700 Subject: [PATCH 11/14] Undo IsBuiltin, some things continued to be referenced with that changed --- src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index e38f983e5..778bfb055 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -271,7 +271,7 @@ private void MergeStub() { var memberType = member?.GetPythonType(); var stubMemberType = stubMember.GetPythonType(); - if (sourceType.IsBuiltin || stubType.IsBuiltin) { + if (builtins.Equals(memberType?.DeclaringModule) || builtins.Equals(stubMemberType?.DeclaringModule)) { continue; // Leave builtins alone. } From 540676fdf1253b4d156c25b6a99ea409b32a640b Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 5 Aug 2019 13:02:13 -0700 Subject: [PATCH 12/14] Move reload logic down into PythonAnalyzer, remove added public inferface items --- .../Analyzer/Definitions/IPythonAnalyzer.cs | 6 ++++-- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 21 ++++++++++++------- .../Ast/Impl/Modules/BuiltinsPythonModule.cs | 4 ---- .../Types/Definitions/IBuiltinPythonModule.cs | 2 -- .../Impl/Implementation/Server.cs | 16 ++------------ 5 files changed, 20 insertions(+), 29 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs index 970c4bb3f..972766b78 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs @@ -56,10 +56,12 @@ public interface IPythonAnalyzer { /// IReadOnlyList LintModule(IPythonModule module); + /// - /// Removes all the modules from the analysis, except Typeshed and builtin + /// Removes all the modules from the analysis and restarts it. /// - void ResetAnalyzer(); + /// True if everything should be dropped, including closed files and stubs. + Task ResetAnalyzer(bool full); /// /// Returns list of currently loaded modules. diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 84cd2da32..9920de415 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -196,8 +196,19 @@ public IReadOnlyList LintModule(IPythonModule module) { return optionsProvider?.Options?.LintingEnabled == false ? Array.Empty() : result; } - public void ResetAnalyzer() { + public async Task ResetAnalyzer(bool full) { + if (full) { + var interpreter = _services.GetService(); + var builtins = interpreter.ModuleResolution.BuiltinsModule; + builtins.SetAst(builtins.Analysis.Ast); + + await interpreter.TypeshedResolution.ReloadAsync(); + await interpreter.ModuleResolution.ReloadAsync(); + } + lock (_syncObj) { + _forceGCOnNextSession = _forceGCOnNextSession || full; + _analysisEntries.Split(kvp => kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out var entriesToRemove); _analysisEntries.Clear(); foreach (var (key, entry) in entriesToPreserve) { @@ -206,6 +217,8 @@ public void ResetAnalyzer() { _dependencyResolver.RemoveKeys(entriesToRemove.Select(e => e.Key)); } + + _services.GetService().ReloadAll(); } public IReadOnlyList LoadedModules { @@ -218,12 +231,6 @@ public IReadOnlyList LoadedModules { public event EventHandler AnalysisComplete; - public void GCNextSession() { - lock (_syncObj) { - _forceGCOnNextSession = true; - } - } - internal void RaiseAnalysisComplete(int moduleCount, double msElapsed) => AnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs(moduleCount, msElapsed)); diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index 2cc924d89..095df7033 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -56,10 +56,6 @@ protected override void OnAnalysisComplete() { base.OnAnalysisComplete(); } - public void ResetAst() { - this.SetAst(Analysis.Ast); - } - private void SpecializeTypes() { IPythonType noneType = null; var isV3 = Interpreter.LanguageVersion.Is3x(); diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IBuiltinPythonModule.cs b/src/Analysis/Ast/Impl/Types/Definitions/IBuiltinPythonModule.cs index dc568bb44..28f4d1177 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IBuiltinPythonModule.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IBuiltinPythonModule.cs @@ -33,7 +33,5 @@ namespace Microsoft.Python.Analysis.Types { /// public interface IBuiltinsPythonModule : IPythonModule { IMember GetAnyMember(string name); - - void ResetAst(); } } diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 7fad0f794..e66b09900 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -266,31 +266,19 @@ private void ResetPathWatcher() { } public void NotifyPackagesChanged(CancellationToken cancellationToken = default) { - var interpreter = _services.GetService(); _log?.Log(TraceEventType.Information, Resources.ReloadingModules); - interpreter.ModuleResolution.BuiltinsModule.ResetAst(); - - interpreter.TypeshedResolution.ReloadAsync().ContinueWith(async t => { - await interpreter.ModuleResolution.ReloadAsync(); - - _services.GetService().GCNextSession(); - - RestartAnalysis(); - + _services.GetService().ResetAnalyzer(true).ContinueWith(t => { if (_watchSearchPaths) { ResetPathWatcher(); } _log?.Log(TraceEventType.Information, Resources.Done); _log?.Log(TraceEventType.Information, Resources.AnalysisRestarted); }).DoNotWait(); - } private void RestartAnalysis() { - var analyzer = Services.GetService(); - analyzer.ResetAnalyzer(); - _rdt.ReloadAll(); + _services.GetService().ResetAnalyzer(false).DoNotWait(); } } } From 490991b3b4fcdb1d7e203738a08e89b1c73dd730 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 5 Aug 2019 13:13:05 -0700 Subject: [PATCH 13/14] Mode AddReference to PythonType as override --- src/Analysis/Ast/Impl/Types/LocatedMember.cs | 4 ---- src/Analysis/Ast/Impl/Types/PythonType.cs | 9 +++++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs index 09cd113b6..36899d865 100644 --- a/src/Analysis/Ast/Impl/Types/LocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -53,10 +53,6 @@ public virtual IReadOnlyList References { } public virtual void AddReference(Location location) { - if (DeclaringModule == null || DeclaringModule.ModuleType == ModuleType.Builtins) { - return; - } - lock (this) { // Don't add references to library code. if (location.Module?.ModuleType == ModuleType.User && !location.Equals(Location)) { diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index 18c782d56..ba39b0d60 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core.Diagnostics; @@ -51,6 +52,14 @@ private PythonType(string name, Location location, BuiltinTypeId typeId) : base( #region ILocatedMember public override PythonMemberType MemberType => _typeId.GetMemberId(); + + public override void AddReference(Location location) { + if (DeclaringModule == null || DeclaringModule.ModuleType == ModuleType.Builtins) { + return; + } + + base.AddReference(location); + } #endregion #region IPythonType From ac2bb01f98056d089cb6253649f181fae926ef7b Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 5 Aug 2019 14:09:03 -0700 Subject: [PATCH 14/14] Remove dead code, make all ResetAnalyzer calls full --- .../Analyzer/Definitions/IPythonAnalyzer.cs | 5 +-- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 18 ++++----- .../Impl/Implementation/Server.cs | 38 ++++--------------- 3 files changed, 17 insertions(+), 44 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs index 972766b78..1e77bcb22 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs @@ -58,10 +58,9 @@ public interface IPythonAnalyzer { /// - /// Removes all the modules from the analysis and restarts it. + /// Removes all the modules from the analysis and restarts it, including stubs. /// - /// True if everything should be dropped, including closed files and stubs. - Task ResetAnalyzer(bool full); + Task ResetAnalyzer(); /// /// Returns list of currently loaded modules. diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 9920de415..3ce37b951 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -196,18 +196,16 @@ public IReadOnlyList LintModule(IPythonModule module) { return optionsProvider?.Options?.LintingEnabled == false ? Array.Empty() : result; } - public async Task ResetAnalyzer(bool full) { - if (full) { - var interpreter = _services.GetService(); - var builtins = interpreter.ModuleResolution.BuiltinsModule; - builtins.SetAst(builtins.Analysis.Ast); - - await interpreter.TypeshedResolution.ReloadAsync(); - await interpreter.ModuleResolution.ReloadAsync(); - } + public async Task ResetAnalyzer() { + var interpreter = _services.GetService(); + var builtins = interpreter.ModuleResolution.BuiltinsModule; + builtins.SetAst(builtins.Analysis.Ast); + + await interpreter.TypeshedResolution.ReloadAsync(); + await interpreter.ModuleResolution.ReloadAsync(); lock (_syncObj) { - _forceGCOnNextSession = _forceGCOnNextSession || full; + _forceGCOnNextSession = true; _analysisEntries.Split(kvp => kvp.Value.Module is IBuiltinsPythonModule, out var entriesToPreserve, out var entriesToRemove); _analysisEntries.Clear(); diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index e66b09900..beffdcfac 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -178,9 +178,9 @@ public void DidChangeConfiguration(DidChangeConfigurationParams @params, Cancell _disposableBag.ThrowIfDisposed(); switch (@params.settings) { case ServerSettings settings: { - if (HandleConfigurationChanges(settings)) { - RestartAnalysis(); - } + Settings = settings; + _symbolHierarchyMaxSymbols = Settings.analysis.symbolsHierarchyMaxSymbols; + _completionSource.Options = Settings.completion; break; } default: @@ -199,27 +199,6 @@ private void DisplayStartupInfo() { : Resources.InitializingForPythonInterpreter.FormatInvariant(_interpreter.Configuration.InterpreterPath)); } - private bool HandleConfigurationChanges(ServerSettings newSettings) { - var oldSettings = Settings; - Settings = newSettings; - - _symbolHierarchyMaxSymbols = Settings.analysis.symbolsHierarchyMaxSymbols; - _completionSource.Options = Settings.completion; - - if (oldSettings == null) { - return true; - } - - if (!newSettings.analysis.errors.SetEquals(oldSettings.analysis.errors) || - !newSettings.analysis.warnings.SetEquals(oldSettings.analysis.warnings) || - !newSettings.analysis.information.SetEquals(oldSettings.analysis.information) || - !newSettings.analysis.disabled.SetEquals(oldSettings.analysis.disabled)) { - return true; - } - - return false; - } - private IDocumentationSource ChooseDocumentationSource(string[] kinds) { if (kinds == null) { return new PlainTextDocumentationSource(); @@ -261,24 +240,21 @@ private void ResetPathWatcher() { if (_searchPaths == null || !_searchPaths.SequenceEqual(paths)) { _searchPaths = paths; _pathsWatcher?.Dispose(); - _pathsWatcher = new PathsWatcher(_searchPaths, () => NotifyPackagesChanged(), _log); + _pathsWatcher = new PathsWatcher(_searchPaths, NotifyPackagesChanged, _log); } } - public void NotifyPackagesChanged(CancellationToken cancellationToken = default) { + private void NotifyPackagesChanged() { _log?.Log(TraceEventType.Information, Resources.ReloadingModules); - _services.GetService().ResetAnalyzer(true).ContinueWith(t => { + _services.GetService().ResetAnalyzer().ContinueWith(t => { if (_watchSearchPaths) { ResetPathWatcher(); } + _log?.Log(TraceEventType.Information, Resources.Done); _log?.Log(TraceEventType.Information, Resources.AnalysisRestarted); }).DoNotWait(); } - - private void RestartAnalysis() { - _services.GetService().ResetAnalyzer(false).DoNotWait(); - } } }