From 8c80eff98c2b4d482e2afa7f107ae74a5767716e Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 15 Feb 2019 17:15:19 -0800 Subject: [PATCH 1/4] Fix #601 --- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 6 ++--- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 2 +- .../Resolution/MainModuleResolution.cs | 24 +++++++++++++++---- .../Resolution/ModuleResolutionBase.cs | 4 +++- .../Modules/Resolution/TypeshedResolution.cs | 2 ++ src/Core/Impl/Extensions/IOExtensions.cs | 5 ++++ .../Impl/Diagnostics/DiagnosticsService.cs | 13 ++++++++-- 7 files changed, 44 insertions(+), 12 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 5c90c2038..dd6cec795 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -133,14 +133,14 @@ private static void NotifyAnalysisComplete(IDependencyChainNode node, IDocumentA private async Task AnalyzeAsync(IDependencyChainNode node, CancellationToken cancellationToken) { var startTime = DateTime.Now; - //_log?.Log(TraceEventType.Verbose, $"Analysis begins: {node.Document.Name}({node.Document.ModuleType})"); + //_log?.Log(TraceEventType.Verbose, $"Analysis begins: {node.Document.Name} ({node.Document.ModuleType})"); // Store current expected version so we can see if it still // the same at the time the analysis completes. var analysisVersion = node.Analyzable.ExpectedAnalysisVersion; // Make sure the file is parsed ans the AST is up to date. var ast = await node.Document.GetAstAsync(cancellationToken); - //_log?.Log(TraceEventType.Verbose, $"Parse of {node.Document.Name}({node.Document.ModuleType}) complete in {(DateTime.Now - startTime).TotalMilliseconds} ms."); + //_log?.Log(TraceEventType.Verbose, $"Parse of {node.Document.Name} ({node.Document.ModuleType}) complete in {(DateTime.Now - startTime).TotalMilliseconds} ms."); // Now run the analysis. var walker = new ModuleWalker(_services, node.Document, ast); @@ -151,7 +151,7 @@ private async Task AnalyzeAsync(IDependencyChainNode node, Ca // Note that we do not set the new analysis here and rather let // Python analyzer to call NotifyAnalysisComplete. await walker.CompleteAsync(cancellationToken); - _log?.Log(TraceEventType.Verbose, $"Analysis of {node.Document.Name}({node.Document.ModuleType}) complete in {(DateTime.Now - startTime).TotalMilliseconds} ms."); + _log?.Log(TraceEventType.Verbose, $"Analysis of {node.Document.Name} ({node.Document.ModuleType}) complete in {(DateTime.Now - startTime).TotalMilliseconds} ms."); return new DocumentAnalysis(node.Document, analysisVersion, walker.GlobalScope, walker.Eval); } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index d0cc0cea2..6bfcc1e4e 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -385,7 +385,7 @@ private void Parse(CancellationToken cancellationToken) { // Do not report issues with libraries or stubs if (sink != null) { - _diagnosticsService?.Replace(Uri, _parseErrors.Concat(Analysis.Diagnostics)); + _diagnosticsService?.Replace(Uri, _parseErrors); } _parsingTask = null; diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 039736b04..3ea86ba45 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -26,6 +26,7 @@ using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; +using Microsoft.Python.Core.IO; namespace Microsoft.Python.Analysis.Modules.Resolution { internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement { @@ -73,13 +74,21 @@ protected override async Task DoImportAsync(string name, Cancella return null; } + var rdt = _services.GetService(); + IPythonModule module; + if (!string.IsNullOrEmpty(moduleImport.ModulePath) && Uri.TryCreate(moduleImport.ModulePath, UriKind.Absolute, out var uri)) { + module = rdt.GetDocument(uri); + if (module != null) { + return module; + } + } + // If there is a stub, make sure it is loaded and attached // First check stub next to the module. var stub = await GetModuleStubAsync(name, moduleImport.ModulePath, cancellationToken); // If nothing found, try Typeshed. stub = stub ?? await _interpreter.TypeshedResolution.ImportModuleAsync(moduleImport.IsBuiltin ? name : moduleImport.FullName, cancellationToken); - IPythonModule module; if (moduleImport.IsBuiltin) { _log?.Log(TraceEventType.Verbose, "Import built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); module = new CompiledBuiltinPythonModule(name, stub, _services); @@ -88,11 +97,13 @@ protected override async Task DoImportAsync(string name, Cancella module = new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, _services); } else { _log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); - var rdt = _services.GetService(); - // TODO: handle user code and library module separately. + // Module inside workspace == user code. + var moduleType = moduleImport.ModulePath.IsUnderRoot(_root, _fs.StringComparison) + ? ModuleType.User : ModuleType.Library; + var mco = new ModuleCreationOptions { ModuleName = moduleImport.FullName, - ModuleType = ModuleType.Library, + ModuleType = moduleType, FilePath = moduleImport.ModulePath, Stub = stub }; @@ -185,9 +196,12 @@ private async Task GetModuleStubAsync(string name, string moduleP // Try location of stubs that are in a separate folder next to the package. var stubPath = CurrentPathResolver.GetPossibleModuleStubPaths(name).FirstOrDefault(p => _fs.FileExists(p)); if (!string.IsNullOrEmpty(stubPath)) { - return await CreateStubModuleAsync(name, stubPath, cancellationToken); + return await CreateStubModuleAsync(name, stubPath, cancellationToken); } return null; } + + protected override void ReportModuleNotFound(string name) + => _log?.Log(TraceEventType.Information, $"Import not found: {name}"); } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index b46a3b0ca..ba5ccffa5 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -137,7 +137,7 @@ private async Task DoImportModuleAsync(string name, CancellationT case TryImportModuleResultCode.Success: return result.Module; case TryImportModuleResultCode.ModuleNotFound: - _log?.Log(TraceEventType.Information, $"Import not found: {name}"); + ReportModuleNotFound(name); return null; case TryImportModuleResultCode.NeedRetry: case TryImportModuleResultCode.Timeout: @@ -225,5 +225,7 @@ protected async Task CreateStubModuleAsync(string moduleName, str await module.LoadAndAnalyzeAsync(cancellationToken); return module; } + + protected abstract void ReportModuleNotFound(string name); } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs index 51376d6d6..f4e61a800 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs @@ -137,5 +137,7 @@ private bool IsPackage(string directory) !string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) : _fs.DirectoryExists(directory); + protected override void ReportModuleNotFound(string name) + => _log?.Log(TraceEventType.Verbose, $"Typeshed stub not found for '{name}'"); } } diff --git a/src/Core/Impl/Extensions/IOExtensions.cs b/src/Core/Impl/Extensions/IOExtensions.cs index cfabc6cc4..d1a19d9d2 100644 --- a/src/Core/Impl/Extensions/IOExtensions.cs +++ b/src/Core/Impl/Extensions/IOExtensions.cs @@ -124,5 +124,10 @@ public static void WriteTextWithRetry(this IFileSystem fs, string filePath, stri fs.DeleteFile(filePath); } catch (IOException) { } catch (UnauthorizedAccessException) { } } + + public static bool IsUnderRoot(this string path, string root, StringComparison comparison) { + var normalized = PathUtils.NormalizePath(path); + return normalized.StartsWith(root, comparison); + } } } diff --git a/src/LanguageServer/Impl/Diagnostics/DiagnosticsService.cs b/src/LanguageServer/Impl/Diagnostics/DiagnosticsService.cs index b00e10c6a..026c91790 100644 --- a/src/LanguageServer/Impl/Diagnostics/DiagnosticsService.cs +++ b/src/LanguageServer/Impl/Diagnostics/DiagnosticsService.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Idle; @@ -28,13 +29,20 @@ namespace Microsoft.Python.LanguageServer.Diagnostics { internal sealed class DiagnosticsService : IDiagnosticsService, IDisposable { private readonly Dictionary> _diagnostics = new Dictionary>(); private readonly DisposableBag _disposables = DisposableBag.Create(); + private readonly IServiceContainer _services; private readonly IClientApplication _clientApp; private readonly object _lock = new object(); private DiagnosticsSeverityMap _severityMap = new DiagnosticsSeverityMap(); + private IRunningDocumentTable _rdt; private DateTime _lastChangeTime; private bool _changed; + private IRunningDocumentTable Rdt => _rdt ?? (_rdt = _services.GetService()); + public DiagnosticsService(IServiceContainer services) { + _services = services; + _clientApp = services.GetService(); + var idleTimeService = services.GetService(); if (idleTimeService != null) { @@ -45,7 +53,6 @@ public DiagnosticsService(IServiceContainer services) { .Add(() => idleTimeService.Idle -= OnIdle) .Add(() => idleTimeService.Idle -= OnClosing); } - _clientApp = services.GetService(); } #region IDiagnosticsService @@ -118,7 +125,9 @@ private void PublishDiagnostics() { foreach (var kvp in diagnostics) { var parameters = new PublishDiagnosticsParams { uri = kvp.Key, - diagnostics = kvp.Value.Select(ToDiagnostic).ToArray() + diagnostics = Rdt.GetDocument(kvp.Key)?.IsOpen == true + ? kvp.Value.Select(ToDiagnostic).ToArray() + : Array.Empty() }; _clientApp.NotifyWithParameterObjectAsync("textDocument/publishDiagnostics", parameters).DoNotWait(); } From ec216d34eb7a8e096fde66c870d303bfc31410aa Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 19 Feb 2019 14:01:06 -0800 Subject: [PATCH 2/4] Merge issues --- .../Ast/Impl/Modules/Resolution/MainModuleResolution.cs | 6 ------ .../Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs | 4 +--- .../Ast/Impl/Modules/Resolution/TypeshedResolution.cs | 3 --- 3 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 23b27969e..acf19288a 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -100,7 +100,6 @@ protected override IPythonModule CreateModule(string name) { return stub; } - IPythonModule module; if (moduleImport.IsBuiltin) { _log?.Log(TraceEventType.Verbose, "Create built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); module = new CompiledBuiltinPythonModule(name, stub, _services); @@ -109,8 +108,6 @@ protected override IPythonModule CreateModule(string name) { module = new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, _services); } else { _log?.Log(TraceEventType.Verbose, "Create: ", moduleImport.FullName, moduleImport.ModulePath); - var rdt = _services.GetService(); - _log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); // Module inside workspace == user code. var moduleType = moduleImport.ModulePath.IsUnderRoot(_root, _fs.StringComparison) ? ModuleType.User : ModuleType.Library; @@ -217,8 +214,5 @@ private bool TryCreateModuleStub(string name, string modulePath, out IPythonModu module = !string.IsNullOrEmpty(stubPath) ? new StubPythonModule(name, stubPath, false, _services) : null; return module != null; } - - protected override void ReportModuleNotFound(string name) - => _log?.Log(TraceEventType.Information, $"Import not found: {name}"); } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index c2ccc8df4..95eb766fb 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -159,7 +159,5 @@ public IPythonModule GetOrCreate(string name, ModuleResolutionBase mrb) { } } } - - protected abstract void ReportModuleNotFound(string name); } -} \ No newline at end of file +} diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs index 316f58340..46342fa42 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs @@ -138,8 +138,5 @@ private bool IsPackage(string directory) => ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version) ? !string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) : _fs.DirectoryExists(directory); - - protected override void ReportModuleNotFound(string name) - => _log?.Log(TraceEventType.Verbose, $"Typeshed stub not found for '{name}'"); } } From e0773df5f96abe36803f3e0e44514664dec27e03 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 19 Feb 2019 15:09:36 -0800 Subject: [PATCH 3/4] Merge branch 'master' of https://github.com/Microsoft/python-language-server into 601 --- .../Ast/Impl/Modules/Resolution/MainModuleResolution.cs | 3 --- .../Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs | 2 -- 2 files changed, 5 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index cf65aac27..3c38fc644 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -212,8 +212,5 @@ private bool TryCreateModuleStub(string name, string modulePath, out IPythonModu module = !string.IsNullOrEmpty(stubPath) ? new StubPythonModule(name, stubPath, false, _services) : null; return module != null; } - - protected override void ReportModuleNotFound(string name) - => _log?.Log(TraceEventType.Information, $"Import not found: {name}"); } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index cf2d27d01..95eb766fb 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -159,7 +159,5 @@ public IPythonModule GetOrCreate(string name, ModuleResolutionBase mrb) { } } } - - protected abstract void ReportModuleNotFound(string name); } } From 4cb5a4e0ecf49e7b7a0218b64a675bced7af721f Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 19 Feb 2019 15:10:44 -0800 Subject: [PATCH 4/4] Using --- src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index 3c38fc644..3078a5da6 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -27,7 +27,6 @@ using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; -using Microsoft.Python.Core.IO; using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.IO;