Add dependency resolution to the analyzer

src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs

@@ -74,17 +74,17 @@ public override async Task<bool> WalkAsync(ForStatement node, CancellationToken
             return await base.WalkAsync(node, cancellationToken);
         }
 
-        public override Task<bool> WalkAsync(FromImportStatement node, CancellationToken cancellationToken = default)
-            => ImportHandler.HandleFromImportAsync(node, cancellationToken);
+        public override async Task<bool> WalkAsync(FromImportStatement node, CancellationToken cancellationToken = default)
+            => ImportHandler.HandleFromImport(node, cancellationToken);
 
         public override Task<bool> WalkAsync(GlobalStatement node, CancellationToken cancellationToken = default)
             => NonLocalHandler.HandleGlobalAsync(node, cancellationToken);
 
         public override Task<bool> WalkAsync(IfStatement node, CancellationToken cancellationToken = default)
             => ConditionalHandler.HandleIfAsync(node, cancellationToken);
 
-        public override Task<bool> WalkAsync(ImportStatement node, CancellationToken cancellationToken = default)
-            => ImportHandler.HandleImportAsync(node, cancellationToken);
+        public override async Task<bool> WalkAsync(ImportStatement node, CancellationToken cancellationToken = default)
+            => ImportHandler.HandleImport(node, cancellationToken);
 
         public override Task<bool> WalkAsync(NonlocalStatement node, CancellationToken cancellationToken = default)
             => NonLocalHandler.HandleNonLocalAsync(node, cancellationToken);

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -20,39 +20,10 @@ namespace Microsoft.Python.Analysis.Analyzer {
     /// Represents document that can be analyzed asynchronously.
     /// </summary>
     internal interface IAnalyzable {
-        /// <summary>
-        /// Expected version of the analysis when asynchronous operations complete.
-        /// Typically every change to the document or documents that depend on it
-        /// increment the expected version. At the end of the analysis if the expected
-        /// version is still the same, the analysis is applied to the document and
-        /// becomes available to consumers.
-        /// </summary>
-        int ExpectedAnalysisVersion { get; }
-
-        /// <summary>
-        /// Notifies document that analysis is now pending. Typically document increments 
-        /// the expected analysis version. The method can be called repeatedly without
-        /// calling `CompleteAnalysis` first. The method is invoked for every dependency
-        /// in the chain to ensure that objects know that their dependencies have been
-        /// modified and the current analysis is no longer up to date.
-        /// </summary>
-        void NotifyAnalysisPending();
-
         /// <summary>
         /// Notifies document that its analysis is now complete.
         /// </summary>
         /// <param name="analysis">Document analysis</param>
-        /// <returns>True if analysis was accepted, false if is is out of date.</returns>
-        bool NotifyAnalysisComplete(IDocumentAnalysis analysis);
-
-        /// <summary>
-        /// Notifies module that analysis has been canceled.
-        /// </summary>
-        void NotifyAnalysisCanceled();
-
-        /// <summary>
-        /// Notifies module that analysis has thrown an exception.
-        /// </summary>
-        void NotifyAnalysisFailed(Exception ex);
+        void NotifyAnalysisComplete(IDocumentAnalysis analysis);
     }
 }

src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

@@ -16,17 +16,21 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Parsing.Ast;
 
 namespace Microsoft.Python.Analysis.Analyzer {
     public interface IPythonAnalyzer {
+        Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default);
+
         /// <summary>
-        /// Analyze single document.
+        /// Schedules module for re-analysis
         /// </summary>
-        Task AnalyzeDocumentAsync(IDocument document, CancellationToken cancellationToken);
+        void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int version, CancellationToken cancellationToken = default);
 
         /// <summary>
-        /// Analyze document with dependents.
+        /// 
         /// </summary>
-        Task AnalyzeDocumentDependencyChainAsync(IDocument document, CancellationToken cancellationToken);
+        Task<IDocumentAnalysis> GetAnalysisAsync(IPythonModule module, int waitTime = 200, CancellationToken cancellationToken = default);
     }
 }

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs

@@ -160,9 +160,9 @@ public async Task<IMember> GetValueFromFunctionTypeAsync(IPythonFunctionType fn,
                 fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation(null))) {
 
                 if (fn.IsSpecialized && fn is PythonFunctionType ft) {
-                    foreach (var module in ft.Dependencies) {
+                    foreach (var moduleName in ft.Dependencies) {
                         cancellationToken.ThrowIfCancellationRequested();
-                        await Interpreter.ModuleResolution.ImportModuleAsync(module, cancellationToken);
+                        Interpreter.ModuleResolution.GetOrLoadModule(moduleName);
                     }
                 }
 

src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs

@@ -270,12 +270,12 @@ public override bool Walk(ImportStatement node) {
                 SaveStmt(node, true);
 
                 if (_options.ImportNames) {
-                    foreach (var n in node.Names.MaybeEnumerate()) {
+                    foreach (var n in node.Names) {
                         n?.Walk(this);
                     }
                 }
                 if (_options.ImportAsNames) {
-                    foreach (var n in node.AsNames.MaybeEnumerate()) {
+                    foreach (var n in node.AsNames) {
                         n?.Walk(this);
                     }
                 }
@@ -291,13 +291,14 @@ public override bool Walk(FromImportStatement node) {
                 SaveStmt(node, true);
 
                 if (_options.ImportNames) {
-                    node.Root?.Walk(this);
+                    node.Root.Walk(this);
                 }
 
-                foreach (var n in node.Names.MaybeEnumerate()) {
+                foreach (var n in node.Names) {
                     n?.Walk(this);
                 }
-                foreach (var n in node.AsNames.MaybeEnumerate()) {
+
+                foreach (var n in node.AsNames) {
                     n?.Walk(this);
                 }
 

src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs

@@ -16,7 +16,6 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Threading;
-using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Core.DependencyResolution;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
@@ -26,9 +25,9 @@
 
 namespace Microsoft.Python.Analysis.Analyzer.Handlers {
     internal sealed partial class ImportHandler {
-        public async Task<bool> HandleFromImportAsync(FromImportStatement node, CancellationToken cancellationToken = default) {
+        public bool HandleFromImport(FromImportStatement node, CancellationToken cancellationToken = default) {
             cancellationToken.ThrowIfCancellationRequested();
-            if (node.Root == null || node.Names == null || Module.ModuleType == ModuleType.Specialized) {
+            if (Module.ModuleType == ModuleType.Specialized) {
                 return false;
             }
 
@@ -46,13 +45,13 @@ public async Task<bool> HandleFromImportAsync(FromImportStatement node, Cancella
                     await ImportMembersFromSelfAsync(node, cancellationToken);
                     break;
                 case ModuleImport moduleImport:
-                    await ImportMembersFromModuleAsync(node, moduleImport.FullName, cancellationToken);
+                    ImportMembersFromModule(node, moduleImport.FullName, cancellationToken);
                     break;
                 case PossibleModuleImport possibleModuleImport:
-                    await HandlePossibleImportAsync(possibleModuleImport, possibleModuleImport.PossibleModuleFullName, Eval.GetLoc(node.Root), cancellationToken);
+                    HandlePossibleImport(possibleModuleImport, possibleModuleImport.PossibleModuleFullName, Eval.GetLoc(node.Root));
                     break;
                 case PackageImport packageImports:
-                    await ImportMembersFromPackageAsync(node, packageImports, cancellationToken);
+                    ImportMembersFromPackage(node, packageImports);
                     break;
                 case ImportNotFound notFound:
                     MakeUnresolvedImport(null, notFound.FullName, Eval.GetLoc(node.Root));
@@ -85,17 +84,17 @@ private async Task ImportMembersFromSelfAsync(FromImportStatement node, Cancella
                     // Consider 'from . import path as path' in os.pyi in typeshed.
                     var import = ModuleResolution.CurrentPathResolver.GetModuleImportFromModuleName($"{Module.Name}.{importName}");
                     if (!string.IsNullOrEmpty(import?.FullName)) {
-                        member = await ModuleResolution.ImportModuleAsync(import.FullName, cancellationToken);
+                        member = ModuleResolution.GetOrLoadModule(import.FullName);
                     }
                 }
                 Eval.DeclareVariable(memberName, member ?? Eval.UnknownType, VariableSource.Declaration, Eval.GetLoc(names[i]));
             }
         }
 
-        private async Task ImportMembersFromModuleAsync(FromImportStatement node, string moduleName, CancellationToken cancellationToken = default) {
+        private void ImportMembersFromModule(FromImportStatement node, string moduleName, CancellationToken cancellationToken = default) {
             var names = node.Names;
             var asNames = node.AsNames;
-            var module = await ModuleResolution.ImportModuleAsync(moduleName, cancellationToken);
+            var module = ModuleResolution.GetOrLoadModule(moduleName);
             if (module == null) {
                 return;
             }
@@ -104,7 +103,7 @@ private async Task ImportMembersFromModuleAsync(FromImportStatement node, string
                 // TODO: warn this is not a good style per
                 // TODO: https://docs.python.org/3/faq/programming.html#what-are-the-best-practices-for-using-import-in-a-module
                 // TODO: warn this is invalid if not in the global scope.
-                await HandleModuleImportStarAsync(module, cancellationToken);
+                HandleModuleImportStar(module, cancellationToken);
                 return;
             }
 
@@ -122,7 +121,7 @@ private async Task ImportMembersFromModuleAsync(FromImportStatement node, string
             }
         }
 
-        private async Task HandleModuleImportStarAsync(IPythonModule module, CancellationToken cancellationToken = default) {
+        private void HandleModuleImportStar(IPythonModule module, CancellationToken cancellationToken = default) {
             foreach (var memberName in module.GetMemberNames()) {
                 cancellationToken.ThrowIfCancellationRequested();
 
@@ -135,14 +134,14 @@ private async Task HandleModuleImportStarAsync(IPythonModule module, Cancellatio
 
                 member = member ?? Eval.UnknownType;
                 if (member is IPythonModule m) {
-                    await ModuleResolution.ImportModuleAsync(m.Name, cancellationToken);
+                    ModuleResolution.GetOrLoadModule(m.Name);
                 }
 
                 Eval.DeclareVariable(memberName, member, VariableSource.Import, module.Location);
             }
         }
 
-        private async Task ImportMembersFromPackageAsync(FromImportStatement node, PackageImport packageImport, CancellationToken cancellationToken = default) {
+        private void ImportMembersFromPackage(FromImportStatement node, PackageImport packageImport) {
             var names = node.Names;
             var asNames = node.AsNames;
 
@@ -158,13 +157,10 @@ private async Task ImportMembersFromPackageAsync(FromImportStatement node, Packa
                 var memberName = memberReference.Name;
                 var location = Eval.GetLoc(memberReference);
 
-                ModuleImport moduleImport;
-                IPythonType member;
-                if ((moduleImport = packageImport.Modules.FirstOrDefault(mi => mi.Name.EqualsOrdinal(importName))) != null) {
-                    member = await ModuleResolution.ImportModuleAsync(moduleImport.FullName, cancellationToken);
-                } else {
-                    member = Eval.UnknownType;
-                }
+                var moduleImport = packageImport.Modules.FirstOrDefault(mi => mi.Name.EqualsOrdinal(importName));
+                var member = moduleImport != null 
+                    ? ModuleResolution.GetOrLoadModule(moduleImport.FullName) 
+                    : Eval.UnknownType;
 
                 Eval.DeclareVariable(memberName, member, VariableSource.Import, location);
             }

src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs

@@ -16,7 +16,6 @@
 using System;
 using System.Linq;
 using System.Threading;
-using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Core.DependencyResolution;
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Modules;
@@ -31,9 +30,9 @@ namespace Microsoft.Python.Analysis.Analyzer.Handlers {
     internal sealed partial class ImportHandler : StatementHandler {
         public ImportHandler(AnalysisWalker walker) : base(walker) { }
 
-        public async Task<bool> HandleImportAsync(ImportStatement node, CancellationToken cancellationToken = default) {
+        public bool HandleImport(ImportStatement node, CancellationToken cancellationToken = default) {
             cancellationToken.ThrowIfCancellationRequested();
-            if (node.Names == null || Module.ModuleType == ModuleType.Specialized) {
+            if (Module.ModuleType == ModuleType.Specialized) {
                 return false;
             }
 
@@ -58,10 +57,10 @@ public async Task<bool> HandleImportAsync(ImportStatement node, CancellationToke
                         Eval.DeclareVariable(memberName, Module, VariableSource.Declaration, location);
                         break;
                     case ModuleImport moduleImport:
-                        module = await HandleImportAsync(moduleImport, location, cancellationToken);
+                        module = HandleImport(moduleImport, location);
                         break;
                     case PossibleModuleImport possibleModuleImport:
-                        module = await HandlePossibleImportAsync(possibleModuleImport, possibleModuleImport.PossibleModuleFullName, location, cancellationToken);
+                        module = HandlePossibleImport(possibleModuleImport, possibleModuleImport.PossibleModuleFullName, location);
                         break;
                     default:
                         // TODO: Package import?
@@ -73,22 +72,23 @@ public async Task<bool> HandleImportAsync(ImportStatement node, CancellationToke
                     AssignImportedVariables(module, moduleImportExpression, asNameExpression);
                 }
             }
+
             return false;
         }
 
-        private async Task<IPythonModule> HandleImportAsync(ModuleImport moduleImport, LocationInfo location, CancellationToken cancellationToken) {
-            var module = await ModuleResolution.ImportModuleAsync(moduleImport.FullName, cancellationToken);
-            if (module == null) {
-                MakeUnresolvedImport(moduleImport.FullName, moduleImport.FullName, location);
-                return null;
+        private IPythonModule HandleImport(ModuleImport moduleImport, LocationInfo location) {
+            var module = ModuleResolution.GetOrLoadModule(moduleImport.FullName);
+            if (module != null) {
+                return module;
             }
-            return module;
+
+            MakeUnresolvedImport(moduleImport.FullName, moduleImport.FullName, location);
+            return null;
         }
 
-        private async Task<IPythonModule> HandlePossibleImportAsync(
-            PossibleModuleImport possibleModuleImport, string moduleName, LocationInfo location, CancellationToken cancellationToken) {
+        private IPythonModule HandlePossibleImport(PossibleModuleImport possibleModuleImport, string moduleName, LocationInfo location) {
             var fullName = possibleModuleImport.PrecedingModuleFullName;
-            var module = await ModuleResolution.ImportModuleAsync(possibleModuleImport.PrecedingModuleFullName, cancellationToken);
+            var module = ModuleResolution.GetOrLoadModule(possibleModuleImport.PrecedingModuleFullName);
             if (module == null) {
                 MakeUnresolvedImport(possibleModuleImport.PrecedingModuleFullName, moduleName, location);
                 return null;