diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs index 7f8159a4e..336094021 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs @@ -30,7 +30,6 @@ namespace Microsoft.Python.Analysis.Analyzer { /// internal abstract class AnalysisWalker : PythonWalkerAsync { protected ImportHandler ImportHandler { get; } - protected FromImportHandler FromImportHandler { get; } protected LoopHandler LoopHandler { get; } protected ConditionalHandler ConditionalHandler { get; } protected AssignmentHandler AssignmentHandler { get; } @@ -47,7 +46,6 @@ internal abstract class AnalysisWalker : PythonWalkerAsync { protected AnalysisWalker(ExpressionEval eval) { Eval = eval; ImportHandler = new ImportHandler(this); - FromImportHandler = new FromImportHandler(this); AssignmentHandler = new AssignmentHandler(this); LoopHandler = new LoopHandler(this); ConditionalHandler = new ConditionalHandler(this); @@ -77,7 +75,7 @@ public override async Task WalkAsync(ForStatement node, CancellationToken } public override Task WalkAsync(FromImportStatement node, CancellationToken cancellationToken = default) - => FromImportHandler.HandleFromImportAsync(node, cancellationToken); + => ImportHandler.HandleFromImportAsync(node, cancellationToken); public override Task WalkAsync(GlobalStatement node, CancellationToken cancellationToken = default) => NonLocalHandler.HandleGlobalAsync(node, cancellationToken); diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 38c566ec4..2d11db886 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -13,6 +13,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; + namespace Microsoft.Python.Analysis.Analyzer { /// /// Represents document that can be analyzed asynchronously. @@ -40,8 +42,17 @@ internal interface IAnalyzable { /// Notifies document that its analysis is now complete. /// /// Document analysis - /// (version of the snapshot in the beginning of analysis). /// True if analysis was accepted, false if is is out of date. bool NotifyAnalysisComplete(IDocumentAnalysis analysis); + + /// + /// Notifies module that analysis has been canceled. + /// + void NotifyAnalysisCanceled(); + + /// + /// Notifies module that analysis has thrown an exception. + /// + void NotifyAnalysisFailed(Exception ex); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs new file mode 100644 index 000000000..3c44ba8dc --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs @@ -0,0 +1,70 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer { + public interface IExpressionEvaluator { + /// + /// Opens existing scope for a node. The scope is pushed + /// on the stack and will be removed when the returned + /// disposable is disposed. + /// + IDisposable OpenScope(IScope scope); + + /// + /// Opens existing scope for a node. The scope is pushed + /// on the stack and will be removed when the returned + /// disposable is disposed. + /// + IDisposable OpenScope(IPythonModule module, ScopeStatement scope); + + /// + /// Currently opened (deep-most) scope. + /// + IScope CurrentScope { get; } + + /// + /// Module global scope. + /// + IGlobalScope GlobalScope { get; } + + /// + /// Determines node location in the module source code. + /// + LocationInfo GetLocation(Node node); + + /// + /// Evaluates expression in the currently open scope. + /// + Task GetValueFromExpressionAsync(Expression expr, CancellationToken cancellationToken = default); + + IMember LookupNameInScopes(string name, out IScope scope); + + IPythonType GetTypeFromPepHint(Node node); + IPythonType GetTypeFromString(string typeString); + + PythonAst Ast { get; } + IPythonModule Module { get; } + IPythonInterpreter Interpreter { get; } + IServiceContainer Services { get; } + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs index 37a77b81e..b1f8f8de0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs @@ -13,8 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Analyzer.Expressions; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Documents; @@ -22,19 +26,18 @@ using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { internal sealed class DocumentAnalysis : IDocumentAnalysis { - public static readonly IDocumentAnalysis Empty = new EmptyAnalysis(); - - public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScope, PythonAst ast) { + public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScope, IExpressionEvaluator eval) { Check.ArgumentNotNull(nameof(document), document); Check.ArgumentNotNull(nameof(globalScope), globalScope); Document = document; Version = version; GlobalScope = globalScope; - Ast = ast; + ExpressionEvaluator = eval; } #region IDocumentAnalysis @@ -53,7 +56,7 @@ public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScop /// /// AST that was used in the analysis. /// - public PythonAst Ast { get; } + public PythonAst Ast => ExpressionEvaluator.Ast; /// /// Document/module global scope. @@ -61,39 +64,29 @@ public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScop public IGlobalScope GlobalScope { get; } /// - /// Module top-level members - /// - public IVariableCollection TopLevelVariables => GlobalScope.Variables; - - /// - /// All module members from all scopes. + /// Expression evaluator used in the analysis. /// - public IEnumerable AllVariables - => (GlobalScope as IScope).TraverseBreadthFirst(s => s.Children).SelectMany(s => s.Variables); - - public IEnumerable GetAllAvailableItems(SourceLocation location) => Enumerable.Empty(); - public IEnumerable GetMembers(SourceLocation location) => Enumerable.Empty(); - public IEnumerable GetSignatures(SourceLocation location) => Enumerable.Empty(); - public IEnumerable GetValues(SourceLocation location) => Enumerable.Empty(); + public IExpressionEvaluator ExpressionEvaluator { get; } #endregion + } - private sealed class EmptyAnalysis : IDocumentAnalysis { - public EmptyAnalysis(IDocument document = null) { - Document = document; - GlobalScope = new EmptyGlobalScope(document); - } + public sealed class EmptyAnalysis : IDocumentAnalysis { + private static PythonAst _emptyAst; - public IDocument Document { get; } - public int Version { get; } = -1; - public IGlobalScope GlobalScope { get; } - public PythonAst Ast => null; - public IEnumerable GetAllAvailableItems(SourceLocation location) => Enumerable.Empty(); - public IEnumerable Diagnostics => Enumerable.Empty(); - public IVariableCollection TopLevelVariables => VariableCollection.Empty; - public IEnumerable AllVariables => Enumerable.Empty(); - public IEnumerable GetMembers(SourceLocation location) => Enumerable.Empty(); - public IEnumerable GetSignatures(SourceLocation location) => Enumerable.Empty(); - public IEnumerable GetValues(SourceLocation location) => Enumerable.Empty(); + public EmptyAnalysis(IServiceContainer services, IDocument document) { + Document = document ?? throw new ArgumentNullException(nameof(document)); + GlobalScope = new EmptyGlobalScope(document); + + _emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile()); + ExpressionEvaluator = new ExpressionEval(services, document, Ast); } + + public IDocument Document { get; } + public int Version { get; } = -1; + public IGlobalScope GlobalScope { get; } + public PythonAst Ast => _emptyAst; + public IExpressionEvaluator ExpressionEvaluator { get; } + public IEnumerable Diagnostics => Enumerable.Empty(); } + } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index f4cbc6fa4..fd65b523a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -15,6 +15,7 @@ using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Types; @@ -32,6 +33,12 @@ public async Task GetValueFromCallableAsync(CallExpression expr, Cancel } var target = await GetValueFromExpressionAsync(expr.Target, cancellationToken); + // Try generics + var result = await GetValueFromGenericAsync(target, expr, cancellationToken); + if (result != null) { + return result; + } + // Should only be two types of returns here. First, an bound type // so we can invoke Call over the instance. Second, an type info // so we can create an instance of the type (as in C() where C is class). @@ -70,18 +77,17 @@ public async Task GetValueFromClassCtorAsync(IPythonClassType cls, Call var args = ArgumentSet.Empty; var init = cls.GetMember(@"__init__"); if (init != null) { - var a = new ArgumentSet(init, new PythonInstance(cls), expr, Module, this); + var a = new ArgumentSet(init, 0, new PythonInstance(cls), expr, Module, this); if (a.Errors.Count > 0) { - AddDiagnostics(a.Errors); - } else { - args = await a.EvaluateAsync(cancellationToken); + // AddDiagnostics(Module.Uri, a.Errors); } + args = await a.EvaluateAsync(cancellationToken); } return cls.CreateInstance(cls.Name, GetLoc(expr), args); } public async Task GetValueFromBoundAsync(IPythonBoundType t, CallExpression expr, CancellationToken cancellationToken = default) { - switch(t.Type) { + switch (t.Type) { case IPythonFunctionType fn: return await GetValueFromFunctionTypeAsync(fn, t.Self, expr, cancellationToken); case IPythonPropertyType p: @@ -110,10 +116,6 @@ public async Task GetValueFromInstanceCall(IPythonInstance pi, CallExpr } public async Task GetValueFromFunctionTypeAsync(IPythonFunctionType fn, IPythonInstance instance, CallExpression expr, CancellationToken cancellationToken = default) { - // Determine argument types - var args = new ArgumentSet(fn, instance, expr, this); - args = await args.EvaluateAsync(cancellationToken); - // If order to be able to find matching overload, we need to know // parameter types and count. This requires function to be analyzed. // Since we don't know which overload we will need, we have to @@ -122,11 +124,26 @@ public async Task GetValueFromFunctionTypeAsync(IPythonFunctionType fn, await SymbolTable.EvaluateAsync(o.FunctionDefinition, cancellationToken); } + // Pick the best overload. + FunctionDefinition fd; + ArgumentSet args; + if (fn.Overloads.Count == 1) { + fd = fn.Overloads[0].FunctionDefinition; + args = new ArgumentSet(fn, 0, instance, expr, this); + args = await args.EvaluateAsync(cancellationToken); + } else { + args = await FindOverloadAsync(fn, instance, expr, cancellationToken); + if (args == null) { + return UnknownType; + } + fd = fn.Overloads.Count > 0 ? fn.Overloads[args.OverloadIndex].FunctionDefinition : null; + } + // Re-declare parameters in the function scope since originally // their types might not have been known and now argument set // may contain concrete values. - if (fn.FunctionDefinition != null) { - using (OpenScope(fn.FunctionDefinition, out _)) { + if (fd != null) { + using (OpenScope(fn.DeclaringModule, fn.FunctionDefinition, out _)) { args.DeclareParametersInScope(this); } } @@ -135,10 +152,19 @@ public async Task GetValueFromFunctionTypeAsync(IPythonFunctionType fn, // most probably comes from the derived class which means that // the original 'self' and 'cls' variables are no longer valid // and function has to be re-evaluated with new arguments. + // Note that there is nothing to re-evaluate in stubs. var instanceType = instance?.GetPythonType(); - if (instanceType == null || fn.DeclaringType == null || fn.IsSpecialized || + if (instanceType == null || fn.DeclaringType == null || fn.IsSpecialized || instanceType.IsSpecialized || fn.DeclaringType.IsSpecialized || - instanceType.Equals(fn.DeclaringType)) { + instanceType.Equals(fn.DeclaringType) || + fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].ReturnDocumentation)) { + + if (fn.IsSpecialized && fn is PythonFunctionType ft) { + foreach (var module in ft.Dependencies) { + cancellationToken.ThrowIfCancellationRequested(); + await Interpreter.ModuleResolution.ImportModuleAsync(module, cancellationToken); + } + } var t = instance?.Call(fn.Name, args) ?? fn.Call(null, fn.Name, args); if (!t.IsUnknown()) { @@ -147,7 +173,7 @@ public async Task GetValueFromFunctionTypeAsync(IPythonFunctionType fn, } // Try and evaluate with specific arguments but prevent recursion. - return await TryEvaluateVithArgumentsAsync(fn.FunctionDefinition, args, cancellationToken); + return await TryEvaluateWithArgumentsAsync(fn.DeclaringModule, fd, args, cancellationToken); } public async Task GetValueFromPropertyAsync(IPythonPropertyType p, IPythonInstance instance, CancellationToken cancellationToken = default) { @@ -156,15 +182,15 @@ public async Task GetValueFromPropertyAsync(IPythonPropertyType p, IPyt return instance.Call(p.Name, ArgumentSet.Empty); } - private async Task TryEvaluateVithArgumentsAsync(FunctionDefinition fd, IArgumentSet args, CancellationToken cancellationToken = default) { + private async Task TryEvaluateWithArgumentsAsync(IPythonModule module, FunctionDefinition fd, IArgumentSet args, CancellationToken cancellationToken = default) { // Attempt to evaluate with specific arguments but prevent recursion. IMember result = UnknownType; if (fd != null && !_callEvalStack.Contains(fd)) { - using (OpenScope(fd.Parent, out _)) { + using (OpenScope(module, fd.Parent, out _)) { _callEvalStack.Push(fd); try { // TODO: cache results per function + argument set? - var eval = new FunctionCallEvaluator(fd, this, Interpreter); + var eval = new FunctionCallEvaluator(module, fd, this); result = await eval.EvaluateCallAsync(args, cancellationToken); } finally { _callEvalStack.Pop(); @@ -173,5 +199,62 @@ private async Task TryEvaluateVithArgumentsAsync(FunctionDefinition fd, } return result; } + + private async Task FindOverloadAsync(IPythonFunctionType fn, IPythonInstance instance, CallExpression expr, CancellationToken cancellationToken = default) { + if (fn.Overloads.Count == 1) { + return null; + } + + cancellationToken.ThrowIfCancellationRequested(); + var sets = new List(); + for (var i = 0; i < fn.Overloads.Count; i++) { + var a = new ArgumentSet(fn, i, instance, expr, this); + var args = await a.EvaluateAsync(cancellationToken); + sets.Add(args); + } + + var orderedSets = sets.OrderBy(s => s.Errors.Count); + var noErrorsMatches = sets.OrderBy(s => s.Errors.Count).TakeWhile(e => e.Errors.Count == 0).ToArray(); + var result = noErrorsMatches.Any() + ? noErrorsMatches.FirstOrDefault(args => IsMatch(args, fn.Overloads[args.OverloadIndex].Parameters)) + : null; + + // Optimistically pick the best available. + return result ?? orderedSets.FirstOrDefault(); + } + + private static bool IsMatch(IArgumentSet args, IReadOnlyList parameters) { + // Arguments passed to function are created off the function definition + // and hence match by default. However, if multiple overloads are specified, + // we need to figure out if annotated types match. + // https://docs.python.org/3/library/typing.html#typing.overload + // + // @overload + // def process(response: None) -> None: + // @overload + // def process(response: int) -> Tuple[int, str]: + // + // Note that in overloads there are no * or ** parameters. + // We match loosely by type. + + var d = parameters.ToDictionary(p => p.Name, p => p.Type); + foreach (var a in args.Arguments()) { + if (!d.TryGetValue(a.Key, out var t)) { + return false; + } + + var at = a.Value?.GetPythonType(); + if (t == null && at == null) { + continue; + } + + if (t != null && at != null && !t.Equals(at)) { + return false; + } + } + return true; + } + + } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs index a413f05f4..1ff8f4936 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs @@ -17,7 +17,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Types.Collections; using Microsoft.Python.Analysis.Values; @@ -32,9 +31,10 @@ public async Task GetValueFromIndexAsync(IndexExpression expr, Cancella } var target = await GetValueFromExpressionAsync(expr.Target, cancellationToken); - // If target is a generic type, create specific class - if (target is IGenericType gen) { - return await CreateSpecificFromGenericAsync(gen, expr, cancellationToken); + // Try generics + var result = await GetValueFromGenericAsync(target, expr, cancellationToken); + if (result != null) { + return result; } if (expr.Index is SliceExpression || expr.Index is TupleExpression) { @@ -98,21 +98,5 @@ public async Task GetValueFromGeneratorAsync(GeneratorExpression expres } return UnknownType; } - - - private async Task CreateSpecificFromGenericAsync(IGenericType gen, IndexExpression expr, CancellationToken cancellationToken = default) { - var args = new List(); - if (expr.Index is TupleExpression tex) { - foreach (var item in tex.Items) { - var e = await GetValueFromExpressionAsync(item, cancellationToken); - args.Add(e?.GetPythonType() ?? UnknownType); - } - } else { - var index = await GetValueFromExpressionAsync(expr.Index, cancellationToken); - args.Add(index?.GetPythonType() ?? UnknownType); - } - return gen.CreateSpecificType(args, Module, GetLoc(expr)); - } - } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs new file mode 100644 index 000000000..59471e632 --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs @@ -0,0 +1,142 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Specializations.Typing.Types; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer.Evaluation { + internal sealed partial class ExpressionEval { + /// + /// Creates specific type from expression that involves generic type + /// and the specific type arguments. + /// + /// + /// x = List[T] + /// + /// + /// T = TypeVar('T', Exception) + /// class A(Generic[T]): ... + /// x = A(TypeError) + /// + private async Task GetValueFromGenericAsync(IMember target, Expression expr, CancellationToken cancellationToken = default) { + if (!(target is PythonClassType c && c.IsGeneric()) && !(target is IGenericType)) { + return null; + } + // Evaluate index to check if the result is generic parameter. + // If it is, then this is a declaration expression such as Generic[T] + // rather than specific type instantiation as in List[str]. + + IPythonType[] specificTypes; + switch (expr) { + case IndexExpression indexExpr: { + // Generic[T1, T2, ...] or A[type]() + var indices = await EvaluateIndexAsync(indexExpr, cancellationToken); + // See which ones are generic parameters as defined by TypeVar() + // and which are specific types. Normally there should not be a mix. + var genericTypeArgs = indices.OfType().ToArray(); + specificTypes = indices.Where(i => !(i is IGenericTypeParameter)).OfType().ToArray(); + + if (specificTypes.Length == 0 && genericTypeArgs.Length > 0) { + // The expression is still generic. For example, generic return + // annotation of a class method, such as 'def func(self) -> A[_E]: ...'. + // Leave it alone, we don't want resolve generic with generic. + return null; + } + + if (genericTypeArgs.Length > 0 && genericTypeArgs.Length != indices.Count) { + // TODO: report that some type arguments are not declared with TypeVar. + } + if (specificTypes.Length > 0 && specificTypes.Length != indices.Count) { + // TODO: report that arguments are not specific types or are not declared. + } + + // Optimistically use what we have + if (target is IGenericType gt) { + if (gt.Name.EqualsOrdinal("Generic")) { + if (genericTypeArgs.Length > 0) { + // Generic[T1, T2, ...] expression. Create generic base for the class. + return new GenericClassBaseType(genericTypeArgs, Module, GetLoc(expr)); + } else { + // TODO: report too few type arguments for Generic[]. + return UnknownType; + } + } + + if (specificTypes.Length > 0) { + // If target is a generic type and indexes are specific types, create specific class + return await gt.CreateSpecificTypeAsync(new ArgumentSet(specificTypes), Module, GetLoc(expr), cancellationToken); + } else { + // TODO: report too few type arguments for the Generic[]. + return UnknownType; + } + } + + break; + } + case CallExpression callExpr: + // Alternative instantiation: + // class A(Generic[T]): ... + // x = A(1234) + specificTypes = (await EvaluateICallArgsAsync(callExpr, cancellationToken)).Select(x => x.GetPythonType()).ToArray(); + break; + + default: + return null; + } + + // This is a bit of a hack since we don't have GenericClassType at the moment. + // The reason is that PythonClassType is created before ClassDefinition is walked + // as we resolve classes on demand. Therefore we don't know if class is generic + // or not at the time of the PythonClassType creation. + // TODO: figure out if we could make GenericClassType: PythonClassType, IGenericType instead. + return target is PythonClassType cls + ? await cls.CreateSpecificTypeAsync(new ArgumentSet(specificTypes), Module, GetLoc(expr), cancellationToken) + : null; + } + + private async Task> EvaluateIndexAsync(IndexExpression expr, CancellationToken cancellationToken = default) { + var indices = new List(); + if (expr.Index is TupleExpression tex) { + cancellationToken.ThrowIfCancellationRequested(); + foreach (var item in tex.Items) { + var e = await GetValueFromExpressionAsync(item, cancellationToken); + indices.Add(e); + } + } else { + var index = await GetValueFromExpressionAsync(expr.Index, cancellationToken); + indices.Add(index); + } + return indices; + } + + private async Task> EvaluateICallArgsAsync(CallExpression expr, CancellationToken cancellationToken = default) { + var indices = new List(); + cancellationToken.ThrowIfCancellationRequested(); + foreach (var e in expr.Args.Select(a => a.Expression).ExcludeDefault()) { + var value = await GetValueFromExpressionAsync(e, cancellationToken); + indices.Add(value); + } + return indices; + } + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs new file mode 100644 index 000000000..c62d9f077 --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs @@ -0,0 +1,108 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.IO; +using System.Linq; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer.Evaluation { + /// + /// Helper class that provides methods for looking up variables + /// and types in a chain of scopes during analysis. + /// + internal sealed partial class ExpressionEval : IExpressionEvaluator { + public IPythonType GetTypeFromPepHint(Node node) { + var location = GetLoc(node); + var content = (Module as IDocument)?.Content; + if (string.IsNullOrEmpty(content) || !location.EndLine.HasValue) { + return null; + } + + var ch = '\0'; + var i = node.IndexSpan.End; + // Starting with the end of the node, look for #. + for (; i < content.Length; i++) { + ch = content[i]; + if (ch == '#' || ch == '\r' || ch == '\n') { + break; + } + } + + if (ch != '#') { + return null; + } + + // Skip # and whitespace. + i++; + for (; i < content.Length; i++) { + ch = content[i]; + if (!char.IsWhiteSpace(ch)) { + break; + } + } + + // Must be at 'type:' + if (ch != 't' || i > content.Length - 5) { + return null; + } + + if (content[i + 1] != 'y' || content[i + 2] != 'p' || content[i + 3] != 'e' || content[i + 4] != ':') { + return null; + } + + // Skip 'type:' and locate end of the line. + i += 5; + var hintStart = i; + for (; i < content.Length; i++) { + if (content[i] == '\r' || content[i] == '\n') { + break; + } + } + + if (i == hintStart) { + return null; + } + + // Type alone is not a valid syntax, so we need to simulate the annotation. + var typeString = content.Substring(hintStart, i - hintStart); + return GetTypeFromString(typeString); + } + + public IPythonType GetTypeFromString(string typeString) { + // Type alone is not a valid syntax, so we need to simulate the annotation. + typeString = $"x: {typeString}"; + using (var sr = new StringReader(typeString)) { + var sink = new CollectingErrorSink(); + var parser = Parser.CreateParser(sr, Module.Interpreter.LanguageVersion, new ParserOptions { ErrorSink = sink }); + var ast = parser.ParseFile(); + var exprStatement = (ast?.Body as SuiteStatement)?.Statements?.FirstOrDefault() as ExpressionStatement; + if (!(Statement.GetExpression(exprStatement) is ExpressionWithAnnotation annExpr) || sink.Errors.Count > 0) { + return null; + } + + var ann = new TypeAnnotation(Ast.LanguageVersion, annExpr.Annotation); + var value = ann.GetValue(new TypeAnnotationConverter(this)); + var t = value.GetPythonType(); + if (!t.IsUnknown()) { + return t; + } + } + return null; + } + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index d43110ca0..99329b895 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -21,6 +21,7 @@ using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Core.Disposables; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Evaluation { @@ -34,26 +35,23 @@ public T GetInScope(string name, IScope scope) where T : class, IMember public IMember GetInScope(string name) => GetInScope(name, CurrentScope); public T GetInScope(string name) where T : class, IMember => GetInScope(name, CurrentScope); - public void DeclareVariable(string name, IMember value, Node expression) - => DeclareVariable(name, value, GetLoc(expression)); + public void DeclareVariable(string name, IMember value, VariableSource source, Node expression) + => DeclareVariable(name, value, source, GetLoc(expression)); - public void DeclareVariable(string name, IMember value, LocationInfo location, bool overwrite = false) { + public void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location, bool overwrite = false) { var member = GetInScope(name); if (member != null) { if (!value.IsUnknown()) { - CurrentScope.DeclareVariable(name, value, location); + CurrentScope.DeclareVariable(name, value, source, location); } } else { - CurrentScope.DeclareVariable(name, value, location); + CurrentScope.DeclareVariable(name, value, source, location); } } [DebuggerStepThrough] public IMember LookupNameInScopes(string name, out IScope scope) => LookupNameInScopes(name, out scope, DefaultLookupOptions); - [DebuggerStepThrough] - public IMember LookupNameInScopes(string name) => LookupNameInScopes(name, DefaultLookupOptions); - [DebuggerStepThrough] public IMember LookupNameInScopes(string name, LookupOptions options) => LookupNameInScopes(name, out _, options); @@ -99,6 +97,17 @@ public async Task GetTypeFromAnnotationAsync(Expression expr, Cance // x: NamedTuple(...) return (await GetValueFromCallableAsync(callExpr, cancellationToken))?.GetPythonType() ?? UnknownType; } + + if (expr is IndexExpression indexExpr) { + // Try generics + var target = await GetValueFromExpressionAsync(indexExpr.Target, cancellationToken); + var result = await GetValueFromGenericAsync(target, indexExpr, cancellationToken); + if(result != null) { + return result.GetPythonType(); + } + } + + cancellationToken.ThrowIfCancellationRequested(); // Look at specialization and typing first var ann = new TypeAnnotation(Ast.LanguageVersion, expr); return ann.GetValue(new TypeAnnotationConverter(this, options)); @@ -109,22 +118,30 @@ public async Task GetTypeFromAnnotationAsync(Expression expr, Cance /// as a child of the specified scope. Scope is pushed on the stack /// and will be removed when returned the disposable is disposed. /// - public IDisposable OpenScope(ScopeStatement node, out Scope fromScope) { + public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scope fromScope) { fromScope = null; + if (node == null) { + return Disposable.Empty; + } + + var gs = module.GlobalScope as Scope ?? GlobalScope; if (node.Parent != null) { - fromScope = (GlobalScope as Scope) + fromScope = gs .TraverseBreadthFirst(s => s.Children.OfType()) .FirstOrDefault(s => s.Node == node.Parent); } - fromScope = fromScope ?? GlobalScope; - var scope = fromScope.Children.OfType().FirstOrDefault(s => s.Node == node); - if (scope == null) { - scope = new Scope(node, fromScope, true); - fromScope.AddChildScope(scope); + fromScope = fromScope ?? gs; + if (fromScope != null) { + var scope = fromScope.Children.OfType().FirstOrDefault(s => s.Node == node); + if (scope == null) { + scope = new Scope(node, fromScope, true); + fromScope.AddChildScope(scope); + } + + _openScopes.Push(scope); + CurrentScope = scope; } - _openScopes.Push(scope); - CurrentScope = scope; return new ScopeTracker(this); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index e01a8ea8c..9e4ba96c1 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -24,9 +24,8 @@ using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; +using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Logging; -using Microsoft.Python.Core.Text; -using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Evaluation { @@ -34,7 +33,7 @@ namespace Microsoft.Python.Analysis.Analyzer.Evaluation { /// Helper class that provides methods for looking up variables /// and types in a chain of scopes during analysis. /// - internal sealed partial class ExpressionEval { + internal sealed partial class ExpressionEval : IExpressionEvaluator { private readonly Stack _openScopes = new Stack(); private readonly IDiagnosticsService _diagnostics; @@ -49,29 +48,43 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs //Log = services.GetService(); _diagnostics = services.GetService(); - - UnknownType = Interpreter.UnknownType ?? - new FallbackBuiltinPythonType(new FallbackBuiltinsModule(Ast.LanguageVersion), BuiltinTypeId.Unknown); } - public PythonAst Ast { get; } - public IPythonModule Module { get; } public LookupOptions DefaultLookupOptions { get; set; } public GlobalScope GlobalScope { get; } public Scope CurrentScope { get; private set; } - public IPythonInterpreter Interpreter => Module.Interpreter; public bool SuppressBuiltinLookup => Module.ModuleType == ModuleType.Builtins; public ILogger Log { get; } - public IServiceContainer Services { get; } public ModuleSymbolTable SymbolTable { get; } = new ModuleSymbolTable(); - public IPythonType UnknownType { get; } + public IPythonType UnknownType => Interpreter.UnknownType; public LocationInfo GetLoc(Node node) => node?.GetLocation(Module, Ast) ?? LocationInfo.Empty; public LocationInfo GetLocOfName(Node node, NameExpression header) => node?.GetLocationOfName(header, Module, Ast) ?? LocationInfo.Empty; + #region IExpressionEvaluator + public PythonAst Ast { get; } + public IPythonModule Module { get; } + public IPythonInterpreter Interpreter => Module.Interpreter; + public IServiceContainer Services { get; } + IScope IExpressionEvaluator.CurrentScope => CurrentScope; + IGlobalScope IExpressionEvaluator.GlobalScope => GlobalScope; + public LocationInfo GetLocation(Node node) => node?.GetLocation(Module, Ast) ?? LocationInfo.Empty; + public Task GetValueFromExpressionAsync(Expression expr, CancellationToken cancellationToken = default) => GetValueFromExpressionAsync(expr, DefaultLookupOptions, cancellationToken); + public IDisposable OpenScope(IScope scope) { + if (!(scope is Scope s)) { + return Disposable.Empty; + } + _openScopes.Push(s); + CurrentScope = s; + return new ScopeTracker(this); + } + + public IDisposable OpenScope(IPythonModule module, ScopeStatement scope) => OpenScope(module, scope, out _); + #endregion + public async Task GetValueFromExpressionAsync(Expression expr, LookupOptions options, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); if (expr == null) { @@ -85,7 +98,7 @@ public async Task GetValueFromExpressionAsync(Expression expr, LookupOp IMember m; switch (expr) { case NameExpression nex: - m = GetValueFromName(nex, options); + m = await GetValueFromNameAsync(nex, options, cancellationToken); break; case MemberExpression mex: m = await GetValueFromMemberAsync(mex, cancellationToken); @@ -130,19 +143,32 @@ public async Task GetValueFromExpressionAsync(Expression expr, LookupOp return m; } - private IMember GetValueFromName(NameExpression expr, LookupOptions options) { - if (string.IsNullOrEmpty(expr?.Name)) { + private async Task GetValueFromNameAsync(NameExpression expr, LookupOptions options, CancellationToken cancellationToken = default) { + if (expr == null || string.IsNullOrEmpty(expr.Name)) { return null; } - var existing = LookupNameInScopes(expr.Name, options); - if (existing != null) { - return existing; - } - if (expr.Name == Module.Name) { return Module; } + + cancellationToken.ThrowIfCancellationRequested(); + var member = LookupNameInScopes(expr.Name, options); + if (member != null) { + switch (member.GetPythonType()) { + case IPythonClassType cls: + await SymbolTable.EvaluateAsync(cls.ClassDefinition, cancellationToken); + break; + case IPythonFunctionType fn: + await SymbolTable.EvaluateAsync(fn.FunctionDefinition, cancellationToken); + break; + case IPythonPropertyType prop: + await SymbolTable.EvaluateAsync(prop.FunctionDefinition, cancellationToken); + break; + } + return member; + } + Log?.Log(TraceEventType.Verbose, $"Unknown name: {expr.Name}"); return UnknownType; } @@ -167,6 +193,15 @@ private async Task GetValueFromMemberAsync(MemberExpression expr, Cance instance = instance ?? m as IPythonInstance; var type = m.GetPythonType(); // Try inner type var value = type?.GetMember(expr.Name); + + // Class type GetMember returns a type. However, class members are + // mostly instances (consider self.x = 1, x is an instance of int). + // However, it is indeed possible to have them as types, like in + // class X ... + // class C: ... + // self.x = X + // which is somewhat rare as compared to self.x = X() but does happen. + switch (value) { case IPythonClassType _: return value; @@ -193,9 +228,12 @@ private async Task GetValueFromConditionalAsync(ConditionalExpression e return trueValue ?? falseValue; } - private void AddDiagnostics(IEnumerable entries) { - foreach (var e in entries) { - _diagnostics?.Add(e); + private void AddDiagnostics(Uri documentUri, IEnumerable entries) { + // Do not add if module is library, etc. Only handle user code. + if (Module.ModuleType == ModuleType.User) { + foreach (var e in entries) { + _diagnostics?.Add(documentUri, e); + } } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs index 244da787c..738d6fc63 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/FunctionCallEvaluator.cs @@ -28,14 +28,14 @@ namespace Microsoft.Python.Analysis.Analyzer.Evaluation { /// internal sealed class FunctionCallEvaluator: AnalysisWalker { private readonly ExpressionEval _eval; + private readonly IPythonModule _declaringModule; private readonly FunctionDefinition _function; - private readonly IPythonInterpreter _interpreter; private IMember _result; - public FunctionCallEvaluator(FunctionDefinition fd, ExpressionEval eval, IPythonInterpreter interpreter): base(eval) { + public FunctionCallEvaluator(IPythonModule declaringModule, FunctionDefinition fd, ExpressionEval eval): base(eval) { + _declaringModule = declaringModule ?? throw new ArgumentNullException(nameof(declaringModule)); _eval = eval ?? throw new ArgumentNullException(nameof(eval)); _function = fd ?? throw new ArgumentNullException(nameof(fd)); - _interpreter = interpreter; } /// @@ -52,7 +52,7 @@ public async Task EvaluateCallAsync(IArgumentSet args, CancellationToke cancellationToken.ThrowIfCancellationRequested(); // Open scope and declare parameters - using (_eval.OpenScope(_function, out _)) { + using (_eval.OpenScope(_declaringModule, _function, out _)) { args.DeclareParametersInScope(_eval); await _function.Body.WalkAsync(this, cancellationToken); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/TypeAnnotationConverter.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/TypeAnnotationConverter.cs index 25366efc2..9cca0ed4f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/TypeAnnotationConverter.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/TypeAnnotationConverter.cs @@ -37,7 +37,7 @@ public override IPythonType Finalize(IPythonType type) { var n = GetName(type); if (!string.IsNullOrEmpty(n)) { - return _eval.LookupNameInScopes(n).GetPythonType(); + return _eval.LookupNameInScopes(n, out _).GetPythonType(); } return type; @@ -50,11 +50,16 @@ public override IPythonType GetTypeMember(IPythonType baseType, string member) => baseType.GetMember(member)?.GetPythonType(); public override IPythonType MakeGeneric(IPythonType baseType, IReadOnlyList args) { - if (!(baseType is IGenericType gt)) { - // TODO: report unhandled generic? - return null; + if (baseType is IGenericType gt) { + return gt.CreateSpecificType(args, _eval.Module, LocationInfo.Empty); + } + if(baseType is IPythonClassType cls && cls.IsGeneric()) { + // Type is not yet known for generic classes. Resolution is delayed + // until specific type is instantiated. + return cls; } - return gt.CreateSpecificType(args, _eval.Module, LocationInfo.Empty); + // TODO: report unhandled generic? + return null; } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs b/src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs index 4434e5fc4..3f4ac60e3 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Expressions/ExpressionFinder.cs @@ -21,7 +21,7 @@ using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Expressions { - internal sealed class ExpressionFinder { + public sealed class ExpressionFinder { public ExpressionFinder(PythonAst ast, FindExpressionOptions options) { Ast = ast; Options = options; @@ -85,7 +85,7 @@ public Node GetExpression(SourceSpan range) { public SourceSpan? GetExpressionSpan(SourceSpan range) => GetExpression(range)?.GetSpan(Ast); private abstract class ExpressionWalker : PythonWalkerWithLocation { - public ExpressionWalker(int location) : base(location) { } + protected ExpressionWalker(int location) : base(location) { } public Node Expression { get; protected set; } public Node Statement { get; protected set; } public ScopeStatement Scope { get; protected set; } @@ -420,26 +420,14 @@ public override bool Walk(ComprehensionFor node) { if (node.IsAsync && !Save(node, true, "async")) { return false; } - if (!Save(node.GetIndexOfFor(_ast), true, "for")) { - return false; - } - if (!Save(node.GetIndexOfIn(_ast), true, "in")) { - return false; - } - return true; + return Save(node.GetIndexOfFor(_ast), true, "for") && Save(node.GetIndexOfIn(_ast), true, "in"); } return false; } public override bool Walk(ConditionalExpression node) { if (base.Walk(node)) { - if (!Save(node.IfIndex, true, "if")) { - return false; - } - if (!Save(node.ElseIndex, true, "else")) { - return false; - } - return true; + return Save(node.IfIndex, true, "if") && Save(node.ElseIndex, true, "else"); } return false; } @@ -455,10 +443,7 @@ public override bool Walk(ForStatement node) { if (!Save(node.InIndex, true, "in")) { return false; } - if (node.Else != null) { - return Save(node.Else.StartIndex, true, "else"); - } - return true; + return node.Else == null || Save(node.Else.StartIndex, true, "else"); } return false; } @@ -490,17 +475,8 @@ public override bool Walk(FunctionDefinition node) { } - public override bool Walk(IfStatement node) { - if (base.Walk(node)) { - if (!Save(node, true, "if")) { - return false; - } - // TODO: elif and if locations - // These cannot be trivially obtained from the node - return true; - } - return false; - } + public override bool Walk(IfStatement node) + => base.Walk(node) && Save(node, true, "if"); public override bool Walk(UnaryExpression node) { if (base.Walk(node)) { @@ -511,27 +487,15 @@ public override bool Walk(UnaryExpression node) { return false; } - public override bool Walk(TryStatement node) { - if (base.Walk(node)) { - if (!Save(node, true, "try")) { - return false; - } - // TODO: except, finally and else locations - // These cannot be trivially obtained from the node - return true; - } - return base.Walk(node); - } + public override bool Walk(TryStatement node) + => base.Walk(node) ? Save(node, true, "try") : base.Walk(node); public override bool Walk(WhileStatement node) { if (base.Walk(node)) { if (!Save(node, true, "while")) { return false; } - if (node.ElseStatement != null) { - return Save(node.ElseStatement.StartIndex, true, "else"); - } - return true; + return node.ElseStatement == null || Save(node.ElseStatement.StartIndex, true, "else"); } return false; } @@ -541,28 +505,15 @@ public override bool Walk(WithStatement node) { if (node.IsAsync && !Save(node, true, "async")) { return false; } - if (!Save(node.GetIndexOfWith(_ast), true, "with")) { - return false; - } - foreach (var item in node.Items.MaybeEnumerate()) { - if (!Save(item.AsIndex, true, "as")) { - return false; - } - } - return true; + return Save(node.GetIndexOfWith(_ast), true, "with") && + node.Items.MaybeEnumerate().All(item => Save(item.AsIndex, true, "as")); } return false; } public override bool Walk(YieldFromExpression node) { if (base.Walk(node)) { - if (!Save(node, true, "yield")) { - return false; - } - if (!Save(node.GetIndexOfFrom(_ast), true, "from")) { - return false; - } - return true; + return Save(node, true, "yield") && Save(node.GetIndexOfFrom(_ast), true, "from"); } return false; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Expressions/FindExpressionOptions.cs b/src/Analysis/Ast/Impl/Analyzer/Expressions/FindExpressionOptions.cs index 7f3972d27..437cd50ee 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Expressions/FindExpressionOptions.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Expressions/FindExpressionOptions.cs @@ -22,7 +22,6 @@ public struct FindExpressionOptions { Members = true, ParameterNames = true, ParenthesisedExpression = true, - Literals = true, ImportNames = true, ImportAsNames = true, ClassDefinitionName = true, @@ -50,10 +49,12 @@ public struct FindExpressionOptions { }; public static FindExpressionOptions Complete => new FindExpressionOptions { Names = true, - MemberName = true, + Members = true, NamedArgumentNames = true, ImportNames = true, - Keywords = true + ImportAsNames = true, + Literals = true, + Errors = true }; public bool Calls { get; set; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs index 440cda767..25ae6e427 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs @@ -13,13 +13,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Handlers { @@ -28,12 +28,21 @@ public AssignmentHandler(AnalysisWalker walker) : base(walker) { } public async Task HandleAssignmentAsync(AssignmentStatement node, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); + if (node.Right is ErrorExpression) { + return; + } + var value = await Eval.GetValueFromExpressionAsync(node.Right, cancellationToken); + // Check PEP hint first + var valueType = Eval.GetTypeFromPepHint(node.Right); + if (valueType != null) { + HandleTypedVariable(valueType, value, node.Left.FirstOrDefault()); + return; + } if (value.IsUnknown()) { Log?.Log(TraceEventType.Verbose, $"Undefined value: {node.Right.ToCodeString(Ast).Trim()}"); } - if (value?.GetPythonType().TypeId == BuiltinTypeId.Ellipsis) { value = Eval.UnknownType; } @@ -44,6 +53,8 @@ public async Task HandleAssignmentAsync(AssignmentStatement node, CancellationTo await texHandler.HandleTupleAssignmentAsync(lhs, node.Right, value, cancellationToken); return; } + + // Process annotations, if any. foreach (var expr in node.Left.OfType()) { // x: List[str] = [...] await HandleAnnotatedExpressionAsync(expr, value, cancellationToken); @@ -70,7 +81,8 @@ public async Task HandleAssignmentAsync(AssignmentStatement node, CancellationTo continue; } - Eval.DeclareVariable(ne.Name, value, Eval.GetLoc(ne)); + var source = value.IsGeneric() ? VariableSource.Generic : VariableSource.Declaration; + Eval.DeclareVariable(ne.Name, value ?? Module.Interpreter.UnknownType, source, Eval.GetLoc(ne)); } } @@ -84,7 +96,10 @@ public async Task HandleAnnotatedExpressionAsync(ExpressionWithAnnotation expr, // x: List[str] // without a value. If value is provided, then this is // x: List[str] = [...] + HandleTypedVariable(variableType, value, expr.Expression); + } + private void HandleTypedVariable(IPythonType variableType, IMember value, Expression expr) { // Check value type for compatibility IMember instance = null; if (value != null) { @@ -97,14 +112,14 @@ public async Task HandleAnnotatedExpressionAsync(ExpressionWithAnnotation expr, instance = value; } } - instance = instance ?? variableType?.CreateInstance(variableType.Name, Eval.GetLoc(expr.Expression), ArgumentSet.Empty) ?? Eval.UnknownType; + instance = instance ?? variableType?.CreateInstance(variableType.Name, Eval.GetLoc(expr), ArgumentSet.Empty) ?? Eval.UnknownType; - if (expr.Expression is NameExpression ne) { - Eval.DeclareVariable(ne.Name, instance, expr.Expression); + if (expr is NameExpression ne) { + Eval.DeclareVariable(ne.Name, instance, VariableSource.Declaration, expr); return; } - if (expr.Expression is MemberExpression m) { + if (expr is MemberExpression m) { // self.x : int = 42 var self = Eval.LookupNameInScopes("self", out var scope); var argType = self?.GetPythonType(); diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs index fae710311..9fc467aac 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs @@ -18,6 +18,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; @@ -80,7 +81,8 @@ public async Task HandleIfAsync(IfStatement node, CancellationToken cancel if (name != null && typeName != null) { var typeId = typeName.GetTypeId(); if (typeId != BuiltinTypeId.Unknown) { - Eval.DeclareVariable(name, new PythonType(typeName, typeId), nex); + var t = new PythonType(typeName, Module, string.Empty, LocationInfo.Empty, typeId); + Eval.DeclareVariable(name, t, VariableSource.Declaration, nex); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index c435d5dc9..dd2705b5c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -20,13 +20,12 @@ using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Handlers { - internal sealed class FromImportHandler: StatementHandler { - public FromImportHandler(AnalysisWalker walker) : base(walker) { } - + internal sealed partial class ImportHandler { public async Task HandleFromImportAsync(FromImportStatement node, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); if (node.Root == null || node.Names == null || Module.ModuleType == ModuleType.Specialized) { @@ -34,14 +33,15 @@ public async Task HandleFromImportAsync(FromImportStatement node, Cancella } var rootNames = node.Root.Names; + IImportSearchResult imports = null; if (rootNames.Count == 1) { - switch (rootNames[0].Name) { - case "__future__": - return false; + var rootName = rootNames[0].Name; + if (rootName.EqualsOrdinal("__future__")) { + return false; } } - var imports = ModuleResolution.CurrentPathResolver.FindImports(Module.FilePath, node); + imports = ModuleResolution.CurrentPathResolver.FindImports(Module.FilePath, node); // If we are processing stub, ignore imports of the original module. // For example, typeshed stub for sys imports sys. if (Module.ModuleType == ModuleType.Stub && imports is ModuleImport mi && mi.Name == Module.Name) { @@ -56,7 +56,7 @@ public async Task HandleFromImportAsync(FromImportStatement node, Cancella await ImportMembersFromModuleAsync(node, moduleImport.FullName, cancellationToken); return false; case PossibleModuleImport possibleModuleImport: - await ImportMembersFromModuleAsync(node, possibleModuleImport.PossibleModuleFullName, cancellationToken); + await HandlePossibleImportAsync(node, possibleModuleImport, cancellationToken); return false; case PackageImport packageImports: await ImportMembersFromPackageAsync(node, packageImports, cancellationToken); @@ -85,7 +85,7 @@ private void ImportMembersFromSelf(FromImportStatement node) { var memberName = memberReference.Name; var member = Module.GetMember(importName); - Eval.DeclareVariable(memberName, member ?? Eval.UnknownType, Eval.GetLoc(names[i])); + Eval.DeclareVariable(memberName, member ?? Eval.UnknownType, VariableSource.Declaration, Eval.GetLoc(names[i])); } } @@ -93,6 +93,9 @@ private async Task ImportMembersFromModuleAsync(FromImportStatement node, string var names = node.Names; var asNames = node.AsNames; var module = await ModuleResolution.ImportModuleAsync(moduleName, cancellationToken); + if (module == null) { + return; + } if (names.Count == 1 && names[0].Name == "*") { // TODO: warn this is not a good style per @@ -102,14 +105,17 @@ private async Task ImportMembersFromModuleAsync(FromImportStatement node, string return; } + Eval.DeclareVariable(module.Name, module, VariableSource.Import, node); + for (var i = 0; i < names.Count; i++) { cancellationToken.ThrowIfCancellationRequested(); - var memberReference = asNames[i] ?? names[i]; - var memberName = memberReference.Name; - - var type = module?.GetMember(memberReference.Name); - Eval.DeclareVariable(memberName, type, names[i]); + var memberName = names[i].Name; + if (!string.IsNullOrEmpty(memberName)) { + var variableName = asNames[i]?.Name ?? memberName; + var type = module.GetMember(memberName) ?? Interpreter.UnknownType; + Eval.DeclareVariable(variableName, type, VariableSource.Import, names[i]); + } } } @@ -129,7 +135,7 @@ private async Task HandleModuleImportStarAsync(IPythonModule module, Cancellatio await ModuleResolution.ImportModuleAsync(m.Name, cancellationToken); } - Eval.DeclareVariable(memberName, member, module.Location); + Eval.DeclareVariable(memberName, member, VariableSource.Import, module.Location); } } @@ -139,7 +145,7 @@ private async Task ImportMembersFromPackageAsync(FromImportStatement node, Packa if (names.Count == 1 && names[0].Name == "*") { // TODO: Need tracking of previous imports to determine possible imports for namespace package. For now import nothing - Eval.DeclareVariable("*", Eval.UnknownType, Eval.GetLoc(names[0])); + Eval.DeclareVariable("*", Eval.UnknownType, VariableSource.Import, Eval.GetLoc(names[0])); return; } @@ -157,7 +163,7 @@ private async Task ImportMembersFromPackageAsync(FromImportStatement node, Packa member = Eval.UnknownType; } - Eval.DeclareVariable(memberName, member, location); + Eval.DeclareVariable(memberName, member, VariableSource.Import, location); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs index 5e3cb7d90..b964ceb72 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ImportHandler.cs @@ -20,10 +20,11 @@ using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Handlers { - internal sealed class ImportHandler: StatementHandler { + internal sealed partial class ImportHandler: StatementHandler { public ImportHandler(AnalysisWalker walker) : base(walker) { } public async Task HandleImportAsync(ImportStatement node, CancellationToken cancellationToken = default) { @@ -41,8 +42,6 @@ public async Task HandleImportAsync(ImportStatement node, CancellationToke var asNameExpression = node.AsNames[i]; var memberName = asNameExpression?.Name ?? moduleImportExpression.Names[0].Name; - // If we are processing stub, ignore imports of the original module. - // For example, typeshed stub for sys imports sys. var imports = ModuleResolution.CurrentPathResolver.GetImportsFromAbsoluteName(Module.FilePath, importNames, node.ForceAbsolute); if (Module.ModuleType == ModuleType.Stub && imports is ModuleImport mi && mi.Name == Module.Name) { continue; @@ -52,7 +51,7 @@ public async Task HandleImportAsync(ImportStatement node, CancellationToke IPythonModule module = null; switch (imports) { case ModuleImport moduleImport when moduleImport.FullName == Module.Name: - Eval.DeclareVariable(memberName, Module, location); + Eval.DeclareVariable(memberName, Module, VariableSource.Declaration, location); break; case ModuleImport moduleImport: module = await HandleImportAsync(node, moduleImport, cancellationToken); @@ -82,7 +81,7 @@ private async Task HandleImportAsync(ImportStatement node, Module return module; } - private async Task HandlePossibleImportAsync(ImportStatement node, PossibleModuleImport possibleModuleImport, CancellationToken cancellationToken) { + private async Task HandlePossibleImportAsync(Node node, PossibleModuleImport possibleModuleImport, CancellationToken cancellationToken) { var fullName = possibleModuleImport.PrecedingModuleFullName; var module = await ModuleResolution.ImportModuleAsync(possibleModuleImport.PrecedingModuleFullName, cancellationToken); if (module == null) { @@ -109,7 +108,7 @@ private void AssignImportedVariables(IPythonModule module, DottedName moduleImpo // "import fob.oar as baz" is handled as // baz = import_module('fob.oar') if (asNameExpression != null) { - Eval.DeclareVariable(asNameExpression.Name, module, asNameExpression); + Eval.DeclareVariable(asNameExpression.Name, module, VariableSource.Import, asNameExpression); return; } @@ -138,13 +137,13 @@ private void AssignImportedVariables(IPythonModule module, DottedName moduleImpo } if (pythonPackage == null) { - Eval.DeclareVariable(importNames[0].Name, child, importNames[0]); + Eval.DeclareVariable(importNames[0].Name, child, VariableSource.Import, importNames[0]); } else { pythonPackage.AddChildModule(importNames[existingDepth].Name, child); } } private void MakeUnresolvedImport(string name, Node node) - => Eval.DeclareVariable(name, new SentinelModule(name, Eval.Services), Eval.GetLoc(node)); + => Eval.DeclareVariable(name, new SentinelModule(name, Eval.Services), VariableSource.Import, Eval.GetLoc(node)); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs index 8cbd796d5..8dd838035 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs @@ -13,11 +13,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing.Ast; @@ -38,7 +36,7 @@ public async Task HandleForAsync(ForStatement node, CancellationToken cancellati switch (node.Left) { case NameExpression nex: // for x in y: - Eval.DeclareVariable(nex.Name, value, Eval.GetLoc(node.Left)); + Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, Eval.GetLoc(node.Left)); break; case TupleExpression tex: // x = [('abc', 42, True), ('abc', 23, False)] @@ -47,7 +45,7 @@ public async Task HandleForAsync(ForStatement node, CancellationToken cancellati if (value is IPythonIterable valueIterable) { var valueIterator = valueIterable.GetIterator(); foreach (var n in names) { - Eval.DeclareVariable(n, valueIterator?.Next ?? Eval.UnknownType, Eval.GetLoc(node.Left)); + Eval.DeclareVariable(n, valueIterator?.Next ?? Eval.UnknownType, VariableSource.Declaration, Eval.GetLoc(node.Left)); } } else { // TODO: report that expression yields value that does not evaluate to iterable. diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs index a62f68827..7aa5e9969 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs @@ -26,13 +26,17 @@ internal abstract class StatementHandler { protected AnalysisWalker Walker { get; } protected ExpressionEval Eval => Walker.Eval; protected IPythonModule Module => Eval.Module; - protected IModuleResolution ModuleResolution => Module.Interpreter.ModuleResolution; protected ILogger Log => Eval.Log; protected IPythonInterpreter Interpreter => Eval.Interpreter; protected GlobalScope GlobalScope => Eval.GlobalScope; protected PythonAst Ast => Eval.Ast; protected ModuleSymbolTable SymbolTable => Eval.SymbolTable; + protected IModuleResolution ModuleResolution + => Module.ModuleType == ModuleType.Stub + ? Module.Interpreter.TypeshedResolution + : Module.Interpreter.ModuleResolution; + protected StatementHandler(AnalysisWalker walker) { Walker = walker; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/TryExceptHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/TryExceptHandler.cs index ecdc8adeb..6a95acdfb 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/TryExceptHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/TryExceptHandler.cs @@ -13,13 +13,10 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; -using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Handlers { @@ -31,7 +28,7 @@ public async Task HandleTryExceptAsync(TryStatement node, CancellationToke foreach (var handler in node.Handlers.MaybeEnumerate()) { if (handler.Test != null && handler.Target is NameExpression nex) { var value = await Eval.GetValueFromExpressionAsync(handler.Test, cancellationToken); - Eval.DeclareVariable(nex.Name, value, nex); + Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, nex); } await handler.Body.WalkAsync(Walker, cancellationToken); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs index 7e354c7ff..e0bc9eb71 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs @@ -35,7 +35,7 @@ public async Task HandleTupleAssignmentAsync(TupleExpression lhs, Expression rhs if (returnedExpressions[i] != null && !string.IsNullOrEmpty(names[i])) { var v = await Eval.GetValueFromExpressionAsync(returnedExpressions[i], cancellationToken); if (v != null) { - Eval.DeclareVariable(names[i], v, returnedExpressions[i]); + Eval.DeclareVariable(names[i], v, VariableSource.Declaration, returnedExpressions[i]); } } } @@ -50,7 +50,7 @@ public async Task HandleTupleAssignmentAsync(TupleExpression lhs, Expression rhs for (var i = 0; i < Math.Min(names.Length, types.Length); i++) { if (names[i] != null && types[i] != null) { var instance = types[i].CreateInstance(null, Eval.GetLoc(expressions[i]), ArgumentSet.Empty); - Eval.DeclareVariable(names[i], instance, expressions[i]); + Eval.DeclareVariable(names[i], instance, VariableSource.Declaration, expressions[i]); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs index edad9d423..6cf89220e 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs @@ -32,9 +32,15 @@ public async Task HandleWithAsync(WithStatement node, CancellationToken cancella var enter = cmType?.GetMember(node.IsAsync ? @"__aenter__" : @"__enter__")?.GetPythonType(); if (enter != null) { - var context = await Eval.GetValueFromFunctionTypeAsync(enter, null, null, cancellationToken); + var instance = contextManager as IPythonInstance; + var callExpr = item.ContextManager as CallExpression; + var context = await Eval.GetValueFromFunctionTypeAsync(enter, instance, callExpr, cancellationToken); + // If fetching context from __enter__ failed, annotation in the stub may be using + // type from typing that we haven't specialized yet or there may be an issue in + // the stub itself, such as type or incorrect type. Try using context manager then. + context = context ?? contextManager; if (item.Variable is NameExpression nex && !string.IsNullOrEmpty(nex.Name)) { - Eval.DeclareVariable(nex.Name, context, Eval.GetLoc(item)); + Eval.DeclareVariable(nex.Name, context, VariableSource.Declaration, Eval.GetLoc(item)); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 206e8c7cd..f405d97aa 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -89,21 +89,21 @@ private void MergeStub() { // Or the stub can have definitions that scraping had missed. Therefore // merge is the combination of the two with the documentation coming // from the library source of from the scraped module. - foreach (var v in _stubAnalysis.TopLevelVariables) { + foreach (var v in _stubAnalysis.GlobalScope.Variables) { var stubType = v.Value.GetPythonType(); if (stubType.IsUnknown()) { continue; } var sourceVar = Eval.GlobalScope.Variables[v.Name]; - var srcType = sourceVar?.Value.GetPythonType(); + var sourceType = sourceVar?.Value.GetPythonType(); // If types are the classes, merge members. // Otherwise, replace type from one from the stub. - - if (srcType is PythonClassType cls) { - // If class exists, add or replace its members - // with ones from the stub, preserving documentation. + if (sourceType is PythonClassType cls && Module.Equals(cls.DeclaringModule)) { + // If class exists and belongs to this module, add or replace + // its members with ones from the stub, preserving documentation. + // Don't augment types that do not come from this module. foreach (var name in stubType.GetMemberNames()) { var stubMember = stubType.GetMember(name); var member = cls.GetMember(name); @@ -116,9 +116,10 @@ private void MergeStub() { } else { // Re-declare variable with the data from the stub. if (!stubType.IsUnknown()) { - srcType.TransferDocumentation(stubType); + sourceType.TransferDocumentation(stubType); // TODO: choose best type between the scrape and the stub. Stub probably should always win. - Eval.DeclareVariable(v.Name, v.Value, LocationInfo.Empty, overwrite: true); + var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? VariableSource.Declaration; + Eval.DeclareVariable(v.Name, v.Value, source, LocationInfo.Empty, overwrite: true); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 32eb85ea3..1b8960ff8 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -22,18 +22,29 @@ using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.Logging; +using Microsoft.Python.Core.Shell; namespace Microsoft.Python.Analysis.Analyzer { public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable { - private readonly IServiceContainer _services; + private readonly IServiceManager _services; private readonly IDependencyResolver _dependencyResolver; private readonly CancellationTokenSource _globalCts = new CancellationTokenSource(); private readonly ILogger _log; - public PythonAnalyzer(IServiceContainer services) { + public PythonAnalyzer(IServiceManager services, string root) { _services = services; _log = services.GetService(); + _dependencyResolver = services.GetService(); + if (_dependencyResolver == null) { + _dependencyResolver = new DependencyResolver(_services); + _services.AddService(_dependencyResolver); + } + + //var rdt = services.GetService(); + //if (rdt == null) { + // services.AddService(new RunningDocumentTable(root, services)); + //} } public void Dispose() => _globalCts.Cancel(); @@ -45,8 +56,16 @@ public async Task AnalyzeDocumentAsync(IDocument document, CancellationToken can var node = new DependencyChainNode(document); using (var cts = CancellationTokenSource.CreateLinkedTokenSource(_globalCts.Token, cancellationToken)) { node.Analyzable.NotifyAnalysisPending(); - var analysis = await AnalyzeAsync(node, cts.Token).ConfigureAwait(false); - node.Analyzable.NotifyAnalysisComplete(analysis); + try { + var analysis = await AnalyzeAsync(node, cts.Token); + node.Analyzable.NotifyAnalysisComplete(analysis); + } catch(OperationCanceledException) { + node.Analyzable.NotifyAnalysisCanceled(); + throw; + } catch(Exception ex) when(!ex.IsCriticalException()) { + node.Analyzable.NotifyAnalysisFailed(ex); + throw; + } } } @@ -57,12 +76,12 @@ public async Task AnalyzeDocumentDependencyChainAsync(IDocument document, Cancel Check.InvalidOperation(() => _dependencyResolver != null, "Dependency resolver must be provided for the group analysis."); using (var cts = CancellationTokenSource.CreateLinkedTokenSource(_globalCts.Token, cancellationToken)) { - var dependencyRoot = await _dependencyResolver.GetDependencyChainAsync(document, cts.Token).ConfigureAwait(false); + var dependencyRoot = await _dependencyResolver.GetDependencyChainAsync(document, cts.Token); // Notify each dependency that the analysis is now pending NotifyAnalysisPending(dependencyRoot); cts.Token.ThrowIfCancellationRequested(); - await AnalyzeChainAsync(dependencyRoot, cts.Token).ConfigureAwait(false); + await AnalyzeChainAsync(dependencyRoot, cts.Token); } } @@ -75,13 +94,21 @@ private void NotifyAnalysisPending(IDependencyChainNode node) { private async Task AnalyzeChainAsync(IDependencyChainNode node, CancellationToken cancellationToken) { using (var cts = CancellationTokenSource.CreateLinkedTokenSource(_globalCts.Token, cancellationToken)) { - var analysis = await AnalyzeAsync(node, cts.Token).ConfigureAwait(false); + try { + var analysis = await AnalyzeAsync(node, cts.Token); + NotifyAnalysisComplete(node, analysis); + } catch (OperationCanceledException) { + node.Analyzable.NotifyAnalysisCanceled(); + throw; + } catch (Exception ex) when (!ex.IsCriticalException()) { + node.Analyzable.NotifyAnalysisFailed(ex); + throw; + } - NotifyAnalysisComplete(node, analysis); cts.Token.ThrowIfCancellationRequested(); foreach (var c in node.Children) { - await AnalyzeChainAsync(c, cts.Token).ConfigureAwait(false); + await AnalyzeChainAsync(c, cts.Token); } } } @@ -101,16 +128,16 @@ private static void NotifyAnalysisComplete(IDependencyChainNode node, IDocumentA /// of dependencies, it is intended for the single file analysis. /// private async Task AnalyzeAsync(IDependencyChainNode node, CancellationToken cancellationToken) { - var _startTime = DateTime.Now; + var startTime = DateTime.Now; - _log?.Log(TraceEventType.Verbose, $"Analysis begins: {node.Document.Name}({node.Document.ModuleType})"); + //_log?.Log(TraceEventType.Verbose, $"Analysis begins: {node.Document.Name}({node.Document.ModuleType})"); // Store current expected version so we can see if it still // the same at the time the analysis completes. var analysisVersion = node.Analyzable.ExpectedAnalysisVersion; // Make sure the file is parsed ans the AST is up to date. var ast = await node.Document.GetAstAsync(cancellationToken); - _log?.Log(TraceEventType.Verbose, $"Parse of {node.Document.Name}({node.Document.ModuleType}) complete in {(DateTime.Now - _startTime).TotalMilliseconds} ms."); + //_log?.Log(TraceEventType.Verbose, $"Parse of {node.Document.Name}({node.Document.ModuleType}) complete in {(DateTime.Now - startTime).TotalMilliseconds} ms."); // Now run the analysis. var walker = new ModuleWalker(_services, node.Document, ast); @@ -121,8 +148,8 @@ private async Task AnalyzeAsync(IDependencyChainNode node, Ca // Note that we do not set the new analysis here and rather let // Python analyzer to call NotifyAnalysisComplete. await walker.CompleteAsync(cancellationToken); - _log?.Log(TraceEventType.Verbose, $"Analysis of {node.Document.Name}({node.Document.ModuleType}) complete in {(DateTime.Now - _startTime).TotalMilliseconds} ms."); - return new DocumentAnalysis(node.Document, analysisVersion, walker.GlobalScope, walker.Ast); + _log?.Log(TraceEventType.Verbose, $"Analysis of {node.Document.Name}({node.Document.ModuleType}) complete in {(DateTime.Now - startTime).TotalMilliseconds} ms."); + return new DocumentAnalysis(node.Document, analysisVersion, walker.GlobalScope, walker.Eval); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs index 775cbc819..3a758ff63 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs @@ -19,6 +19,7 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Modules.Resolution; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; @@ -29,29 +30,44 @@ namespace Microsoft.Python.Analysis.Analyzer { /// /// Describes Python interpreter associated with the analysis. /// - internal sealed class PythonInterpreter : IPythonInterpreter { - private ModuleResolution _moduleResolution; + public sealed class PythonInterpreter : IPythonInterpreter { + private MainModuleResolution _moduleResolution; + private TypeshedResolution _stubResolution; private readonly object _lock = new object(); - private readonly Dictionary _builtinTypes = new Dictionary() { - {BuiltinTypeId.NoneType, new PythonType("NoneType", BuiltinTypeId.NoneType)}, - {BuiltinTypeId.Unknown, new PythonType("Unknown", BuiltinTypeId.Unknown)} - }; + private readonly Dictionary _builtinTypes = new Dictionary(); private PythonInterpreter(InterpreterConfiguration configuration) { Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); LanguageVersion = Configuration.Version.ToLanguageVersion(); - UnknownType = _builtinTypes[BuiltinTypeId.Unknown]; + } + + private async Task LoadBuiltinTypesAsync(string root, IServiceManager sm, CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); + + sm.AddService(this); + _moduleResolution = new MainModuleResolution(root, sm); + await _moduleResolution.InitializeAsync(cancellationToken); + + var builtinModule = _moduleResolution.BuiltinsModule; + lock (_lock) { + _builtinTypes[BuiltinTypeId.NoneType] + = new PythonType("NoneType", builtinModule, string.Empty, LocationInfo.Empty, BuiltinTypeId.NoneType); + _builtinTypes[BuiltinTypeId.Unknown] + = UnknownType = new PythonType("Unknown", builtinModule, string.Empty, LocationInfo.Empty); + } + await _moduleResolution.LoadBuiltinTypesAsync(cancellationToken); + + _stubResolution = new TypeshedResolution(sm); + await _stubResolution.InitializeAsync(cancellationToken); } public static async Task CreateAsync(InterpreterConfiguration configuration, string root, IServiceManager sm, CancellationToken cancellationToken = default) { var pi = new PythonInterpreter(configuration); - sm.AddService(pi); - pi._moduleResolution = new ModuleResolution(root, sm); - // Load builtins - await pi._moduleResolution.LoadBuiltinTypesAsync(cancellationToken); + await pi.LoadBuiltinTypesAsync(root, sm, cancellationToken); + // Specialize typing - await TypingModule.CreateAsync(sm, cancellationToken); + TypingModule.Create(sm); return pi; } @@ -68,7 +84,12 @@ public static async Task CreateAsync(InterpreterConfiguratio /// /// Module resolution service. /// - public IModuleResolution ModuleResolution => _moduleResolution; + public IModuleManagement ModuleResolution => _moduleResolution; + + /// + /// Stub resolution service. + /// + public IModuleResolution TypeshedResolution => _stubResolution; /// /// Gets a well known built-in type such as int, list, dict, etc... @@ -109,7 +130,7 @@ public IPythonType GetBuiltinType(BuiltinTypeId id) { /// /// Unknown type. /// - public IPythonType UnknownType { get; } + public IPythonType UnknownType { get; private set; } public void NotifyImportableModulesChanged() => ModuleResolution.ReloadAsync().DoNotWait(); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs index a4a09cd44..500f1b1d8 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs @@ -19,6 +19,7 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; @@ -38,13 +39,12 @@ public async Task EvaluateClassAsync(CancellationToken cancellationToken = defau cancellationToken.ThrowIfCancellationRequested(); // Open class scope chain - using (Eval.OpenScope(_classDef, out var outerScope)) { + using (Eval.OpenScope(Module, _classDef, out var outerScope)) { var instance = Eval.GetInScope(_classDef.Name, outerScope); if (!(instance?.GetPythonType() is PythonClassType classInfo)) { if (instance != null) { // TODO: warning that variable is already declared of a different type. } - // May be odd case like class inside a class. return; } @@ -61,10 +61,10 @@ public async Task EvaluateClassAsync(CancellationToken cancellationToken = defau bases.Add(b.GetPythonType()); } } - _class.SetBases(Interpreter, bases); + _class.SetBases(bases); // Declare __class__ variable in the scope. - Eval.DeclareVariable("__class__", _class, _classDef); + Eval.DeclareVariable("__class__", _class, VariableSource.Declaration, _classDef); await ProcessClassBody(cancellationToken); } @@ -84,7 +84,7 @@ private async Task ProcessClassBody(CancellationToken cancellationToken = defaul // Process imports foreach (var s in GetStatements(_classDef)) { - await FromImportHandler.HandleFromImportAsync(s, cancellationToken); + await ImportHandler.HandleFromImportAsync(s, cancellationToken); } foreach (var s in GetStatements(_classDef)) { await ImportHandler.HandleImportAsync(s, cancellationToken); @@ -139,7 +139,8 @@ private async Task EvaluateInnerClassesAsync(ClassDefinition cd, CancellationTok private void UpdateClassMembers() { // Add members from this file - _class.AddMembers(Eval.CurrentScope.Variables, false); + var members = Eval.CurrentScope.Variables.Where(v => v.Source == VariableSource.Declaration || v.Source == VariableSource.Import); + _class.AddMembers(members, false); // Add members from stub var stubClass = Eval.Module.Stub?.GetMember(_class.Name); _class.AddMembers(stubClass, false); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index e0dcfd3b6..e9929825c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -23,6 +23,7 @@ using Microsoft.Python.Analysis.Extensions; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Analysis.Values.Collections; using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; @@ -73,7 +74,7 @@ public override async Task EvaluateAsync(CancellationToken cancellationToken = d } } - using (Eval.OpenScope(FunctionDefinition, out _)) { + using (Eval.OpenScope(_function.DeclaringModule, FunctionDefinition, out _)) { await DeclareParametersAsync(cancellationToken); if (annotationType.IsUnknown() || Module.ModuleType == ModuleType.User) { // Return type from the annotation is sufficient for libraries @@ -92,7 +93,7 @@ public override async Task WalkAsync(AssignmentStatement node, Cancellatio switch (lhs) { case MemberExpression memberExp when memberExp.Target is NameExpression nameExp1: { if (_function.DeclaringType.GetPythonType() is PythonClassType t && nameExp1.Name == "self") { - t.AddMembers(new[] { new KeyValuePair(memberExp.Name, value) }, true); + t.AddMembers(new[] { new KeyValuePair(memberExp.Name, value) }, false); } continue; } @@ -131,7 +132,7 @@ private async Task DeclareParametersAsync(CancellationToken cancellationToken = // Actual parameter type will be determined when method is invoked. // The reason is that if method might be called on a derived class. // Declare self or cls in this scope. - Eval.DeclareVariable(p0.Name, _self, p0.NameExpression); + Eval.DeclareVariable(p0.Name, new PythonInstance(_self), VariableSource.Declaration, p0.NameExpression); // Set parameter info. var pi = new ParameterInfo(Ast, p0, _self); pi.SetType(_self); @@ -185,7 +186,7 @@ private async Task DeclareParameterAsync(Parameter p, int index, ParameterInfo p } } - Eval.DeclareVariable(p.Name, paramType, p.NameExpression); + Eval.DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression); } private async Task EvaluateInnerFunctionsAsync(FunctionDefinition fd, CancellationToken cancellationToken = default) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index 0628e32fa..8dacfd1c5 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -49,29 +49,38 @@ private SymbolCollector(ModuleSymbolTable table, ExpressionEval eval) { public override Task WalkAsync(ClassDefinition cd, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - var classInfo = CreateClass(cd); - _eval.DeclareVariable(cd.Name, classInfo, GetLoc(cd)); - _table.Add(new ClassEvaluator(_eval, cd)); - // Open class scope - _scopes.Push(_eval.OpenScope(cd, out _)); + if (!string.IsNullOrEmpty(cd.NameExpression?.Name)) { + var classInfo = CreateClass(cd); + _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration, GetLoc(cd)); + _table.Add(new ClassEvaluator(_eval, cd)); + // Open class scope + _scopes.Push(_eval.OpenScope(_eval.Module, cd, out _)); + } return Task.FromResult(true); } public override Task PostWalkAsync(ClassDefinition cd, CancellationToken cancellationToken = default) { - _scopes.Pop().Dispose(); + if (!string.IsNullOrEmpty(cd.NameExpression?.Name)) { + _scopes.Pop().Dispose(); + } return base.PostWalkAsync(cd, cancellationToken); } public override Task WalkAsync(FunctionDefinition fd, CancellationToken cancellationToken = default) { cancellationToken.ThrowIfCancellationRequested(); - AddFunctionOrProperty(fd); - // Open function scope - _scopes.Push(_eval.OpenScope(fd, out _)); + if (!string.IsNullOrEmpty(fd.NameExpression?.Name)) { + AddFunctionOrProperty(fd); + // Open function scope + _scopes.Push(_eval.OpenScope(_eval.Module, fd, out _)); + } + return Task.FromResult(true); } public override Task PostWalkAsync(FunctionDefinition fd, CancellationToken cancellationToken = default) { - _scopes.Pop().Dispose(); + if (!string.IsNullOrEmpty(fd.NameExpression?.Name)) { + _scopes.Pop().Dispose(); + } return base.PostWalkAsync(fd, cancellationToken); } @@ -94,7 +103,7 @@ private void AddFunctionOrProperty(FunctionDefinition fd) { private IMember AddFunction(FunctionDefinition node, IPythonType declaringType, LocationInfo loc) { if (!(_eval.LookupNameInScopes(node.Name, LookupOptions.Local) is PythonFunctionType existing)) { existing = new PythonFunctionType(node, _eval.Module, declaringType, loc); - _eval.DeclareVariable(node.Name, existing, loc); + _eval.DeclareVariable(node.Name, existing, VariableSource.Declaration, loc); } AddOverload(node, existing, o => existing.AddOverload(o)); return existing; @@ -119,8 +128,7 @@ private void AddOverload(FunctionDefinition node, IPythonClassMember function, A // Do not evaluate parameter types just yet. During light-weight top-level information // collection types cannot be determined as imports haven't been processed. var location = _eval.GetLocOfName(node, node.NameExpression); - var returnDoc = node.ReturnAnnotation?.ToCodeString(_eval.Ast); - var overload = new PythonFunctionOverload(node, _eval.Module, location, returnDoc); + var overload = new PythonFunctionOverload(node, function, _eval.Module, location); addOverload(overload); _table.Add(new FunctionEvaluator(_eval, node, overload, function)); } @@ -156,7 +164,7 @@ private bool TryAddProperty(FunctionDefinition node, IPythonType declaringType, private PythonPropertyType AddProperty(FunctionDefinition node, IPythonModule declaringModule, IPythonType declaringType, bool isAbstract, LocationInfo loc) { if (!(_eval.LookupNameInScopes(node.Name, LookupOptions.Local) is PythonPropertyType existing)) { existing = new PythonPropertyType(node, declaringModule, declaringType, isAbstract, loc); - _eval.DeclareVariable(node.Name, existing, loc); + _eval.DeclareVariable(node.Name, existing, VariableSource.Declaration, loc); } AddOverload(node, existing, o => existing.AddOverload(o)); return existing; diff --git a/src/Analysis/Ast/Impl/Definitions/IDocumentAnalysis.cs b/src/Analysis/Ast/Impl/Definitions/IDocumentAnalysis.cs index ce4548ae5..fb1055c94 100644 --- a/src/Analysis/Ast/Impl/Definitions/IDocumentAnalysis.cs +++ b/src/Analysis/Ast/Impl/Definitions/IDocumentAnalysis.cs @@ -13,11 +13,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis { @@ -48,45 +46,8 @@ public interface IDocumentAnalysis { IGlobalScope GlobalScope { get; } /// - /// Module top-level variables + /// Expression evaluator used in the analysis. /// - IVariableCollection TopLevelVariables { get; } - - /// - /// All module members from all scopes. - /// - IEnumerable AllVariables { get; } - - /// - /// Evaluates a given expression and returns a list of members which - /// exist in the expression. - /// - /// If the expression is an empty string returns all available members - /// at that location. - /// - /// The location in the file where the expression should be evaluated. - IEnumerable GetMembers(SourceLocation location); - - /// - /// Evaluates the given expression in at the provided line number and returns the values - /// that the expression can evaluate to. - /// - /// The location in the file where the expression should be evaluated. - IEnumerable GetValues(SourceLocation location); - - /// - /// Gets information about the available signatures for the given expression. - /// - /// The location in the file. - IEnumerable GetSignatures(SourceLocation location); - - /// - /// Gets the available items at the given location. This includes - /// built-in variables, global variables, and locals. - /// - /// - /// The location in the file where the available members should be looked up. - /// - IEnumerable GetAllAvailableItems(SourceLocation location); + IExpressionEvaluator ExpressionEvaluator { get; } } } diff --git a/src/Analysis/Ast/Impl/Definitions/IPythonInterpreter.cs b/src/Analysis/Ast/Impl/Definitions/IPythonInterpreter.cs index 6802c7c28..d70c696fd 100644 --- a/src/Analysis/Ast/Impl/Definitions/IPythonInterpreter.cs +++ b/src/Analysis/Ast/Impl/Definitions/IPythonInterpreter.cs @@ -50,9 +50,14 @@ public interface IPythonInterpreter { IPythonType UnknownType { get; } /// - /// Module resolution service. + /// Regular module resolution service. /// - IModuleResolution ModuleResolution { get; } + IModuleManagement ModuleResolution { get; } + + /// + /// Stub resolution service. + /// + IModuleResolution TypeshedResolution { get; } /// /// Tells analyzer that module set has changed. Client application that tracks changes diff --git a/src/Analysis/Ast/Impl/Diagnostics/IDiagnosticsService.cs b/src/Analysis/Ast/Impl/Diagnostics/IDiagnosticsService.cs index 85275688b..133e2fc55 100644 --- a/src/Analysis/Ast/Impl/Diagnostics/IDiagnosticsService.cs +++ b/src/Analysis/Ast/Impl/Diagnostics/IDiagnosticsService.cs @@ -13,14 +13,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; -using Microsoft.Python.Core.Text; -using Microsoft.Python.Parsing; namespace Microsoft.Python.Analysis.Diagnostics { public interface IDiagnosticsService { IReadOnlyList Diagnostics { get; } - void Add(DiagnosticsEntry entry); - void Add(string message, SourceSpan span, string errorCode, Severity severity); + void Add(Uri documentUri, DiagnosticsEntry entry); + int PublishingDelay { get; set; } } } diff --git a/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs b/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs index 196b8473b..5429b855d 100644 --- a/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs +++ b/src/Analysis/Ast/Impl/Documents/Definitions/IDocument.cs @@ -25,7 +25,7 @@ namespace Microsoft.Python.Analysis.Documents { /// /// Represent document (file) loaded for the analysis. /// - public interface IDocument: IPythonModule { + public interface IDocument: IPythonModule, IDisposable { /// /// Module content version (increments after every change). /// @@ -65,7 +65,12 @@ public interface IDocument: IPythonModule { /// Updates document content with the list of changes. /// /// - void Update(IEnumerable changes); + void Update(IEnumerable changes); + + /// + /// Resets document buffer to the provided content. + /// + void Reset(string content); /// /// Provides collection of parsing errors, if any. diff --git a/src/Analysis/Ast/Impl/Documents/Definitions/IRunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/Definitions/IRunningDocumentTable.cs index d775a1a4a..6164f3125 100644 --- a/src/Analysis/Ast/Impl/Documents/Definitions/IRunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/Definitions/IRunningDocumentTable.cs @@ -26,12 +26,13 @@ namespace Microsoft.Python.Analysis.Documents { /// public interface IRunningDocumentTable: IEnumerable { /// - /// Adds file to the list of available documents. + /// Opens document. Adds file to the list of available documents + /// unless it was already loaded via indirect import. /// /// Document URI. /// Document content /// Optional file path, if different from the URI. - IDocument AddDocument(Uri uri, string content, string filePath = null); + IDocument OpenDocument(Uri uri, string content, string filePath = null); /// /// Adds library module to the list of available documents. @@ -39,9 +40,10 @@ public interface IRunningDocumentTable: IEnumerable { IDocument AddModule(ModuleCreationOptions mco); /// - /// Removes document from the table. + /// Closes document. Document is removed from + /// the table if there are no more references to it. /// - void RemoveDocument(Uri uri); + void CloseDocument(Uri uri); /// /// Fetches document by its URI. Returns null if document is not loaded. diff --git a/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs b/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs index 78f344d94..f8fa67c76 100644 --- a/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs +++ b/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs @@ -37,42 +37,11 @@ public void Reset(int version, string content) { } } - public void Update(IEnumerable sets) { - foreach (var set in sets) { - Update(set); - } - } - - public void Update(DocumentChangeSet set) { - Check.InvalidOperation(() => _ownerThreadId == Thread.CurrentThread.ManagedThreadId, - "Document buffer update must be done from the thread that created it"); - - if (!set.Changes.Any(c => c.WholeBuffer)) { - if (Version >= 0) { - if (set.FromVersion < Version) { - return; - } - if (set.FromVersion > Version) { - throw new InvalidOperationException("missing prior versions"); - } - } - if (set.FromVersion >= set.ToVersion) { - throw new InvalidOperationException("cannot reduce version without resetting buffer"); - } - } - + public void Update(IEnumerable changes) { var lastStart = int.MaxValue; var lineLoc = SplitLines(_sb).ToArray(); - foreach (var change in set.Changes) { - if (change.WholeBuffer) { - _sb.Clear(); - if (!string.IsNullOrEmpty(change.InsertedText)) { - _sb.Append(change.InsertedText); - } - continue; - } - + foreach (var change in changes) { var start = NewLineLocation.LocationToIndex(lineLoc, change.ReplacedSpan.Start, _sb.Length); if (start > lastStart) { throw new InvalidOperationException("changes must be in reverse order of start location"); @@ -87,8 +56,7 @@ public void Update(DocumentChangeSet set) { _sb.Insert(start, change.InsertedText); } } - - Version = set.ToVersion; + Version++; } private static IEnumerable SplitLines(StringBuilder text) { diff --git a/src/Analysis/Ast/Impl/Documents/DocumentChange.cs b/src/Analysis/Ast/Impl/Documents/DocumentChange.cs index a0d0b0de2..efc2e329c 100644 --- a/src/Analysis/Ast/Impl/Documents/DocumentChange.cs +++ b/src/Analysis/Ast/Impl/Documents/DocumentChange.cs @@ -20,7 +20,6 @@ namespace Microsoft.Python.Analysis.Documents { public sealed class DocumentChange { public string InsertedText { get; set; } public SourceSpan ReplacedSpan { get; set; } - public bool WholeBuffer { get; set; } public static DocumentChange Insert(string text, SourceLocation start) => new DocumentChange { InsertedText = text, ReplacedSpan = new SourceSpan(start, start) }; diff --git a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs index 17bb34120..0bda5802e 100644 --- a/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs +++ b/src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs @@ -16,11 +16,11 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Core; -using Microsoft.Python.Core.IO; namespace Microsoft.Python.Analysis.Documents { /// @@ -29,19 +29,24 @@ namespace Microsoft.Python.Analysis.Documents { /// the running document table in Visual Studio, see /// "https://docs.microsoft.com/en-us/visualstudio/extensibility/internals/running-document-table"/> /// - internal sealed class RunningDocumentTable : IRunningDocumentTable, IDisposable { - private readonly Dictionary _documentsByUri = new Dictionary(); - private readonly Dictionary _documentsByName = new Dictionary(); + public sealed class RunningDocumentTable : IRunningDocumentTable, IDisposable { + private readonly Dictionary _documentsByUri = new Dictionary(); + private readonly Dictionary _documentsByName = new Dictionary(); private readonly IServiceContainer _services; - private readonly IFileSystem _fs; - private readonly IModuleResolution _moduleResolution; + private readonly object _lock = new object(); private readonly string _workspaceRoot; + private IModuleManagement _moduleManagement; + private IModuleManagement ModuleManagement => _moduleManagement ?? (_moduleManagement = _services.GetService().ModuleResolution); + + private class DocumentEntry { + public IDocument Document; + public int LockCount; + } + public RunningDocumentTable(string workspaceRoot, IServiceContainer services) { _workspaceRoot = workspaceRoot; _services = services; - _fs = services.GetService(); - _moduleResolution = services.GetService().ModuleResolution; } public event EventHandler Opened; @@ -53,75 +58,111 @@ public RunningDocumentTable(string workspaceRoot, IServiceContainer services) { /// Document URI. /// Document content /// Optional file path, if different from the URI. - public IDocument AddDocument(Uri uri, string content, string filePath = null) { - var document = FindDocument(null, uri); - if (document != null) { - return OpenDocument(document, ModuleLoadOptions.Open); + public IDocument OpenDocument(Uri uri, string content, string filePath = null) { + var justOpened = false; + DocumentEntry entry; + lock (_lock) { + entry = FindDocument(null, uri); + if (entry == null) { + var mco = new ModuleCreationOptions { + ModuleName = Path.GetFileNameWithoutExtension(uri.LocalPath), + Content = content, + FilePath = filePath, + Uri = uri, + ModuleType = ModuleType.User + }; + entry = CreateDocument(mco); + } + justOpened = TryOpenDocument(entry, content); } - - var mco = new ModuleCreationOptions { - ModuleName = Path.GetFileNameWithoutExtension(uri.LocalPath), - Content = content, - FilePath = filePath, - Uri = uri, - ModuleType = ModuleType.User, - LoadOptions = ModuleLoadOptions.Open - }; - return CreateDocument(mco); + if (justOpened) { + Opened?.Invoke(this, new DocumentEventArgs(entry.Document)); + } + return entry.Document; } /// /// Adds library module to the list of available documents. /// public IDocument AddModule(ModuleCreationOptions mco) { - if (mco.Uri == null) { - mco.FilePath = mco.FilePath ?? throw new ArgumentNullException(nameof(mco.FilePath)); - if (!Uri.TryCreate(mco.FilePath, UriKind.Absolute, out var uri)) { - throw new ArgumentException("Unable to determine URI from the file path."); + lock (_lock) { + if (mco.Uri == null) { + mco.FilePath = mco.FilePath ?? throw new ArgumentNullException(nameof(mco.FilePath)); + if (!Uri.TryCreate(mco.FilePath, UriKind.Absolute, out var uri)) { + throw new ArgumentException("Unable to determine URI from the file path."); + } + + mco.Uri = uri; } - mco.Uri = uri; + + var entry = FindDocument(mco.FilePath, mco.Uri) ?? CreateDocument(mco); + entry.LockCount++; + return entry.Document; + } + } + + public IDocument GetDocument(Uri documentUri) { + lock (_lock) { + return _documentsByUri.TryGetValue(documentUri, out var entry) ? entry.Document : null; } + } - return FindDocument(mco.FilePath, mco.Uri) ?? CreateDocument(mco); + public IDocument GetDocument(string name) { + lock (_lock) { + return _documentsByName.TryGetValue(name, out var entry) ? entry.Document : null; + } } - public IDocument GetDocument(Uri documentUri) - => _documentsByUri.TryGetValue(documentUri, out var document) ? document : null; + public IEnumerator GetEnumerator() => _documentsByUri.Values.Select(e => e.Document).GetEnumerator(); - public IDocument GetDocument(string name) - => _documentsByName.TryGetValue(name, out var document) ? document : null; + public void CloseDocument(Uri documentUri) { + var justClosed = false; + DocumentEntry entry; + lock (_lock) { + if (_documentsByUri.TryGetValue(documentUri, out entry)) { + Debug.Assert(entry.LockCount >= 1); - public IEnumerator GetEnumerator() => _documentsByUri.Values.GetEnumerator(); + if (entry.Document.IsOpen) { + entry.Document.IsOpen = false; + justClosed = true; + } - public void RemoveDocument(Uri documentUri) { - if (_documentsByUri.TryGetValue(documentUri, out var document)) { - _documentsByUri.Remove(documentUri); - _documentsByName.Remove(document.Name); - document.IsOpen = false; - Closed?.Invoke(this, new DocumentEventArgs(document)); - // TODO: Remove from module resolution? + entry.LockCount--; + + if (entry.LockCount == 0) { + _documentsByUri.Remove(documentUri); + _documentsByName.Remove(entry.Document.Name); + entry.Document.Dispose(); + } + // TODO: Remove from module resolution? + } + } + if(justClosed) { + Closed?.Invoke(this, new DocumentEventArgs(entry.Document)); } } IEnumerator IEnumerable.GetEnumerator() => _documentsByUri.Values.GetEnumerator(); public void Dispose() { - foreach (var d in _documentsByUri.Values.OfType()) { - d.Dispose(); + lock(_lock) { + foreach (var d in _documentsByUri.Values.OfType()) { + d.Dispose(); + } } } - private IDocument FindDocument(string moduleName, Uri uri) { - if (uri != null && _documentsByUri.TryGetValue(uri, out var document)) { - return document; + private DocumentEntry FindDocument(string moduleName, Uri uri) { + if (uri != null && _documentsByUri.TryGetValue(uri, out var entry)) { + return entry; } - if (!string.IsNullOrEmpty(moduleName) && _documentsByName.TryGetValue(moduleName, out document)) { - return document; + if (!string.IsNullOrEmpty(moduleName) && _documentsByName.TryGetValue(moduleName, out entry)) { + return entry; } return null; } - private IDocument CreateDocument(ModuleCreationOptions mco) { + private DocumentEntry CreateDocument(ModuleCreationOptions mco) { IDocument document; switch (mco.ModuleType) { case ModuleType.Stub: @@ -141,19 +182,22 @@ private IDocument CreateDocument(ModuleCreationOptions mco) { throw new InvalidOperationException($"CreateDocument does not support module type {mco.ModuleType}"); } - _documentsByUri[document.Uri] = document; - _documentsByName[mco.ModuleName] = document; + var entry = new DocumentEntry { Document = document, LockCount = 0 }; + _documentsByUri[document.Uri] = entry; + _documentsByName[mco.ModuleName] = entry; - _moduleResolution.AddModulePath(document.FilePath); - return OpenDocument(document, mco.LoadOptions); + ModuleManagement.AddModulePath(document.FilePath); + return entry; } - private IDocument OpenDocument(IDocument document, ModuleLoadOptions options) { - if ((options & ModuleLoadOptions.Open) == ModuleLoadOptions.Open) { - document.IsOpen = true; - Opened?.Invoke(this, new DocumentEventArgs(document)); + private bool TryOpenDocument(DocumentEntry entry, string content) { + if (!entry.Document.IsOpen) { + entry.Document.Reset(content); + entry.Document.IsOpen = true; + entry.LockCount++; + return true; } - return document; + return false; } } } diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs new file mode 100644 index 000000000..924665968 --- /dev/null +++ b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs @@ -0,0 +1,63 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core.Text; + +namespace Microsoft.Python.Analysis { + public static class AnalysisExtensions { + public static IScope FindScope(this IDocumentAnalysis analysis, SourceLocation location) + => analysis.GlobalScope.FindScope(analysis.Ast, location); + + /// + /// Provides ability to specialize function return type manually. + /// + public static void SpecializeFunction(this IDocumentAnalysis analysis, string name, IMember returnValue) { + var f = analysis.GetOrCreateFunction(name); + if (f != null) { + foreach (var o in f.Overloads.OfType()) { + o.SetReturnValue(returnValue, true); + } + } + } + + /// + /// Provides ability to dynamically calculate function return type. + /// + public static void SpecializeFunction(this IDocumentAnalysis analysis, string name, ReturnValueProvider returnTypeCallback, string[] dependencies = null) { + var f = analysis.GetOrCreateFunction(name); + if (f != null) { + foreach (var o in f.Overloads.OfType()) { + o.SetReturnValueProvider(returnTypeCallback); + } + f.Specialize(dependencies); + } + } + + private static PythonFunctionType GetOrCreateFunction(this IDocumentAnalysis analysis, string name) { + // We DO want to replace class by function. Consider type() in builtins. + // 'type()' in code is a function call, not a type class instantiation. + if (!(analysis.GlobalScope.Variables[name]?.Value is PythonFunctionType f)) { + f = PythonFunctionType.ForSpecialization(name, analysis.Document); + f.AddOverload(new PythonFunctionOverload(name, analysis.Document, LocationInfo.Empty)); + analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration, LocationInfo.Empty); + } + return f; + } + } +} diff --git a/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs index 5d070afda..f3d8c2db5 100644 --- a/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ArgumentSetExtensions.cs @@ -19,8 +19,8 @@ using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Types.Collections; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Analysis.Values.Collections; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis { public static class ArgumentSetExtensions { @@ -33,6 +33,14 @@ public static IReadOnlyList> Arguments(this IArgument public static T Argument(this IArgumentSet args, int index) where T : class => args.Arguments[index].Value as T; + public static T GetArgumentValue(this IArgumentSet args, string name) where T : class { + var value = args.Arguments.FirstOrDefault(a => name.Equals(a.Name))?.Value; + if (value == null && args.DictionaryArgument?.Arguments != null && args.DictionaryArgument.Arguments.TryGetValue(name, out var m)) { + return m as T; + } + return value as T; + } + internal static void DeclareParametersInScope(this IArgumentSet args, ExpressionEval eval) { if (eval == null) { return; @@ -44,19 +52,19 @@ internal static void DeclareParametersInScope(this IArgumentSet args, Expression foreach (var a in args.Arguments) { if (a.Value is IMember m) { - eval.DeclareVariable(a.Name, m, LocationInfo.Empty, true); + eval.DeclareVariable(a.Name, m, VariableSource.Declaration, a.Location, false); } } if (args.ListArgument != null && !string.IsNullOrEmpty(args.ListArgument.Name)) { var type = new PythonCollectionType(null, BuiltinTypeId.List, eval.Interpreter, false); var list = new PythonCollection(type, LocationInfo.Empty, args.ListArgument.Values); - eval.DeclareVariable(args.ListArgument.Name, list, LocationInfo.Empty, true); + eval.DeclareVariable(args.ListArgument.Name, list, VariableSource.Declaration, args.ListArgument.Location, false); } if (args.DictionaryArgument != null) { foreach (var kvp in args.DictionaryArgument.Arguments) { - eval.DeclareVariable(kvp.Key, kvp.Value, LocationInfo.Empty, true); + eval.DeclareVariable(kvp.Key, kvp.Value, VariableSource.Declaration, args.DictionaryArgument.Location, false); } } } diff --git a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs index 10e145188..d09fc5f92 100644 --- a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs @@ -13,6 +13,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; +using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Expressions; using Microsoft.Python.Core.Text; @@ -31,5 +33,49 @@ public static string GetDocumentation(this ScopeStatement node) { var ce = docExpr?.Expression as ConstantExpression; return ce?.Value as string; } + + public static bool IsInsideComment(this PythonAst ast, SourceLocation location) { + var match = Array.BinarySearch(ast.CommentLocations, location); + // If our index = -1, it means we're before the first comment + if (match == -1) { + return false; + } + + if (match < 0) { + // If we couldn't find an exact match for this position, get the nearest + // matching comment before this point + match = ~match - 1; + } + + if (match >= ast.CommentLocations.Length) { + Debug.Fail("Failed to find nearest preceding comment in AST"); + return false; + } + + if (ast.CommentLocations[match].Line != location.Line) { + return false; + } + + return ast.CommentLocations[match].Column < location.Column; + } + + public static bool IsInsideString(this PythonAst ast, SourceLocation location) { + var index = ast.LocationToIndex(location); + return ast.FindExpression(index, new FindExpressionOptions {Literals = true}) != null; + } + + public static bool IsInParameter(this FunctionDefinition fd, PythonAst tree, SourceLocation location) { + var index = tree.LocationToIndex(location); + if (index < fd.StartIndex + || (fd.Body != null && index >= fd.Body.StartIndex) + || (fd.NameExpression != null && index > fd.NameExpression.EndIndex)) { + // Not within the def line + return false; + } + return fd.Parameters.Any(p => { + var paramName = p.GetVerbatimImage(tree) ?? p.Name; + return index >= p.StartIndex && index <= p.StartIndex + paramName.Length; + }); + } } } diff --git a/src/Analysis/Ast/Impl/Extensions/DiagnosticsServiceExtensions.cs b/src/Analysis/Ast/Impl/Extensions/DiagnosticsServiceExtensions.cs new file mode 100644 index 000000000..f0d0411c7 --- /dev/null +++ b/src/Analysis/Ast/Impl/Extensions/DiagnosticsServiceExtensions.cs @@ -0,0 +1,26 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing; + +namespace Microsoft.Python.Analysis { + public static class DiagnosticsServiceExtensions { + public static void Add(this IDiagnosticsService ds, Uri documentUri, string message, SourceSpan span, string errorCode, Severity severity) + => ds.Add(documentUri, new DiagnosticsEntry(message, span, errorCode, severity)); + } +} diff --git a/src/Analysis/Ast/Impl/Extensions/MemberContainerExtensions.cs b/src/Analysis/Ast/Impl/Extensions/MemberContainerExtensions.cs index 6dd541324..9dafdb2cf 100644 --- a/src/Analysis/Ast/Impl/Extensions/MemberContainerExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/MemberContainerExtensions.cs @@ -13,8 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; +using System.Linq; + namespace Microsoft.Python.Analysis.Types { public static class MemberContainerExtensions { public static T GetMember(this IMemberContainer mc, string name) where T: class, IPythonType => mc.GetMember(name) as T; + public static IEnumerable GetMembers(this IMemberContainer mc) => mc.GetMemberNames().Select(mc.GetMember); } } diff --git a/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs b/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs index fe6c2d439..ae87c5d80 100644 --- a/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing.Ast; @@ -36,6 +37,20 @@ public static IPythonType GetPythonType(this IMember m) public static T GetPythonType(this IMember m) where T : class, IPythonType => m is IPythonType pt ? pt as T : (m as IPythonInstance)?.Type as T; + public static bool IsGeneric(this IMember m) { + var t = m.GetPythonType(); + if(t is IGenericType || t is IGenericTypeParameter) { + return true; + } + if (t is IPythonClassType c && c.IsGeneric()) { + return true; + } + if (m?.MemberType == PythonMemberType.Generic) { + return true; + } + return m is IVariable v && v.Value.MemberType == PythonMemberType.Generic; + } + public static bool TryGetConstant(this IMember m, out T value) { if (m is IPythonConstant c && c.TryGetValue(out var v)) { value = v; diff --git a/src/LanguageServer/Impl/Implementation/CodeActionProvider.cs b/src/Analysis/Ast/Impl/Extensions/ModuleResolutionExtensions.cs similarity index 64% rename from src/LanguageServer/Impl/Implementation/CodeActionProvider.cs rename to src/Analysis/Ast/Impl/Extensions/ModuleResolutionExtensions.cs index 93ced1191..b23aecd38 100644 --- a/src/LanguageServer/Impl/Implementation/CodeActionProvider.cs +++ b/src/Analysis/Ast/Impl/Extensions/ModuleResolutionExtensions.cs @@ -14,18 +14,14 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Threading.Tasks; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; -namespace Microsoft.Python.LanguageServer.Implementation { - internal class CodeActionProvider { - private readonly Server _server; - - public CodeActionProvider(Server server) { - _server = server; +namespace Microsoft.Python.Analysis { + public static class ModuleResolutionExtensions { + public static IPythonModule ImportModule(this IModuleResolution m, string name, int timeout) { + m.ImportModuleAsync(name).Wait(timeout); + return m.GetImportedModule(name); } - - public Task GetCodeActionsAsync(CodeActionParams parameters) - => Task.FromResult(Array.Empty()); } } diff --git a/src/Analysis/Ast/Impl/Extensions/ModuleLoadOptionsExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs similarity index 55% rename from src/Analysis/Ast/Impl/Extensions/ModuleLoadOptionsExtensions.cs rename to src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs index 0d8f0b8ae..ef8dd7d93 100644 --- a/src/Analysis/Ast/Impl/Extensions/ModuleLoadOptionsExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonClassExtensions.cs @@ -13,15 +13,13 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using Microsoft.Python.Analysis.Modules; +using System.Linq; +using Microsoft.Python.Analysis.Specializations.Typing; +using Microsoft.Python.Analysis.Types; -namespace Microsoft.Python.Analysis.Extensions { - public static class ModuleLoadOptionsExtensions { - public static bool ShouldLoad(this ModuleLoadOptions o) - => (o & ModuleLoadOptions.Load) == ModuleLoadOptions.Load; - public static bool ShouldParse(this ModuleLoadOptions o) - => (o & ModuleLoadOptions.Ast) == ModuleLoadOptions.Ast; - public static bool ShouldAnalyze(this ModuleLoadOptions o) - => (o & ModuleLoadOptions.Analyze) == ModuleLoadOptions.Analyze; +namespace Microsoft.Python.Analysis { + public static class PythonClassExtensions { + public static bool IsGeneric(this IPythonClassType cls) + => cls.Bases != null && cls.Bases.Any(b => b is IGenericType || b is IGenericClassBaseType); } } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonConstantExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonConstantExtensions.cs new file mode 100644 index 000000000..e3b951b4d --- /dev/null +++ b/src/Analysis/Ast/Impl/Extensions/PythonConstantExtensions.cs @@ -0,0 +1,32 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Parsing; + +namespace Microsoft.Python.Analysis { + public static class PythonConstantExtensions { + public static string GetString(this IPythonConstant c) { + switch (c.Value) { + case string s: + return s; + case AsciiString ascii: + return ascii.String; + default: + return null; + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs index f7a9eabe0..a17d4dfad 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonTypeExtensions.cs @@ -24,11 +24,14 @@ public static bool IsUnknown(this IPythonType value) => public static bool IsGenericParameter(this IPythonType value) => value is IGenericTypeParameter; + public static bool IsGeneric(this IPythonType value) + => value is IGenericTypeParameter || value is IGenericType || (value is IPythonClassType c && c.IsGeneric()); + public static void TransferDocumentation(this IPythonType src, IPythonType dst) { if (src != null && dst is PythonType pt) { pt.TrySetTypeId(dst.TypeId); var documentation = src.Documentation; - if (!string.IsNullOrEmpty(documentation)) { + if (string.IsNullOrEmpty(pt.Documentation) && !string.IsNullOrEmpty(documentation)) { pt.SetDocumentationProvider(_ => documentation); } } diff --git a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs index e2eb08826..3e42839d6 100644 --- a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs @@ -14,11 +14,91 @@ // permissions and limitations under the License. using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public static class ScopeExtensions { public static bool IsClassScope(this IScope scope) => scope.Node is ClassDefinition; public static bool IsFunctionScope(this IScope scope) => scope.Node is FunctionDefinition; + + public static int GetBodyStartIndex(this IScope scope, PythonAst ast) { + switch (scope.Node) { + case ClassDefinition cd: + return cd.HeaderIndex; + case FunctionDefinition fd: + return fd.HeaderIndex; + default: + return ast.LocationToIndex(scope.Node.GetStart(ast)); + } + } + + public static IScope FindScope(this IScope parent, PythonAst ast, SourceLocation location) { + var children = parent.Children; + var index = ast.LocationToIndex(location); + IScope candidate = null; + + for (var i = 0; i < children.Count; ++i) { + if (children[i].Node is FunctionDefinition fd && fd.IsInParameter(ast, location)) { + // In parameter name scope, so consider the function scope. + candidate = children[i]; + continue; + } + + var start = children[i].GetBodyStartIndex(ast); + if (start > index) { + // We've gone past index completely so our last candidate is + // the best one. + break; + } + + var end = children[i].Node.EndIndex; + if (i + 1 < children.Count) { + var nextStart = children[i + 1].Node.StartIndex; + if (nextStart > end) { + end = nextStart; + } + } + + if (index <= end || (candidate == null && i + 1 == children.Count)) { + candidate = children[i]; + } + } + + if (candidate == null) { + return parent; + } + + var scopeIndent = GetParentScopeIndent(candidate, ast); + if (location.Column <= scopeIndent) { + // Candidate is at deeper indentation than location and the + // candidate is scoped, so return the parent instead. + return parent; + } + + // Recurse to check children of candidate scope + var child = FindScope(candidate, ast, location); + + if (child.Node is FunctionDefinition fd1 && fd1.IsLambda && child.Node.EndIndex < index) { + // Do not want to extend a lambda function's scope to the end of + // the parent scope. + return parent; + } + + return child; + } + + private static int GetParentScopeIndent(IScope scope, PythonAst ast) { + switch (scope.Node) { + case ClassDefinition cd: + // Return column of "class" statement + return cd.GetStart(ast).Column; + case FunctionDefinition fd when !fd.IsLambda: + // Return column of "def" statement + return fd.GetStart(ast).Column; + default: + return -1; + } + } } } diff --git a/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj b/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj index 0d264f7df..805e064e4 100644 --- a/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj +++ b/src/Analysis/Ast/Impl/Microsoft.Python.Analysis.csproj @@ -7,9 +7,8 @@ - 1701;1702;1998;$(NoWarn) + 1701;1702;$(NoWarn) 7.2 diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index 334a68d12..7c2e6d0dd 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -18,6 +18,7 @@ using System.Linq; using Microsoft.Python.Analysis.Specializations; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.IO; using Microsoft.Python.Parsing; @@ -33,7 +34,7 @@ internal sealed class BuiltinsPythonModule : CompiledPythonModule, IBuiltinsPyth private IPythonType _boolType; public BuiltinsPythonModule(string moduleName, string filePath, IServiceContainer services) - : base(moduleName, ModuleType.Builtins, filePath, null, services, ModuleLoadOptions.None) { } // TODO: builtins stub + : base(moduleName, ModuleType.Builtins, filePath, null, services) { } // TODO: builtins stub public override IMember GetMember(string name) => _hiddenNames.Contains(name) ? null : base.GetMember(name); @@ -48,6 +49,9 @@ protected override void OnAnalysisComplete() { lock (AnalysisLock) { SpecializeTypes(); SpecializeFunctions(); + foreach (var n in GetMemberNames()) { + GetMember(n).GetPythonType()?.MakeReadOnly(); + } } } @@ -111,47 +115,47 @@ private void SpecializeTypes() { _hiddenNames.Add("__builtin_module_names__"); if (_boolType != null) { - Analysis.GlobalScope.DeclareVariable("True", _boolType, LocationInfo.Empty); - Analysis.GlobalScope.DeclareVariable("False", _boolType, LocationInfo.Empty); + Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Declaration, LocationInfo.Empty); + Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Declaration, LocationInfo.Empty); } if (noneType != null) { - Analysis.GlobalScope.DeclareVariable("None", noneType, LocationInfo.Empty); + Analysis.GlobalScope.DeclareVariable("None", noneType, VariableSource.Declaration, LocationInfo.Empty); } foreach (var n in GetMemberNames()) { var t = GetMember(n).GetPythonType(); if (t.TypeId == BuiltinTypeId.Unknown && t.MemberType != PythonMemberType.Unknown) { - (t as PythonType)?.TrySetTypeId(BuiltinTypeId.Type); + if (t is PythonType pt) { + pt.TrySetTypeId(BuiltinTypeId.Type); + } } } } private void SpecializeFunctions() { // TODO: deal with commented out functions. - SpecializeFunction("abs", BuiltinsSpecializations.Identity); - SpecializeFunction("cmp", Interpreter.GetBuiltinType(BuiltinTypeId.Int)); - SpecializeFunction("dir", BuiltinsSpecializations.ListOfStrings); - SpecializeFunction("eval", Interpreter.GetBuiltinType(BuiltinTypeId.Object)); - SpecializeFunction("globals", BuiltinsSpecializations.DictStringToObject); - SpecializeFunction(@"isinstance", _boolType); - SpecializeFunction(@"issubclass", _boolType); - SpecializeFunction(@"iter", BuiltinsSpecializations.Iterator); - SpecializeFunction("locals", BuiltinsSpecializations.DictStringToObject); - //SpecializeFunction(_builtinName, "max", ReturnUnionOfInputs); - //SpecializeFunction(_builtinName, "min", ReturnUnionOfInputs); - SpecializeFunction("next", BuiltinsSpecializations.Next); - //SpecializeFunction(_builtinName, "open", SpecialOpen); - SpecializeFunction("ord", Interpreter.GetBuiltinType(BuiltinTypeId.Int)); - SpecializeFunction("pow", BuiltinsSpecializations.Identity); - SpecializeFunction("range", BuiltinsSpecializations.Range); - SpecializeFunction("type", BuiltinsSpecializations.TypeInfo); + Analysis.SpecializeFunction("abs", BuiltinsSpecializations.Identity); + Analysis.SpecializeFunction("cmp", Interpreter.GetBuiltinType(BuiltinTypeId.Int)); + Analysis.SpecializeFunction("dir", BuiltinsSpecializations.ListOfStrings); + Analysis.SpecializeFunction("eval", Interpreter.GetBuiltinType(BuiltinTypeId.Object)); + Analysis.SpecializeFunction("globals", BuiltinsSpecializations.DictStringToObject); + Analysis.SpecializeFunction(@"isinstance", _boolType); + Analysis.SpecializeFunction(@"issubclass", _boolType); + Analysis.SpecializeFunction(@"iter", BuiltinsSpecializations.Iterator); + Analysis.SpecializeFunction("locals", BuiltinsSpecializations.DictStringToObject); + Analysis.SpecializeFunction("next", BuiltinsSpecializations.Next); + Analysis.SpecializeFunction("open", BuiltinsSpecializations.Open, new[] { "io" }); + Analysis.SpecializeFunction("ord", Interpreter.GetBuiltinType(BuiltinTypeId.Int)); + Analysis.SpecializeFunction("pow", BuiltinsSpecializations.Identity); + Analysis.SpecializeFunction("range", BuiltinsSpecializations.Range); + Analysis.SpecializeFunction("type", BuiltinsSpecializations.TypeInfo); //SpecializeFunction(_builtinName, "range", RangeConstructor); //SpecializeFunction(_builtinName, "sorted", ReturnsListOfInputIterable); - SpecializeFunction("sum", BuiltinsSpecializations.CollectionItem); + Analysis.SpecializeFunction("sum", BuiltinsSpecializations.CollectionItem); //SpecializeFunction(_builtinName, "super", SpecialSuper); - SpecializeFunction("vars", BuiltinsSpecializations.DictStringToObject); + Analysis.SpecializeFunction("vars", BuiltinsSpecializations.DictStringToObject); } } } diff --git a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs index acfd75cba..318cce697 100644 --- a/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/CompiledPythonModule.cs @@ -25,9 +25,8 @@ namespace Microsoft.Python.Analysis.Modules { internal class CompiledPythonModule : PythonModule { protected IModuleCache ModuleCache => Interpreter.ModuleResolution.ModuleCache; - public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, - IServiceContainer services, ModuleLoadOptions options = ModuleLoadOptions.Analyze) - : base(moduleName, filePath, moduleType, options, stub, services) { } + public CompiledPythonModule(string moduleName, ModuleType moduleType, string filePath, IPythonModule stub, IServiceContainer services) + : base(moduleName, filePath, moduleType, stub, services) { } public override string Documentation => GetMember("__doc__").TryGetConstant(out var s) ? s : string.Empty; @@ -52,18 +51,16 @@ protected virtual IEnumerable GetScrapeArguments(IPythonInterpreter inte return args; } - protected override string LoadContent(ModuleLoadOptions options) { - var code = string.Empty; - if ((options & ModuleLoadOptions.Load) == ModuleLoadOptions.Load) { - code = ModuleCache.ReadCachedModule(FilePath); - if (string.IsNullOrEmpty(code)) { - if (!FileSystem.FileExists(Interpreter.Configuration.InterpreterPath)) { - return string.Empty; - } - - code = ScrapeModule(); - SaveCachedCode(code); + protected override string LoadContent() { + // Exceptions are handled in the base + var code = ModuleCache.ReadCachedModule(FilePath); + if (string.IsNullOrEmpty(code)) { + if (!FileSystem.FileExists(Interpreter.Configuration.InterpreterPath)) { + return string.Empty; } + + code = ScrapeModule(); + SaveCachedCode(code); } return code; } @@ -84,21 +81,21 @@ private string ScrapeModule() { )) { proc.StartInfo.StandardOutputEncoding = Encoding.UTF8; proc.OnOutputLine = s => sb.AppendLine(s); - proc.OnErrorLine = s => Log?.Log(TraceEventType.Error, "Scrape", s); + proc.OnErrorLine = s => Log?.Log(TraceEventType.Warning, "Scrape", s); - Log?.Log(TraceEventType.Information, "Scrape", proc.FileName, proc.Arguments); + Log?.Log(TraceEventType.Verbose, "Scrape", proc.FileName, proc.Arguments); proc.Start(); var exitCode = proc.Wait(60000); if (exitCode == null) { proc.Kill(); - Log?.Log(TraceEventType.Error, "ScrapeTimeout", proc.FileName, proc.Arguments); + Log?.Log(TraceEventType.Warning, "ScrapeTimeout", proc.FileName, proc.Arguments); return string.Empty; } if (exitCode != 0) { - Log?.Log(TraceEventType.Error, "Scrape", "ExitCode", exitCode); + Log?.Log(TraceEventType.Warning, "Scrape", "ExitCode", exitCode); return string.Empty; } } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs new file mode 100644 index 000000000..a5997bc29 --- /dev/null +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs @@ -0,0 +1,56 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading; +using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Analysis.Types; + +namespace Microsoft.Python.Analysis.Modules { + /// + /// Represents module resolution and search subsystem. + /// + public interface IModuleManagement: IModuleResolution { + /// + /// Locates module by path. + /// + /// + /// + ModulePath FindModule(string filePath); + + IReadOnlyCollection GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken = default); + + IModuleCache ModuleCache { get; } + + void AddModulePath(string path); + + /// + /// Provides ability to specialize module by replacing module import by + /// implementation in code. Real module + /// content is loaded and analyzed only for class/functions definitions + /// so the original documentation can be extracted. + /// + /// Module to specialize. + /// Specialized module constructor. + /// Original (library) module loaded as stub, if any. + IPythonModule SpecializeModule(string name, Func specializationConstructor); + + /// + /// Returns specialized module, if any. + /// + IPythonModule GetSpecializedModule(string name); + } +} diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs index c896bc270..950506809 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IModuleResolution.cs @@ -13,27 +13,25 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Core.DependencyResolution; -using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Types; namespace Microsoft.Python.Analysis.Modules { + /// + /// Represents basic module resolution and search subsystem. + /// public interface IModuleResolution { + /// + /// Builtins module name. + /// string BuiltinModuleName { get; } - Task> GetSearchPathsAsync(CancellationToken cancellationToken = default); - Task> GetImportableModulesAsync(CancellationToken cancellationToken = default); - Task> GetImportableModulesAsync(IEnumerable searchPaths, CancellationToken cancellationToken = default); - ModulePath FindModule(string filePath); - IReadOnlyCollection GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken = default); /// - /// Determines if directory contains Python package. + /// Builtins module. /// - bool IsPackage(string directory); + IBuiltinsPythonModule BuiltinsModule { get; } /// /// Path resolver providing file resolution in module imports. @@ -47,31 +45,11 @@ public interface IModuleResolution { Task ImportModuleAsync(string name, CancellationToken cancellationToken = default); /// - /// Builtins module. + /// Returns an IPythonModule for a given module name. Returns null if + /// the module has not been imported. /// - IBuiltinsPythonModule BuiltinsModule { get; } - - IModuleCache ModuleCache { get; } + IPythonModule GetImportedModule(string name); Task ReloadAsync(CancellationToken token = default); - - void AddModulePath(string path); - - /// - /// Provides ability to specialize module by replacing module import by - /// implementation in code. Real module - /// content is loaded and analyzed only for class/functions definitions - /// so the original documentation can be extracted. - /// - /// Module to specialize. - /// Specialized module constructor. - /// Cancellation token. - /// Original (library) module loaded as stub. - Task SpecializeModuleAsync(string name, Func specializationConstructor, CancellationToken cancellationToken = default); - - /// - /// Returns specialized module, if any. - /// - IPythonModule GetSpecializedModule(string name); } } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs index 7731b29be..b49e555dc 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/ModuleCreationOptions.cs @@ -47,10 +47,5 @@ public sealed class ModuleCreationOptions { /// Module stub, if any. /// public IPythonModule Stub { get; set; } - - /// - /// Module loading options. - /// - public ModuleLoadOptions LoadOptions { get; set; } = ModuleLoadOptions.Analyze; } } diff --git a/src/Analysis/Ast/Impl/Modules/FallbackBuiltinModule.cs b/src/Analysis/Ast/Impl/Modules/FallbackBuiltinModule.cs deleted file mode 100644 index 6d54c2092..000000000 --- a/src/Analysis/Ast/Impl/Modules/FallbackBuiltinModule.cs +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Parsing; - -namespace Microsoft.Python.Analysis.Modules { - internal sealed class FallbackBuiltinsModule : PythonModule, IBuiltinsPythonModule { - public readonly PythonLanguageVersion LanguageVersion; - private readonly Dictionary _cachedInstances; - - public FallbackBuiltinsModule(PythonLanguageVersion version) - : base(BuiltinTypeId.Unknown.GetModuleName(version), ModuleType.Builtins, null) { - LanguageVersion = version; - _cachedInstances = new Dictionary(); - } - - private IPythonType GetOrCreate(BuiltinTypeId typeId) { - if (typeId.IsVirtualId()) { - switch (typeId) { - case BuiltinTypeId.Str: - typeId = BuiltinTypeId.Str; - break; - case BuiltinTypeId.StrIterator: - typeId = BuiltinTypeId.StrIterator; - break; - default: - typeId = BuiltinTypeId.Unknown; - break; - } - } - - lock (_cachedInstances) { - if (!_cachedInstances.TryGetValue(typeId, out var value)) { - _cachedInstances[typeId] = value = new FallbackBuiltinPythonType(this, typeId); - } - return value; - } - } - - public IMember GetAnyMember(string name) { - foreach (BuiltinTypeId typeId in Enum.GetValues(typeof(BuiltinTypeId))) { - if (typeId.GetTypeName(LanguageVersion) == name) { - return GetOrCreate(typeId); - } - } - return GetOrCreate(BuiltinTypeId.Unknown); - } - } - - class FallbackBuiltinPythonType : PythonType { - public FallbackBuiltinPythonType(FallbackBuiltinsModule declaringModule, BuiltinTypeId typeId) : - base(typeId.GetModuleName(declaringModule.LanguageVersion), declaringModule, declaringModule.Documentation, null) { - TypeId = typeId; - } - - public override PythonMemberType MemberType => PythonMemberType.Class; - public override BuiltinTypeId TypeId { get; } - } -} diff --git a/src/Analysis/Ast/Impl/Modules/ModuleCache.cs b/src/Analysis/Ast/Impl/Modules/ModuleCache.cs index b9d19afb0..e4c30565d 100644 --- a/src/Analysis/Ast/Impl/Modules/ModuleCache.cs +++ b/src/Analysis/Ast/Impl/Modules/ModuleCache.cs @@ -34,15 +34,15 @@ internal sealed class ModuleCache : IModuleCache { private readonly bool _skipCache; private bool _loggedBadDbPath; - private string ModuleCachePath => _interpreter.Configuration.ModuleCachePath; + private string ModuleCachePath => _interpreter.Configuration.DatabasePath; public ModuleCache(IPythonInterpreter interpreter, IServiceContainer services) { _interpreter = interpreter; _services = services; _fs = services.GetService(); _log = services.GetService(); - _skipCache = string.IsNullOrEmpty(_interpreter.Configuration.ModuleCachePath); - SearchPathCachePath = Path.Combine(_interpreter.Configuration.ModuleCachePath, "database.path"); + _skipCache = string.IsNullOrEmpty(_interpreter.Configuration.DatabasePath); + SearchPathCachePath = Path.Combine(_interpreter.Configuration.DatabasePath, "database.path"); } public string SearchPathCachePath { get; } @@ -65,14 +65,13 @@ public async Task ImportFromCacheAsync(string name, CancellationToken var rdt = _services.GetService(); var mco = new ModuleCreationOptions { - ModuleName = name, + ModuleName = name, ModuleType = ModuleType.Stub, - FilePath = cache, - LoadOptions = ModuleLoadOptions.Analyze + FilePath = cache }; var module = rdt.AddModule(mco); - await module.LoadAndAnalyzeAsync(cancellationToken).ConfigureAwait(false); + await module.LoadAndAnalyzeAsync(cancellationToken); return module; } @@ -119,32 +118,51 @@ public string ReadCachedModule(string filePath) { return string.Empty; } - var path = GetCacheFilePath(filePath); - if (string.IsNullOrEmpty(path)) { + var cachePath = GetCacheFilePath(filePath); + if (string.IsNullOrEmpty(cachePath)) { return string.Empty; } - var fileIsOkay = false; + var cachedFileExists = false; + var cachedFileOlderThanAssembly = false; + var cachedFileOlderThanSource = false; + Exception exception = null; + try { - var cacheTime = _fs.GetLastWriteTimeUtc(path); - var sourceTime = _fs.GetLastWriteTimeUtc(filePath); - if (sourceTime <= cacheTime) { - var assemblyTime = _fs.GetLastWriteTimeUtc(GetType().Assembly.Location); - if (assemblyTime <= cacheTime) { - fileIsOkay = true; + cachedFileExists = _fs.FileExists(cachePath); + if (cachedFileExists) { + // Source path is fake for scraped/compiled modules. + // The time will be very old, which is good. + var sourceTime = _fs.GetLastWriteTimeUtc(filePath); + var cacheTime = _fs.GetLastWriteTimeUtc(cachePath); + + cachedFileOlderThanSource = cacheTime < sourceTime; + if (!cachedFileOlderThanSource) { + var assemblyTime = _fs.GetLastWriteTimeUtc(GetType().Assembly.Location); + if (assemblyTime > cacheTime) { + cachedFileOlderThanAssembly = true; + } else { + return _fs.ReadAllText(cachePath); + } } } } catch (Exception ex) when (!ex.IsCriticalException()) { + exception = ex; } - if (fileIsOkay) { - try { - return _fs.ReadAllText(filePath); - } catch (IOException) { } catch (UnauthorizedAccessException) { } + var reason = "Unknown"; + if (!cachedFileExists) { + reason = "Cached file does not exist"; + } else if (cachedFileOlderThanAssembly) { + reason = "Cached file is older than the assembly."; + } else if (cachedFileOlderThanSource) { + reason = $"Cached file is older than the source {filePath}."; + } else { + reason = $"Exception during cache file check {exception.Message}."; } - _log?.Log(TraceEventType.Verbose, "Invalidate cached module", path); - _fs.DeleteFileWithRetries(path); + _log?.Log(TraceEventType.Verbose, $"Invalidate cached module {cachePath}. Reason: {reason}"); + _fs.DeleteFileWithRetries(cachePath); return string.Empty; } @@ -152,8 +170,13 @@ public void WriteCachedModule(string filePath, string code) { var cache = GetCacheFilePath(filePath); if (!string.IsNullOrEmpty(cache)) { _log?.Log(TraceEventType.Verbose, "Write cached module: ", cache); - _fs.WriteTextWithRetry(cache, code); + // Don't block analysis on cache writes. + CacheWritingTask = Task.Run(() => _fs.WriteTextWithRetry(cache, code)); + CacheWritingTask.DoNotWait(); } } + + // For tests synchronization + internal Task CacheWritingTask { get; private set; } = Task.CompletedTask; } } diff --git a/src/Analysis/Ast/Impl/Modules/ModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/ModuleResolution.cs deleted file mode 100644 index 9128af8e3..000000000 --- a/src/Analysis/Ast/Impl/Modules/ModuleResolution.cs +++ /dev/null @@ -1,475 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Analysis.Core.DependencyResolution; -using Microsoft.Python.Analysis.Core.Interpreter; -using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Specializations; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core; -using Microsoft.Python.Core.IO; -using Microsoft.Python.Core.Logging; - -namespace Microsoft.Python.Analysis.Modules { - internal sealed class ModuleResolution : IModuleResolution { - private static readonly IReadOnlyDictionary _emptyModuleSet = EmptyDictionary.Instance; - private readonly ConcurrentDictionary _modules = new ConcurrentDictionary(); - private readonly IReadOnlyList _typeStubPaths; - private readonly IServiceContainer _services; - private readonly IPythonInterpreter _interpreter; - private readonly IFileSystem _fs; - private readonly ILogger _log; - private readonly bool _requireInitPy; - private readonly string _root; - - private PathResolver _pathResolver; - private IReadOnlyDictionary _searchPathPackages; - private IReadOnlyList _searchPaths; - - private InterpreterConfiguration Configuration => _interpreter.Configuration; - - public ModuleResolution(string root, IServiceContainer services) { - _root = root; - _services = services; - _interpreter = services.GetService(); - _fs = services.GetService(); - _log = services.GetService(); - - _requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_interpreter.Configuration.Version); - // TODO: merge with user-provided stub paths - _typeStubPaths = GetTypeShedPaths(_interpreter.Configuration?.TypeshedPath).ToArray(); - } - - internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken = default) { - // Add names from search paths - await ReloadAsync(cancellationToken); - - // Initialize built-in - var moduleName = BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion); - var modulePath = ModuleCache.GetCacheFilePath(_interpreter.Configuration.InterpreterPath ?? "python.exe"); - - var b = new BuiltinsPythonModule(moduleName, modulePath, _services); - _modules[BuiltinModuleName] = BuiltinsModule = b; - await b.LoadAndAnalyzeAsync(cancellationToken); - - // Add built-in module names - var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__"); - if (builtinModuleNamesMember.TryGetConstant(out var s)) { - var builtinModuleNames = s.Split(',').Select(n => n.Trim()); - _pathResolver.SetBuiltins(builtinModuleNames); - } - } - - public IModuleCache ModuleCache { get; private set; } - public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion); - - /// - /// Path resolver providing file resolution in module imports. - /// - public PathResolverSnapshot CurrentPathResolver => _pathResolver.CurrentSnapshot; - - /// - /// Builtins module. - /// - public IBuiltinsPythonModule BuiltinsModule { get; private set; } - - public async Task> GetImportableModulesAsync(CancellationToken cancellationToken) { - if (_searchPathPackages != null) { - return _searchPathPackages; - } - - var packageDict = await GetImportableModulesAsync(Configuration.SearchPaths, cancellationToken).ConfigureAwait(false); - if (!packageDict.Any()) { - return _emptyModuleSet; - } - - _searchPathPackages = packageDict; - return packageDict; - } - - public async Task> GetSearchPathsAsync(CancellationToken cancellationToken = default) { - if (_searchPaths != null) { - return _searchPaths; - } - - _searchPaths = await GetInterpreterSearchPathsAsync(cancellationToken).ConfigureAwait(false); - Debug.Assert(_searchPaths != null, "Should have search paths"); - _searchPaths = _searchPaths.Concat(Configuration.SearchPaths ?? Array.Empty()).ToArray(); - _log?.Log(TraceEventType.Information, "SearchPaths", _searchPaths.Cast().ToArray()); - return _searchPaths; - } - - public async Task> GetImportableModulesAsync(IEnumerable searchPaths, CancellationToken cancellationToken = default) { - var packageDict = new Dictionary(); - - foreach (var searchPath in searchPaths.MaybeEnumerate()) { - IReadOnlyCollection packages = null; - if (_fs.FileExists(searchPath)) { - packages = GetPackagesFromZipFile(searchPath, cancellationToken); - } else if (_fs.DirectoryExists(searchPath)) { - packages = await Task.Run(() - => GetPackagesFromDirectory(searchPath, cancellationToken), cancellationToken).ConfigureAwait(false); - } - foreach (var package in packages.MaybeEnumerate()) { - cancellationToken.ThrowIfCancellationRequested(); - packageDict[package] = searchPath; - } - } - - return packageDict; - } - - private async Task> GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default) { - if (!_fs.FileExists(Configuration.InterpreterPath)) { - return Array.Empty(); - } - - _log?.Log(TraceEventType.Information, "GetCurrentSearchPaths", Configuration.InterpreterPath, ModuleCache.SearchPathCachePath); - try { - var paths = await PythonLibraryPath.GetDatabaseSearchPathsAsync(Configuration, ModuleCache.SearchPathCachePath).ConfigureAwait(false); - cancellationToken.ThrowIfCancellationRequested(); - return paths.MaybeEnumerate().Select(p => p.Path).ToArray(); - } catch (InvalidOperationException) { - return Array.Empty(); - } - } - - private async Task TryImportModuleAsync(string name, CancellationToken cancellationToken = default) { - if (string.IsNullOrEmpty(name)) { - return TryImportModuleResult.ModuleNotFound; - } - if (name == BuiltinModuleName) { - return new TryImportModuleResult(BuiltinsModule); - } - - Debug.Assert(!name.EndsWithOrdinal("."), $"{name} should not end with '.'"); - // Return any existing module - if (_modules.TryGetValue(name, out var module) && module != null) { - if (module is SentinelModule) { - // TODO: we can't just wait here or we hang. There are two cases: - // a. Recursion on the same analysis chain (A -> B -> A) - // b. Call from another chain (A -> B -> C and D -> B -> E). - // TODO: Both should be resolved at the dependency chain level. - _log?.Log(TraceEventType.Warning, $"Recursive import: {name}"); - } - return new TryImportModuleResult(module); - } - - // Set up a sentinel so we can detect recursive imports - var sentinelValue = new SentinelModule(name, _services); - if (!_modules.TryAdd(name, sentinelValue)) { - // Try to get the new module, in case we raced with a .Clear() - if (_modules.TryGetValue(name, out module) && !(module is SentinelModule)) { - return new TryImportModuleResult(module); - } - // If we reach here, the race is too complicated to recover - // from. Signal the caller to try importing again. - _log?.Log(TraceEventType.Warning, $"Retry import: {name}"); - return TryImportModuleResult.NeedRetry; - } - - // Do normal searches - try { - module = await ImportFromSearchPathsAsync(name, cancellationToken).ConfigureAwait(false); - } catch (OperationCanceledException) { - _log?.Log(TraceEventType.Error, $"Import timeout {name}"); - return TryImportModuleResult.Timeout; - } - - module = module ?? await ModuleCache.ImportFromCacheAsync(name, cancellationToken); - - // Replace our sentinel - if (!_modules.TryUpdate(name, module, sentinelValue)) { - // Try to get the new module, in case we raced - if (_modules.TryGetValue(name, out module) && !(module is SentinelModule)) { - return new TryImportModuleResult(module); - } - // If we reach here, the race is too complicated to recover - // from. Signal the caller to try importing again. - _log?.Log(TraceEventType.Warning, $"Retry import: {name}"); - return TryImportModuleResult.NeedRetry; - } - - return new TryImportModuleResult(module); - } - - public IReadOnlyCollection GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken) { - return ModulePath.GetModulesInPath( - searchPath, - recurse: false, - includePackages: true, - requireInitPy: _requireInitPy - ).Select(mp => mp.ModuleName).Where(n => !string.IsNullOrEmpty(n)).TakeWhile(_ => !cancellationToken.IsCancellationRequested).ToList(); - } - - public async Task ImportModuleAsync(string name, CancellationToken cancellationToken = default) { - if (name == BuiltinModuleName) { - return BuiltinsModule; - } - - for (var retries = 5; retries > 0; --retries) { - cancellationToken.ThrowIfCancellationRequested(); - - // The call should be cancelled by the cancellation token, but since we - // are blocking here we wait for slightly longer. Timeouts are handled - // gracefully by TryImportModuleAsync(), so we want those to trigger if - // possible, but if all else fails then we'll abort and treat it as an - // error. - // (And if we've got a debugger attached, don't time out at all.) - TryImportModuleResult result; - try { - result = await TryImportModuleAsync(name, cancellationToken); - } catch (OperationCanceledException) { - _log?.Log(TraceEventType.Error, $"Import timeout: {name}"); - Debug.Fail("Import timeout"); - return null; - } - - switch (result.Status) { - case TryImportModuleResultCode.Success: - return result.Module; - case TryImportModuleResultCode.ModuleNotFound: - _log?.Log(TraceEventType.Information, $"Import not found: {name}"); - return null; - case TryImportModuleResultCode.NeedRetry: - case TryImportModuleResultCode.Timeout: - break; - case TryImportModuleResultCode.NotSupported: - _log?.Log(TraceEventType.Error, $"Import not supported: {name}"); - return null; - } - } - // Never succeeded, so just log the error and fail - _log?.Log(TraceEventType.Error, $"Retry import failed: {name}"); - return null; - } - - public async Task ReloadAsync(CancellationToken cancellationToken = default) { - ModuleCache = new ModuleCache(_interpreter, _services); - - _pathResolver = new PathResolver(_interpreter.LanguageVersion); - - var addedRoots = _pathResolver.SetRoot(_root); - ReloadModulePaths(addedRoots); - - var interpreterPaths = await GetSearchPathsAsync(cancellationToken); - addedRoots = _pathResolver.SetInterpreterSearchPaths(interpreterPaths); - ReloadModulePaths(addedRoots); - - addedRoots = _pathResolver.SetUserSearchPaths(_interpreter.Configuration.SearchPaths); - ReloadModulePaths(addedRoots); - } - - public void AddModulePath(string path) => _pathResolver.TryAddModulePath(path, out var _); - - /// - /// Determines whether the specified directory is an importable package. - /// - public bool IsPackage(string directory) - => ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version) ? - !string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) : - _fs.DirectoryExists(directory); - - public ModulePath FindModule(string filePath) { - var bestLibraryPath = string.Empty; - - foreach (var p in Configuration.SearchPaths) { - if (PathEqualityComparer.Instance.StartsWith(filePath, p)) { - if (p.Length > bestLibraryPath.Length) { - bestLibraryPath = p; - } - } - } - - var mp = ModulePath.FromFullPath(filePath, bestLibraryPath); - return mp; - } - - /// - /// Provides ability to specialize module by replacing module import by - /// implementation in code. Real module - /// content is loaded and analyzed only for class/functions definitions - /// so the original documentation can be extracted. - /// - /// Module to specialize. - /// Specialized module constructor. - /// Cancellation token. - /// Original (library) module loaded as stub. - public async Task SpecializeModuleAsync(string name, Func specializationConstructor, CancellationToken cancellationToken = default) { - var import = CurrentPathResolver.GetModuleImportFromModuleName(name); - if (!string.IsNullOrEmpty(import?.ModulePath)) { - var module = specializationConstructor(import.ModulePath); - _modules[name] = module; - await module.LoadAndAnalyzeAsync(cancellationToken); - return module; - } - return null; - } - - /// - /// Returns specialized module, if any. - /// - public IPythonModule GetSpecializedModule(string name) - => _modules.TryGetValue(name, out var m) && m is SpecializedModule ? m : null; - - private async Task ImportFromSearchPathsAsync(string name, CancellationToken cancellationToken) { - var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name); - if (moduleImport == null) { - _log?.Log(TraceEventType.Verbose, "Import not found: ", name); - return null; - } - // If there is a stub, make sure it is loaded and attached - var stub = await ImportFromTypeStubsAsync(moduleImport.IsBuiltin ? name : moduleImport.FullName, cancellationToken); - IPythonModule module; - - if (moduleImport.IsBuiltin) { - _log?.Log(TraceEventType.Verbose, "Import built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); - module = new CompiledBuiltinPythonModule(name, stub, _services); - } else if (moduleImport.IsCompiled) { - _log?.Log(TraceEventType.Verbose, "Import compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath); - module = new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, _services); - } else { - _log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); - var rdt = _services.GetService(); - // TODO: handle user code and library module separately. - var mco = new ModuleCreationOptions { - ModuleName = moduleImport.FullName, - ModuleType = ModuleType.Library, - FilePath = moduleImport.ModulePath, - Stub = stub, - LoadOptions = ModuleLoadOptions.Analyze - }; - module = rdt.AddModule(mco); - } - - await module.LoadAndAnalyzeAsync(cancellationToken).ConfigureAwait(false); - return module; - } - - private async Task ImportFromTypeStubsAsync(string name, CancellationToken cancellationToken = default) { - var mp = FindModuleInSearchPath(_typeStubPaths, null, name); - if (mp != null) { - if (mp.Value.IsCompiled) { - _log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", mp.Value.FullName, mp.Value.SourceFile); - return null; - } - return await CreateStubModuleAsync(mp.Value.FullName, mp.Value.SourceFile, cancellationToken); - } - - var i = name.IndexOf('.'); - if (i == 0) { - Debug.Fail("Invalid module name"); - return null; - } - - var stubPath = CurrentPathResolver.GetPossibleModuleStubPaths(name).FirstOrDefault(p => _fs.FileExists(p)); - return stubPath != null ? await CreateStubModuleAsync(name, stubPath, cancellationToken) : null; - } - - private async Task CreateStubModuleAsync(string moduleName, string filePath, CancellationToken cancellationToken = default) { - _log?.Log(TraceEventType.Verbose, "Import type stub", moduleName, filePath); - var module = new StubPythonModule(moduleName, filePath, _services); - await module.LoadAndAnalyzeAsync(cancellationToken); - return module; - } - - private ModulePath? FindModuleInSearchPath(IReadOnlyList searchPaths, IReadOnlyDictionary packages, string name) { - if (searchPaths == null || searchPaths.Count == 0) { - return null; - } - - _log?.Log(TraceEventType.Verbose, "FindModule", name, "system", string.Join(", ", searchPaths)); - - var i = name.IndexOf('.'); - var firstBit = i < 0 ? name : name.Remove(i); - - ModulePath mp; - Func isPackage = IsPackage; - if (firstBit.EndsWithOrdinal("-stubs", ignoreCase: true)) { - isPackage = _fs.DirectoryExists; - } - - var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version); - if (packages != null && packages.TryGetValue(firstBit, out var searchPath) && !string.IsNullOrEmpty(searchPath)) { - if (ModulePath.FromBasePathAndName_NoThrow(searchPath, name, isPackage, null, requireInitPy, out mp, out _, out _, out _)) { - return mp; - } - } - - foreach (var sp in searchPaths.MaybeEnumerate()) { - if (ModulePath.FromBasePathAndName_NoThrow(sp, name, isPackage, null, requireInitPy, out mp, out _, out _, out _)) { - return mp; - } - } - - return null; - } - - private async Task> GetModuleStubsAsync(string moduleName, string modulePath, CancellationToken cancellationToken = default) { - cancellationToken.ThrowIfCancellationRequested(); - - // Also search for type stub packages if enabled and we are not a blacklisted module - if (_typeStubPaths.Count > 0 && moduleName != "typing") { - var tsModule = await ImportFromTypeStubsAsync(moduleName, cancellationToken); - if (tsModule != null) { - // TODO: What about custom stub files? - return new[] { tsModule }; - } - } - return Array.Empty(); - } - - private IEnumerable GetTypeShedPaths(string typeshedRootPath) { - if (string.IsNullOrEmpty(typeshedRootPath)) { - yield break; - } - - var stdlib = Path.Combine(typeshedRootPath, "stdlib"); - var thirdParty = Path.Combine(typeshedRootPath, "third_party"); - - var v = Configuration.Version; - foreach (var subdir in new[] { v.ToString(), v.Major.ToString(), "2and3" }) { - yield return Path.Combine(stdlib, subdir); - } - - foreach (var subdir in new[] { v.ToString(), v.Major.ToString(), "2and3" }) { - yield return Path.Combine(thirdParty, subdir); - } - } - - private static IReadOnlyCollection GetPackagesFromZipFile(string searchPath, CancellationToken cancellationToken) { - // TODO: Search zip files for packages - return new string[0]; - } - - private void ReloadModulePaths(in IEnumerable rootPaths) { - foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(p))) { - _pathResolver.TryAddModulePath(modulePath, out _); - } - } - - // For tests - internal void AddUnimportableModule(string moduleName) - => _modules[moduleName] = new SentinelModule(moduleName, _services); - } -} diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 60bc74cae..05192c8c5 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -24,7 +24,7 @@ using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Extensions; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -41,27 +41,35 @@ namespace Microsoft.Python.Analysis.Modules { /// to AST and the module analysis. /// [DebuggerDisplay("{Name} : {ModuleType}")] - public class PythonModule : IDocument, IAnalyzable, IDisposable, IEquatable { + public class PythonModule : IDocument, IAnalyzable, IEquatable { + protected enum State { + None, + Loading, + Loaded, + Parsing, + Parsed, + Analyzing, + Analyzed + } + + private readonly AsyncLocal _awaiting = new AsyncLocal(); private readonly DocumentBuffer _buffer = new DocumentBuffer(); private readonly CancellationTokenSource _allProcessingCts = new CancellationTokenSource(); private IReadOnlyList _parseErrors = Array.Empty(); - private ModuleLoadOptions _options; - private string _documentation = string.Empty; - + private string _documentation; // Must be null initially. private TaskCompletionSource _analysisTcs; private CancellationTokenSource _linkedAnalysisCts; // cancellation token combined with the 'dispose' cts private CancellationTokenSource _parseCts; private CancellationTokenSource _linkedParseCts; // combined with 'dispose' cts private Task _parsingTask; private PythonAst _ast; - private bool _loaded; protected ILogger Log { get; } protected IFileSystem FileSystem { get; } protected IServiceContainer Services { get; } - protected IDocumentAnalysis Analysis { get; private set; } = DocumentAnalysis.Empty; protected object AnalysisLock { get; } = new object(); + protected State ContentState { get; set; } = State.None; protected PythonModule(string name, ModuleType moduleType, IServiceContainer services) { Check.ArgumentNotNull(nameof(name), name); @@ -71,15 +79,15 @@ protected PythonModule(string name, ModuleType moduleType, IServiceContainer ser Log = services?.GetService(); Interpreter = services?.GetService(); + Analysis = new EmptyAnalysis(services, this); } - protected PythonModule(string moduleName, string filePath, ModuleType moduleType, ModuleLoadOptions loadOptions, IPythonModule stub, IServiceContainer services) : + protected PythonModule(string moduleName, string filePath, ModuleType moduleType, IPythonModule stub, IServiceContainer services) : this(new ModuleCreationOptions { ModuleName = moduleName, FilePath = filePath, ModuleType = moduleType, - Stub = stub, - LoadOptions = loadOptions + Stub = stub }, services) { } internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer services) @@ -97,7 +105,10 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s FilePath = creationOptions.FilePath ?? uri?.LocalPath; Stub = creationOptions.Stub; - InitializeContent(creationOptions.Content, creationOptions.LoadOptions); + if (ModuleType == ModuleType.Specialized || ModuleType == ModuleType.Unresolved) { + ContentState = State.Analyzed; + } + InitializeContent(creationOptions.Content); } #region IPythonType @@ -135,7 +146,25 @@ public virtual string Documentation { #region IMemberContainer public virtual IMember GetMember(string name) => Analysis.GlobalScope.Variables[name]?.Value; - public virtual IEnumerable GetMemberNames() => Analysis.GlobalScope.Variables.Names; + public virtual IEnumerable GetMemberNames() { + // Try __all__ since it contains exported members + var all = Analysis.GlobalScope.Variables["__all__"]; + if (all?.Value is IPythonCollection collection) { + return collection.Contents + .OfType() + .Select(c => c.GetString()) + .ExcludeDefault() + .Where(s => !string.IsNullOrEmpty(s)); + } + + // __all__ is not declared. Try filtering by origin: + // drop imported modules and generics. + return Analysis.GlobalScope.Variables + .Where(v => v.Value?.MemberType != PythonMemberType.Generic + && !(v.Value?.GetPythonType() is PythonModule) + && !(v.Value?.GetPythonType().DeclaringModule is TypingModule && !(this is TypingModule))) + .Select(v => v.Name); + } #endregion #region IPythonFile @@ -144,6 +173,8 @@ public virtual string Documentation { #endregion #region IPythonModule + public IDocumentAnalysis Analysis { get; private set; } + public IPythonInterpreter Interpreter { get; } /// @@ -164,36 +195,31 @@ public virtual string Documentation { /// analysis until later time, when module members are actually needed. /// public virtual Task LoadAndAnalyzeAsync(CancellationToken cancellationToken = default) { - InitializeContent(null, ModuleLoadOptions.Analyze); + if (_awaiting.Value) { + return Task.FromResult(Analysis); + } + _awaiting.Value = true; + InitializeContent(null); return GetAnalysisAsync(cancellationToken); } - protected virtual string LoadContent(ModuleLoadOptions options) { - if (options.ShouldLoad() && ModuleType != ModuleType.Unresolved) { - return FileSystem.ReadAllText(FilePath); + protected virtual string LoadContent() { + if (ContentState < State.Loading) { + ContentState = State.Loading; + try { + var code = FileSystem.ReadAllText(FilePath); + ContentState = State.Loaded; + return code; + } catch (IOException) { } catch (UnauthorizedAccessException) { } } return null; // Keep content as null so module can be loaded later. } - private void InitializeContent(string content, ModuleLoadOptions newOptions) { + private void InitializeContent(string content) { lock (AnalysisLock) { - if (!_loaded) { - if (!newOptions.ShouldLoad()) { - return; - } - content = content ?? LoadContent(newOptions); - _buffer.Reset(0, content); - _loaded = true; - } - - IsOpen = (newOptions & ModuleLoadOptions.Open) == ModuleLoadOptions.Open; - newOptions = newOptions | (IsOpen ? ModuleLoadOptions.Analyze : 0); - - var change = (_options ^ newOptions); - var startAnalysis = change.ShouldAnalyze() && _analysisTcs?.Task == null; - var startParse = change.ShouldParse() && _parsingTask == null; - - _options = newOptions; + LoadContent(content); + var startParse = ContentState < State.Parsing && _parsingTask == null; + var startAnalysis = startParse | (ContentState < State.Analyzing && _analysisTcs?.Task == null); if (startAnalysis) { _analysisTcs = new TaskCompletionSource(); @@ -204,6 +230,16 @@ private void InitializeContent(string content, ModuleLoadOptions newOptions) { } } } + + private void LoadContent(string content) { + if (ContentState < State.Loading) { + try { + content = content ?? LoadContent(); + _buffer.Reset(0, content); + ContentState = State.Loaded; + } catch (IOException) { } catch (UnauthorizedAccessException) { } + } + } #endregion #region ILocatedMember @@ -250,11 +286,16 @@ protected virtual void Dispose(bool disposing) { /// public async Task GetAstAsync(CancellationToken cancellationToken = default) { Task t = null; - while (t != _parsingTask) { - cancellationToken.ThrowIfCancellationRequested(); - t = _parsingTask; + while (true) { + lock (AnalysisLock) { + if(t == _parsingTask) { + break; + } + cancellationToken.ThrowIfCancellationRequested(); + t = _parsingTask; + } try { - await t; + await (t ?? Task.CompletedTask); break; } catch (OperationCanceledException) { // Parsing as canceled, try next task. @@ -271,7 +312,7 @@ public async Task GetAstAsync(CancellationToken cancellationToken = d /// public IEnumerable GetParseErrors() => _parseErrors.ToArray(); - public void Update(IEnumerable changes) { + public void Update(IEnumerable changes) { lock (AnalysisLock) { ExpectedAnalysisVersion++; @@ -289,11 +330,24 @@ public void Update(IEnumerable changes) { } } + public void Reset(string content) { + lock (AnalysisLock) { + if (content != Content) { + InitializeContent(content); + } + } + } + private void Parse() { + _awaiting.Value = false; + _parseCts?.Cancel(); _parseCts = new CancellationTokenSource(); + _linkedParseCts?.Dispose(); _linkedParseCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, _parseCts.Token); + + ContentState = State.Parsing; _parsingTask = Task.Run(() => Parse(_linkedParseCts.Token), _linkedParseCts.Token); } @@ -302,7 +356,7 @@ private void Parse(CancellationToken cancellationToken) { int version; Parser parser; - Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name}"); + //Log?.Log(TraceEventType.Verbose, $"Parse begins: {Name}"); lock (AnalysisLock) { version = _buffer.Version; @@ -314,7 +368,7 @@ private void Parse(CancellationToken cancellationToken) { var ast = parser.ParseFile(); - Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name}"); + //Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name}"); lock (AnalysisLock) { cancellationToken.ThrowIfCancellationRequested(); @@ -324,12 +378,14 @@ private void Parse(CancellationToken cancellationToken) { _ast = ast; _parseErrors = sink.Diagnostics; _parsingTask = null; + ContentState = State.Parsed; } NewAst?.Invoke(this, EventArgs.Empty); - if ((_options & ModuleLoadOptions.Analyze) == ModuleLoadOptions.Analyze) { + if (ContentState < State.Analyzing) { Log?.Log(TraceEventType.Verbose, $"Analysis queued: {Name}"); + ContentState = State.Analyzing; _linkedAnalysisCts?.Dispose(); _linkedAnalysisCts = CancellationTokenSource.CreateLinkedTokenSource(_allProcessingCts.Token, cancellationToken); @@ -377,23 +433,26 @@ public void NotifyAnalysisPending() { // and then here. This is normal. ExpectedAnalysisVersion++; _analysisTcs = _analysisTcs ?? new TaskCompletionSource(); - Log?.Log(TraceEventType.Verbose, $"Analysis pending: {Name}"); + //Log?.Log(TraceEventType.Verbose, $"Analysis pending: {Name}"); } } public virtual bool NotifyAnalysisComplete(IDocumentAnalysis analysis) { lock (AnalysisLock) { - Log?.Log(TraceEventType.Verbose, $"Analysis complete: {Name}, Version: {analysis.Version}, Expected: {ExpectedAnalysisVersion}"); + // Log?.Log(TraceEventType.Verbose, $"Analysis complete: {Name}, Version: {analysis.Version}, Expected: {ExpectedAnalysisVersion}"); if (analysis.Version == ExpectedAnalysisVersion) { Analysis = analysis; + GlobalScope = analysis.GlobalScope; + // Derived classes can override OnAnalysisComplete if they want // to perform additional actions on the completed analysis such // as declare additional variables, etc. OnAnalysisComplete(); + _analysisTcs.TrySetResult(analysis); _analysisTcs = null; + ContentState = State.Analyzed; - GlobalScope = analysis.GlobalScope; NewAnalysis?.Invoke(this, EventArgs.Empty); return true; } @@ -402,6 +461,9 @@ public virtual bool NotifyAnalysisComplete(IDocumentAnalysis analysis) { } } + public void NotifyAnalysisCanceled() => _analysisTcs?.TrySetCanceled(); + public void NotifyAnalysisFailed(Exception ex) => _analysisTcs?.TrySetException(ex); + protected virtual void OnAnalysisComplete() { } #endregion @@ -410,10 +472,7 @@ protected virtual void OnAnalysisComplete() { } public Task GetAnalysisAsync(CancellationToken cancellationToken = default) { lock (AnalysisLock) { - if ((_options & ModuleLoadOptions.Analyze) != ModuleLoadOptions.Analyze) { - return Task.FromResult(DocumentAnalysis.Empty); - } - return Analysis.Version == ExpectedAnalysisVersion ? Task.FromResult(Analysis) : _analysisTcs.Task; + return _analysisTcs?.Task ?? Task.FromResult(Analysis); } } #endregion @@ -463,43 +522,6 @@ private string TryGetDocFromModuleInitFile() { return string.Empty; } - /// - /// Provides ability to specialize function return type manually. - /// - protected void SpecializeFunction(string name, IMember returnValue) { - var f = GetOrCreateFunction(name); - if (f != null) { - foreach (var o in f.Overloads.OfType()) { - o.SetReturnValue(returnValue, true); - } - } - } - - /// - /// Provides ability to dynamically calculate function return type. - /// - internal void SpecializeFunction(string name, ReturnValueProvider returnTypeCallback) { - var f = GetOrCreateFunction(name); - if (f != null) { - foreach (var o in f.Overloads.OfType()) { - o.SetReturnValueProvider(returnTypeCallback); - } - f.Specialize(); - } - } - - private PythonFunctionType GetOrCreateFunction(string name) { - var f = Analysis.GlobalScope.Variables[name]?.Value as PythonFunctionType; - // We DO want to replace class by function. Consider type() in builtins. - // 'type()' in code is a function call, not a type class instantiation. - if (f == null) { - f = PythonFunctionType.ForSpecialization(name, this); - f.AddOverload(new PythonFunctionOverload(name, this, LocationInfo.Empty)); - Analysis.GlobalScope.DeclareVariable(name, f, LocationInfo.Empty); - } - return f; - } - public bool Equals(IPythonModule other) => Name.Equals(other?.Name) && FilePath.Equals(other?.FilePath); } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonPackage.cs b/src/Analysis/Ast/Impl/Modules/PythonPackage.cs index c0afd16c2..cf0548001 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonPackage.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonPackage.cs @@ -17,6 +17,7 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; namespace Microsoft.Python.Analysis.Modules { @@ -43,7 +44,7 @@ public void AddChildModule(string name, IPythonModule module) { protected override void OnAnalysisComplete() { foreach (var childModuleName in GetChildrenModuleNames()) { var name = $"{Name}.{childModuleName}"; - Analysis.GlobalScope.DeclareVariable(name, this, LocationInfo.Empty); + Analysis.GlobalScope.DeclareVariable(name, this, VariableSource.Declaration, LocationInfo.Empty); } base.OnAnalysisComplete(); } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs new file mode 100644 index 000000000..56fd6c779 --- /dev/null +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -0,0 +1,193 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Core.DependencyResolution; +using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Modules.Resolution { + internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement { + private readonly ConcurrentDictionary _specialized = new ConcurrentDictionary(); + private IReadOnlyList _searchPaths; + + public MainModuleResolution(string root, IServiceContainer services) + : base(root, services) { } + + internal async Task InitializeAsync(CancellationToken cancellationToken = default) { + // Add names from search paths + await ReloadAsync(cancellationToken); + cancellationToken.ThrowIfCancellationRequested(); + + // Initialize built-in + var moduleName = BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion); + var modulePath = ModuleCache.GetCacheFilePath(_interpreter.Configuration.InterpreterPath ?? "python.exe"); + + var b = new BuiltinsPythonModule(moduleName, modulePath, _services); + _modules[BuiltinModuleName] = BuiltinsModule = b; + } + + public async Task> GetSearchPathsAsync(CancellationToken cancellationToken = default) { + if (_searchPaths != null) { + return _searchPaths; + } + + _searchPaths = await GetInterpreterSearchPathsAsync(cancellationToken); + Debug.Assert(_searchPaths != null, "Should have search paths"); + _searchPaths = _searchPaths != null + ? _searchPaths.Concat(Configuration.SearchPaths ?? Array.Empty()).ToArray() + : Array.Empty(); + + _log?.Log(TraceEventType.Information, "SearchPaths:"); + foreach (var s in _searchPaths) { + _log?.Log(TraceEventType.Information, $" {s}"); + } + return _searchPaths; + } + + protected override async Task DoImportAsync(string name, CancellationToken cancellationToken = default) { + var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name); + if (moduleImport == null) { + _log?.Log(TraceEventType.Verbose, "Import not found: ", name); + return null; + } + + // If there is a stub, make sure it is loaded and attached + // First check stub next to the module. + var stub = await GetModuleStubAsync(name, moduleImport.ModulePath, cancellationToken); + // If nothing found, try Typeshed. + stub = stub ?? await _interpreter.TypeshedResolution.ImportModuleAsync(moduleImport.IsBuiltin ? name : moduleImport.FullName, cancellationToken); + + IPythonModule module; + if (moduleImport.IsBuiltin) { + _log?.Log(TraceEventType.Verbose, "Import built-in compiled (scraped) module: ", name, Configuration.InterpreterPath); + module = new CompiledBuiltinPythonModule(name, stub, _services); + } else if (moduleImport.IsCompiled) { + _log?.Log(TraceEventType.Verbose, "Import compiled (scraped): ", moduleImport.FullName, moduleImport.ModulePath, moduleImport.RootPath); + module = new CompiledPythonModule(moduleImport.FullName, ModuleType.Compiled, moduleImport.ModulePath, stub, _services); + } else { + _log?.Log(TraceEventType.Verbose, "Import: ", moduleImport.FullName, moduleImport.ModulePath); + var rdt = _services.GetService(); + // TODO: handle user code and library module separately. + var mco = new ModuleCreationOptions { + ModuleName = moduleImport.FullName, + ModuleType = ModuleType.Library, + FilePath = moduleImport.ModulePath, + Stub = stub + }; + module = rdt.AddModule(mco); + } + + await module.LoadAndAnalyzeAsync(cancellationToken); + return module; + } + + private async Task> GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default) { + if (!_fs.FileExists(Configuration.InterpreterPath)) { + return Array.Empty(); + } + + _log?.Log(TraceEventType.Information, "GetCurrentSearchPaths", Configuration.InterpreterPath, ModuleCache.SearchPathCachePath); + try { + var paths = await PythonLibraryPath.GetDatabaseSearchPathsAsync(Configuration, ModuleCache.SearchPathCachePath); + cancellationToken.ThrowIfCancellationRequested(); + return paths.MaybeEnumerate().Select(p => p.Path).ToArray(); + } catch (InvalidOperationException) { + return Array.Empty(); + } + } + + /// + /// Provides ability to specialize module by replacing module import by + /// implementation in code. Real module + /// content is loaded and analyzed only for class/functions definitions + /// so the original documentation can be extracted. + /// + /// Module to specialize. + /// Specialized module constructor. + /// Original (library) module loaded as stub. + public IPythonModule SpecializeModule(string name, Func specializationConstructor) { + var import = CurrentPathResolver.GetModuleImportFromModuleName(name); + var module = specializationConstructor(import?.ModulePath); + _specialized[name] = module; + return module; + } + + /// + /// Returns specialized module, if any. + /// + public IPythonModule GetSpecializedModule(string name) + => _specialized.TryGetValue(name, out var module) ? module : null; + + internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken = default) { + await BuiltinsModule.LoadAndAnalyzeAsync(cancellationToken); + + // Add built-in module names + var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__"); + if (builtinModuleNamesMember.TryGetConstant(out var s)) { + var builtinModuleNames = s.Split(',').Select(n => n.Trim()); + _pathResolver.SetBuiltins(builtinModuleNames); + } + } + + public override async Task ReloadAsync(CancellationToken cancellationToken = default) { + ModuleCache = new ModuleCache(_interpreter, _services); + + _pathResolver = new PathResolver(_interpreter.LanguageVersion); + + var addedRoots = _pathResolver.SetRoot(_root); + ReloadModulePaths(addedRoots); + + var interpreterPaths = await GetSearchPathsAsync(cancellationToken); + addedRoots = _pathResolver.SetInterpreterSearchPaths(interpreterPaths); + + ReloadModulePaths(addedRoots); + cancellationToken.ThrowIfCancellationRequested(); + + addedRoots = _pathResolver.SetUserSearchPaths(_interpreter.Configuration.SearchPaths); + ReloadModulePaths(addedRoots); + } + + // For tests + internal void AddUnimportableModule(string moduleName) + => _modules[moduleName] = new SentinelModule(moduleName, _services); + + private async Task GetModuleStubAsync(string name, string modulePath, CancellationToken cancellationToken = default) { + // First check stub next to the module. + if (!string.IsNullOrEmpty(modulePath)) { + var pyiPath = Path.ChangeExtension(modulePath, "pyi"); + if (_fs.FileExists(pyiPath)) { + return await CreateStubModuleAsync(name, pyiPath, cancellationToken); + } + } + + // Try location of stubs that are in a separate folder next to the package. + var stubPath = CurrentPathResolver.GetPossibleModuleStubPaths(name).FirstOrDefault(p => _fs.FileExists(p)); + if (!string.IsNullOrEmpty(stubPath)) { + return await CreateStubModuleAsync(name, stubPath, cancellationToken); + } + return null; + } + } +} diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs new file mode 100644 index 000000000..d7e01b06d --- /dev/null +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -0,0 +1,221 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Core.DependencyResolution; +using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Core.IO; +using Microsoft.Python.Core.Logging; + +namespace Microsoft.Python.Analysis.Modules.Resolution { + internal abstract class ModuleResolutionBase { + protected readonly ConcurrentDictionary _modules = new ConcurrentDictionary(); + protected readonly IServiceContainer _services; + protected readonly IPythonInterpreter _interpreter; + protected readonly IFileSystem _fs; + protected readonly ILogger _log; + protected readonly bool _requireInitPy; + protected string _root; + + protected PathResolver _pathResolver; + + protected InterpreterConfiguration Configuration => _interpreter.Configuration; + + protected ModuleResolutionBase(string root, IServiceContainer services) { + _root = root; + _services = services; + _interpreter = services.GetService(); + _fs = services.GetService(); + _log = services.GetService(); + + _requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(_interpreter.Configuration.Version); + } + + public IModuleCache ModuleCache { get; protected set; } + public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(_interpreter.LanguageVersion); + + /// + /// Path resolver providing file resolution in module imports. + /// + public PathResolverSnapshot CurrentPathResolver => _pathResolver.CurrentSnapshot; + + /// + /// Builtins module. + /// + public IBuiltinsPythonModule BuiltinsModule { get; protected set; } + + public abstract Task ReloadAsync(CancellationToken cancellationToken = default); + protected abstract Task DoImportAsync(string name, CancellationToken cancellationToken = default); + + public IReadOnlyCollection GetPackagesFromDirectory(string searchPath, CancellationToken cancellationToken) { + return ModulePath.GetModulesInPath( + searchPath, + recurse: false, + includePackages: true, + requireInitPy: _requireInitPy + ).Select(mp => mp.ModuleName).Where(n => !string.IsNullOrEmpty(n)).TakeWhile(_ => !cancellationToken.IsCancellationRequested).ToList(); + } + + public IPythonModule GetImportedModule(string name) + => _modules.TryGetValue(name, out var module) ? module : null; + + public void AddModulePath(string path) => _pathResolver.TryAddModulePath(path, out var _); + + public ModulePath FindModule(string filePath) { + var bestLibraryPath = string.Empty; + + foreach (var p in Configuration.SearchPaths) { + if (PathEqualityComparer.Instance.StartsWith(filePath, p)) { + if (p.Length > bestLibraryPath.Length) { + bestLibraryPath = p; + } + } + } + return ModulePath.FromFullPath(filePath, bestLibraryPath); + } + + public async Task ImportModuleAsync(string name, CancellationToken cancellationToken = default) { + if (name == BuiltinModuleName) { + return BuiltinsModule; + } + var module = _interpreter.ModuleResolution.GetSpecializedModule(name); + if (module != null) { + return module; + } + return await DoImportModuleAsync(name, cancellationToken); + } + + private async Task DoImportModuleAsync(string name, CancellationToken cancellationToken = default) { + for (var retries = 5; retries > 0; --retries) { + cancellationToken.ThrowIfCancellationRequested(); + + // The call should be cancelled by the cancellation token, but since we + // are blocking here we wait for slightly longer. Timeouts are handled + // gracefully by TryImportModuleAsync(), so we want those to trigger if + // possible, but if all else fails then we'll abort and treat it as an + // error. + // (And if we've got a debugger attached, don't time out at all.) + TryImportModuleResult result; + try { + result = await TryImportModuleAsync(name, cancellationToken); + } catch (OperationCanceledException) { + _log?.Log(TraceEventType.Error, $"Import timeout: {name}"); + Debug.Fail("Import timeout"); + return null; + } + + switch (result.Status) { + case TryImportModuleResultCode.Success: + return result.Module; + case TryImportModuleResultCode.ModuleNotFound: + _log?.Log(TraceEventType.Information, $"Import not found: {name}"); + return null; + case TryImportModuleResultCode.NeedRetry: + case TryImportModuleResultCode.Timeout: + break; + case TryImportModuleResultCode.NotSupported: + _log?.Log(TraceEventType.Error, $"Import not supported: {name}"); + return null; + } + } + // Never succeeded, so just log the error and fail + _log?.Log(TraceEventType.Error, $"Retry import failed: {name}"); + return null; + } + + private async Task TryImportModuleAsync(string name, CancellationToken cancellationToken = default) { + if (string.IsNullOrEmpty(name)) { + return TryImportModuleResult.ModuleNotFound; + } + if (name == BuiltinModuleName) { + return new TryImportModuleResult(BuiltinsModule); + } + + Debug.Assert(!name.EndsWithOrdinal("."), $"{name} should not end with '.'"); + // Return any existing module + if (_modules.TryGetValue(name, out var module) && module != null) { + if (module is SentinelModule) { + // TODO: we can't just wait here or we hang. There are two cases: + // a. Recursion on the same analysis chain (A -> B -> A) + // b. Call from another chain (A -> B -> C and D -> B -> E). + // TODO: Both should be resolved at the dependency chain level. + //_log?.Log(TraceEventType.Warning, $"Recursive import: {name}"); + } + return new TryImportModuleResult(module); + } + + // Set up a sentinel so we can detect recursive imports + var sentinelValue = new SentinelModule(name, _services); + if (!_modules.TryAdd(name, sentinelValue)) { + // Try to get the new module, in case we raced with a .Clear() + if (_modules.TryGetValue(name, out module) && !(module is SentinelModule)) { + return new TryImportModuleResult(module); + } + // If we reach here, the race is too complicated to recover + // from. Signal the caller to try importing again. + _log?.Log(TraceEventType.Warning, $"Retry import: {name}"); + return TryImportModuleResult.NeedRetry; + } + + // Do normal searches + try { + module = await DoImportAsync(name, cancellationToken); + } catch (OperationCanceledException) { + _log?.Log(TraceEventType.Error, $"Import timeout {name}"); + return TryImportModuleResult.Timeout; + } + + if (ModuleCache != null) { + module = module ?? await ModuleCache.ImportFromCacheAsync(name, cancellationToken); + } + + // Replace our sentinel + if (!_modules.TryUpdate(name, module, sentinelValue)) { + // Try to get the new module, in case we raced + if (_modules.TryGetValue(name, out module) && !(module is SentinelModule)) { + return new TryImportModuleResult(module); + } + // If we reach here, the race is too complicated to recover + // from. Signal the caller to try importing again. + _log?.Log(TraceEventType.Warning, $"Retry import: {name}"); + return TryImportModuleResult.NeedRetry; + } + + return new TryImportModuleResult(module); + } + + protected void ReloadModulePaths(in IEnumerable rootPaths) { + foreach (var modulePath in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(p))) { + _pathResolver.TryAddModulePath(modulePath, out _); + } + } + + protected async Task CreateStubModuleAsync(string moduleName, string filePath, CancellationToken cancellationToken = default) { + _log?.Log(TraceEventType.Verbose, "Import type stub", moduleName, filePath); + var module = new StubPythonModule(moduleName, filePath, _services); + await module.LoadAndAnalyzeAsync(cancellationToken); + return module; + } + } +} diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs new file mode 100644 index 000000000..ef8850ff1 --- /dev/null +++ b/src/Analysis/Ast/Impl/Modules/Resolution/TypeshedResolution.cs @@ -0,0 +1,141 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis.Core.DependencyResolution; +using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Modules.Resolution { + internal sealed class TypeshedResolution : ModuleResolutionBase, IModuleResolution { + private readonly IReadOnlyList _typeStubPaths; + + public TypeshedResolution(IServiceContainer services) : base(null, services) { + _modules[BuiltinModuleName] = BuiltinsModule = _interpreter.ModuleResolution.BuiltinsModule; + _root = _interpreter.Configuration?.TypeshedPath; + // TODO: merge with user-provided stub paths + _typeStubPaths = GetTypeShedPaths(_interpreter.Configuration?.TypeshedPath).ToArray(); + + _log?.Log(TraceEventType.Verbose, @"Typeshed paths:"); + foreach (var p in _typeStubPaths) { + _log?.Log(TraceEventType.Verbose, $" {p}"); + } + } + + internal Task InitializeAsync(CancellationToken cancellationToken = default) + => ReloadAsync(cancellationToken); + + protected override async Task DoImportAsync(string name, CancellationToken cancellationToken = default) { + var mp = FindModuleInSearchPath(_typeStubPaths, null, name); + if (mp != null) { + if (mp.Value.IsCompiled) { + _log?.Log(TraceEventType.Warning, "Unsupported native module in stubs", mp.Value.FullName, mp.Value.SourceFile); + return null; + } + return await CreateStubModuleAsync(mp.Value.FullName, mp.Value.SourceFile, cancellationToken); + } + + var i = name.IndexOf('.'); + if (i == 0) { + Debug.Fail("Invalid module name"); + return null; + } + + var stubPath = CurrentPathResolver.GetPossibleModuleStubPaths(name).FirstOrDefault(p => _fs.FileExists(p)); + return stubPath != null ? await CreateStubModuleAsync(name, stubPath, cancellationToken) : null; + } + + public override Task ReloadAsync(CancellationToken cancellationToken = default) { + _pathResolver = new PathResolver(_interpreter.LanguageVersion); + + var addedRoots = _pathResolver.SetRoot(_root); + ReloadModulePaths(addedRoots); + + addedRoots = _pathResolver.SetInterpreterSearchPaths(_typeStubPaths); + ReloadModulePaths(addedRoots); + + cancellationToken.ThrowIfCancellationRequested(); + return Task.CompletedTask; + } + + private IEnumerable GetTypeShedPaths(string typeshedRootPath) { + if (string.IsNullOrEmpty(typeshedRootPath)) { + yield break; + } + + var stdlib = Path.Combine(typeshedRootPath, "stdlib"); + var thirdParty = Path.Combine(typeshedRootPath, "third_party"); + + var v = Configuration.Version; + var subdirs = new List { v.Major.ToString(), "2and3" }; + for (var i = 1; i < v.Minor; i++) { + subdirs.Add($"{v.Major}.{i}"); + } + + // For 3: all between 3 and current version inclusively + 2and3 + foreach (var subdir in subdirs) { + yield return Path.Combine(stdlib, subdir); + } + + foreach (var subdir in subdirs) { + yield return Path.Combine(thirdParty, subdir); + } + } + + private ModulePath? FindModuleInSearchPath(IReadOnlyList searchPaths, IReadOnlyDictionary packages, string name) { + if (searchPaths == null || searchPaths.Count == 0) { + return null; + } + + var i = name.IndexOf('.'); + var firstBit = i < 0 ? name : name.Remove(i); + + ModulePath mp; + Func isPackage = IsPackage; + if (firstBit.EndsWithOrdinal("-stubs", ignoreCase: true)) { + isPackage = _fs.DirectoryExists; + } + + var requireInitPy = ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version); + if (packages != null && packages.TryGetValue(firstBit, out var searchPath) && !string.IsNullOrEmpty(searchPath)) { + if (ModulePath.FromBasePathAndName_NoThrow(searchPath, name, isPackage, null, requireInitPy, out mp, out _, out _, out _)) { + return mp; + } + } + + if (searchPaths.MaybeEnumerate() + .Any(sp => ModulePath.FromBasePathAndName_NoThrow(sp, name, isPackage, null, requireInitPy, out mp, out _, out _, out _))) { + return mp; + } + return null; + } + + /// + /// Determines whether the specified directory is an importable package. + /// + private bool IsPackage(string directory) + => ModulePath.PythonVersionRequiresInitPyFiles(Configuration.Version) ? + !string.IsNullOrEmpty(ModulePath.GetPackageInitPy(directory)) : + _fs.DirectoryExists(directory); + + } +} diff --git a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs index 77fe8e2db..d9d03fa1d 100644 --- a/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs +++ b/src/Analysis/Ast/Impl/Modules/SpecializedModule.cs @@ -31,15 +31,11 @@ namespace Microsoft.Python.Analysis.Modules { /// public abstract class SpecializedModule : PythonModule { protected SpecializedModule(string name, string modulePath, IServiceContainer services) - : base(name, modulePath, ModuleType.Specialized, ModuleLoadOptions.Analyze, null, services) { } + : base(name, modulePath, ModuleType.Specialized, null, services) { } - protected override string LoadContent(ModuleLoadOptions options) { - try { - if (FileSystem.FileExists(FilePath)) { - return FileSystem.ReadAllText(FilePath); - } - } catch (IOException) { } catch (UnauthorizedAccessException) { } - return string.Empty; + protected override string LoadContent() { + // Exceptions are handled in the base + return FileSystem.FileExists(FilePath) ? FileSystem.ReadAllText(FilePath) : string.Empty; } } } diff --git a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs index 0d2a8f51d..e3b2f7bb2 100644 --- a/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/StubPythonModule.cs @@ -28,13 +28,9 @@ public StubPythonModule(string moduleName, string stubPath, IServiceContainer se : base(moduleName, ModuleType.Stub, stubPath, null, services) { } - protected override string LoadContent(ModuleLoadOptions options) { - try { - if (FileSystem.FileExists(FilePath) && (options & ModuleLoadOptions.Load) == ModuleLoadOptions.Load) { - return FileSystem.ReadAllText(FilePath); - } - } catch (IOException) { } catch(UnauthorizedAccessException) { } - return string.Empty; + protected override string LoadContent() { + // Exceptions are handled in the base + return FileSystem.FileExists(FilePath) ? FileSystem.ReadAllText(FilePath) : string.Empty; } protected override IEnumerable GetScrapeArguments(IPythonInterpreter factory) => Enumerable.Empty(); diff --git a/src/Analysis/Ast/Impl/Properties/AssemblyInfo.cs b/src/Analysis/Ast/Impl/Properties/AssemblyInfo.cs index cfcc08c5c..52c4e3665 100644 --- a/src/Analysis/Ast/Impl/Properties/AssemblyInfo.cs +++ b/src/Analysis/Ast/Impl/Properties/AssemblyInfo.cs @@ -16,3 +16,4 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs index 98ba65bd3..a7debce60 100644 --- a/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs +++ b/src/Analysis/Ast/Impl/Specializations/BuiltinsSpecializations.cs @@ -24,13 +24,18 @@ namespace Microsoft.Python.Analysis.Specializations { public static class BuiltinsSpecializations { - public static ReturnValueProvider Identity - => (module, overload, location, args) => args.Count > 0 ? args[0] : null; + public static IMember Identity(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + var args = argSet.Values(); + return args.Count > 0 ? args[0] : null; + } - public static ReturnValueProvider TypeInfo - => (module, overload, location, args) => args.Count > 0 ? args[0].GetPythonType() : null; + public static IMember TypeInfo(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + var args = argSet.Values(); + return args.Count > 0 ? args[0].GetPythonType() : module.Interpreter.GetBuiltinType(BuiltinTypeId.Type); + } - public static IMember Iterator(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IReadOnlyList args) { + public static IMember Iterator(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + var args = argSet.Values(); if (args.Count > 0) { if (args[0] is IPythonCollection seq) { return seq.GetIterator(); @@ -43,22 +48,24 @@ public static IMember Iterator(IPythonModule module, IPythonFunctionOverload ove return null; } - public static IMember List(IPythonInterpreter interpreter, IPythonFunctionOverload overload, LocationInfo location, IReadOnlyList args) - => PythonCollectionType.CreateList(interpreter, location, args); + public static IMember List(IPythonInterpreter interpreter, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) + => PythonCollectionType.CreateList(interpreter, location, argSet); - public static IMember ListOfStrings(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IReadOnlyList args) { + public static IMember ListOfStrings(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { var type = new TypingListType("List", module.Interpreter.GetBuiltinType(BuiltinTypeId.Str), module.Interpreter, false); return new TypingList(type, location); } - public static IMember DictStringToObject(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IReadOnlyList args) { + public static IMember DictStringToObject(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { var str = module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var obj = module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); var type = new TypingDictionaryType("Dict", str, obj, module.Interpreter, false); return new TypingDictionary(type, location); } - public static ReturnValueProvider Next - => (module, overload, location, args) => args.Count > 0 && args[0] is IPythonIterator it ? it.Next : null; + public static IMember Next(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + var args = argSet.Values(); + return args.Count > 0 && args[0] is IPythonIterator it ? it.Next : null; + } public static IMember __iter__(IPythonInterpreter interpreter, BuiltinTypeId contentTypeId) { var fn = new PythonFunctionType(@"__iter__", interpreter.ModuleResolution.BuiltinsModule, null, string.Empty, LocationInfo.Empty); @@ -68,15 +75,32 @@ public static IMember __iter__(IPythonInterpreter interpreter, BuiltinTypeId con return fn; } - public static IMember Range(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IReadOnlyList args) { + public static IMember Range(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + var args = argSet.Values(); if (args.Count > 0) { var type = new PythonCollectionType(null, BuiltinTypeId.List, module.Interpreter, false); - return new PythonCollection(type, location, new [] {args[0]}); + return new PythonCollection(type, location, new[] { args[0] }); } return null; } - public static ReturnValueProvider CollectionItem - => (module, overload, location, args) => args.Count > 0 && args[0] is PythonCollection c ? c.Contents.FirstOrDefault() : null; + public static IMember CollectionItem(IPythonModule module, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + var args = argSet.Values(); + return args.Count > 0 && args[0] is PythonCollection c ? c.Contents.FirstOrDefault() : null; + } + + public static IMember Open(IPythonModule declaringModule, IPythonFunctionOverload overload, LocationInfo location, IArgumentSet argSet) { + var mode = argSet.GetArgumentValue("mode"); + + var bytes = false; + if (mode != null) { + var modeString = mode.GetString(); + bytes = modeString != null && modeString.Contains("b"); + } + + var io = declaringModule.Interpreter.ModuleResolution.GetImportedModule("io"); + var ioBase = io?.GetMember(bytes ? "BufferedIOBase" : "TextIOWrapper")?.GetPythonType(); + return ioBase != null ? new PythonInstance(ioBase) : null; + } } } diff --git a/src/Analysis/Ast/Impl/Documents/DocumentChangeSet.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBaseType.cs similarity index 60% rename from src/Analysis/Ast/Impl/Documents/DocumentChangeSet.cs rename to src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBaseType.cs index ebd2fd0e8..8c0afd9a2 100644 --- a/src/Analysis/Ast/Impl/Documents/DocumentChangeSet.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBaseType.cs @@ -14,18 +14,12 @@ // permissions and limitations under the License. using System.Collections.Generic; -using System.Linq; -namespace Microsoft.Python.Analysis.Documents { - public sealed class DocumentChangeSet { - public DocumentChangeSet(int fromVersion, int toVersion, IEnumerable changes) { - FromVersion = fromVersion; - ToVersion = toVersion; - Changes = changes.ToArray(); - } - - public int FromVersion { get; } - public int ToVersion { get; } - public IReadOnlyCollection Changes { get; } +namespace Microsoft.Python.Analysis.Specializations.Typing { + /// + /// Represents Generic[T1, T2, ...]. Used as a base class to generic classes. + /// + public interface IGenericClassBaseType { + IReadOnlyList TypeArgs { get; } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs index 1a653b86a..8e0f0ee2b 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericType.cs @@ -21,11 +21,13 @@ namespace Microsoft.Python.Analysis.Specializations.Typing { /// Represents generic type, such as class or function. /// Generic type is a template for the actual type. /// - public interface IGenericType: IPythonType { + public interface IGenericType : IPythonTemplateType { /// - /// Creates instance of a type information with the specific - /// type arguments from the generic template. + /// Type parameters such as in Tuple[T1, T2. ...] or + /// Generic[_T1, _T2, ...] as returned by TypeVar. /// + IReadOnlyList Parameters { get; } + IPythonType CreateSpecificType(IReadOnlyList typeArguments, IPythonModule declaringModule, LocationInfo location = null); } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBaseType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBaseType.cs new file mode 100644 index 000000000..48676c160 --- /dev/null +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBaseType.cs @@ -0,0 +1,28 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Types; + +namespace Microsoft.Python.Analysis.Specializations.Typing.Types { + internal sealed class GenericClassBaseType: PythonClassType, IGenericClassBaseType { + internal GenericClassBaseType(IReadOnlyList typeArgs, IPythonModule declaringModule, LocationInfo location) + : base("Generic", declaringModule, location) { + TypeArgs = typeArgs; + } + + public IReadOnlyList TypeArgs { get; } + } +} diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs index de9f6382d..1fc67e787 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericType.cs @@ -15,28 +15,59 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; namespace Microsoft.Python.Analysis.Specializations.Typing.Types { + internal delegate IPythonType SpecificTypeConstructor( + IReadOnlyList typeArgs, + IPythonModule declaringModule, + LocationInfo location); + /// /// Base class for generic types and type declarations. /// internal class GenericType : IGenericType { - private readonly Func, IPythonModule, LocationInfo, IPythonType> _typeConstructor; + private readonly SpecificTypeConstructor _specificTypeConstructor; + + /// + /// Constructs generic type with generic parameters. Typically used + /// in generic classes such as when handling Generic[_T] base. + /// + public GenericType(string name, IPythonModule declaringModule, IReadOnlyList parameters) + : this(name, declaringModule) { + Parameters = parameters ?? throw new ArgumentNullException(nameof(parameters)); + } - public GenericType(string name, IPythonModule declaringModule, - Func, IPythonModule, LocationInfo, IPythonType> typeConstructor) { + /// + /// Constructs generic type with dynamic type constructor. + /// Typically used in type specialization scenarios. + /// + public GenericType(string name, IPythonModule declaringModule, SpecificTypeConstructor specificTypeConstructor) + : this(name, declaringModule) { + _specificTypeConstructor = specificTypeConstructor ?? throw new ArgumentNullException(nameof(specificTypeConstructor)); + } + private GenericType(string name, IPythonModule declaringModule) { Name = name ?? throw new ArgumentNullException(nameof(name)); DeclaringModule = declaringModule ?? throw new ArgumentNullException(nameof(declaringModule)); - _typeConstructor = typeConstructor ?? throw new ArgumentNullException(nameof(typeConstructor)); } + /// + /// Type parameters such as in Tuple[T1, T2. ...] or + /// Generic[_T1, _T2, ...] as returned by TypeVar. + /// + public IReadOnlyList Parameters { get; } = Array.Empty(); + + /// + /// Creates instance of a type information with the specific + /// type arguments from a generic template. + /// public IPythonType CreateSpecificType(IReadOnlyList typeArguments, IPythonModule declaringModule, LocationInfo location = null) - => _typeConstructor(typeArguments, declaringModule, location); + => _specificTypeConstructor(typeArguments, declaringModule, location); #region IPythonType public string Name { get; } @@ -56,13 +87,17 @@ public IMember CreateInstance(string typeName, LocationInfo location, IArgumentS throw new ArgumentException(@"Generic type instance construction arguments must be all of IPythonType", nameof(args)); } var specific = CreateSpecificType(types, DeclaringModule, location); - return specific == null - ? DeclaringModule.Interpreter.UnknownType + return specific == null + ? DeclaringModule.Interpreter.UnknownType : specific.CreateInstance(typeName, location, null); } public virtual IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) => DeclaringModule.Interpreter.UnknownType; public virtual IMember Index(IPythonInstance instance, object index) => DeclaringModule.Interpreter.UnknownType; + + public Task CreateSpecificTypeAsync(IArgumentSet typeArguments, IPythonModule declaringModule, LocationInfo location, CancellationToken cancellationToken = default) + => Task.FromResult(CreateSpecificType(typeArguments.Arguments.Select(a => a.Value).OfType().ToArray(), declaringModule, location)); + #endregion } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs index bd1779456..6faf70b7a 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericTypeParameter.cs @@ -33,7 +33,8 @@ public GenericTypeParameter(string name, IPythonModule declaringModule, IReadOnl public override bool IsSpecialized => true; - public static IPythonType FromTypeVar(IReadOnlyList args, IPythonModule declaringModule, LocationInfo location) { + public static IPythonType FromTypeVar(IArgumentSet argSet, IPythonModule declaringModule, LocationInfo location) { + var args = argSet.Values(); if (args.Count == 0) { // TODO: report that at least one argument is required. return declaringModule.Interpreter.UnknownType; @@ -45,7 +46,11 @@ public static IPythonType FromTypeVar(IReadOnlyList args, IPythonModule return declaringModule.Interpreter.UnknownType; } - var constraints = args.Skip(1).Select(a => a.GetPythonType()).ToArray(); + var constraints = args.Skip(1).Select(a => { + // Type constraints may be specified as type name strings. + var typeString = (a as IPythonConstant)?.GetString(); + return !string.IsNullOrEmpty(typeString) ? argSet.Eval.GetTypeFromString(typeString) : a.GetPythonType(); + }).ToArray(); if (constraints.Any(c => c.IsUnknown())) { // TODO: report that some constraints could be be resolved. } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs index 63460fa70..6946be738 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/Types/TypingIteratorType.cs @@ -43,7 +43,6 @@ public TypingIteratorType(IPythonType itemType, BuiltinTypeId iteratorType, IPyt /// public TypingIteratorType(IReadOnlyList itemTypes, BuiltinTypeId iteratorType, IPythonInterpreter interpreter) : base(iteratorType, interpreter) { - Check.ArgumentOutOfRange(nameof(itemTypes), () => itemTypes.Count == 0); ItemTypes = itemTypes; Name = $"Iterator[{CodeFormatter.FormatSequence(string.Empty, '(', itemTypes)}]"; } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs index f308d734a..10dd98583 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs @@ -30,10 +30,10 @@ internal sealed class TypingModule : SpecializedModule { private TypingModule(string modulePath, IServiceContainer services) : base("typing", modulePath, services) { } - public static async Task CreateAsync(IServiceContainer services, CancellationToken cancellationToken = default) { + public static IPythonModule Create(IServiceContainer services) { var interpreter = services.GetService(); - var module = await interpreter.ModuleResolution - .SpecializeModuleAsync("typing", modulePath => new TypingModule(modulePath, services), cancellationToken) as TypingModule; + var module = interpreter.ModuleResolution + .SpecializeModule("typing", modulePath => new TypingModule(modulePath, services)) as TypingModule; module?.SpecializeMembers(); return module; } @@ -49,7 +49,8 @@ private void SpecializeMembers() { var o = new PythonFunctionOverload(fn.Name, this, _ => fn.Location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, location, args) => GenericTypeParameter.FromTypeVar(args, interpreter, location)); + o.SetReturnValueProvider((interpreter, overload, location, args) + => GenericTypeParameter.FromTypeVar(args, interpreter, location)); fn.AddOverload(o); _members["TypeVar"] = fn; @@ -59,7 +60,7 @@ private void SpecializeMembers() { o = new PythonFunctionOverload(fn.Name, this, _ => fn.Location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, location, args) => CreateTypeAlias(args)); + o.SetReturnValueProvider((interpreter, overload, location, args) => CreateTypeAlias(args.Values())); fn.AddOverload(o); _members["NewType"] = fn; @@ -68,7 +69,10 @@ private void SpecializeMembers() { o = new PythonFunctionOverload(fn.Name, this, _ => fn.Location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. - o.SetReturnValueProvider((interpreter, overload, location, args) => args.Count == 1 ? args[0] : Interpreter.UnknownType); + o.SetReturnValueProvider((interpreter, overload, location, args) => { + var a = args.Values(); + return a.Count == 1 ? a[0] : Interpreter.UnknownType; + }); fn.AddOverload(o); _members["Type"] = fn; @@ -127,7 +131,7 @@ private void SpecializeMembers() { fn = new PythonFunctionType("NamedTuple", this, null, GetMemberDocumentation, GetMemberLocation); o = new PythonFunctionOverload(fn.Name, this, _ => fn.Location); - o.SetReturnValueProvider((interpreter, overload, location, args) => CreateNamedTuple(args)); + o.SetReturnValueProvider((interpreter, overload, location, args) => CreateNamedTuple(args.Values())); fn.AddOverload(o); _members["NamedTuple"] = fn; @@ -142,10 +146,12 @@ private void SpecializeMembers() { var anyStrArgs = Interpreter.LanguageVersion.Is3x() ? new IMember[] { anyStrName, str, bytes } : new IMember[] { anyStrName, str, unicode }; - _members["AnyStr"] = GenericTypeParameter.FromTypeVar(anyStrArgs, this, LocationInfo.Empty); + _members["AnyStr"] = GenericTypeParameter.FromTypeVar(new ArgumentSet(anyStrArgs), this, LocationInfo.Empty); _members["Optional"] = new GenericType("Optional", this, (typeArgs, module, location) => CreateOptional(typeArgs)); _members["Type"] = new GenericType("Type", this, (typeArgs, module, location) => CreateType(typeArgs)); + + _members["Generic"] = new GenericType("Generic", this, (typeArgs, module, location) => CreateGenericBase(typeArgs, module)); } @@ -288,5 +294,21 @@ private IPythonType CreateType(IReadOnlyList typeArgs) { // TODO: report wrong number of arguments return Interpreter.UnknownType; } + + private IPythonType CreateGenericBase(IReadOnlyList typeArgs, IPythonModule declaringModule) { + // Handle Generic[_T1, _T2, ...]. _T1, et al are IGenericTypeParameter from TypeVar. + // Hold the parameter until concrete type is provided at the time + // of the class instantiation. + if (typeArgs.Count > 0) { + var genericTypes = typeArgs.OfType().ToArray(); + if (genericTypes.Length == typeArgs.Count) { + return new GenericType("Generic", declaringModule, genericTypes); + } else { + // TODO: report some type arguments are undefined. + } + } + // TODO: report wrong number of arguments + return Interpreter.UnknownType; + } } } diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index 96f315ffb..460127466 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -18,12 +18,11 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Extensions; -using Microsoft.Python.Analysis.Types.Collections; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Analysis.Values.Collections; using Microsoft.Python.Core; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; @@ -36,7 +35,6 @@ namespace Microsoft.Python.Analysis.Types { internal sealed class ArgumentSet : IArgumentSet { private readonly List _arguments = new List(); private readonly List _errors = new List(); - private readonly ExpressionEval _eval; private readonly ListArg _listArgument; private readonly DictArg _dictArgument; private bool _evaluated; @@ -47,11 +45,24 @@ internal sealed class ArgumentSet : IArgumentSet { public IListArgument ListArgument => _listArgument; public IDictionaryArgument DictionaryArgument => _dictArgument; public IReadOnlyList Errors => _errors; + public int OverloadIndex { get; } + public IExpressionEvaluator Eval { get; } + private ArgumentSet() { } - public ArgumentSet(IPythonFunctionType fn, IPythonInstance instance, CallExpression callExpr, ExpressionEval eval) : - this(fn, instance, callExpr, eval.Module, eval) { } + public ArgumentSet(IReadOnlyList typeArgs) { + _arguments = typeArgs.Select(t => new Argument(t, LocationInfo.Empty)).ToList(); + _evaluated = true; + } + + public ArgumentSet(IReadOnlyList memberArgs) { + _arguments = memberArgs.Select(t => new Argument(t, LocationInfo.Empty)).ToList(); + _evaluated = true; + } + + public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, ExpressionEval eval) : + this(fn, overloadIndex, instance, callExpr, eval.Module, eval) { } /// /// Creates set of arguments for a function call based on the call expression @@ -59,15 +70,19 @@ public ArgumentSet(IPythonFunctionType fn, IPythonInstance instance, CallExpress /// for arguments, but not actual values. on how to /// get values for actual parameters. /// - /// Function to call. + /// Function type. + /// Function overload to call. /// Type instance the function is bound to. For derived classes it is different from the declared type. /// Call expression that invokes the function. /// Module that contains the call expression. /// Evaluator that can calculate values of arguments from their respective expressions. - public ArgumentSet(IPythonFunctionType fn, IPythonInstance instance, CallExpression callExpr, IPythonModule module, ExpressionEval eval) { - _eval = eval; + public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance instance, CallExpression callExpr, IPythonModule module, ExpressionEval eval) { + Eval = eval; + OverloadIndex = overloadIndex; + + var overload = fn.Overloads[overloadIndex]; + var fd = overload.FunctionDefinition; - var fd = fn.FunctionDefinition; if (fd == null || fn.IsSpecialized) { // Typically specialized function, like TypeVar() that does not actually have AST definition. // Make the arguments from the call expression. If argument does not have name, @@ -75,11 +90,14 @@ public ArgumentSet(IPythonFunctionType fn, IPythonInstance instance, CallExpress _arguments = new List(); for (var i = 0; i < callExpr.Args.Count; i++) { var name = callExpr.Args[i].Name; + var location = fd != null && i < fd.Parameters.Length ? fd.Parameters[i].GetLocation(fn.DeclaringModule) : LocationInfo.Empty; if (string.IsNullOrEmpty(name)) { name = fd != null && i < fd.Parameters.Length ? fd.Parameters[i].Name : null; } name = name ?? $"arg{i}"; - _arguments.Add(new Argument(name, ParameterKind.Normal) { Expression = callExpr.Args[i].Expression }); + _arguments.Add(new Argument(name, ParameterKind.Normal, location) { + Expression = callExpr.Args[i].Expression + }); } return; } @@ -98,7 +116,7 @@ public ArgumentSet(IPythonFunctionType fn, IPythonInstance instance, CallExpress // had values assigned to them are marked as 'filled'.Slots which have // no value assigned to them yet are considered 'empty'. - var slots = fd.Parameters.Select(p => new Argument(p.Name, p.Kind)).ToArray(); + var slots = fd.Parameters.Select(p => new Argument(p.Name, p.Kind, p.GetLocation(module))).ToArray(); // Locate sequence argument, if any var sa = slots.Where(s => s.Kind == ParameterKind.List).ToArray(); if (sa.Length > 1) { @@ -112,8 +130,8 @@ public ArgumentSet(IPythonFunctionType fn, IPythonInstance instance, CallExpress return; } - _listArgument = sa.Length == 1 && sa[0].Name.Length > 0 ? new ListArg(sa[0].Name) : null; - _dictArgument = da.Length == 1 ? new DictArg(da[0].Name) : null; + _listArgument = sa.Length == 1 && sa[0].Name.Length > 0 ? new ListArg(sa[0].Name, sa[0].Expression, sa[0].Location) : null; + _dictArgument = da.Length == 1 ? new DictArg(da[0].Name, da[0].Expression, da[0].Location) : null; // Class methods var formalParamIndex = 0; @@ -248,68 +266,89 @@ public ArgumentSet(IPythonFunctionType fn, IPythonInstance instance, CallExpress } public async Task EvaluateAsync(CancellationToken cancellationToken = default) { - if (_evaluated || _eval == null) { - return this; - } + if (_evaluated || Eval == null) { + return this; + } - foreach (var a in _arguments.Where(x => x.Value == null)) { - a.Value = await _eval.GetValueFromExpressionAsync(a.Expression, cancellationToken); - } + foreach (var a in _arguments.Where(x => x.Value == null)) { + a.Value = await Eval.GetValueFromExpressionAsync(a.Expression, cancellationToken); + } - if (_listArgument != null) { - foreach (var e in _listArgument.Expressions) { - var value = await _eval.GetValueFromExpressionAsync(e, cancellationToken); - _listArgument._Values.Add(value); - } + if (_listArgument != null) { + foreach (var e in _listArgument.Expressions) { + var value = await Eval.GetValueFromExpressionAsync(e, cancellationToken); + _listArgument._Values.Add(value); } + } - if (_dictArgument != null) { - foreach (var e in _dictArgument.Expressions) { - var value = await _eval.GetValueFromExpressionAsync(e.Value, cancellationToken); - _dictArgument._Args[e.Key] = value; - } + if (_dictArgument != null) { + foreach (var e in _dictArgument.Expressions) { + var value = await Eval.GetValueFromExpressionAsync(e.Value, cancellationToken); + _dictArgument._Args[e.Key] = value; } - - _evaluated = true; - return this; } + _evaluated = true; + return this; + } + private sealed class Argument : IArgument { public string Name { get; } public object Value { get; internal set; } public ParameterKind Kind { get; } public Expression Expression { get; set; } + public LocationInfo Location { get; } - public Argument(string name, ParameterKind kind) { + public Argument(string name, ParameterKind kind, LocationInfo location) { Name = name; Kind = kind; + Location = location; + } + + public Argument(IPythonType type, LocationInfo location) : this(type.Name, type, location) { } + public Argument(IMember member, LocationInfo location) : this(string.Empty, member, location) { } + + private Argument(string name, object value, LocationInfo location) { + Name = name; + Value = value; + Location = location; } } private sealed class ListArg : IListArgument { public string Name { get; } + public Expression Expression { get; } + public LocationInfo Location { get; } + public IReadOnlyList Values => _Values; public IReadOnlyList Expressions => _Expressions; public List _Values { get; } = new List(); public List _Expressions { get; } = new List(); - public ListArg(string name) { + public ListArg(string name, Expression expression, LocationInfo location) { Name = name; + Expression = expression; + Location = location; } } private sealed class DictArg : IDictionaryArgument { public string Name { get; } + public Expression Expression { get; } + public LocationInfo Location { get; } + public IReadOnlyDictionary Arguments => _Args; public IReadOnlyDictionary Expressions => _Expressions; public Dictionary _Args { get; } = new Dictionary(); public Dictionary _Expressions { get; } = new Dictionary(); - public DictArg(string name) { + public DictArg(string name, Expression expression, LocationInfo location) { Name = name; + Expression = expression; + Location = location; } } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs index bb9d37d08..7ce73f711 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs @@ -15,30 +15,129 @@ // permissions and limitations under the License. using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { + /// + /// Function call argument. + /// public interface IArgument { + /// + /// Argument name. + /// string Name { get; } + + /// + /// Expression that evaluates to the value of the argument. + /// Function call parameter. + /// Expression Expression { get; } + + /// + /// Location of the argument in the function definition. + /// Not the same as location of the . + /// + LocationInfo Location { get; } + + /// + /// Value of the argument. + /// object Value { get; } } + /// + /// List argument, such as *args. + /// public interface IListArgument { + /// + /// Argument name. + /// string Name { get; } + + /// + /// Expression that evaluates to the value of the argument. + /// Function call parameter. + /// + Expression Expression { get; } + + /// + /// Location of the argument in the function definition. + /// Not the same as location of the . + /// + LocationInfo Location { get; } + + /// + /// Expressions that evaluate to the elements of the list. + /// IReadOnlyList Expressions { get; } + + /// + /// Values of the elements of the list. + /// IReadOnlyList Values { get; } } + /// + /// Dictionary argument, such as **kwargs. + /// public interface IDictionaryArgument { + /// + /// Argument name. + /// string Name { get; } + + /// + /// Expression that evaluates to the value of the argument. + /// Function call parameter. + /// + Expression Expression { get; } + + /// + /// Location of the argument in the function definition. + /// Not the same as location of the . + /// + LocationInfo Location { get; } + + /// + /// Dictionary arguments. + /// IReadOnlyDictionary Arguments { get; } + + /// + /// Expressions that evaluate to arguments. + /// Function call parameters. + /// IReadOnlyDictionary Expressions { get; } } + /// + /// Describes set of arguments for a function call. + /// public interface IArgumentSet { + /// + /// Regular arguments + /// IReadOnlyList Arguments { get; } + + /// + /// List argument, such as *args. + /// IListArgument ListArgument { get; } + + /// + /// Dictionary argument, such as **kwargs. + /// IDictionaryArgument DictionaryArgument { get; } + + /// + /// Specifies which function overload to call. + /// + int OverloadIndex { get; } + + /// + /// Evaluator associated with the set. + /// + IExpressionEvaluator Eval { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs index e44ca95f1..046c52886 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonClassType.cs @@ -21,8 +21,19 @@ namespace Microsoft.Python.Analysis.Types { /// Represents Python class type definition. /// public interface IPythonClassType : IPythonType { + /// + /// Class definition node in the AST. + /// ClassDefinition ClassDefinition { get; } + + /// + /// Python Method Resolution Order (MRO). + /// IReadOnlyList Mro { get; } + + /// + /// Base types. + /// IReadOnlyList Bases { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs index bd8be66cd..a284a982f 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionOverload.cs @@ -26,6 +26,11 @@ public interface IPythonFunctionOverload { /// FunctionDefinition FunctionDefinition { get; } + /// + /// he corresponding function or property. + /// + IPythonClassMember ClassMember { get; } + /// /// Function name. /// @@ -35,7 +40,7 @@ public interface IPythonFunctionOverload { /// Overload documentation. /// string Documentation { get; } - + /// /// Overload parameters. /// @@ -56,5 +61,10 @@ public interface IPythonFunctionOverload { /// Function definition is decorated with @overload. /// bool IsOverload { get; } + + /// + /// Return type as determined from evaluation or from the return type annotation. + /// + IMember StaticReturnValue { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionType.cs index 7ab68acd6..dce37b589 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonFunctionType.cs @@ -46,6 +46,11 @@ public interface IPythonFunctionType : IPythonClassMember { /// bool IsStub { get; } + /// + /// Function an unbound class method. + /// + bool IsUnbound { get; } + /// /// List of function overloads /// diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs index 2e3fbcb4f..b2b67ba62 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonModule.cs @@ -23,6 +23,11 @@ namespace Microsoft.Python.Analysis.Types { /// Represents a Python module. /// public interface IPythonModule : IPythonType, IPythonFile, ILocatedMember { + /// + /// Module analysis. + /// + IDocumentAnalysis Analysis { get; } + /// /// Interpreter associated with the module. /// diff --git a/src/LanguageServer/Impl/Definitions/ITextChangeNotifications.cs b/src/Analysis/Ast/Impl/Types/Definitions/IPythonTemplateType.cs similarity index 51% rename from src/LanguageServer/Impl/Definitions/ITextChangeNotifications.cs rename to src/Analysis/Ast/Impl/Types/Definitions/IPythonTemplateType.cs index 23d2e525e..7e1452340 100644 --- a/src/LanguageServer/Impl/Definitions/ITextChangeNotifications.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IPythonTemplateType.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -14,15 +13,20 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Python.LanguageServer.Definitions { - public interface IDocumentChangeNotifications { - Task DidChangeConfiguration(DidChangeConfigurationParams @params); - Task DidOpenTextDocument(DidOpenTextDocumentParams @params, CancellationToken token); - Task DidChangeTextDocument(DidChangeTextDocumentParams @params, CancellationToken token); - Task DidSaveTextDocument(DidSaveTextDocumentParams @params, CancellationToken token); - Task DidCloseTextDocument(DidCloseTextDocumentParams @params, CancellationToken token); +namespace Microsoft.Python.Analysis.Types { + /// + /// Represents type that can be a template for specific types. + /// The base type of generics. + /// + public interface IPythonTemplateType: IPythonType { + /// + /// Creates instance of a type information with the specific + /// type arguments from a generic template. + /// + Task CreateSpecificTypeAsync(IArgumentSet typeArguments, IPythonModule declaringModule, LocationInfo location, CancellationToken cancellationToken = default); } } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/LocationInfo.cs b/src/Analysis/Ast/Impl/Types/Definitions/LocationInfo.cs index 97dfffb4d..375e522c6 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/LocationInfo.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/LocationInfo.cs @@ -23,9 +23,7 @@ public sealed class LocationInfo : IEquatable { [DebuggerDisplay("{StartLine}, {StartColumn} - {EndLine}, {EndColumn}")] public static readonly LocationInfo Empty = new LocationInfo(); - private LocationInfo() { - FilePath = string.Empty; - } + private LocationInfo() : this(string.Empty, null, 1, 1) { } public LocationInfo(string path, Uri documentUri, int line, int column) : this(path, documentUri, line, column, null, null) { diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index 359d939f9..a36fb0971 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -18,8 +18,11 @@ using System.Diagnostics; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types.Collections; +using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Analysis.Values.Collections; using Microsoft.Python.Core; @@ -27,15 +30,14 @@ namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("Class {Name}")] - internal sealed class PythonClassType : PythonType, IPythonClassType, IEquatable { + internal class PythonClassType : PythonType, IPythonClassType, IPythonTemplateType, IEquatable { private readonly object _lock = new object(); - + private readonly AsyncLocal _processing = new AsyncLocal(); private IReadOnlyList _mro; - private readonly AsyncLocal _isProcessing = new AsyncLocal(); // For tests - internal PythonClassType(string name, IPythonModule declaringModule) - : base(name, declaringModule, string.Empty, LocationInfo.Empty, BuiltinTypeId.Type) { } + internal PythonClassType(string name, IPythonModule declaringModule, LocationInfo location = null) + : base(name, declaringModule, string.Empty, location ?? LocationInfo.Empty, BuiltinTypeId.Type) { } public PythonClassType( ClassDefinition classDefinition, @@ -74,7 +76,7 @@ public override IMember GetMember(string name) { return member; } } - if (Push()) { + if (Push(this)) { try { foreach (var m in Mro.Reverse()) { if (m == this) { @@ -91,11 +93,13 @@ public override IMember GetMember(string name) { public override string Documentation { get { + // Try doc from the type (class definition AST node). var doc = base.Documentation; + // Try bases. if (string.IsNullOrEmpty(doc) && Bases != null) { - // Try bases doc = Bases.FirstOrDefault(b => !string.IsNullOrEmpty(b?.Documentation))?.Documentation; } + // Try docs __init__. if (string.IsNullOrEmpty(doc)) { doc = GetMember("__init__")?.GetPythonType()?.Documentation; } @@ -108,7 +112,7 @@ public override IMember CreateInstance(string typeName, LocationInfo location, I // Specializations switch (typeName) { case "list": - return PythonCollectionType.CreateList(DeclaringModule.Interpreter, location, args); + return PythonCollectionType.CreateList(DeclaringModule.Interpreter, location, args); case "dict": { // self, then contents var contents = args.Values().Skip(1).FirstOrDefault(); @@ -145,7 +149,7 @@ public IReadOnlyList Mro { } #endregion - internal void SetBases(IPythonInterpreter interpreter, IEnumerable bases) { + internal void SetBases(IEnumerable bases) { lock (_lock) { if (Bases != null) { return; // Already set @@ -219,9 +223,91 @@ internal static IReadOnlyList CalculateMro(IPythonType cls, HashSet } } - private bool Push() => !_isProcessing.Value && (_isProcessing.Value = true); - private void Pop() => _isProcessing.Value = false; + private bool Push(IPythonClassType cls) { + if (_processing.Value == null) { + _processing.Value = cls; + return true; + } + return false; + } + + private void Pop() => _processing.Value = null; public bool Equals(IPythonClassType other) => Name == other?.Name && DeclaringModule.Equals(other?.DeclaringModule); + + public async Task CreateSpecificTypeAsync(IArgumentSet args, IPythonModule declaringModule, LocationInfo location, CancellationToken cancellationToken = default) { + cancellationToken.ThrowIfCancellationRequested(); + + var genericBases = Bases.Where(b => b is IGenericType).ToArray(); + // TODO: handle optional generics as class A(Generic[_T1], Optional[Generic[_T2]]) + if (genericBases.Length != args.Arguments.Count) { + // TODO: report parameters mismatch. + } + + // Create concrete type + var bases = args.Arguments.Select(a => a.Value).OfType().ToArray(); + var specificName = CodeFormatter.FormatSequence(Name, '[', bases); + var classType = new PythonClassType(specificName, declaringModule); + + // Prevent reentrancy when resolving generic class where + // method may be returning instance of type of the same class. + if (!Push(classType)) { + return _processing.Value; + } + + try { + // Optimistically use what is available even if there is an argument mismatch. + // TODO: report unresolved types? + classType.SetBases(bases); + + // Add members from the template class (this one). + classType.AddMembers(this, true); + + // Resolve return types of methods, if any were annotated as generics + var members = classType.GetMemberNames() + .Except(new[] { "__class__", "__bases__", "__base__" }) + .ToDictionary(n => n, n => classType.GetMember(n)); + + foreach (var m in members) { + switch (m.Value) { + case IPythonFunctionType fn: { + foreach (var o in fn.Overloads.OfType()) { + var returnType = o.StaticReturnValue.GetPythonType(); + if (returnType is PythonClassType cls && cls.IsGeneric()) { + // -> A[_E] + if (!cls.Equals(classType)) { + // Prevent reentrancy + var specificReturnValue = await cls.CreateSpecificTypeAsync(args, declaringModule, location, cancellationToken); + o.SetReturnValue(new PythonInstance(specificReturnValue, location), true); + } + } else if (returnType is IGenericTypeParameter) { + // -> _T + var b = bases.FirstOrDefault(); + if (b != null) { + o.SetReturnValue(new PythonInstance(b, location), true); + } + } + } + break; + } + case IPythonTemplateType tt: { + var specificType = await tt.CreateSpecificTypeAsync(args, declaringModule, location, cancellationToken); + classType.AddMember(m.Key, specificType, true); + break; + } + case IPythonInstance inst: { + if (inst.GetPythonType() is IPythonTemplateType tt && tt.IsGeneric()) { + var specificType = await tt.CreateSpecificTypeAsync(args, declaringModule, location, cancellationToken); + classType.AddMember(m.Key, new PythonInstance(specificType, location), true); + } + break; + } + } + } + } finally { + Pop(); + } + return classType; + } } } diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index d05daefa9..000bf85d0 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -16,7 +16,8 @@ using System; using System.Collections.Generic; using System.Linq; -using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Parsing.Ast; @@ -35,7 +36,7 @@ public delegate IMember ReturnValueProvider( IPythonModule declaringModule, IPythonFunctionOverload overload, LocationInfo location, - IReadOnlyList args); + IArgumentSet args); internal sealed class PythonFunctionOverload : IPythonFunctionOverload, ILocatedMember { private readonly Func _locationProvider; @@ -51,20 +52,21 @@ internal sealed class PythonFunctionOverload : IPythonFunctionOverload, ILocated private Func _documentationProvider; private bool _fromAnnotation; - public PythonFunctionOverload(string name, IPythonModule declaringModule, LocationInfo location, string returnDocumentation = null) - : this(name, declaringModule, _ => location ?? LocationInfo.Empty, returnDocumentation) { + public PythonFunctionOverload(string name, IPythonModule declaringModule, LocationInfo location) + : this(name, declaringModule, _ => location ?? LocationInfo.Empty) { _declaringModule = declaringModule; } - public PythonFunctionOverload(FunctionDefinition fd, IPythonModule declaringModule, LocationInfo location, string returnDocumentation = null) - : this(fd.Name, declaringModule, _ => location, returnDocumentation) { + public PythonFunctionOverload(FunctionDefinition fd, IPythonClassMember classMember, IPythonModule declaringModule, LocationInfo location) + : this(fd.Name, declaringModule, _ => location) { FunctionDefinition = fd; + ClassMember = classMember; + var ast = (declaringModule as IDocument)?.Analysis.Ast; + ReturnDocumentation = ast != null ? fd.ReturnAnnotation?.ToCodeString(ast) : null; } - public PythonFunctionOverload(string name, IPythonModule declaringModule, - Func locationProvider, string returnDocumentation = null) { + public PythonFunctionOverload(string name, IPythonModule declaringModule, Func locationProvider) { Name = name ?? throw new ArgumentNullException(nameof(name)); - ReturnDocumentation = returnDocumentation; _declaringModule = declaringModule; _locationProvider = locationProvider; } @@ -100,11 +102,9 @@ internal void SetReturnValue(IMember value, bool fromAnnotation) { internal void SetReturnValueProvider(ReturnValueProvider provider) => _returnValueProvider = provider; - internal IMember StaticReturnValue { get; private set; } - - #region IPythonFunctionOverload public FunctionDefinition FunctionDefinition { get; } + public IPythonClassMember ClassMember { get; } public string Name { get; } public string Documentation { @@ -122,16 +122,35 @@ public string Documentation { public IReadOnlyList Parameters { get; private set; } = Array.Empty(); public LocationInfo Location => _locationProvider?.Invoke(Name) ?? LocationInfo.Empty; public PythonMemberType MemberType => PythonMemberType.Function; + public IMember StaticReturnValue { get; private set; } public IMember GetReturnValue(LocationInfo callLocation, IArgumentSet args) { if (!_fromAnnotation) { // First try supplied specialization callback. - var rt = _returnValueProvider?.Invoke(_declaringModule, this, callLocation, args.Values()); + var rt = _returnValueProvider?.Invoke(_declaringModule, this, callLocation, args); if (!rt.IsUnknown()) { return rt; } } - return StaticReturnValue; + + // If function returns generic, try to return the incoming argument + // TODO: improve this, the heuristic is pretty basic and tailored to simple func(_T) -> _T + IMember retValue = null; + if (StaticReturnValue.GetPythonType() is IGenericTypeParameter) { + if (args.Arguments.Count > 0) { + retValue = args.Arguments[0].Value as IMember; + } + + if (retValue == null) { + // Try returning the constraint + var name = StaticReturnValue.GetPythonType()?.Name; + var typeDefVar = _declaringModule.Analysis.GlobalScope.Variables[name]; + if (typeDefVar?.Value is IGenericTypeParameter gtp) { + retValue = gtp.Constraints.FirstOrDefault(); + } + } + } + return retValue ?? StaticReturnValue; } #endregion diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index bf5a0a7f7..889b0805f 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -29,6 +29,7 @@ internal class PythonFunctionType : PythonType, IPythonFunctionType { private readonly object _lock = new object(); private bool _isAbstract; private bool _isSpecialized; + private string[] _dependencies = Array.Empty(); /// /// Creates function for specializations @@ -99,8 +100,8 @@ public override PythonMemberType MemberType public override IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) { // Now we can go and find overload with matching arguments. - var overload = FindOverload(args); - return overload?.GetReturnValue(instance?.Location ?? LocationInfo.Empty, args) ?? DeclaringModule.Interpreter.UnknownType; + var overload = Overloads[args.OverloadIndex]; + return overload?.GetReturnValue(instance?.Location ?? LocationInfo.Empty, args); } internal override void SetDocumentationProvider(Func provider) { @@ -123,6 +124,7 @@ internal override void SetDocumentationProvider(Func provider) { public bool IsOverload { get; private set; } public bool IsStub { get; internal set; } + public bool IsUnbound => DeclaringType == null; public IReadOnlyList Overloads => _overloads.ToArray(); #endregion @@ -133,7 +135,12 @@ internal override void SetDocumentationProvider(Func provider) { new KeyValuePair((DeclaringType as IHasQualifiedName)?.FullyQualifiedName ?? DeclaringType?.Name ?? DeclaringModule?.Name, Name); #endregion - internal void Specialize() => _isSpecialized = true; + internal void Specialize(string[] dependencies) { + _isSpecialized = true; + _dependencies = dependencies ?? Array.Empty(); + } + + internal IEnumerable Dependencies => _dependencies; internal void AddOverload(IPythonFunctionOverload overload) { lock (_lock) { @@ -175,48 +182,6 @@ private void ProcessDecorators(FunctionDefinition fd) { } } - private IPythonFunctionOverload FindOverload(IArgumentSet args) { - // Find best overload match. Of only one, use it. - if (Overloads.Count == 1) { - return Overloads[0]; - } - // Try matching parameters - return Overloads.FirstOrDefault(o => IsMatch(args, o.Parameters)); - } - - public static bool IsMatch(IArgumentSet args, IReadOnlyList parameters) { - // Arguments passed to function are created off the function definition - // and hence match by default. However, if multiple overloads are specified, - // we need to figure out if annotated types match. - // https://docs.python.org/3/library/typing.html#typing.overload - // - // @overload - // def process(response: None) -> None: - // @overload - // def process(response: int) -> Tuple[int, str]: - // - // Note that in overloads there are no * or ** parameters. - // We match loosely by type. - - var d = parameters.ToDictionary(p => p.Name, p => p.Type); - foreach (var a in args.Arguments()) { - if (!d.TryGetValue(a.Key, out var t)) { - return false; - } - - var at = a.Value?.GetPythonType(); - if (t == null && at == null) { - continue; - } - - if (t != null && at != null && !t.Equals(at)) { - return false; - } - } - return true; - } - - /// /// Represents unbound method, such in C.f where C is class rather than the instance. /// @@ -233,6 +198,7 @@ public PythonUnboundMethod(IPythonFunctionType function) : base(function, functi public bool IsClassMethod => _pf.IsClassMethod; public bool IsOverload => _pf.IsOverload; public bool IsStub => _pf.IsStub; + public bool IsUnbound => true; public IReadOnlyList Overloads => _pf.Overloads; public override BuiltinTypeId TypeId => BuiltinTypeId.Function; diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index 54dff441b..84c544749 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -24,9 +24,11 @@ namespace Microsoft.Python.Analysis.Types { internal class PythonType : IPythonType, ILocatedMember, IHasQualifiedName, IEquatable { private readonly object _lock = new object(); private readonly Func _locationProvider; + private readonly string _name; private Func _documentationProvider; private Dictionary _members; private BuiltinTypeId _typeId; + private bool _readonly; protected IReadOnlyDictionary Members => WritableMembers; @@ -47,19 +49,17 @@ public PythonType( Func documentationProvider, Func locationProvider, BuiltinTypeId typeId = BuiltinTypeId.Unknown - ) : this(name, typeId) { + ) { + _name = name ?? throw new ArgumentNullException(nameof(name)); DeclaringModule = declaringModule; _documentationProvider = documentationProvider; _locationProvider = locationProvider; - } - - public PythonType(string name, BuiltinTypeId typeId) { - Name = name ?? throw new ArgumentNullException(nameof(name)); _typeId = typeId; } #region IPythonType - public virtual string Name { get; } + + public virtual string Name => TypeId == BuiltinTypeId.Ellipsis ? "..." : _name; public virtual string Documentation => _documentationProvider?.Invoke(Name); public IPythonModule DeclaringModule { get; } public virtual PythonMemberType MemberType => _typeId.GetMemberId(); @@ -84,7 +84,7 @@ public virtual IMember CreateInstance(string typeName, LocationInfo location, IA /// Instance of the type. /// Member name to call, if applicable. /// Call arguments. - public virtual IMember Call(IPythonInstance instance, string memberName, IArgumentSet argSet) + public virtual IMember Call(IPythonInstance instance, string memberName, IArgumentSet argSet) => instance?.Call(memberName, argSet) ?? UnknownType; /// @@ -122,16 +122,20 @@ internal bool TrySetTypeId(BuiltinTypeId typeId) { internal void AddMembers(IEnumerable variables, bool overwrite) { lock (_lock) { - foreach (var v in variables.Where(m => overwrite || !Members.ContainsKey(m.Name))) { - WritableMembers[v.Name] = v.Value.GetPythonType(); + if (!_readonly) { + foreach (var v in variables.Where(m => overwrite || !Members.ContainsKey(m.Name))) { + WritableMembers[v.Name] = v.Value; + } } } } internal void AddMembers(IEnumerable> members, bool overwrite) { lock (_lock) { - foreach (var kv in members.Where(m => overwrite || !Members.ContainsKey(m.Key))) { - WritableMembers[kv.Key] = kv.Value; + if (!_readonly) { + foreach (var kv in members.Where(m => overwrite || !Members.ContainsKey(m.Key))) { + WritableMembers[kv.Key] = kv.Value; + } } } } @@ -146,15 +150,18 @@ internal void AddMembers(IPythonClassType cls, bool overwrite) { internal IMember AddMember(string name, IMember member, bool overwrite) { lock (_lock) { - if (overwrite || !Members.ContainsKey(name)) { - WritableMembers[name] = member; + if (!_readonly) { + if (overwrite || !Members.ContainsKey(name)) { + WritableMembers[name] = member; + } } return member; } } - internal bool IsHidden => ContainsMember("__hidden__"); + internal void MakeReadOnly() => _readonly = true; + internal bool IsHidden => ContainsMember("__hidden__"); protected bool ContainsMember(string name) => Members.ContainsKey(name); protected IMember UnknownType => DeclaringModule.Interpreter.UnknownType; diff --git a/src/Analysis/Ast/Impl/Types/PythonUnionType.cs b/src/Analysis/Ast/Impl/Types/PythonUnionType.cs index 3e13f3485..46e91169e 100644 --- a/src/Analysis/Ast/Impl/Types/PythonUnionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonUnionType.cs @@ -47,13 +47,18 @@ public string Name { } } - public IPythonModule DeclaringModule => null; + public IPythonModule DeclaringModule { + get { lock (_lock) { return _types.First().DeclaringModule; } } + } + public BuiltinTypeId TypeId => BuiltinTypeId.Type; public PythonMemberType MemberType => PythonMemberType.Union; public string Documentation => Name; + public bool IsBuiltin { get { lock (_lock) { return _types.All(t => t.IsBuiltin); } } } + public bool IsAbstract => false; public bool IsSpecialized => true; @@ -61,9 +66,9 @@ public IMember CreateInstance(string typeName, LocationInfo location, IArgumentS => new PythonUnion(this, location); public IMember Call(IPythonInstance instance, string memberName, IArgumentSet args) - => DeclaringModule?.Interpreter.UnknownType ?? this; + => DeclaringModule.Interpreter.UnknownType; public IMember Index(IPythonInstance instance, object index) - => DeclaringModule?.Interpreter.UnknownType ?? this; + => DeclaringModule.Interpreter.UnknownType; #endregion #region IPythonUnionType diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs index df1f2cdeb..df13dabaf 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs @@ -33,6 +33,6 @@ public interface IScope { IVariableCollection Variables { get; } IVariableCollection NonLocals { get; } IVariableCollection Globals { get; } - void DeclareVariable(string name, IMember value, LocationInfo location); + void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location); } } diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs b/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs index c4ebe7557..b8e4976c8 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs @@ -20,7 +20,17 @@ namespace Microsoft.Python.Analysis.Values { /// Represents a variable. /// public interface IVariable: ILocatedMember { + /// + /// Variable name. + /// string Name { get; } + /// + /// Variable source. + /// + VariableSource Source { get; } + /// + /// Variable value. + /// IMember Value { get; set; } } } diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IVariableCollection.cs b/src/Analysis/Ast/Impl/Values/Definitions/IVariableCollection.cs index 0ae5b635a..530bf74fb 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IVariableCollection.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IVariableCollection.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System.Collections.Generic; -using Microsoft.Python.Analysis.Types; namespace Microsoft.Python.Analysis.Values { public interface IVariableCollection: IReadOnlyCollection { diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleLoadOptions.cs b/src/Analysis/Ast/Impl/Values/Definitions/VariableSource.cs similarity index 56% rename from src/Analysis/Ast/Impl/Modules/Definitions/ModuleLoadOptions.cs rename to src/Analysis/Ast/Impl/Values/Definitions/VariableSource.cs index 0fa7c340c..07eee6fc2 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/ModuleLoadOptions.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/VariableSource.cs @@ -13,35 +13,30 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; - -namespace Microsoft.Python.Analysis.Modules { - [Flags] - public enum ModuleLoadOptions { - /// - /// Do nothing. Typically this is a placeholder or empty module. - /// - None, - +namespace Microsoft.Python.Analysis.Values { + /// + /// Describes variable source. Used in filtering variables during completion + /// so the list does not show what the imported module has imported itself + /// or what generic variables it has declared for internal purposes. + /// + public enum VariableSource { /// - /// Just load the document, do not parse or analyze. + /// Variable is user declaration. /// - Load = 1, - + Declaration, /// - /// Load and parse. Do not analyze. + /// Variable is import from another module. /// - Ast = Load | 2, - + Import, /// - /// Load, parse and analyze. + /// Variable is a generic type definition. /// - Analyze = Ast | 4, + Generic, /// - /// The document is opened in the editor. - /// This implies Ast and Analysis. + /// Variable is as reference to existing variable + /// declared as nonlocal or global. /// - Open = Analyze | 8 + Locality } } diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index 8815f27c7..1e87f1c53 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -67,12 +67,12 @@ public IEnumerable EnumerateTowardsGlobal { } public IEnumerable EnumerateFromGlobal => EnumerateTowardsGlobal.Reverse(); - public void DeclareVariable(string name, IMember value, LocationInfo location) - => (_variables ?? (_variables = new VariableCollection())).DeclareVariable(name, value, location); + public void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location) + => (_variables ?? (_variables = new VariableCollection())).DeclareVariable(name, value, source, location); public void DeclareNonLocal(string name, LocationInfo location) - => (_nonLocals ?? (_nonLocals = new VariableCollection())).DeclareVariable(name, null, location); + => (_nonLocals ?? (_nonLocals = new VariableCollection())).DeclareVariable(name, null, VariableSource.Locality, location); public void DeclareGlobal(string name, LocationInfo location) - => (_globals ?? (_globals = new VariableCollection())).DeclareVariable(name, null, location); + => (_globals ?? (_globals = new VariableCollection())).DeclareVariable(name, null, VariableSource.Locality, location); #endregion public void AddChildScope(Scope s) => (_childScopes ?? (_childScopes = new List())).Add(s); @@ -97,7 +97,7 @@ public EmptyGlobalScope(IPythonModule module) { public IVariableCollection NonLocals => VariableCollection.Empty; public IVariableCollection Globals => VariableCollection.Empty; - public void DeclareVariable(string name, IMember value, LocationInfo location) { } + public void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location) { } } } diff --git a/src/Analysis/Ast/Impl/Values/Variable.cs b/src/Analysis/Ast/Impl/Values/Variable.cs index 3af65884d..e18beb614 100644 --- a/src/Analysis/Ast/Impl/Values/Variable.cs +++ b/src/Analysis/Ast/Impl/Values/Variable.cs @@ -19,17 +19,15 @@ namespace Microsoft.Python.Analysis.Values { [DebuggerDisplay("{DebuggerDisplay}")] internal sealed class Variable : IVariable { - public Variable(string name, IMember value, LocationInfo location = null) { + public Variable(string name, IMember value, VariableSource source, LocationInfo location = null) { Name = name; Value = value; - if (location != null) { - Location = location; - } else { - Location = value is ILocatedMember lm ? lm.Location : LocationInfo.Empty; - } + Source = source; + Location = location ?? (value is ILocatedMember lm ? lm.Location : LocationInfo.Empty); } public string Name { get; } + public VariableSource Source { get; } public IMember Value { get; set; } public LocationInfo Location { get; } public PythonMemberType MemberType => PythonMemberType.Variable; diff --git a/src/Analysis/Ast/Impl/Values/VariableCollection.cs b/src/Analysis/Ast/Impl/Values/VariableCollection.cs index b3d73bfe2..7c1b48a88 100644 --- a/src/Analysis/Ast/Impl/Values/VariableCollection.cs +++ b/src/Analysis/Ast/Impl/Values/VariableCollection.cs @@ -20,6 +20,7 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Values { [DebuggerDisplay("Count: {Count}")] @@ -45,7 +46,9 @@ internal sealed class VariableCollection : IVariableCollection { public IEnumerable GetMemberNames() => _variables.Keys.ToArray(); #endregion - internal void DeclareVariable(string name, IMember value, LocationInfo location) - => _variables[name] = new Variable(name, value, location); + internal void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location) { + name = !string.IsNullOrWhiteSpace(name) ? name : throw new ArgumentException(nameof(name)); + _variables[name] = new Variable(name, value, source, location); + } } } diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index 8fea80db7..51682bc1d 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -13,9 +13,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Threading; using System.Threading.Tasks; using FluentAssertions; @@ -31,7 +33,6 @@ using Microsoft.Python.Core.Services; using Microsoft.Python.Core.Shell; using Microsoft.Python.Core.Tests; -using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Tests; using TestUtilities; @@ -54,11 +55,11 @@ protected virtual ServiceManager CreateServiceManager() { protected string GetAnalysisTestDataFilesPath() => TestData.GetPath(Path.Combine("TestData", "AstAnalysis")); - internal async Task CreateServicesAsync(string root, InterpreterConfiguration configuration = null) { + protected async Task CreateServicesAsync(string root, InterpreterConfiguration configuration = null) { configuration = configuration ?? PythonVersions.LatestAvailable; configuration.AssertInstalled(); - Trace.TraceInformation("Cache Path: " + configuration.ModuleCachePath); - configuration.ModuleCachePath = TestData.GetAstAnalysisCachePath(configuration.Version, true); + Trace.TraceInformation("Cache Path: " + configuration.DatabasePath); + configuration.DatabasePath = TestData.GetAstAnalysisCachePath(configuration.Version, true); configuration.SearchPaths = new[] { GetAnalysisTestDataFilesPath() }; configuration.TypeshedPath = TestData.GetDefaultTypeshedPath(); @@ -71,7 +72,7 @@ internal async Task CreateServicesAsync(string root, Interprete sm.AddService(dependencyResolver); TestLogger.Log(TraceEventType.Information, "Create PythonAnalyzer"); - var analyzer = new PythonAnalyzer(sm); + var analyzer = new PythonAnalyzer(sm, root); sm.AddService(analyzer); TestLogger.Log(TraceEventType.Information, "Create PythonInterpreter"); @@ -85,22 +86,24 @@ internal async Task CreateServicesAsync(string root, Interprete return sm; } - internal async Task GetAnalysisAsync( + protected Task GetAnalysisAsync(string code, PythonLanguageVersion version, string modulePath = null) + => GetAnalysisAsync(code, PythonVersions.GetRequiredCPythonConfiguration(version), modulePath); + + protected async Task GetAnalysisAsync( string code, InterpreterConfiguration configuration = null, - string moduleName = null, string modulePath = null) { var moduleUri = TestData.GetDefaultModuleUri(); modulePath = modulePath ?? TestData.GetDefaultModulePath(); - moduleName = Path.GetFileNameWithoutExtension(modulePath); + var moduleName = Path.GetFileNameWithoutExtension(modulePath); var moduleDirectory = Path.GetDirectoryName(modulePath); var services = await CreateServicesAsync(moduleDirectory, configuration); return await GetAnalysisAsync(code, services, moduleName, modulePath); } - internal async Task GetAnalysisAsync( + protected async Task GetAnalysisAsync( string code, IServiceContainer services, string moduleName = null, @@ -113,15 +116,14 @@ internal async Task GetAnalysisAsync( IDocument doc; var rdt = services.GetService(); if (rdt != null) { - doc = rdt.AddDocument(moduleUri, code, modulePath); + doc = rdt.OpenDocument(moduleUri, code, modulePath); } else { var mco = new ModuleCreationOptions { ModuleName = moduleName, Content = code, FilePath = modulePath, Uri = moduleUri, - ModuleType = ModuleType.User, - LoadOptions = ModuleLoadOptions.Analyze + ModuleType = ModuleType.User }; doc = new PythonModule(mco, services); } @@ -145,14 +147,27 @@ public Task GetDependencyChainAsync(IDocument document, Ca } protected sealed class DiagnosticsService : IDiagnosticsService { - private readonly List _diagnostics = new List(); - - public IReadOnlyList Diagnostics => _diagnostics; + private readonly Dictionary> _diagnostics = new Dictionary>(); + private readonly object _lock = new object(); + + public IReadOnlyList Diagnostics { + get { + lock (_lock) { + return _diagnostics.Values.SelectMany().ToArray(); + } + } + } - public void Add(DiagnosticsEntry entry) => _diagnostics.Add(entry); + public void Add(Uri documentUri, DiagnosticsEntry entry) { + lock (_lock) { + if (!_diagnostics.TryGetValue(documentUri, out var list)) { + _diagnostics[documentUri] = list = new List(); + } + list.Add(entry); + } + } - public void Add(string message, SourceSpan span, string errorCode, Severity severity) - => Add(new DiagnosticsEntry(message, span, errorCode, severity)); + public int PublishingDelay { get; set; } } } } diff --git a/src/Analysis/Ast/Test/ArgumentSetTests.cs b/src/Analysis/Ast/Test/ArgumentSetTests.cs index 7490d781d..1e2714638 100644 --- a/src/Analysis/Ast/Test/ArgumentSetTests.cs +++ b/src/Analysis/Ast/Test/ArgumentSetTests.cs @@ -348,14 +348,14 @@ private async Task GetArgSetAsync(string code, string funcName = "f var analysis = await GetAnalysisAsync(code); var f = analysis.Should().HaveFunction(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f, null, call, analysis.Document, null); + return new ArgumentSet(f, 0, null, call, analysis.Document, null); } private async Task GetUnboundArgSetAsync(string code, string funcName = "f") { var analysis = await GetAnalysisAsync(code); var f = analysis.Should().HaveVariable(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f.Value.GetPythonType(), null, call, analysis.Document, null); + return new ArgumentSet(f.Value.GetPythonType(), 0, null, call, analysis.Document, null); } private async Task GetClassArgSetAsync(string code, string className = "A", string funcName = "f") { @@ -363,7 +363,7 @@ private async Task GetClassArgSetAsync(string code, string classNam var cls = analysis.Should().HaveClass(className).Which; var f = cls.Should().HaveMethod(funcName).Which; var call = GetCall(analysis.Ast); - return new ArgumentSet(f, new PythonInstance(cls), call, analysis.Document, null); + return new ArgumentSet(f, 0, new PythonInstance(cls), call, analysis.Document, null); } private CallExpression GetCall(PythonAst ast) { diff --git a/src/Analysis/Ast/Test/ClassesTests.cs b/src/Analysis/Ast/Test/ClassesTests.cs index 0bd2bfc5a..14c22bbdb 100644 --- a/src/Analysis/Ast/Test/ClassesTests.cs +++ b/src/Analysis/Ast/Test/ClassesTests.cs @@ -92,12 +92,12 @@ public async Task Mro() { var E = new PythonClassType("E", m); var F = new PythonClassType("F", m); - F.SetBases(interpreter, new[] { O }); - E.SetBases(interpreter, new[] { O }); - D.SetBases(interpreter, new[] { O }); - C.SetBases(interpreter, new[] { D, F }); - B.SetBases(interpreter, new[] { D, E }); - A.SetBases(interpreter, new[] { B, C }); + F.SetBases(new[] { O }); + E.SetBases(new[] { O }); + D.SetBases(new[] { O }); + C.SetBases(new[] { D, F }); + B.SetBases(new[] { D, E }); + A.SetBases(new[] { B, C }); PythonClassType.CalculateMro(A).Should().Equal(new[] { "A", "B", "C", "D", "E", "F", "O" }, (p, n) => p.Name == n); PythonClassType.CalculateMro(B).Should().Equal(new[] { "B", "D", "E", "O" }, (p, n) => p.Name == n); diff --git a/src/Analysis/Ast/Test/DocumentBufferTests.cs b/src/Analysis/Ast/Test/DocumentBufferTests.cs index 32d527be3..0a273ce55 100644 --- a/src/Analysis/Ast/Test/DocumentBufferTests.cs +++ b/src/Analysis/Ast/Test/DocumentBufferTests.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System; +using System.Collections.Generic; using FluentAssertions; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.Text; @@ -32,7 +33,7 @@ def g(y): return y * 2 "); - doc.Update(new DocumentChangeSet(0, 1, new[] { + doc.Update(new List { // We *should* batch adjacent insertions, but we should also // work fine even if we don't. Note that the insertion point // tracks backwards with previous insertions in the same version. @@ -43,33 +44,29 @@ def g(y): DocumentChange.Insert("(", new SourceLocation(2, 11)), DocumentChange.Insert("g", new SourceLocation(2, 11)), DocumentChange.Insert(" ", new SourceLocation(2, 11)) - })); + }); doc.Text.Should().Contain("return g(x)"); Assert.AreEqual(1, doc.Version); doc.Update(new[] { - new DocumentChangeSet(1, 2, new [] { - DocumentChange.Delete(new SourceLocation(2, 14), new SourceLocation(2, 15)) - }), - new DocumentChangeSet(2, 3, new [] { - DocumentChange.Insert("x * 2", new SourceLocation(2, 14)) - }) + DocumentChange.Delete(new SourceLocation(2, 14), new SourceLocation(2, 15)), + DocumentChange.Insert("x * 2", new SourceLocation(2, 14)) }); doc.Text.Should().Contain("return g(x * 2)"); - doc.Update(new DocumentChangeSet(3, 4, new[] { + doc.Update(new[] { DocumentChange.Replace(new SourceLocation(2, 18), new SourceLocation(2, 19), "300") - })); + }); doc.Text.Should().Contain("return g(x * 300)"); - doc.Update(new DocumentChangeSet(4, 5, new[] { + doc.Update(new[] { // Changes are out of order, but we should fix that automatically DocumentChange.Delete(new SourceLocation(2, 13), new SourceLocation(2, 22)), DocumentChange.Insert("#", new SourceLocation(2, 7)) - })); + }); doc.Text.Should().Contain("re#turn g"); } @@ -77,31 +74,19 @@ def g(y): public void ResetDocumentBuffer() { var doc = new DocumentBuffer(); - doc.Reset(0, ""); - - Assert.AreEqual("", doc.Text.ToString()); + doc.Reset(0, string.Empty); + Assert.AreEqual(string.Empty, doc.Text); - doc.Update(new[] { new DocumentChangeSet(0, 1, new[] { + doc.Update(new[] { DocumentChange.Insert("text", SourceLocation.MinValue) - }) }); - - Assert.AreEqual("text", doc.Text.ToString()); - - try { - doc.Update(new[] { new DocumentChangeSet(1, 0, new[] { - DocumentChange.Delete(SourceLocation.MinValue, SourceLocation.MinValue.AddColumns(4)) - }) }); - Assert.Fail("expected InvalidOperationException"); - } catch (InvalidOperationException) { - } - Assert.AreEqual("text", doc.Text.ToString()); + }); + + Assert.AreEqual("text", doc.Text); Assert.AreEqual(1, doc.Version); - doc.Update(new[] { new DocumentChangeSet(1, 0, new[] { - new DocumentChange { WholeBuffer = true } - }) }); + doc.Reset(0, @"abcdef"); - Assert.AreEqual("", doc.Text.ToString()); + Assert.AreEqual(@"abcdef", doc.Text); Assert.AreEqual(0, doc.Version); } } diff --git a/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs b/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs index 96e708c9e..a30be146a 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs @@ -26,7 +26,7 @@ using Microsoft.Python.Parsing; namespace Microsoft.Python.Analysis.Tests.FluentAssertions { - internal static class AssertionsUtilities { + public static class AssertionsUtilities { public static bool Is3X(IScope scope) => scope.GlobalScope.Module.Interpreter.LanguageVersion.Is3x(); diff --git a/src/Analysis/Ast/Test/FluentAssertions/RangeAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/RangeAssertions.cs index fed68aad7..05c3ac0bb 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/RangeAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/RangeAssertions.cs @@ -19,7 +19,7 @@ using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; namespace Microsoft.Python.Analysis.Tests.FluentAssertions { - internal sealed class RangeAssertions { + public sealed class RangeAssertions { public Range? Subject { get; } public RangeAssertions(Range? range) { diff --git a/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs b/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs index 186daa985..269948ba9 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/VariableAssertions.cs @@ -63,6 +63,13 @@ public AndWhichConstraint HaveMember(string name, s return new AndWhichConstraint(this, m); } + public AndWhichConstraint NotHaveMember(string name, string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + var m = Value.GetPythonType().GetMember(name); + m.GetPythonType().IsUnknown().Should().BeTrue(); + return new AndWhichConstraint(this, Subject); + } + public AndWhichConstraint HaveMembers(IEnumerable names, string because = "", params object[] reasonArgs) { NotBeNull(because, reasonArgs); Value.Should().HaveMembers(names, because, reasonArgs); diff --git a/src/Analysis/Ast/Test/FunctionTests.cs b/src/Analysis/Ast/Test/FunctionTests.cs index 5cca02654..c8ff51d7c 100644 --- a/src/Analysis/Ast/Test/FunctionTests.cs +++ b/src/Analysis/Ast/Test/FunctionTests.cs @@ -163,6 +163,25 @@ def f(a, b) -> str: .HaveVariable("y").OfType(BuiltinTypeId.Str); } + [TestMethod, Priority(0)] + public async Task ReturnValueAnnotatedQuoted() { + const string code = @" +def f(a, b) -> 'int': + return a + b + +x = f('x', 'y') +y = f(1, 2) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveFunction("f") + .Which.Should().HaveSingleOverload() + .Which.Should().HaveReturnType(BuiltinTypeId.Int); + + analysis.Should() + .HaveVariable("x").OfType(BuiltinTypeId.Int).And + .HaveVariable("y").OfType(BuiltinTypeId.Int); + } + [TestMethod, Priority(0)] public async Task BadMethod() { const string code = @" diff --git a/src/Analysis/Ast/Test/ImportTests.cs b/src/Analysis/Ast/Test/ImportTests.cs index afa93f2a2..448708991 100644 --- a/src/Analysis/Ast/Test/ImportTests.cs +++ b/src/Analysis/Ast/Test/ImportTests.cs @@ -80,14 +80,6 @@ public async Task FromImportSpecificValues() { analysis.Should().HaveVariable("D").OfType(BuiltinTypeId.Dict); } - [TestMethod, Priority(0)] - public async Task ImportNonExistingModule() { - var code = await File.ReadAllTextAsync(Path.Combine(GetAnalysisTestDataFilesPath(), "Imports.py")); - var analysis = await GetAnalysisAsync(code); - - analysis.GlobalScope.Variables.Names.Should().OnlyContain("version_info", "a_made_up_module"); - } - [TestMethod, Priority(0)] public async Task FromImportReturnTypes() { const string code = @"from ReturnValues import * diff --git a/src/Analysis/Ast/Test/LibraryTests.cs b/src/Analysis/Ast/Test/LibraryTests.cs index 2e25d6b40..06c99e205 100644 --- a/src/Analysis/Ast/Test/LibraryTests.cs +++ b/src/Analysis/Ast/Test/LibraryTests.cs @@ -16,8 +16,10 @@ using System.IO; using System.Threading.Tasks; using FluentAssertions; +using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -61,5 +63,43 @@ public async Task Datetime() { @"astimezone", @"isocalendar", @"resolution", @"fromordinal", @"fromtimestamp", @"min", @"max", @"date", @"utcnow", "combine", "replace", "second"); } + + [TestMethod, Priority(0)] + public async Task Requests() { + const string code = @" +import requests +x = requests.get('microsoft.com') +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var v = analysis.GlobalScope.Variables["requests"]; + v.Should().NotBeNull(); + if (v.Value.GetPythonType().ModuleType == ModuleType.Unresolved) { + Assert.Inconclusive("'requests' package is not installed."); + } + + var r = analysis.Should().HaveVariable("x").OfType("Response").Which; + r.Should().HaveMember("encoding").Which.Should().HaveType(BuiltinTypeId.Str); + } + + [TestMethod, Priority(0)] + public async Task OpenBinaryFile() { + const string code = @" +with open('foo.txt', 'wb') as file: + file +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Should().HaveVariable("file").OfType("BufferedIOBase"); + } + + [TestMethod, Priority(0)] + public async Task OpenTextFile() { + const string code = @" +with open('foo.txt', 'w') as file: + file +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + // TODO: change to TextIOBase when TextIO is specialized. + analysis.Should().HaveVariable("file").OfType("TextIOWrapper"); + } } } diff --git a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj index 2772e926b..345a3ce1d 100644 --- a/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj +++ b/src/Analysis/Ast/Test/Microsoft.Python.Analysis.Tests.csproj @@ -19,11 +19,6 @@ - - - - - diff --git a/src/Analysis/Ast/Test/PepHintTests.cs b/src/Analysis/Ast/Test/PepHintTests.cs new file mode 100644 index 000000000..8e1e04bcb --- /dev/null +++ b/src/Analysis/Ast/Test/PepHintTests.cs @@ -0,0 +1,80 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Tests.Utilities.FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class PepHintTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + // https://www.python.org/dev/peps/pep-0526/ + [TestMethod, Priority(0)] + public async Task BasicHints() { + const string code = @" +from typing import List, Dict +import datetime + +a = ... # type: str +x = ... # type: int +primes = [] # type: List[int] +stats = {} # type: Dict[str, int] + +class Response: # truncated + encoding = ... # type: str + cookies = ... # type: int + elapsed = ... # type: datetime.timedelta +"; + var analysis = await GetAnalysisAsync(code); + + analysis.Should().HaveVariable("a").OfType(BuiltinTypeId.Str); + analysis.Should().HaveVariable("x").OfType(BuiltinTypeId.Int); + analysis.Should().HaveVariable("primes").OfType("List[int]"); + analysis.Should().HaveVariable("stats").OfType("Dict[str, int]"); + + var cls = analysis.Should().HaveVariable("Response").Which; + cls.Value.Should().BeAssignableTo(); + + var c = cls.Value.GetPythonType(); + c.Should().HaveMember("encoding") + .Which.Should().HaveType(BuiltinTypeId.Str); + c.Should().HaveMember("cookies") + .Which.Should().HaveType(BuiltinTypeId.Int); + + var dt = analysis.GlobalScope.Variables["datetime"]; + dt.Should().NotBeNull(); + var timedelta = dt.Value.GetPythonType().GetMember(@"timedelta"); + timedelta.IsUnknown().Should().BeFalse(); + + c.Should().HaveMember("elapsed") + .Which.Should().HaveSameMembersAs(timedelta); + } + } +} diff --git a/src/Analysis/Ast/Test/ScrapeTests.cs b/src/Analysis/Ast/Test/ScrapeTests.cs index 32019c4b8..647ed39d2 100644 --- a/src/Analysis/Ast/Test/ScrapeTests.cs +++ b/src/Analysis/Ast/Test/ScrapeTests.cs @@ -24,6 +24,7 @@ using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Modules.Resolution; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core.IO; @@ -94,13 +95,13 @@ private async Task CompiledBuiltinScrapeAsync(InterpreterConfiguration configura continue; } - Console.WriteLine("Importing {0} from {1}", mp.ModuleName, mp.SourceFile); + Console.WriteLine(@"Importing {0} from {1}", mp.ModuleName, mp.SourceFile); var mod = await interpreter.ModuleResolution.ImportModuleAsync(mp.ModuleName); Assert.IsInstanceOfType(mod, typeof(CompiledPythonModule)); + await ((ModuleCache)interpreter.ModuleResolution.ModuleCache).CacheWritingTask; var modPath = interpreter.ModuleResolution.ModuleCache.GetCacheFilePath(pyd); Assert.IsTrue(File.Exists(modPath), "No cache file created"); - var moduleCache = File.ReadAllText(modPath); var doc = (IDocument)mod; var ast = await doc.GetAstAsync(); @@ -109,7 +110,7 @@ private async Task CompiledBuiltinScrapeAsync(InterpreterConfiguration configura foreach (var err in errors) { Console.WriteLine(err); } - Assert.AreEqual(0, errors.Count(), "Parse errors occurred"); + Assert.AreEqual(0, errors.Length, "Parse errors occurred"); var imports = ((SuiteStatement)ast.Body).Statements @@ -261,7 +262,7 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params var anyParseError = false; foreach (var m in skip) { - ((ModuleResolution)interpreter.ModuleResolution).AddUnimportableModule(m); + ((MainModuleResolution)interpreter.ModuleResolution).AddUnimportableModule(m); } var set = modules diff --git a/src/Analysis/Ast/Test/TypeshedTests.cs b/src/Analysis/Ast/Test/TypeshedTests.cs index fb210de3a..9a91f965a 100644 --- a/src/Analysis/Ast/Test/TypeshedTests.cs +++ b/src/Analysis/Ast/Test/TypeshedTests.cs @@ -60,7 +60,7 @@ import sys [TestMethod, Priority(0)] public async Task TypeShedJsonMakeScanner() { - var code = @"import _json + const string code = @"import _json scanner = _json.make_scanner()"; var analysis = await GetAnalysisAsync(code); @@ -92,7 +92,7 @@ public async Task MergeStubs() { public async Task TypeStubConditionalDefine() { var seen = new HashSet(); - var code = @"import sys + const string code = @"import sys if sys.version_info < (2, 7): LT_2_7 : bool = ... @@ -135,5 +135,31 @@ public async Task TypeStubConditionalDefine() { } } } + + [TestMethod, Priority(0)] + public async Task TypeShedNoTypeLeaks() { + const string code = @"import json +json.dump() +x = 1"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("json") + .Which.Should().HaveMember("dump"); + + analysis.Should().HaveVariable("x") + .Which.Should().HaveMember("bit_length") + .And.NotHaveMember("SIGABRT"); + } + + [TestMethod, Priority(0)] + public async Task VersionCheck() { + const string code = @" +import asyncio +loop = asyncio.get_event_loop() +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should() + .HaveVariable("loop") + .Which.Value.Should().HaveMembers("add_reader", "add_writer", "call_at", "close"); + } } } diff --git a/src/Analysis/Ast/Test/TypingTests.cs b/src/Analysis/Ast/Test/TypingTests.cs index 19b68f89a..382f5e3f0 100644 --- a/src/Analysis/Ast/Test/TypingTests.cs +++ b/src/Analysis/Ast/Test/TypingTests.cs @@ -24,6 +24,7 @@ using Microsoft.Python.Analysis.Types; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; +using Microsoft.Python.Parsing.Tests; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -145,6 +146,21 @@ from typing import TypeVar analysis.Should().HaveVariable("T").OfType(typeof(IGenericTypeParameter)); } + [TestMethod, Priority(0)] + public async Task TypeVarStringConstraint() { + const string code = @" +from typing import TypeVar +import io + +T = TypeVar('T', bound='io.TextIOWrapper') +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("T") + .Which.Value.Should().HaveDocumentation("TypeVar('T', TextIOWrapper)"); + analysis.Should().HaveVariable("T").OfType(typeof(IGenericTypeParameter)); + } + + [TestMethod, Priority(0)] [Ignore] public async Task TypeVarFunc() { @@ -414,7 +430,6 @@ def ls() -> List[tuple]: .Should().HaveType(BuiltinTypeId.Tuple); } - [TestMethod, Priority(0)] public async Task DictContent() { const string code = @" @@ -620,6 +635,126 @@ from typing import AnyStr .And.HaveVariable("y").OfType("AnyStr"); } + [TestMethod, Priority(0)] + public async Task GenericTypeInstance() { + const string code = @" +from typing import List + +l = List[str]() +x = l[0] +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("l").Which + .Should().HaveType("List[str]"); + analysis.Should().HaveVariable("x").Which + .Should().HaveType(BuiltinTypeId.Str); + } + + + [TestMethod, Priority(0)] + public async Task GenericClassBase1() { + const string code = @" +from typing import TypeVar, Generic + +_E = TypeVar('_E', bound=Exception) + +class A(Generic[_E]): ... + +class B(Generic[_E]): + a: A[_E] + def func(self) -> A[_E]: ... + +b = B[TypeError]() +x = b.func() +y = b.a +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Should().HaveVariable("b") + .Which.Should().HaveMembers("args", @"with_traceback"); + + analysis.Should().HaveVariable("x") + .Which.Should().HaveType("A[TypeError]") // TODO: should be A[TypeError] + .Which.Should().HaveMembers("args", @"with_traceback"); + + analysis.Should().HaveVariable("y") + .Which.Should().HaveType("A[TypeError]") // TODO: should be A[[TypeError] + .Which.Should().HaveMembers("args", @"with_traceback"); + } + + [TestMethod, Priority(0)] + public async Task GenericClassBaseForwardRef() { + const string code = @" +from typing import TypeVar, Generic + +_E = TypeVar('_E', bound=Exception) + +class B(Generic[_E]): + a: A[_E] + def func(self) -> A[_E]: ... + +class A(Generic[_E]): ... + +b = B[TypeError]() +x = b.func() +y = b.a +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Should().HaveVariable("b") + .Which.Should().HaveMembers("args", @"with_traceback"); + + analysis.Should().HaveVariable("x") + .Which.Should().HaveType("A[TypeError]") // TODO: should be A[TypeError] + .Which.Should().HaveMembers("args", @"with_traceback"); + + analysis.Should().HaveVariable("y") + .Which.Should().HaveType("A[TypeError]") // TODO: should be A[[TypeError] + .Which.Should().HaveMembers("args", @"with_traceback"); + } + + [TestMethod, Priority(0)] + public async Task GenericClassBase2() { + const string code = @" +from typing import TypeVar, Generic + +_T = TypeVar('_T') + +class Box(Generic[_T]): + def __init__(self, v: _T): + self.v = v + + def get(self) -> _T: + return self.v + +boxed = Box[int]() +x = boxed.get() +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Should().HaveVariable("x") + .Which.Should().HaveType(BuiltinTypeId.Int); + } + + [TestMethod, Priority(0)] + public async Task GenericClassBase3() { + const string code = @" +from typing import TypeVar, Generic + +_T = TypeVar('_T') + +class Box(Generic[_T]): + def __init__(self, v: _T): + self.v = v + + def get(self) -> _T: + return self.v + +boxed = Box(1234) +x = boxed.get() +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Should().HaveVariable("x") + .Which.Should().HaveType(BuiltinTypeId.Int); + } + [TestMethod, Priority(0)] public void AnnotationParsing() { AssertTransform("List", "NameOp:List"); @@ -657,7 +792,7 @@ private static void AssertConvert(string expr, string expected = null) { private static TypeAnnotation Parse(string expr, PythonLanguageVersion version = PythonLanguageVersion.V36) { var errors = new CollectingErrorSink(); - var ops = new ParserOptions {ErrorSink = errors}; + var ops = new ParserOptions { ErrorSink = errors }; var p = Parser.CreateParser(new StringReader(expr), version, ops); var ast = p.ParseTopExpression(); if (errors.Errors.Any()) { diff --git a/src/Analysis/Core/Impl/Interpreter/InterpreterConfiguration.cs b/src/Analysis/Core/Impl/Interpreter/InterpreterConfiguration.cs index 5211ccaf4..b692db6c0 100644 --- a/src/Analysis/Core/Impl/Interpreter/InterpreterConfiguration.cs +++ b/src/Analysis/Core/Impl/Interpreter/InterpreterConfiguration.cs @@ -48,7 +48,7 @@ public InterpreterConfiguration( LibraryPath = libPath ?? string.Empty; SitePackagesPath = sitePackagesPath ?? string.Empty; TypeshedPath = typeshedPath ?? string.Empty; - ModuleCachePath = moduleCachePath ?? string.Empty; + DatabasePath = moduleCachePath ?? string.Empty; } private static string Read(IReadOnlyDictionary d, string k) @@ -61,7 +61,7 @@ private InterpreterConfiguration(IReadOnlyDictionary properties) PathEnvironmentVariable = Read(properties, nameof(PathEnvironmentVariable)); LibraryPath = Read(properties, nameof(LibraryPath)); TypeshedPath = Read(properties, nameof(TypeshedPath)); - ModuleCachePath = Read(properties, nameof(ModuleCachePath)); + DatabasePath = Read(properties, nameof(DatabasePath)); Architecture = InterpreterArchitecture.TryParse(Read(properties, nameof(Architecture))); try { Version = Version.Parse(Read(properties, nameof(Version))); @@ -86,7 +86,7 @@ public void WriteToDictionary(IDictionary properties) { properties[nameof(PathEnvironmentVariable)] = PathEnvironmentVariable; properties[nameof(LibraryPath)] = LibraryPath; properties[nameof(TypeshedPath)] = TypeshedPath; - properties[nameof(ModuleCachePath)] = ModuleCachePath; + properties[nameof(DatabasePath)] = DatabasePath; properties[nameof(Architecture)] = Architecture.ToString(); if (Version != null) { properties[nameof(Version)] = Version.ToString(); @@ -184,7 +184,7 @@ public void SwitchToFullDescription() { /// /// Module cache folder. /// - public string ModuleCachePath { get; set; } + public string DatabasePath { get; set; } public static bool operator ==(InterpreterConfiguration x, InterpreterConfiguration y) => x?.Equals(y) ?? ReferenceEquals(y, null); diff --git a/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs b/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs index 3026de091..2972b5d0f 100644 --- a/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs +++ b/src/Analysis/Core/Impl/Interpreter/PythonLibraryPath.cs @@ -110,7 +110,7 @@ public static async Task> GetDatabaseSearchPathsAsync(I } try { - paths = await GetUncachedDatabaseSearchPathsAsync(config.InterpreterPath).ConfigureAwait(false); + paths = await GetUncachedDatabaseSearchPathsAsync(config.InterpreterPath); if (!string.IsNullOrEmpty(cachePath)) { WriteDatabaseSearchPaths(cachePath, paths); } @@ -164,7 +164,7 @@ public static async Task> GetUncachedDatabaseSearchPaths using (var cts = new CancellationTokenSource(30000)) { int exitCode; try { - exitCode = await proc.WaitAsync(cts.Token).ConfigureAwait(false); + exitCode = await proc.WaitAsync(cts.Token); } catch (OperationCanceledException) { proc.Kill(); exitCode = -1; diff --git a/src/Core/Test/TestLogger.cs b/src/Core/Test/TestLogger.cs index 28cd09563..4f633e2d2 100644 --- a/src/Core/Test/TestLogger.cs +++ b/src/Core/Test/TestLogger.cs @@ -14,31 +14,22 @@ // permissions and limitations under the License. using System; -using System.Collections.Concurrent; using System.Diagnostics; using System.IO; using System.Text; -using System.Threading; -using System.Threading.Tasks; using Microsoft.Python.Core.Logging; using TestUtilities; namespace Microsoft.Python.Core.Tests { public sealed class TestLogger : ILogger, IDisposable { - private readonly ConcurrentQueue> _messages = new ConcurrentQueue>(); - private readonly ManualResetEventSlim _itemsAvailable = new ManualResetEventSlim(false); - private readonly CancellationTokenSource _cts = new CancellationTokenSource(); - private readonly object _lock = new object(); private readonly FileStream _file = null; public TestLogger() { //var path = Path.Combine(Path.GetTempPath(), "python_analysis.log"); //_file = File.OpenWrite(path); - Task.Run(() => Worker()).DoNotWait(); } public void Dispose() { - _cts.Cancel(); _file?.Close(); _file?.Dispose(); } @@ -48,10 +39,22 @@ public void Dispose() { public void Log(TraceEventType eventType, string message) { var m = $"[{TestEnvironmentImpl.Elapsed()}]: {message}"; - lock (_lock) { - _messages.Enqueue(new Tuple(eventType, m)); - _itemsAvailable.Set(); + switch (eventType) { + case TraceEventType.Error: + case TraceEventType.Critical: + Trace.TraceError(m); + break; + case TraceEventType.Warning: + Trace.TraceWarning(m); + break; + case TraceEventType.Information: + Trace.TraceInformation(m); + break; + case TraceEventType.Verbose: + Trace.TraceInformation($"LOG: {m}"); + break; } + WriteToFile(m); } private void WriteToFile(string s) { @@ -70,43 +73,5 @@ public void Log(TraceEventType eventType, params object[] parameters) { } Log(eventType, sb.ToString().FormatUI(parameters)); } - - private void Worker() { - while (!_cts.IsCancellationRequested) { - Tuple t; - lock (_lock) { - if (!_messages.TryDequeue(out t)) { - _itemsAvailable.Reset(); - } - } - - if (t == null) { - try { - _itemsAvailable.Wait(_cts.Token); - } catch (OperationCanceledException) { - break; - } - continue; - } - - var m = t.Item2; - switch (t.Item1) { - case TraceEventType.Error: - case TraceEventType.Critical: - Trace.TraceError(m); - break; - case TraceEventType.Warning: - Trace.TraceWarning(m); - break; - case TraceEventType.Information: - Trace.TraceInformation(m); - break; - case TraceEventType.Verbose: - Trace.TraceInformation($"LOG: {m}"); - break; - } - WriteToFile(m); - } - } } } diff --git a/src/LanguageServer/Impl/Completion/ClassDefinitionCompletion.cs b/src/LanguageServer/Impl/Completion/ClassDefinitionCompletion.cs new file mode 100644 index 000000000..47366fabe --- /dev/null +++ b/src/LanguageServer/Impl/Completion/ClassDefinitionCompletion.cs @@ -0,0 +1,52 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class ClassDefinitionCompletion { + public static bool NoCompletions(ClassDefinition cd, CompletionContext context, out bool addMetadataArg) { + addMetadataArg = false; + + if (cd.HeaderIndex > cd.StartIndex && context.Position > cd.HeaderIndex) { + return false; + } + + if (context.Position == cd.HeaderIndex) { + return true; + } + + if (cd.Bases.Length > 0 && context.Position >= cd.Bases[0].StartIndex) { + foreach (var p in cd.Bases.Reverse()) { + if (context.Position >= p.StartIndex) { + if (p.Name == null && context.Ast.LanguageVersion.Is3x() && cd.Bases.All(b => b.Name != @"metaclass")) { + addMetadataArg = true; + } + + return false; + } + } + } + + if (cd.NameExpression != null && cd.NameExpression.StartIndex > cd.KeywordEndIndex && context.Position >= cd.NameExpression.StartIndex) { + return true; + } + + return context.Position > cd.KeywordEndIndex; + } + } +} diff --git a/src/LanguageServer/Impl/Completion/CompletionContext.cs b/src/LanguageServer/Impl/Completion/CompletionContext.cs new file mode 100644 index 000000000..b40f8a44b --- /dev/null +++ b/src/LanguageServer/Impl/Completion/CompletionContext.cs @@ -0,0 +1,51 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using Microsoft.Python.Analysis; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal sealed class CompletionContext { + private TokenSource _ts; + + public IDocumentAnalysis Analysis { get; } + public PythonAst Ast => Analysis.Ast; + public SourceLocation Location { get; } + public int Position { get; } + public TokenSource TokenSource => _ts ?? (_ts = new TokenSource(Analysis.Document, Position)); + public CompletionItemSource ItemSource { get; } + + public CompletionContext(IDocumentAnalysis analysis, SourceLocation location, CompletionItemSource itemSource) { + Location = location; + Analysis = analysis; + Position = Ast.LocationToIndex(location); + ItemSource = itemSource; + } + + public SourceLocation IndexToLocation(int index) => Ast.IndexToLocation(index); + + public SourceSpan GetApplicableSpanFromLastToken(Node containingNode) { + if (containingNode != null && Position >= containingNode.EndIndex) { + var token = TokenSource.Tokens.LastOrDefault(); + if (token.Key.End >= Position) { + return TokenSource.GetTokenSpan(token.Key); + } + } + return default; + } + } +} diff --git a/src/LanguageServer/Impl/Completion/CompletionItemSource.cs b/src/LanguageServer/Impl/Completion/CompletionItemSource.cs new file mode 100644 index 000000000..f21874b95 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/CompletionItemSource.cs @@ -0,0 +1,83 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.LanguageServer.Protocol; + +namespace Microsoft.Python.LanguageServer.Completion { + internal class CompletionItemSource { + public static readonly CompletionItem FromKeyword = CreateCompletionItem("from", CompletionItemKind.Keyword); + public static readonly CompletionItem ImportKeyword = CreateCompletionItem("import", CompletionItemKind.Keyword); + public static readonly CompletionItem MetadataArg = CreateCompletionItem(@"metaclass=", CompletionItemKind.TypeParameter); + public static readonly CompletionItem AsKeyword = CreateCompletionItem("as", CompletionItemKind.Keyword); + public static readonly CompletionItem InKeyword = CreateCompletionItem("in", CompletionItemKind.Keyword); + public static readonly CompletionItem WithKeyword = CreateCompletionItem("with", CompletionItemKind.Keyword); + public static readonly CompletionItem Star = CreateCompletionItem("*", CompletionItemKind.Keyword); + + private readonly IDocumentationSource _docSource; + private readonly ServerSettings.PythonCompletionOptions _options; + + public CompletionItemSource(IDocumentationSource docSource, ServerSettings.PythonCompletionOptions options) { + _docSource = docSource; + _options = options; + } + + public CompletionItem CreateCompletionItem(string text, IMember member, string label = null) + => CreateCompletionItem(text, ToCompletionItemKind(member?.MemberType ?? PythonMemberType.Class), member, label); + + public CompletionItem CreateCompletionItem(string text, CompletionItemKind kind, IMember member, string label = null) { + var t = member?.GetPythonType(); + var docFormat = _docSource.DocumentationFormat; + + if (_options.addBrackets && (kind == CompletionItemKind.Constructor || kind == CompletionItemKind.Function || kind == CompletionItemKind.Method)) { + text += "($0)"; + docFormat = InsertTextFormat.Snippet; + } + + return new CompletionItem { + label = label ?? text, + insertText = text, + insertTextFormat = docFormat, + // Place regular items first, advanced entries last + sortText = char.IsLetter(text, 0) ? "1" : "2", + kind = kind, + documentation = !t.IsUnknown() ? _docSource.GetHover(label ?? text, member) : null + }; + } + + public static CompletionItem CreateCompletionItem(string text, CompletionItemKind kind) + => new CompletionItem { + label = text, insertText = text, insertTextFormat = InsertTextFormat.PlainText, + sortText = char.IsLetter(text, 0) ? "1" : "2", kind = kind + }; + + private static CompletionItemKind ToCompletionItemKind(PythonMemberType memberType) { + switch (memberType) { + case PythonMemberType.Unknown: return CompletionItemKind.None; + case PythonMemberType.Class: return CompletionItemKind.Class; + case PythonMemberType.Instance: return CompletionItemKind.Value; + case PythonMemberType.Function: return CompletionItemKind.Function; + case PythonMemberType.Method: return CompletionItemKind.Method; + case PythonMemberType.Module: return CompletionItemKind.Module; + case PythonMemberType.Property: return CompletionItemKind.Property; + case PythonMemberType.Union: return CompletionItemKind.Struct; + case PythonMemberType.Variable: return CompletionItemKind.Variable; + case PythonMemberType.Generic: return CompletionItemKind.TypeParameter; + } + return CompletionItemKind.None; + } + } +} diff --git a/src/LanguageServer/Impl/Completion/CompletionResult.cs b/src/LanguageServer/Impl/Completion/CompletionResult.cs new file mode 100644 index 000000000..9a490d8b6 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/CompletionResult.cs @@ -0,0 +1,35 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; + +namespace Microsoft.Python.LanguageServer.Completion { + internal sealed class CompletionResult { + public static readonly CompletionResult Empty = new CompletionResult(Enumerable.Empty()); + + public IReadOnlyList Completions { get; } + public SourceSpan? ApplicableSpan { get; } + + public CompletionResult(IEnumerable completions, SourceSpan? applicableSpan) : this(completions) { + ApplicableSpan = applicableSpan; + } + public CompletionResult(IEnumerable completions) { + Completions = completions.ToArray(); + } + } +} diff --git a/src/LanguageServer/Impl/Completion/CompletionSource.cs b/src/LanguageServer/Impl/Completion/CompletionSource.cs new file mode 100644 index 000000000..5ab711280 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/CompletionSource.cs @@ -0,0 +1,96 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Analyzer.Expressions; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal sealed class CompletionSource { + private readonly CompletionItemSource _itemSource; + + public CompletionSource(IDocumentationSource docSource, ServerSettings.PythonCompletionOptions completionSettings) { + _itemSource = new CompletionItemSource(docSource, completionSettings); + } + + public async Task GetCompletionsAsync(IDocumentAnalysis analysis, SourceLocation location, CancellationToken cancellationToken = default) { + var context = new CompletionContext(analysis, location, _itemSource); + + ExpressionLocator.FindExpression(analysis.Ast, location, + FindExpressionOptions.Complete, out var expression, out var statement, out var scope); + + switch (expression) { + case MemberExpression me when me.Target != null && me.DotIndex > me.StartIndex && context.Position > me.DotIndex: + return new CompletionResult(await ExpressionCompletion.GetCompletionsFromMembersAsync(me.Target, scope, context, cancellationToken)); + case ConstantExpression ce when ce.Value is double || ce.Value is float: + // no completions on integer ., the user is typing a float + return CompletionResult.Empty; + case ConstantExpression ce when ce.Value is string: + // no completions in strings + return CompletionResult.Empty; + case null when context.Ast.IsInsideComment(context.Location): + return CompletionResult.Empty; + case null when context.Ast.IsInsideString(context.Location): + return CompletionResult.Empty; + } + + if (statement is ImportStatement import) { + var result = ImportCompletion.TryGetCompletions(import, context); + if (result != null) { + return result; + } + } + if (statement is FromImportStatement fromImport) { + var result = ImportCompletion.GetCompletionsInFromImport(fromImport, context); + if (result != null) { + return result; + } + } + + switch (statement) { + case FunctionDefinition fd when FunctionDefinitionCompletion.TryGetCompletionsForOverride(fd, context, null, out var result): + return result; + case FunctionDefinition fd when FunctionDefinitionCompletion.NoCompletions(fd, context.Position, context.Ast): + return CompletionResult.Empty; + case ClassDefinition cd: + if (!ClassDefinitionCompletion.NoCompletions(cd, context, out var addMetadataArg)) { + var result = await TopLevelCompletion.GetCompletionsAsync(statement, scope, context, cancellationToken); + return addMetadataArg + ? new CompletionResult(result.Completions.Append(CompletionItemSource.MetadataArg), result.ApplicableSpan) + : result; + } + return CompletionResult.Empty; + case ForStatement forStatement when ForCompletion.TryGetCompletions(forStatement, context, out var result): + return result; + case WithStatement withStatement when WithCompletion.TryGetCompletions(withStatement, context, out var result): + return result; + case RaiseStatement raiseStatement when RaiseCompletion.TryGetCompletions(raiseStatement, context, out var result): + return result; + case TryStatementHandler tryStatement when ExceptCompletion.TryGetCompletions(tryStatement, context, out var result): + return result; + default: { + var result = await ErrorExpressionCompletion.GetCompletionsAsync(scope, statement, expression, context, cancellationToken); + return result == CompletionResult.Empty + ? await TopLevelCompletion.GetCompletionsAsync(statement, scope, context, cancellationToken) + : result; + } + } + } + } +} diff --git a/src/LanguageServer/Impl/Completion/ErrorExpressionCompletion.cs b/src/LanguageServer/Impl/Completion/ErrorExpressionCompletion.cs new file mode 100644 index 000000000..612c11628 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/ErrorExpressionCompletion.cs @@ -0,0 +1,106 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class ErrorExpressionCompletion { + public static async Task GetCompletionsAsync(ScopeStatement scope, Node statement, Node expression, CompletionContext context, CancellationToken cancellationToken = default) { + if (!(expression is ErrorExpression)) { + return CompletionResult.Empty; + } + + if (statement is AssignmentStatement assign && expression == assign.Right) { + return CompletionResult.Empty; + } + + var tokens = context.TokenSource.Tokens.Reverse().ToArray(); + var es = new ExpressionSource(context.TokenSource); + + string code; + SourceLocation location; + var items = Enumerable.Empty(); + SourceSpan? applicableSpan; + Expression e; + + var lastToken = tokens.FirstOrDefault(); + var nextLast = tokens.ElementAtOrDefault(1).Value?.Kind ?? TokenKind.EndOfFile; + switch (lastToken.Value.Kind) { + case TokenKind.Dot: + code = es.ReadExpression(tokens.Skip(1)); + applicableSpan = new SourceSpan(context.Location, context.Location); + e = GetExpressionFromText(code, context, out var s1, out _); + items = await ExpressionCompletion.GetCompletionsFromMembersAsync(e, s1, context, cancellationToken); + break; + + case TokenKind.KeywordDef when lastToken.Key.End < context.Position && scope is FunctionDefinition fd: { + applicableSpan = new SourceSpan(context.Location, context.Location); + location = context.TokenSource.GetTokenSpan(lastToken.Key).Start; + if (FunctionDefinitionCompletion.TryGetCompletionsForOverride(fd, context, location, out var result)) { + items = result.Completions; + } + + break; + } + + case TokenKind.Name when nextLast == TokenKind.Dot: + code = es.ReadExpression(tokens.Skip(2)); + applicableSpan = new SourceSpan(context.TokenSource.GetTokenSpan(lastToken.Key).Start, context.Location); + e = GetExpressionFromText(code, context, out var s2, out _); + items = await ExpressionCompletion.GetCompletionsFromMembersAsync(e, s2, context, cancellationToken); + break; + + case TokenKind.Name when nextLast == TokenKind.KeywordDef && scope is FunctionDefinition fd: { + applicableSpan = new SourceSpan(context.TokenSource.GetTokenSpan(lastToken.Key).Start, context.Location); + location = context.TokenSource.GetTokenSpan(tokens.ElementAt(1).Key).Start; + if (FunctionDefinitionCompletion.TryGetCompletionsForOverride(fd, context, location, out var result)) { + items = result.Completions; + } + break; + } + + case TokenKind.KeywordFor: + case TokenKind.KeywordAs: + return lastToken.Key.Start <= context.Position && context.Position <= lastToken.Key.End ? null : CompletionResult.Empty; + + default: + return CompletionResult.Empty; + } + + return new CompletionResult(items, applicableSpan); + } + + private static Expression GetExpressionFromText(string text, CompletionContext context, out IScope scope, out PythonAst expressionAst) { + scope = context.Analysis.FindScope(context.Location); + using (var reader = new StringReader(text)) { + var parser = Parser.CreateParser(reader, context.Ast.LanguageVersion, new ParserOptions()); + expressionAst = parser.ParseTopExpression(); + return Statement.GetExpression(expressionAst.Body); + } + } + } +} diff --git a/src/LanguageServer/Impl/Completion/ExceptCompletion.cs b/src/LanguageServer/Impl/Completion/ExceptCompletion.cs new file mode 100644 index 000000000..50c543d1c --- /dev/null +++ b/src/LanguageServer/Impl/Completion/ExceptCompletion.cs @@ -0,0 +1,42 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class ExceptCompletion { + public static bool TryGetCompletions(TryStatementHandler tryStatement, CompletionContext context, out CompletionResult result) { + result = CompletionResult.Empty; + + // except Test as Target + if (tryStatement.Target != null && context.Position >= tryStatement.Target.StartIndex) { + return true; + } + + if (tryStatement.Test is TupleExpression || tryStatement.Test is null) { + return false; + } + + if (context.Position <= tryStatement.Test.EndIndex) { + return false; + } + + var applicableSpan = context.GetApplicableSpanFromLastToken(tryStatement); + result = new CompletionResult(Enumerable.Repeat(CompletionItemSource.AsKeyword, 1), applicableSpan); + return true; + } + } +} diff --git a/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs b/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs new file mode 100644 index 000000000..27ed98e52 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/ExpressionCompletion.cs @@ -0,0 +1,57 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class ExpressionCompletion { + public static Task> GetCompletionsFromMembersAsync(Expression e, IScope scope, CompletionContext context, CancellationToken cancellationToken = default) { + using (context.Analysis.ExpressionEvaluator.OpenScope(scope)) { + return GetItemsFromExpressionAsync(e, context, cancellationToken); + } + } + + public static Task> GetCompletionsFromMembersAsync(Expression e, ScopeStatement scope, CompletionContext context, CancellationToken cancellationToken = default) { + using (context.Analysis.ExpressionEvaluator.OpenScope(context.Analysis.Document, scope)) { + return GetItemsFromExpressionAsync(e, context, cancellationToken); + } + } + + private static async Task> GetItemsFromExpressionAsync(Expression e, CompletionContext context, CancellationToken cancellationToken = default) { + var value = await context.Analysis.ExpressionEvaluator.GetValueFromExpressionAsync(e, cancellationToken); + if (!value.IsUnknown()) { + var items = new List(); + var type = value.GetPythonType(); + var names = type.GetMemberNames().ToArray(); + foreach (var t in names) { + var m = type.GetMember(t); + if(m is IVariable v && v.Source != VariableSource.Declaration) { + continue; + } + items.Add(context.ItemSource.CreateCompletionItem(t, m)); + } + return items; + } + return Enumerable.Empty(); + } + } +} diff --git a/src/LanguageServer/Impl/Completion/ExpressionLocator.cs b/src/LanguageServer/Impl/Completion/ExpressionLocator.cs new file mode 100644 index 000000000..e8aacb22f --- /dev/null +++ b/src/LanguageServer/Impl/Completion/ExpressionLocator.cs @@ -0,0 +1,59 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Analyzer.Expressions; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class ExpressionLocator { + + public static void FindExpression(PythonAst ast, SourceLocation position, FindExpressionOptions options, out Node expression, out Node statement, out ScopeStatement scope) { + var finder = new ExpressionFinder(ast, options); + + var index = ast.LocationToIndex(position); + finder.Get(index, index, out expression, out statement, out scope); + + var col = position.Column; + while (CanBackUp(ast, expression, statement, scope, col)) { + col -= 1; + index -= 1; + finder.Get(index, index, out expression, out statement, out scope); + } + + expression = expression ?? (statement as ExpressionStatement)?.Expression; + } + + private static bool CanBackUp(PythonAst tree, Node node, Node statement, ScopeStatement scope, int column) { + if (node != null || !((statement as ExpressionStatement)?.Expression is ErrorExpression)) { + return false; + } + + var top = 1; + if (scope != null) { + var scopeStart = scope.GetStart(tree); + if (scope.Body != null) { + top = scope.Body.GetEnd(tree).Line == scopeStart.Line + ? scope.Body.GetStart(tree).Column + : scopeStart.Column; + } else { + top = scopeStart.Column; + } + } + + return column > top; + } + } +} diff --git a/src/LanguageServer/Impl/Completion/ExpressionSource.cs b/src/LanguageServer/Impl/Completion/ExpressionSource.cs new file mode 100644 index 000000000..b203ee722 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/ExpressionSource.cs @@ -0,0 +1,98 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing; + +namespace Microsoft.Python.LanguageServer.Completion { + internal sealed class ExpressionSource { + private readonly TokenSource _tokenSource; + + public ExpressionSource(TokenSource tokenSource) { + _tokenSource = tokenSource; + } + + public string ReadExpression(IEnumerable> tokens) { + var expr = ReadExpressionTokens(tokens); + return string.Join("", expr.Select(e => e.VerbatimImage ?? e.Image)); + } + + private IEnumerable ReadExpressionTokens(IEnumerable> tokens) { + var nesting = 0; + var exprTokens = new Stack(); + var currentLine = -1; + + foreach (var t in tokens) { + var p = _tokenSource.GetTokenSpan(t.Key).Start; + if (p.Line > currentLine) { + currentLine = p.Line; + } else if (p.Line < currentLine && nesting == 0) { + break; + } + + exprTokens.Push(t.Value); + + switch (t.Value.Kind) { + case TokenKind.RightParenthesis: + case TokenKind.RightBracket: + case TokenKind.RightBrace: + nesting += 1; + break; + case TokenKind.LeftParenthesis: + case TokenKind.LeftBracket: + case TokenKind.LeftBrace: + if (--nesting < 0) { + exprTokens.Pop(); + return exprTokens; + } + + break; + + case TokenKind.Comment: + exprTokens.Pop(); + break; + + case TokenKind.Name: + case TokenKind.Constant: + case TokenKind.Dot: + case TokenKind.Ellipsis: + case TokenKind.MatMultiply: + case TokenKind.KeywordAwait: + break; + + case TokenKind.Assign: + case TokenKind.LeftShiftEqual: + case TokenKind.RightShiftEqual: + case TokenKind.BitwiseAndEqual: + case TokenKind.BitwiseOrEqual: + case TokenKind.ExclusiveOrEqual: + exprTokens.Pop(); + return exprTokens; + + default: + if (t.Value.Kind >= TokenKind.FirstKeyword || nesting == 0) { + exprTokens.Pop(); + return exprTokens; + } + break; + } + } + + return exprTokens; + } + } +} diff --git a/src/LanguageServer/Impl/Completion/ForCompletion.cs b/src/LanguageServer/Impl/Completion/ForCompletion.cs new file mode 100644 index 000000000..16a08168f --- /dev/null +++ b/src/LanguageServer/Impl/Completion/ForCompletion.cs @@ -0,0 +1,56 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class ForCompletion { + public static bool TryGetCompletions(ForStatement forStatement, CompletionContext context, out CompletionResult result) { + result = null; + + if (forStatement.Left == null) { + return false; + } + + if (forStatement.InIndex > forStatement.StartIndex) { + if (context.Position > forStatement.InIndex + 2) { + return false; + } + + if (context.Position >= forStatement.InIndex) { + var applicableSpan = new SourceSpan( + context.IndexToLocation(forStatement.InIndex), + context.IndexToLocation(forStatement.InIndex + 2) + ); + result = new CompletionResult(Enumerable.Repeat(CompletionItemSource.InKeyword, 1), applicableSpan); + return true; + } + } + + if (forStatement.Left.StartIndex > forStatement.StartIndex && + forStatement.Left.EndIndex > forStatement.Left.StartIndex && + context.Position > forStatement.Left.EndIndex) { + + var applicableSpan = context.GetApplicableSpanFromLastToken(forStatement); + result = new CompletionResult(Enumerable.Repeat(CompletionItemSource.InKeyword, 1), applicableSpan); + return true; + } + + return forStatement.ForIndex >= forStatement.StartIndex && context.Position > forStatement.ForIndex + 3; + } + } +} diff --git a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs new file mode 100644 index 000000000..9e3802dc1 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs @@ -0,0 +1,179 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class FunctionDefinitionCompletion { + public static bool TryGetCompletionsForOverride(FunctionDefinition function, CompletionContext context, SourceLocation? location, out CompletionResult result) { + result = CompletionResult.Empty; + if (!string.IsNullOrEmpty(function.NameExpression?.Name) && context.Position > function.NameExpression.EndIndex) { + return false; + } + + if (function.Parent is ClassDefinition cd && function.NameExpression != null && context.Position > function.NameExpression.StartIndex) { + var loc = function.GetStart(context.Ast); + var overrideable = GetOverrideable(context, location).ToArray(); + overrideable = !string.IsNullOrEmpty(function.Name) + ? overrideable.Where(o => o.Name.StartsWithOrdinal(function.Name)).ToArray() + : overrideable; + var items = overrideable.Select(o => ToOverrideCompletionItem(o, cd, context, new string(' ', loc.Column - 1))); + result = new CompletionResult(items); + return true; + } + + return false; + } + + private static CompletionItem ToOverrideCompletionItem(IPythonFunctionOverload o, ClassDefinition cd, CompletionContext context, string indent) { + return new CompletionItem { + label = o.Name, + insertText = MakeOverrideCompletionString(indent, o, cd.Name, context), + insertTextFormat = InsertTextFormat.PlainText, + kind = CompletionItemKind.Method + }; + } + + private static string MakeOverrideParameter(IParameterInfo paramInfo, string defaultValue) { + if (paramInfo.IsParamArray) { + return $"*{paramInfo.Name}"; + } + if (paramInfo.IsKeywordDict) { + return $"**{paramInfo.Name}"; + } + return !string.IsNullOrEmpty(paramInfo.DefaultValueString) ? $"{paramInfo.Name}={defaultValue}" : paramInfo.Name; + } + + private static string MakeOverrideCompletionString(string indentation, IPythonFunctionOverload overload, string className, CompletionContext context) { + var sb = new StringBuilder(); + + var first = overload.Parameters.FirstOrDefault(); + var fn = overload.ClassMember as IPythonFunctionType; + var skipFirstParameters = fn?.IsStatic == true ? overload.Parameters : overload.Parameters.Skip(1); + + sb.AppendLine(overload.Name + "(" + string.Join(", ", overload.Parameters.Select(p => MakeOverrideParameter(p, p.DefaultValueString))) + "):"); + sb.Append(indentation); + + if (overload.Parameters.Count > 0) { + var parameterString = string.Join(", ", skipFirstParameters.Select(p => MakeOverrideParameter(p, p.Name))); + + if (context.Ast.LanguageVersion.Is3x()) { + sb.AppendFormat("return super().{0}({1})", + overload.Name, + parameterString); + } else if (!string.IsNullOrEmpty(className)) { + sb.AppendFormat("return super({0}, {1}).{2}({3})", + className, + first?.Name ?? string.Empty, + overload.Name, + parameterString); + } else { + sb.Append("pass"); + } + } else { + sb.Append("pass"); + } + + return sb.ToString(); + } + + /// + /// Gets information about methods defined on base classes but not directly on the current class. + /// + private static IEnumerable GetOverrideable(CompletionContext context, SourceLocation? location = null) { + var result = new List(); + + var scope = context.Analysis.FindScope(location ?? context.Location); + if (!(scope?.Node is ClassDefinition cls)) { + return result; + } + var handled = new HashSet(scope.Children.Select(child => child.Name)); + + var classVar = scope.OuterScope.Variables[cls.Name]; + var classType = classVar?.GetValue()?.GetPythonType(); + if (classType?.Mro == null) { + return result; + } + + foreach (var baseClassType in classType.Mro.Skip(1).OfType()) { + + var functions = baseClassType.GetMemberNames().Select(n => baseClassType.GetMember(n)).OfType(); + foreach (var f in functions.Where(f => f.Overloads.Count > 0)) { + var overload = f.Overloads.Aggregate( + (best, o) => o.Parameters.Count > best.Parameters.Count ? o : best + ); + + if (handled.Contains(overload.Name)) { + continue; + } + + handled.Add(overload.Name); + result.Add(overload); + } + } + + return result; + } + + public static bool NoCompletions(FunctionDefinition fd, int position, PythonAst ast) { + // Here we work backwards through the various parts of the definitions. + // When we find that Index is within a part, we return either the available + // completions + if (fd.HeaderIndex > fd.StartIndex && position > fd.HeaderIndex) { + return false; + } + + if (position == fd.HeaderIndex) { + return true; + } + + foreach (var p in fd.Parameters.Reverse()) { + if (position >= p.StartIndex) { + if (p.Annotation != null) { + return position < p.Annotation.StartIndex; + } + + if (p.DefaultValue != null) { + return position < p.DefaultValue.StartIndex; + } + } + } + + if (fd.NameExpression != null) { + if (fd.NameExpression.StartIndex > fd.KeywordEndIndex && position >= fd.NameExpression.StartIndex) { + return true; + } + + var defLine = ast.IndexToLocation(fd.KeywordEndIndex).Line; + var posLine = ast.IndexToLocation(position).Line; + if (fd.NameExpression.StartIndex == fd.NameExpression.EndIndex && defLine == posLine) { + return false; + } + } + + return position > fd.KeywordEndIndex; + } + } +} diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs new file mode 100644 index 000000000..3b458dc68 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs @@ -0,0 +1,217 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Python.Analysis.Core.DependencyResolution; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal class ImportCompletion { + public static CompletionResult TryGetCompletions(ImportStatement import, CompletionContext context) { + // No names, so if we are at the end. Return available modules + if (import.Names.Count == 0 && context.Position > import.KeywordEndIndex) { + return new CompletionResult(GetAllImportableModules(context)); + } + + foreach (var (name, asName) in ZipLongest(import.Names, import.AsNames).Reverse()) { + if (asName != null && context.Position >= asName.StartIndex) { + return CompletionResult.Empty; + } + + if (name != null && context.Position >= name.StartIndex) { + if (context.Position > name.EndIndex && name.EndIndex > name.StartIndex) { + var applicableSpan = context.GetApplicableSpanFromLastToken(import); + return new CompletionResult(Enumerable.Repeat(CompletionItemSource.AsKeyword, 1), applicableSpan); + } + + return new CompletionResult(GetImportsFromModuleName(name.Names, context)); + } + } + return null; + } + + public static CompletionResult GetCompletionsInFromImport(FromImportStatement fromImport, CompletionContext context) { + // No more completions after '*', ever! + if (fromImport.Names != null && fromImport.Names.Any(n => n?.Name == "*" && context.Position > n.EndIndex)) { + return CompletionResult.Empty; + } + + var document = context.Analysis.Document; + var mres = document.Interpreter.ModuleResolution; + + foreach (var (name, asName) in ZipLongest(fromImport.Names, fromImport.AsNames).Reverse()) { + if (asName != null && context.Position >= asName.StartIndex) { + return CompletionResult.Empty; + } + + if (name != null) { + if (context.Position > name.EndIndex && name.EndIndex > name.StartIndex) { + var applicableSpan = context.GetApplicableSpanFromLastToken(fromImport); + return new CompletionResult(Enumerable.Repeat(CompletionItemSource.AsKeyword, 1), applicableSpan); + } + + if (context.Position >= name.StartIndex) { + var applicableSpan = name.GetSpan(context.Ast); + return new CompletionResult(GetModuleMembers(fromImport.Root.Names, context), applicableSpan); + } + } + } + + if (fromImport.ImportIndex > fromImport.StartIndex && context.Position > fromImport.ImportIndex + 6) { + var importSearchResult = mres.CurrentPathResolver.FindImports(document.FilePath, fromImport); + var result = GetResultFromSearch(importSearchResult, context); + if (result != CompletionResult.Empty) { + return result; + } + } + + if (fromImport.ImportIndex > 0 && context.Position >= fromImport.ImportIndex) { + var applicableSpan = new SourceSpan( + context.IndexToLocation(fromImport.ImportIndex), + context.IndexToLocation(Math.Min(fromImport.ImportIndex + 6, fromImport.EndIndex)) + ); + return new CompletionResult(Enumerable.Repeat(CompletionItemSource.ImportKeyword, 1), applicableSpan); + } + + if (context.Position > fromImport.Root.EndIndex && fromImport.Root.EndIndex > fromImport.Root.StartIndex) { + SourceSpan? applicableSpan = null; + if (context.Position > fromImport.EndIndex) { + // Only end up here for "from ... imp", and "imp" is not counted + // as part of our span + var token = context.TokenSource.Tokens.LastOrDefault(); + if (token.Key.End >= context.Position) { + applicableSpan = context.TokenSource.GetTokenSpan(token.Key); + } + } + + return new CompletionResult(Enumerable.Repeat(CompletionItemSource.ImportKeyword, 1), applicableSpan); + } + + if (context.Position >= fromImport.Root.StartIndex) { + return new CompletionResult(GetImportsFromModuleName(fromImport.Root.Names, context)); + } + + return context.Position > fromImport.KeywordEndIndex + ? new CompletionResult(GetAllImportableModules(context)) + : null; + } + + private static IEnumerable GetImportsFromModuleName(IEnumerable nameExpressions, CompletionContext context) { + IReadOnlyList items; + var names = nameExpressions.TakeWhile(n => n.StartIndex <= context.Position).Select(n => n.Name).ToArray(); + if (names.Length <= 1) { + var mres = context.Analysis.Document.Interpreter.ModuleResolution; + var importable = mres.CurrentPathResolver.GetAllModuleNames(); + items = importable.Select(m => CompletionItemSource.CreateCompletionItem(m, CompletionItemKind.Module)).ToArray(); + } else { + items = GetChildModules(names, context); + } + return items; + } + + private static IEnumerable GetModuleMembers(IEnumerable nameExpressions, CompletionContext context) { + var fullName = string.Join(".", nameExpressions.Select(n => n.Name)); + var mres = context.Analysis.Document.Interpreter.ModuleResolution; + + var module = mres.GetImportedModule(fullName); + return module != null + ? module.GetMemberNames().Select(n => context.ItemSource.CreateCompletionItem(n, module.GetMember(n))) + : Enumerable.Empty(); + } + + private static IEnumerable GetAllImportableModules(CompletionContext context) { + var mres = context.Analysis.Document.Interpreter.ModuleResolution; + var modules = mres.CurrentPathResolver.GetAllModuleNames(); + return modules.Select(n => CompletionItemSource.CreateCompletionItem(n, CompletionItemKind.Module)); + } + + private static CompletionResult GetResultFromSearch(IImportSearchResult importSearchResult, CompletionContext context) { + var document = context.Analysis.Document; + var mres = document.Interpreter.ModuleResolution; + + IPythonModule module; + switch (importSearchResult) { + case ModuleImport moduleImports: + module = mres.GetImportedModule(moduleImports.Name); + break; + case PossibleModuleImport possibleModuleImport: + module = mres.GetImportedModule(possibleModuleImport.PossibleModuleFullName); + break; + case PackageImport packageImports: + return new CompletionResult(packageImports.Modules + .Select(m => mres.GetImportedModule(m.FullName)) + .ExcludeDefault() + .Select(m => CompletionItemSource.CreateCompletionItem(m.Name, CompletionItemKind.Module)) + .Prepend(CompletionItemSource.Star)); + default: + return CompletionResult.Empty; + } + + if (module != null) { + var items = module.GetMemberNames() + .Select(n => context.ItemSource.CreateCompletionItem(n, module.GetMember(n))) + .Prepend(CompletionItemSource.Star); + return new CompletionResult(items); + } + return CompletionResult.Empty; + } + + private static IReadOnlyList GetChildModules(string[] names, CompletionContext context) { + if (!names.Any()) { + return Array.Empty(); + } + + var mres = context.Analysis.Document.Interpreter.ModuleResolution; + var fullName = string.Join(".", names.Take(names.Length - 1)); + + var import = mres.CurrentPathResolver.GetModuleImportFromModuleName(fullName); + if (string.IsNullOrEmpty(import?.ModulePath)) { + return Array.Empty(); + } + + var packages = mres.GetPackagesFromDirectory(Path.GetDirectoryName(import.ModulePath)); + return packages.Select(n => CompletionItemSource.CreateCompletionItem(n, CompletionItemKind.Module)).ToArray(); + } + + private static IEnumerable<(T1, T2)> ZipLongest(IEnumerable src1, IEnumerable src2) { + using (var e1 = src1?.GetEnumerator()) + using (var e2 = src2?.GetEnumerator()) { + bool b1 = e1?.MoveNext() ?? false, b2 = e2?.MoveNext() ?? false; + while (b1 && b2) { + yield return (e1.Current, e2.Current); + b1 = e1.MoveNext(); + b2 = e2.MoveNext(); + } + + while (b1) { + yield return (e1.Current, default); + b1 = e1.MoveNext(); + } + + while (b2) { + yield return (default, e2.Current); + b2 = e2.MoveNext(); + } + } + } + } +} diff --git a/src/LanguageServer/Impl/Completion/PythonKeywords.cs b/src/LanguageServer/Impl/Completion/PythonKeywords.cs new file mode 100644 index 000000000..c0a8c29a1 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/PythonKeywords.cs @@ -0,0 +1,136 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Parsing; + +namespace Microsoft.Python.LanguageServer.Completion { + /// + /// The canonical source of keyword names for Python. + /// + internal static class PythonKeywords { + /// + /// Returns true if the specified identifier is a keyword in a + /// particular version of Python. + /// + public static bool IsKeyword(string keyword, PythonLanguageVersion version = PythonLanguageVersion.None) + => All(version).Contains(keyword, StringComparer.Ordinal); + + /// + /// Returns true if the specified identifier is a statement keyword in a + /// particular version of Python. + /// + public static bool IsStatementKeyword(string keyword, PythonLanguageVersion version = PythonLanguageVersion.None) + => Statement(version).Contains(keyword, StringComparer.Ordinal); + + /// + /// Returns true if the specified identifier is a statement keyword and + /// never an expression in a particular version of Python. + /// + public static bool IsOnlyStatementKeyword(string keyword, PythonLanguageVersion version = PythonLanguageVersion.None) + => Statement(version).Except(Expression(version)).Contains(keyword, StringComparer.Ordinal); + + /// + /// Returns a sequence of all keywords in a particular version of + /// Python. + /// + public static IEnumerable All(PythonLanguageVersion version = PythonLanguageVersion.None) + => Expression(version).Union(Statement(version)); + + /// + /// Returns a sequence of all keywords usable in an expression in a + /// particular version of Python. + /// + public static IEnumerable Expression(PythonLanguageVersion version = PythonLanguageVersion.None) { + yield return "and"; + yield return "as"; + if (version.IsNone() || version >= PythonLanguageVersion.V35) { + yield return "await"; + } + yield return "else"; + yield return "False"; // Not technically a keyword in Python 2.x, but may as well be + yield return "for"; + if (version.IsNone() || version >= PythonLanguageVersion.V33) { + yield return "from"; + } + yield return "if"; + yield return "in"; + yield return "is"; + yield return "lambda"; + yield return "None"; + yield return "not"; + yield return "or"; + yield return "True"; // Not technically a keyword in Python 2.x, but may as well be + yield return "yield"; + } + + /// + /// Returns sequence of all keywords usable as a statement in a + /// particular version of Python. + /// + public static IEnumerable Statement(PythonLanguageVersion version = PythonLanguageVersion.None) { + yield return "assert"; + if (version.IsNone() || version >= PythonLanguageVersion.V35) { + yield return "async"; + yield return "await"; + } + yield return "break"; + yield return "continue"; + yield return "class"; + yield return "def"; + yield return "del"; + if (version.IsNone() || version.Is2x()) { + yield return "exec"; + } + yield return "if"; + yield return "elif"; + yield return "except"; + yield return "finally"; + yield return "for"; + yield return "from"; + yield return "global"; + yield return "import"; + if (version.IsNone() || version.Is3x()) { + yield return "nonlocal"; + } + yield return "pass"; + if (version.IsNone() || version.Is2x()) { + yield return "print"; + } + yield return "raise"; + yield return "return"; + yield return "try"; + yield return "while"; + yield return "with"; + yield return "yield"; + } + + /// + /// Returns a sequence of all keywords that are invalid outside of + /// function definitions in a particular version of Python. + /// + public static IEnumerable InvalidOutsideFunction(PythonLanguageVersion version = PythonLanguageVersion.None) { + if (version.IsNone() || version >= PythonLanguageVersion.V35) { + yield return "await"; + } + yield return "return"; + if (version.IsNone() || version >= PythonLanguageVersion.V25) { + yield return "yield"; + } + } + } +} diff --git a/src/LanguageServer/Impl/Completion/RaiseCompletion.cs b/src/LanguageServer/Impl/Completion/RaiseCompletion.cs new file mode 100644 index 000000000..04388c10e --- /dev/null +++ b/src/LanguageServer/Impl/Completion/RaiseCompletion.cs @@ -0,0 +1,54 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class RaiseCompletion { + public static bool TryGetCompletions(RaiseStatement raiseStatement, CompletionContext context, out CompletionResult result) { + result = null; + + // raise Type, Value, Traceback with Cause + if (raiseStatement.Cause != null && context.Position >= raiseStatement.CauseFieldStartIndex) { + return false; + } + + if (raiseStatement.Traceback != null && context.Position >= raiseStatement.TracebackFieldStartIndex) { + return false; + } + + if (raiseStatement.Value != null && context.Position >= raiseStatement.ValueFieldStartIndex) { + return false; + } + + if (raiseStatement.ExceptType == null) { + return false; + } + + if (context.Position <= raiseStatement.ExceptType.EndIndex) { + return false; + } + + if (context.Ast.LanguageVersion.Is3x()) { + var applicableSpan = context.GetApplicableSpanFromLastToken(raiseStatement); + result = new CompletionResult(Enumerable.Repeat(CompletionItemSource.FromKeyword, 1), applicableSpan); + } + + return true; + } + } +} diff --git a/src/LanguageServer/Impl/Completion/TokenSource.cs b/src/LanguageServer/Impl/Completion/TokenSource.cs new file mode 100644 index 000000000..ed4c4c775 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/TokenSource.cs @@ -0,0 +1,52 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.IO; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing; + +namespace Microsoft.Python.LanguageServer.Completion { + internal sealed class TokenSource { + public IEnumerable> Tokens { get; } + public NewLineLocation[] TokenNewlines { get; } + + public SourceSpan GetTokenSpan(IndexSpan span) { + return new SourceSpan( + NewLineLocation.IndexToLocation(TokenNewlines, span.Start), + NewLineLocation.IndexToLocation(TokenNewlines, span.End) + ); + } + + public TokenSource(IDocument document, int toIndex) { + var tokens = new List>(); + Tokenizer tokenizer; + using (var reader = new StringReader(document.Content)) { + tokenizer = new Tokenizer(document.Interpreter.LanguageVersion, options: TokenizerOptions.GroupingRecovery); + tokenizer.Initialize(reader); + for (var t = tokenizer.GetNextToken(); + t.Kind != TokenKind.EndOfFile && tokenizer.TokenSpan.Start < toIndex; + t = tokenizer.GetNextToken()) { + tokens.Add(new KeyValuePair(tokenizer.TokenSpan, t)); + } + } + + Tokens = tokens; + TokenNewlines = tokenizer.GetLineLocations(); + } + + } +} diff --git a/src/LanguageServer/Impl/Completion/TopLevelCompletion.cs b/src/LanguageServer/Impl/Completion/TopLevelCompletion.cs new file mode 100644 index 000000000..fbf28310b --- /dev/null +++ b/src/LanguageServer/Impl/Completion/TopLevelCompletion.cs @@ -0,0 +1,185 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Analyzer.Expressions; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class TopLevelCompletion { + public static async Task GetCompletionsAsync(Node statement, ScopeStatement scopeStatement, CompletionContext context, CancellationToken cancellationToken = default) { + SourceSpan? applicableSpan = null; + var eval = context.Analysis.ExpressionEvaluator; + + var options = GetOptions(statement, context.Position, out var span); + if (span.HasValue) { + applicableSpan = new SourceSpan(context.IndexToLocation(span.Value.Start), context.IndexToLocation(span.Value.End)); + } + + var scope = context.Analysis.FindScope(context.Location); + IEnumerable items; + using (eval.OpenScope(scope)) { + // Get variables declared in the module. + var variables = eval.CurrentScope.EnumerateTowardsGlobal.SelectMany(s => s.Variables).ToArray(); + items = variables.Select(v => context.ItemSource.CreateCompletionItem(v.Name, v)); + } + + // Get builtins + var builtins = context.Analysis.Document.Interpreter.ModuleResolution.BuiltinsModule; + var builtinItems = builtins.GetMemberNames() + .Select(n => { + var m = builtins.GetMember(n); + if ((options & CompletionListOptions.ExceptionsOnly) == CompletionListOptions.ExceptionsOnly && !IsExceptionType(m.GetPythonType())) { + return null; + } + return context.ItemSource.CreateCompletionItem(n, m); + }).ExcludeDefault(); + items = items.Concat(builtinItems); + + // Add possible function arguments. + var finder = new ExpressionFinder(context.Ast, new FindExpressionOptions { Calls = true }); + if (finder.GetExpression(context.Position) is CallExpression callExpr && callExpr.GetArgumentAtIndex(context.Ast, context.Position, out _)) { + var value = await eval.GetValueFromExpressionAsync(callExpr.Target, cancellationToken); + if (value?.GetPythonType() is IPythonFunctionType ft) { + var arguments = ft.Overloads.SelectMany(o => o.Parameters).Select(p => p?.Name) + .Where(n => !string.IsNullOrEmpty(n)) + .Distinct() + .Except(callExpr.Args.MaybeEnumerate().Select(a => a.Name).Where(n => !string.IsNullOrEmpty(n))) + .Select(n => CompletionItemSource.CreateCompletionItem($"{n}=", CompletionItemKind.Variable)) + .ToArray(); + + items = items.Concat(arguments).ToArray(); + } + } + + var keywords = GetKeywordItems(context, options, scopeStatement); + items = items.Concat(keywords); + + return new CompletionResult(items, applicableSpan); + } + + [Flags] + enum CompletionListOptions { + NoKeywords = 1, + ExceptionsOnly = 2, + StatementKeywords = 4, + ExpressionKeywords = 8, + AllKeywords = StatementKeywords | ExpressionKeywords + } + + private static CompletionListOptions GetOptions(Node statement, int index, out IndexSpan? span) { + span = null; + + switch (statement) { + // Disallow keywords, unless we're between the end of decorators and the + // end of the "[async] def" keyword. + case FunctionDefinition fd when index > fd.KeywordEndIndex || fd.Decorators != null && index < fd.Decorators.EndIndex: + case ClassDefinition cd when index > cd.KeywordEndIndex || cd.Decorators != null && index < cd.Decorators.EndIndex: + return CompletionListOptions.NoKeywords; + + case TryStatementHandler tryStatement when tryStatement.Test is TupleExpression || index >= tryStatement.Test.StartIndex: + return CompletionListOptions.ExceptionsOnly; + + // Always allow keywords in non-keyword statements + case ExpressionStatement _: + case ImportStatement _: + case FromImportStatement _: + case null: + return CompletionListOptions.AllKeywords; + + // Allow keywords at start of assignment, but not in subsequent names + case AssignmentStatement ss: + var firstAssign = ss.Left?.FirstOrDefault(); + return firstAssign == null || index <= firstAssign.EndIndex + ? CompletionListOptions.AllKeywords : CompletionListOptions.ExpressionKeywords; + + // Allow keywords when we are in another keyword + case Statement s when index <= s.KeywordEndIndex: + var keywordStart = s.KeywordEndIndex - s.KeywordLength; + if (index >= keywordStart) { + span = new IndexSpan(keywordStart, s.KeywordLength); + } else if ((s as IMaybeAsyncStatement)?.IsAsync == true) { + // Must be in the "async" at the start of the keyword + span = new IndexSpan(s.StartIndex, "async".Length); + } + return CompletionListOptions.AllKeywords; + + case RaiseStatement raise when raise.ExceptType != null && index >= raise.ExceptType.StartIndex || index > raise.KeywordEndIndex: + return CompletionListOptions.ExpressionKeywords | CompletionListOptions.ExceptionsOnly; + + // TryStatementHandler is 'except', but not a Statement subclass + case TryStatementHandler except when index <= except.KeywordEndIndex: + var exceptKeywordStart = except.KeywordEndIndex - except.KeywordLength; + if (index >= exceptKeywordStart) { + span = new IndexSpan(exceptKeywordStart, except.KeywordLength); + } + return CompletionListOptions.AllKeywords; + + // Allow keywords in function body (we'd have a different statement if we were deeper) + case FunctionDefinition fd when index >= fd.HeaderIndex: + return CompletionListOptions.AllKeywords; + + // Allow keywords within with blocks, but not in their definition + case WithStatement ws: + return index >= ws.HeaderIndex || index <= ws.KeywordEndIndex + ? CompletionListOptions.AllKeywords : CompletionListOptions.ExpressionKeywords; + + default: + return CompletionListOptions.ExpressionKeywords; + } + } + private static IEnumerable GetKeywordItems(CompletionContext context, CompletionListOptions options, ScopeStatement scope) { + var keywords = Enumerable.Empty(); + + if ((options & CompletionListOptions.ExpressionKeywords) == CompletionListOptions.ExpressionKeywords) { + // keywords available in any context + keywords = PythonKeywords.Expression(context.Ast.LanguageVersion); + } + + if ((options & CompletionListOptions.StatementKeywords) == CompletionListOptions.StatementKeywords) { + keywords = keywords.Union(PythonKeywords.Statement(context.Ast.LanguageVersion)); + } + + if (!(scope is FunctionDefinition)) { + keywords = keywords.Except(PythonKeywords.InvalidOutsideFunction(context.Ast.LanguageVersion)); + } + + return keywords.Select(kw => CompletionItemSource.CreateCompletionItem(kw, CompletionItemKind.Keyword)); + } + + private static bool IsExceptionType(IPythonType type) { + if (type.Name.IndexOf("error", StringComparison.OrdinalIgnoreCase) >= 0 || + type.Name.IndexOf("exception", StringComparison.OrdinalIgnoreCase) >= 0) { + return true; + } + + if (type is IPythonClassType cls) { + var baseCls = type.DeclaringModule.Interpreter.ModuleResolution.BuiltinsModule.GetMember("BaseException") as IPythonType; + return cls.Mro.Any(b => b is IPythonClassType c && c.Bases.Contains(baseCls)); + } + return false; + } + } +} diff --git a/src/LanguageServer/Impl/Completion/WithCompletion.cs b/src/LanguageServer/Impl/Completion/WithCompletion.cs new file mode 100644 index 000000000..fdf07fef9 --- /dev/null +++ b/src/LanguageServer/Impl/Completion/WithCompletion.cs @@ -0,0 +1,58 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Completion { + internal static class WithCompletion { + public static bool TryGetCompletions(WithStatement withStatement, CompletionContext context, out CompletionResult result) { + result = CompletionResult.Empty; + + if (context.Position > withStatement.HeaderIndex && withStatement.HeaderIndex > withStatement.StartIndex) { + return false; + } + + foreach (var item in withStatement.Items.Reverse().MaybeEnumerate()) { + if (item.AsIndex > item.StartIndex) { + if (context.Position > item.AsIndex + 2) { + return true; + } + + if (context.Position >= item.AsIndex) { + var applicableSpan = new SourceSpan(context.IndexToLocation(item.AsIndex), context.IndexToLocation(item.AsIndex + 2)); + result = new CompletionResult(Enumerable.Repeat(CompletionItemSource.AsKeyword,1), applicableSpan); + return true; + } + } + + if (item.ContextManager != null && !(item.ContextManager is ErrorExpression)) { + if (context.Position > item.ContextManager.EndIndex && item.ContextManager.EndIndex > item.ContextManager.StartIndex) { + result = result = new CompletionResult(Enumerable.Repeat(CompletionItemSource.AsKeyword, 1)); + return true; + } + + if (context.Position >= item.ContextManager.StartIndex) { + return false; + } + } + } + + return false; + } + } +} diff --git a/src/LanguageServer/Impl/Definitions/CallbackEventArgs.cs b/src/LanguageServer/Impl/Definitions/CallbackEventArgs.cs index 5ff38902d..38fb4c9e7 100644 --- a/src/LanguageServer/Impl/Definitions/CallbackEventArgs.cs +++ b/src/LanguageServer/Impl/Definitions/CallbackEventArgs.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -16,24 +15,19 @@ using System; using System.Threading.Tasks; +using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer { - public abstract class CallbackEventArgs : EventArgs where T : struct { - private TaskCompletionSource _task; + public abstract class CallbackEventArgs : EventArgs where T : class { + private readonly TaskCompletionSource _task; internal CallbackEventArgs(TaskCompletionSource task) { _task = task; } - public T? @params { get; set; } - - public void SetResult() { - _task.TrySetResult(null); - } - - public void SetError(ResponseError error) { - _task.TrySetException(new LanguageServerException(error.code, error.message)); - } + public T @params { get; set; } + public void SetResult() => _task.TrySetResult(null); + public void SetError(ResponseError error) => _task.TrySetException(new LanguageServerException(error.code, error.message)); } public abstract class CallbackEventArgs : EventArgs where T : class where U : class { @@ -44,13 +38,7 @@ internal CallbackEventArgs(TaskCompletionSource task) { } public T @params { get; set; } - - public void SetResult(U result) { - _task.TrySetResult(result); - } - - public void SetError(ResponseError error) { - _task.TrySetException(new LanguageServerException(error.code, error.message)); - } + public void SetResult(U result) => _task.TrySetResult(result); + public void SetError(ResponseError error) => _task.TrySetException(new LanguageServerException(error.code, error.message)); } } diff --git a/src/LanguageServer/Impl/Definitions/EditorOperationException.cs b/src/LanguageServer/Impl/Definitions/EditorOperationException.cs index a84e4c144..63bdeb8ed 100644 --- a/src/LanguageServer/Impl/Definitions/EditorOperationException.cs +++ b/src/LanguageServer/Impl/Definitions/EditorOperationException.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -14,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. - using System; namespace Microsoft.Python.LanguageServer { diff --git a/src/LanguageServer/Impl/Definitions/IDocumentReader.cs b/src/LanguageServer/Impl/Definitions/IDocumentReader.cs deleted file mode 100644 index 1073946e1..000000000 --- a/src/LanguageServer/Impl/Definitions/IDocumentReader.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using Microsoft.Python.Core.Text; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.LanguageServer { - public interface IDocumentReader { - string ReadToEnd(); - string Read(int start, int count); - } - - public static class DocumentReaderExtensions { - public static string ReadLinearSpan(this IDocumentReader reader, IndexSpan span) - => reader.Read(span.Start, span.Length); - public static string ReadRange(this IDocumentReader reader, Range range, PythonAst ast) - => reader.ReadLinearSpan(range.ToIndexSpan(ast)); - public static string ReadSourceSpan(this IDocumentReader reader, SourceSpan span, PythonAst ast) - => reader.ReadLinearSpan(span.ToIndexSpan(ast)); - } -} diff --git a/src/LanguageServer/Impl/Definitions/IDocumentationSource.cs b/src/LanguageServer/Impl/Definitions/IDocumentationSource.cs new file mode 100644 index 000000000..1dce4db25 --- /dev/null +++ b/src/LanguageServer/Impl/Definitions/IDocumentationSource.cs @@ -0,0 +1,27 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.LanguageServer.Protocol; + +namespace Microsoft.Python.LanguageServer { + public interface IDocumentationSource { + InsertTextFormat DocumentationFormat { get; } + MarkupContent GetHover(string name, IMember member); + string GetSignatureString(IPythonFunctionType ft, int overloadIndex = 0); + MarkupContent FormatParameterDocumentation(IParameterInfo parameter); + MarkupContent FormatDocumentation(string documentation); + } +} diff --git a/src/LanguageServer/Impl/Definitions/ILanguageServerExtensionProvider.cs b/src/LanguageServer/Impl/Definitions/ILanguageServerExtensionProvider.cs index aff42f70f..7df43d305 100644 --- a/src/LanguageServer/Impl/Definitions/ILanguageServerExtensionProvider.cs +++ b/src/LanguageServer/Impl/Definitions/ILanguageServerExtensionProvider.cs @@ -17,6 +17,8 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Core; +using Microsoft.Python.LanguageServer.Extensibility; namespace Microsoft.Python.LanguageServer.Extensions { /// @@ -27,10 +29,7 @@ public interface ILanguageServerExtensionProvider { /// /// Called when the extension is loaded for a language server. /// - Task CreateAsync( - IPythonLanguageServer server, - IReadOnlyDictionary properties, - CancellationToken cancellationToken - ); + Task CreateAsync(IServiceContainer services, + IReadOnlyDictionary properties, CancellationToken cancellationToken = default); } } diff --git a/src/LanguageServer/Impl/Definitions/IPythonLanguageServer.cs b/src/LanguageServer/Impl/Definitions/IPythonLanguageServer.cs deleted file mode 100644 index ae102c698..000000000 --- a/src/LanguageServer/Impl/Definitions/IPythonLanguageServer.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Core.Logging; -using Microsoft.PythonTools.Analysis; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.LanguageServer { - public interface IPythonLanguageServer : IPythonLanguageServerProtocol { - ILogger Logger { get; } - Task ReloadModulesAsync(CancellationToken token); - PythonAst GetCurrentAst(Uri documentUri); - Task GetAstAsync(Uri documentUri, CancellationToken token); - Task GetAnalysisAsync(Uri documentUri, CancellationToken token); - IProjectEntry GetProjectEntry(Uri documentUri); - IEnumerable GetLoadedFiles(); - } -} diff --git a/src/LanguageServer/Impl/Definitions/IPythonLanguageServerProtocol.cs b/src/LanguageServer/Impl/Definitions/IPythonLanguageServerProtocol.cs deleted file mode 100644 index 4d0efdab0..000000000 --- a/src/LanguageServer/Impl/Definitions/IPythonLanguageServerProtocol.cs +++ /dev/null @@ -1,50 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.Python.LanguageServer { - /// - /// Implements part of the language protocol relevant to the extensibility. - /// - /// - /// - /// https://microsoft.github.io/language-server-protocol/specification - /// - public interface IPythonLanguageServerProtocol { - Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken token); - Task ExecuteCommand(ExecuteCommandParams @params, CancellationToken token); - Task Completion(CompletionParams @params, CancellationToken token); - Task CompletionItemResolve(CompletionItem item, CancellationToken token); - Task Hover(TextDocumentPositionParams @params, CancellationToken token); - Task SignatureHelp(TextDocumentPositionParams @params, CancellationToken token); - Task GotoDefinition(TextDocumentPositionParams @params, CancellationToken token); - Task FindReferences(ReferencesParams @params, CancellationToken token); - Task DocumentHighlight(TextDocumentPositionParams @params, CancellationToken token); - Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken token); - Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken); - Task CodeAction(CodeActionParams @params, CancellationToken token); - Task CodeLens(TextDocumentPositionParams @params, CancellationToken token); - Task CodeLensResolve(CodeLens item, CancellationToken token); - Task DocumentLink(DocumentLinkParams @params, CancellationToken token); - Task DocumentLinkResolve(DocumentLink item, CancellationToken token); - Task DocumentFormatting(DocumentFormattingParams @params, CancellationToken token); - Task DocumentRangeFormatting(DocumentRangeFormattingParams @params, CancellationToken token); - Task DocumentOnTypeFormatting(DocumentOnTypeFormattingParams @params, CancellationToken token); - Task Rename(RenameParams @params, CancellationToken cancellationToken); - } -} diff --git a/src/LanguageServer/Impl/Definitions/Messages.cs b/src/LanguageServer/Impl/Definitions/Messages.cs deleted file mode 100644 index f8aec2d94..000000000 --- a/src/LanguageServer/Impl/Definitions/Messages.cs +++ /dev/null @@ -1,372 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Runtime.InteropServices; -using System.Threading.Tasks; -using Microsoft.Python.Core.Text; -using Microsoft.PythonTools.Analysis; - -namespace Microsoft.Python.LanguageServer { - [Serializable] - public struct InitializeParams { - public int? processId; - public string rootPath; - public Uri rootUri; - public PythonInitializationOptions initializationOptions; - public ClientCapabilities capabilities; - public TraceLevel trace; - } - - [Serializable] - public class LanguageServerException : Exception { - public const int UnknownDocument = 1; - public const int UnsupportedDocumentType = 2; - public const int MismatchedVersion = 3; - public const int UnknownExtension = 4; - - public int Code => (int)Data["Code"]; - - public sealed override System.Collections.IDictionary Data => base.Data; - - public LanguageServerException(int code, string message) : base(message) { - Data["Code"] = code; - } - - public LanguageServerException(int code, string message, Exception innerException) : base(message, innerException) { - Data["Code"] = code; - } - } - - [Serializable] - public struct InitializeResult { - public ServerCapabilities? capabilities; - } - - [Serializable] - public struct InitializedParams { } - - public sealed class ShowMessageEventArgs : EventArgs { - public MessageType type { get; set; } - public string message { get; set; } - } - - [Serializable] - public class ShowMessageRequestParams { - public MessageType type; - public string message; - public MessageActionItem[] actions; - } - - public sealed class CommandEventArgs: EventArgs { - public string command; - public object[] arguments; - } - - [Serializable] - public struct RegistrationParams { - public Registration[] registrations; - } - - [ComVisible(false)] - public sealed class RegisterCapabilityEventArgs : CallbackEventArgs { - internal RegisterCapabilityEventArgs(TaskCompletionSource task) : base(task) { } - } - - [Serializable] - public struct UnregistrationParams { - public Unregistration[] unregistrations; - } - - [ComVisible(false)] - public sealed class UnregisterCapabilityEventArgs : CallbackEventArgs { - internal UnregisterCapabilityEventArgs(TaskCompletionSource task) : base(task) { } - } - - [Serializable] - public struct DidChangeConfigurationParams { - public object settings; - } - - [Serializable] - public struct DidChangeWatchedFilesParams { - public FileEvent[] changes; - } - - [Serializable] - public struct WorkspaceSymbolParams { - public string query; - } - - [Serializable] - public struct ExecuteCommandParams { - public string command; - public object[] arguments; - } - - [Serializable] - public class ApplyWorkspaceEditParams { - /// - /// An optional label of the workspace edit.This label is - /// presented in the user interface for example on an undo - /// stack to undo the workspace edit. - /// - public string label; - public WorkspaceEdit edit; - } - - [Serializable] - public class ApplyWorkspaceEditResponse { - public bool applied; - } - - [ComVisible(false)] - public sealed class ApplyWorkspaceEditEventArgs : CallbackEventArgs { - internal ApplyWorkspaceEditEventArgs(TaskCompletionSource task) : base(task) { } - } - - [Serializable] - public struct DidOpenTextDocumentParams { - public TextDocumentItem textDocument; - } - - [Serializable] - public struct DidChangeTextDocumentParams { - public VersionedTextDocumentIdentifier textDocument; - public TextDocumentContentChangedEvent[] contentChanges; - - // Defaults to true, but can be set to false to suppress analysis - public bool? _enqueueForAnalysis; - } - - [Serializable] - public struct WillSaveTextDocumentParams { - public TextDocumentIdentifier textDocument; - public TextDocumentSaveReason reason; - } - - [Serializable] - public struct DidSaveTextDocumentParams { - public TextDocumentIdentifier textDocument; - public string content; - } - - [Serializable] - public struct DidCloseTextDocumentParams { - public TextDocumentIdentifier textDocument; - } - - public sealed class PublishDiagnosticsEventArgs : EventArgs { - public Uri uri { get; set; } - public IReadOnlyList diagnostics { get; set; } - - /// - /// The version the ranges in the diagnostics apply to. - /// - public int? _version { get; set; } - } - - [Serializable] - public struct TextDocumentPositionParams { - public TextDocumentIdentifier textDocument; - public Position position; - - /// - /// The intended version that position applies to. The request may fail if - /// the server cannot map correctly. - /// - public int? _version; - /// - /// Override the expression to evaluate. If omitted, uses the context at the - /// specified position. - /// - public string _expr; - } - - [Serializable] - public struct CompletionParams { - public TextDocumentIdentifier textDocument; - public Position position; - public CompletionContext? context; - - /// - /// The intended version that position applies to. The request may fail if - /// the server cannot map correctly. - /// - public int? _version; - /// - /// Override the expression to evaluate. If omitted, uses the context at the - /// specified position. - /// - public string _expr; - } - - [Serializable] - public struct CompletionContext { - public CompletionTriggerKind triggerKind; - public string triggerCharacter; - - public bool _intersection; - //public bool? _statementKeywords; - //public bool? _expressionKeywords; - //public bool? _includeAllModules; - //public bool? _includeArgumentNames; - public CompletionItemKind? _filterKind; - } - - [Serializable] - public struct ReferencesParams { - public TextDocumentIdentifier textDocument; - public Position position; - public ReferenceContext? context; - - /// - /// The intended version that range applies to. The request may fail if - /// the server cannot map correctly. - /// - public int? _version; - - /// - /// Override the expression to evaluate. If omitted, uses the context at the - /// specified position. - /// - public string _expr; - } - - public struct ReferenceContext { - public bool includeDeclaration; - - public bool _includeValues; - } - - [Serializable] - public struct DocumentSymbolParams { - public TextDocumentIdentifier textDocument; - } - - [Serializable] - public struct CodeActionParams { - public TextDocumentIdentifier textDocument; - public Range range; - public CodeActionContext? context; - - /// - /// The intended version that range applies to. The request may fail if - /// the server cannot map correctly. - /// - public int? _version; - } - - [Serializable] - public struct CodeActionContext { - public Diagnostic[] diagnostics; - - /// - /// The intended version that diagnostic locations apply to. The request may - /// fail if the server cannot map correctly. - /// - public int? _version; - } - - [Serializable] - public struct DocumentLinkParams { - public TextDocumentIdentifier textDocument; - } - - [Serializable] - public struct DocumentFormattingParams { - public TextDocumentIdentifier textDocument; - public FormattingOptions options; - } - - [Serializable] - public struct DocumentRangeFormattingParams { - public TextDocumentIdentifier textDocument; - public Range range; - public FormattingOptions options; - - /// - /// The intended version that range applies to. The request may fail if - /// the server cannot map correctly. - /// - public int? _version; - } - - [Serializable] - public struct DocumentOnTypeFormattingParams { - public TextDocumentIdentifier textDocument; - public Position position; - public string ch; - public FormattingOptions options; - - /// - /// The intended version that range applies to. The request may fail if - /// the server cannot map correctly. - /// - public int? _version; - } - - [Serializable] - public struct RenameParams { - public TextDocumentIdentifier textDocument; - public Position position; - public string newName; - - /// - /// The intended version that position applies to. The request may fail if - /// the server cannot map correctly. - /// - public int? _version; - } - - [Serializable] - public class PythonAnalysisExtensionParams { - public string assembly; - public string typeName; - public Dictionary properties; - } - - [Serializable] - public class ExtensionCommandParams { - public string extensionName; - public string command; - public Dictionary properties; - } - - [Serializable] - public class ExtensionCommandResult { - public IReadOnlyDictionary properties; - } - - public sealed class FileFoundEventArgs : EventArgs { - public Uri uri { get; set; } - } - - public sealed class ParseCompleteEventArgs : EventArgs { - public Uri uri { get; set; } - public int version { get; set; } - } - - public sealed class AnalysisQueuedEventArgs : EventArgs { - public Uri uri { get; set; } - } - - public sealed class AnalysisCompleteEventArgs : EventArgs { - public Uri uri { get; set; } - public int version { get; set; } - } -} diff --git a/src/LanguageServer/Impl/Definitions/ServerSettings.cs b/src/LanguageServer/Impl/Definitions/ServerSettings.cs index a86f2d987..86820b48d 100644 --- a/src/LanguageServer/Impl/Definitions/ServerSettings.cs +++ b/src/LanguageServer/Impl/Definitions/ServerSettings.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -16,14 +15,13 @@ using System; using System.Collections.Generic; -using Microsoft.PythonTools.Analysis; +using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer { public class ServerSettings { public class PythonAnalysisOptions { - private Dictionary _map = new Dictionary(); + private readonly Dictionary _map = new Dictionary(); - public bool openFilesOnly; public int symbolsHierarchyDepthLimit = 10; public int symbolsHierarchyMaxSymbols = 1000; diff --git a/src/LanguageServer/Impl/Diagnostics/DiagnosticsPublisher.cs b/src/LanguageServer/Impl/Diagnostics/DiagnosticsService.cs similarity index 50% rename from src/LanguageServer/Impl/Diagnostics/DiagnosticsPublisher.cs rename to src/LanguageServer/Impl/Diagnostics/DiagnosticsService.cs index cd5b58ab6..77fadecaf 100644 --- a/src/LanguageServer/Impl/Diagnostics/DiagnosticsPublisher.cs +++ b/src/LanguageServer/Impl/Diagnostics/DiagnosticsService.cs @@ -16,26 +16,24 @@ using System; using System.Collections.Generic; using System.Linq; +using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Idle; -using Microsoft.Python.Core.Shell; -using Microsoft.PythonTools.Analysis; -using Newtonsoft.Json; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing; using StreamJsonRpc; namespace Microsoft.Python.LanguageServer.Diagnostics { - internal sealed class DiagnosticsPublisher : IDisposable { - private readonly Dictionary _pendingDiagnostic = new Dictionary(); - private readonly DisposableBag _disposables = DisposableBag.Create(); + + internal sealed class DiagnosticsService : IDiagnosticsService, IDisposable { + private readonly Dictionary> _pendingDiagnostics = new Dictionary>(); + private readonly DisposableBag _disposables = DisposableBag.Create(); private readonly JsonRpc _rpc; private readonly object _lock = new object(); private DateTime _lastChangeTime; - public DiagnosticsPublisher(Implementation.Server server, IServiceContainer services) { - var s = server; - s.OnPublishDiagnostics += OnPublishDiagnostics; - + public DiagnosticsService(IServiceContainer services) { var idleTimeService = services.GetService(); idleTimeService.Idle += OnIdle; idleTimeService.Closing += OnClosing; @@ -43,60 +41,83 @@ public DiagnosticsPublisher(Implementation.Server server, IServiceContainer serv _rpc = services.GetService(); _disposables - .Add(() => s.OnPublishDiagnostics -= OnPublishDiagnostics) .Add(() => idleTimeService.Idle -= OnIdle) .Add(() => idleTimeService.Idle -= OnClosing); } + #region IDiagnosticsService + public IReadOnlyList Diagnostics { + get { + lock(_lock) { + return _pendingDiagnostics.Values.SelectMany().ToArray(); + } + } + } + + public void Add(Uri documentUri, DiagnosticsEntry entry) { + lock(_lock) { + if(!_pendingDiagnostics.TryGetValue(documentUri, out var list)) { + _pendingDiagnostics[documentUri] = list = new List(); + } + list.Add(entry); + _lastChangeTime = DateTime.Now; + } + } + public int PublishingDelay { get; set; } + #endregion public void Dispose() => _disposables.TryDispose(); private void OnClosing(object sender, EventArgs e) => Dispose(); private void OnIdle(object sender, EventArgs e) { - if (_pendingDiagnostic.Count > 0 && (DateTime.Now - _lastChangeTime).TotalMilliseconds > PublishingDelay) { + if (_pendingDiagnostics.Count > 0 && (DateTime.Now - _lastChangeTime).TotalMilliseconds > PublishingDelay) { PublishPendingDiagnostics(); } } - private void OnPublishDiagnostics(object sender, PublishDiagnosticsEventArgs e) { - lock (_lock) { - // If list is empty (errors got fixed), publish immediately, - // otherwise throttle so user does not get spurious squiggles - // while typing normally. - var diags = e.diagnostics.ToArray(); - _pendingDiagnostic[e.uri] = diags; - if (diags.Length == 0) { - PublishPendingDiagnostics(); - } - _lastChangeTime = DateTime.Now; - } - } - private void PublishPendingDiagnostics() { - List> list; + List>> list; lock (_lock) { - list = _pendingDiagnostic.ToList(); - _pendingDiagnostic.Clear(); + list = _pendingDiagnostics.ToList(); + _pendingDiagnostics.Clear(); } foreach (var kvp in list) { var parameters = new PublishDiagnosticsParams { uri = kvp.Key, - diagnostics = kvp.Value.Where(d => d.severity != DiagnosticSeverity.Unspecified).ToArray() + diagnostics = kvp.Value.Select(x => ToDiagnostic(kvp.Key, x)).ToArray() }; _rpc.NotifyWithParameterObjectAsync("textDocument/publishDiagnostics", parameters).DoNotWait(); } } - [JsonObject] - private class PublishDiagnosticsParams { - [JsonProperty] - public Uri uri; - [JsonProperty] - public Diagnostic[] diagnostics; + private static Diagnostic ToDiagnostic(Uri uri, DiagnosticsEntry e) { + DiagnosticSeverity s; + switch (e.Severity) { + case Severity.Warning: + s = DiagnosticSeverity.Warning; + break; + case Severity.Information: + s = DiagnosticSeverity.Information; + break; + case Severity.Hint: + s = DiagnosticSeverity.Hint; + break; + default: + s = DiagnosticSeverity.Error; + break; + } + + return new Diagnostic { + range = e.SourceSpan, + severity = s, + source = "Python", + code = e.ErrorCode, + message = e.Message, + }; } } } diff --git a/src/LanguageServer/Impl/Extensibility/Extensibility.cs b/src/LanguageServer/Impl/Extensibility/Extensibility.cs new file mode 100644 index 000000000..f4aa0c0bd --- /dev/null +++ b/src/LanguageServer/Impl/Extensibility/Extensibility.cs @@ -0,0 +1,38 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; + +namespace Microsoft.Python.LanguageServer.Extensibility { + [Serializable] + public class PythonAnalysisExtensionParams { + public string assembly; + public string typeName; + public Dictionary properties; + } + + [Serializable] + public class ExtensionCommandParams { + public string extensionName; + public string command; + public Dictionary properties; + } + + [Serializable] + public class ExtensionCommandResult { + public IReadOnlyDictionary properties; + } +} diff --git a/src/LanguageServer/Impl/Extensions/ICompletionExtension.cs b/src/LanguageServer/Impl/Extensibility/ICompletionExtension.cs similarity index 67% rename from src/LanguageServer/Impl/Extensions/ICompletionExtension.cs rename to src/LanguageServer/Impl/Extensibility/ICompletionExtension.cs index 369b3f5d0..e82e79f19 100644 --- a/src/LanguageServer/Impl/Extensions/ICompletionExtension.cs +++ b/src/LanguageServer/Impl/Extensibility/ICompletionExtension.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -14,15 +13,16 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing.Ast; -using Microsoft.PythonTools.Analysis; -namespace Microsoft.Python.LanguageServer.Extensions { +namespace Microsoft.Python.LanguageServer.Extensibility { public interface ICompletionExtension { - Task HandleCompletionAsync(Uri documentUri, IModuleAnalysis analysis, PythonAst tree, SourceLocation location, CompletionList completions, CancellationToken token); + Task HandleCompletionAsync(IDocument document, IDocumentAnalysis analysis, PythonAst tree, SourceLocation location, CompletionList completions, CancellationToken token); } } diff --git a/src/LanguageServer/Impl/Extensions/ILanguageServerExtension.cs b/src/LanguageServer/Impl/Extensibility/ILanguageServerExtension.cs similarity index 89% rename from src/LanguageServer/Impl/Extensions/ILanguageServerExtension.cs rename to src/LanguageServer/Impl/Extensibility/ILanguageServerExtension.cs index 581454c5f..fcbc94595 100644 --- a/src/LanguageServer/Impl/Extensions/ILanguageServerExtension.cs +++ b/src/LanguageServer/Impl/Extensibility/ILanguageServerExtension.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -20,7 +19,7 @@ using System.Threading.Tasks; using Microsoft.Python.Core; -namespace Microsoft.Python.LanguageServer.Extensions { +namespace Microsoft.Python.LanguageServer.Extensibility { public interface ILanguageServerExtension: IDisposable { string Name { get; } Task Initialize(IServiceContainer services, CancellationToken token); diff --git a/src/LanguageServer/Impl/Implementation/BlockFormatter.cs b/src/LanguageServer/Impl/Formatting/BlockFormatter.cs similarity index 98% rename from src/LanguageServer/Impl/Implementation/BlockFormatter.cs rename to src/LanguageServer/Impl/Formatting/BlockFormatter.cs index b13356946..30e820c8a 100644 --- a/src/LanguageServer/Impl/Implementation/BlockFormatter.cs +++ b/src/LanguageServer/Impl/Formatting/BlockFormatter.cs @@ -23,8 +23,9 @@ using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; -namespace Microsoft.Python.LanguageServer.Implementation { +namespace Microsoft.Python.LanguageServer.Formatting { /// /// Port of BlockFormatProviders in vscode-python. /// diff --git a/src/LanguageServer/Impl/Implementation/LineFormatter.cs b/src/LanguageServer/Impl/Formatting/LineFormatter.cs similarity index 99% rename from src/LanguageServer/Impl/Implementation/LineFormatter.cs rename to src/LanguageServer/Impl/Formatting/LineFormatter.cs index 4fed7a0f8..3276a6a66 100644 --- a/src/LanguageServer/Impl/Implementation/LineFormatter.cs +++ b/src/LanguageServer/Impl/Formatting/LineFormatter.cs @@ -22,9 +22,10 @@ using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; using Microsoft.Python.Parsing; -namespace Microsoft.Python.LanguageServer.Implementation { +namespace Microsoft.Python.LanguageServer.Formatting { /// /// LineFormatter formats lines of code to generally conform with PEP8. /// @@ -578,7 +579,7 @@ public TokenExt Next() { var tokenExt = new TokenExt( token, - _tokenizer.PreceedingWhiteSpace, + _tokenizer.PrecedingWhiteSpace, tokenSpan, line, isMultiLine, diff --git a/src/LanguageServer/Impl/Implementation/CompletionAnalysis.cs b/src/LanguageServer/Impl/Implementation/CompletionAnalysis.cs deleted file mode 100644 index 9bda0b3b0..000000000 --- a/src/LanguageServer/Impl/Implementation/CompletionAnalysis.cs +++ /dev/null @@ -1,1018 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using Microsoft.Python.Analysis.Core.DependencyResolution; -using Microsoft.Python.Core; -using Microsoft.Python.Core.Diagnostics; -using Microsoft.Python.Core.Logging; -using Microsoft.Python.Core.Text; -using Microsoft.Python.Parsing; -using Microsoft.Python.Parsing.Ast; -using Microsoft.PythonTools.Analysis; -using Microsoft.PythonTools.Analysis.Documentation; -using Microsoft.PythonTools.Analysis.Values; -using Microsoft.PythonTools.Interpreter; - -namespace Microsoft.Python.LanguageServer.Implementation { - class CompletionAnalysis { - private readonly Node _statement; - private readonly ScopeStatement _scope; - private readonly ILogger _log; - private readonly PathResolverSnapshot _pathResolver; - private readonly DocumentationBuilder _textBuilder; - private readonly Func _openDocument; - private readonly bool _addBrackets; - - public CompletionAnalysis( - IModuleAnalysis analysis, - PythonAst tree, - SourceLocation position, - GetMemberOptions opts, - ServerSettings.PythonCompletionOptions completionSettings, - DocumentationBuilder textBuilder, - ILogger log, - Func openDocument - ) { - Analysis = analysis ?? throw new ArgumentNullException(nameof(analysis)); - Tree = tree ?? throw new ArgumentNullException(nameof(tree)); - Position = position; - Index = Tree.LocationToIndex(Position); - Options = opts; - _pathResolver = analysis.ProjectState.CurrentPathResolver; - _textBuilder = textBuilder; - _log = log; - _openDocument = openDocument; - _addBrackets = completionSettings.addBrackets; - - var finder = new ExpressionFinder(Tree, new GetExpressionOptions { - Names = true, - Members = true, - NamedArgumentNames = true, - ImportNames = true, - ImportAsNames = true, - Literals = true, - Errors = true - }); - - finder.Get(Index, Index, out var node, out _statement, out _scope); - - var index = Index; - var col = Position.Column; - while (CanBackUp(Tree, node, _statement, _scope, col)) { - col -= 1; - index -= 1; - finder.Get(index, index, out node, out _statement, out _scope); - } - - Node = node ?? (_statement as ExpressionStatement)?.Expression; - } - - private static bool CanBackUp(PythonAst tree, Node node, Node statement, ScopeStatement scope, int column) { - if (node != null || !((statement as ExpressionStatement)?.Expression is ErrorExpression)) { - return false; - } - - var top = 1; - if (scope != null) { - var scopeStart = scope.GetStart(tree); - if (scope.Body != null) { - top = scope.Body.GetEnd(tree).Line == scopeStart.Line - ? scope.Body.GetStart(tree).Column - : scopeStart.Column; - } else { - top = scopeStart.Column; - } - } - - return column > top; - } - - private static readonly IEnumerable Empty = Enumerable.Empty(); - - public IModuleAnalysis Analysis { get; } - public PythonAst Tree { get; } - public SourceLocation Position { get; } - public int Index { get; } - public GetMemberOptions Options { get; set; } - public SourceSpan? ApplicableSpan { get; set; } - - public bool? ShouldCommitByDefault { get; set; } - public bool? ShouldAllowSnippets { get; set; } - - public Node Node { get; private set; } - public Node Statement => _statement; - public ScopeStatement Scope => _scope; - - /// - /// The node that members were returned for, if any. - /// - public Expression ParentExpression { get; private set; } - - - private IReadOnlyList> _tokens; - private NewLineLocation[] _tokenNewlines; - - private IEnumerable> Tokens { - get { - EnsureTokens(); - return _tokens; - } - } - - private SourceSpan GetTokenSpan(IndexSpan span) { - EnsureTokens(); - return new SourceSpan( - NewLineLocation.IndexToLocation(_tokenNewlines, span.Start), - NewLineLocation.IndexToLocation(_tokenNewlines, span.End) - ); - } - - private void EnsureTokens() { - if (_tokens != null) { - return; - } - - var reader = _openDocument?.Invoke(); - if (reader == null) { - _log.Log(TraceEventType.Verbose, "Cannot get completions at error node without sources"); - _tokens = Array.Empty>(); - _tokenNewlines = Array.Empty(); - return; - } - - var tokens = new List>(); - Tokenizer tokenizer; - using (reader) { - tokenizer = new Tokenizer(Tree.LanguageVersion, options: TokenizerOptions.GroupingRecovery); - tokenizer.Initialize(reader); - for (var t = tokenizer.GetNextToken(); - t.Kind != TokenKind.EndOfFile && tokenizer.TokenSpan.Start < Index; - t = tokenizer.GetNextToken()) { - tokens.Add(new KeyValuePair(tokenizer.TokenSpan, t)); - } - } - - _tokens = tokens; - _tokenNewlines = tokenizer.GetLineLocations(); - } - - - public IEnumerable GetCompletionsFromString(string expr) { - Check.ArgumentNotNullOrEmpty(nameof(expr), expr); - _log.Log(TraceEventType.Verbose, $"Completing expression '{expr}'"); - return Analysis.GetMembers(expr, Position, Options).Select(ToCompletionItem); - } - - public IEnumerable GetCompletions() { - switch (Node) { - case MemberExpression me when me.Target != null && me.DotIndex > me.StartIndex && Index > me.DotIndex: - return GetCompletionsFromMembers(me); - case ConstantExpression ce when ce.Value != null: - case null when IsInsideComment(): - return null; - } - - switch (Statement) { - case ImportStatement import when TryGetCompletionsInImport(import, out var result): - return result; - case FromImportStatement fromImport when TryGetCompletionsInFromImport(fromImport, out var result): - return result; - case FunctionDefinition fd when TryGetCompletionsForOverride(fd, out var result): - return result; - case FunctionDefinition fd when NoCompletionsInFunctionDefinition(fd): - return null; - case ClassDefinition cd: - if (NoCompletionsInClassDefinition(cd, out var addMetadataArg)) { - return null; - } - - return addMetadataArg - ? GetCompletionsFromTopLevel().Append(MetadataArgCompletion) - : GetCompletionsFromTopLevel(); - - case ForStatement forStatement when TryGetCompletionsInForStatement(forStatement, out var result): - return result; - case WithStatement withStatement when TryGetCompletionsInWithStatement(withStatement, out var result): - return result; - case RaiseStatement raiseStatement when TryGetCompletionsInRaiseStatement(raiseStatement, out var result): - return result; - case TryStatementHandler tryStatement when TryGetCompletionsInExceptStatement(tryStatement, out var result): - return result; - default: - return GetCompletionsFromError() ?? GetCompletionsFromTopLevel(); - } - } - - private static IEnumerable Once(CompletionItem item) { - yield return item; - } - - private static IEnumerable<(T1, T2)> ZipLongest(IEnumerable src1, IEnumerable src2) { - using (var e1 = src1?.GetEnumerator()) - using (var e2 = src2?.GetEnumerator()) { - bool b1 = e1?.MoveNext() ?? false, b2 = e2?.MoveNext() ?? false; - while (b1 && b2) { - yield return (e1.Current, e2.Current); - b1 = e1.MoveNext(); - b2 = e2.MoveNext(); - } - - while (b1) { - yield return (e1.Current, default(T2)); - b1 = e1.MoveNext(); - } - - while (b2) { - yield return (default(T1), e2.Current); - b2 = e2.MoveNext(); - } - } - } - - private IEnumerable GetCompletionsFromMembers(MemberExpression me) { - _log.Log(TraceEventType.Verbose, - $"Completing expression {me.Target.ToCodeString(Tree, CodeFormattingOptions.Traditional)}"); - ParentExpression = me.Target; - if (!string.IsNullOrEmpty(me.Name)) { - Node = new NameExpression(me.Name); - Node.SetLoc(me.NameHeader, me.NameHeader + me.Name.Length); - } else { - Node = null; - } - - ShouldCommitByDefault = true; - return Analysis.GetMembers(me.Target, Position, Options | GetMemberOptions.ForEval) - .Select(ToCompletionItem); - } - - private IEnumerable GetModules(string[] names, bool includeMembers) { - if (names.Any()) { - return Analysis.ProjectState - .GetModuleMembers(Analysis.InterpreterContext, names, includeMembers) - .Select(ToCompletionItem); - } - - return GetModules(); - } - - private IEnumerable GetModules() - => Analysis.ProjectState.GetModules().Select(ToCompletionItem); - - private IEnumerable GetModulesFromNode(DottedName name, bool includeMembers = false) => GetModules(GetNamesFromDottedName(name), includeMembers); - - private string[] GetNamesFromDottedName(DottedName name) => name.Names.TakeWhile(n => Index > n.EndIndex).Select(n => n.Name).ToArray(); - - private void SetApplicableSpanToLastToken(Node containingNode) { - if (containingNode != null && Index >= containingNode.EndIndex) { - var token = Tokens.LastOrDefault(); - if (token.Key.End >= Index) { - ApplicableSpan = GetTokenSpan(token.Key); - } - } - } - - private bool TryGetCompletionsInImport(ImportStatement import, out IEnumerable result) { - result = null; - - // No names, so if we're at the end return modules - if (import.Names.Count == 0 && Index > import.KeywordEndIndex) { - result = GetModules(); - return true; - } - - foreach (var (name, asName) in ZipLongest(import.Names, import.AsNames).Reverse()) { - if (asName != null && Index >= asName.StartIndex) { - return true; - } - - if (name != null && Index >= name.StartIndex) { - if (Index > name.EndIndex && name.EndIndex > name.StartIndex) { - SetApplicableSpanToLastToken(import); - result = Once(AsKeywordCompletion); - } else { - Node = name.Names.LastOrDefault(n => n.StartIndex <= Index && Index <= n.EndIndex); - result = GetModulesFromNode(name); - } - - return true; - } - } - - return false; - } - - private bool TryGetCompletionsInFromImport(FromImportStatement fromImport, out IEnumerable result) { - result = null; - - // No more completions after '*', ever! - if (fromImport.Names != null && fromImport.Names.Any(n => n?.Name == "*" && Index > n.EndIndex)) { - return true; - } - - foreach (var (name, asName) in ZipLongest(fromImport.Names, fromImport.AsNames).Reverse()) { - if (asName != null && Index >= asName.StartIndex) { - return true; - } - - if (name != null) { - if (Index > name.EndIndex && name.EndIndex > name.StartIndex) { - SetApplicableSpanToLastToken(fromImport); - result = Once(AsKeywordCompletion); - return true; - } - - if (Index >= name.StartIndex) { - ApplicableSpan = name.GetSpan(Tree); - var mods = GetModulesFromNode(fromImport.Root, true).ToArray(); - result = mods.Any() && fromImport.Names.Count == 1 ? Once(StarCompletion).Concat(mods) : mods; - return true; - } - } - } - - if (fromImport.ImportIndex > fromImport.StartIndex) { - if (Index > fromImport.ImportIndex + 6 && Analysis.Scope.AnalysisValue is ModuleInfo moduleInfo) { - var importSearchResult = _pathResolver.FindImports(moduleInfo.ProjectEntry.FilePath, fromImport); - switch (importSearchResult) { - case ModuleImport moduleImports when moduleInfo.TryGetModuleReference(moduleImports.FullName, out var moduleReference): - case PossibleModuleImport possibleModuleImport when moduleInfo.TryGetModuleReference(possibleModuleImport.PossibleModuleFullName, out moduleReference): - var module = moduleReference.Module; - if (module != null) { - result = module.GetAllMembers(Analysis.InterpreterContext) - .GroupBy(kvp => kvp.Key) - .Select(g => (IMemberResult)new MemberResult(g.Key, g.SelectMany(kvp => kvp.Value))) - .Select(ToCompletionItem) - .Prepend(StarCompletion); - } - - return true; - - case PackageImport packageImports: - var modules = Analysis.ProjectState.Modules; - result = packageImports.Modules - .Select(m => { - var hasReference = modules.TryGetImportedModule(m.FullName, out var mr); - return (hasReference: hasReference && mr?.Module != null, name: m.Name, module: mr?.AnalysisModule); - }) - .Where(t => t.hasReference) - .Select(t => ToCompletionItem(new MemberResult(t.name, new[] { t.module }))) - .Prepend(StarCompletion); - return true; - default: - return true; - } - } - - if (Index >= fromImport.ImportIndex) { - ApplicableSpan = new SourceSpan( - Tree.IndexToLocation(fromImport.ImportIndex), - Tree.IndexToLocation(Math.Min(fromImport.ImportIndex + 6, fromImport.EndIndex)) - ); - result = Once(ImportKeywordCompletion); - return true; - } - } - - if (Index > fromImport.Root.EndIndex && fromImport.Root.EndIndex > fromImport.Root.StartIndex) { - if (Index > fromImport.EndIndex) { - // Only end up here for "from ... imp", and "imp" is not counted - // as part of our span - var token = Tokens.LastOrDefault(); - if (token.Key.End >= Index) { - ApplicableSpan = GetTokenSpan(token.Key); - } - } - - result = Once(ImportKeywordCompletion); - return true; - } - - if (Index >= fromImport.Root.StartIndex) { - Node = fromImport.Root.Names.MaybeEnumerate() - .LastOrDefault(n => n.StartIndex <= Index && Index <= n.EndIndex); - result = GetModulesFromNode(fromImport.Root); - return true; - } - - if (Index > fromImport.KeywordEndIndex) { - result = GetModules(); - return true; - } - - return false; - } - - private bool TryGetCompletionsForOverride(FunctionDefinition function, out IEnumerable result) { - if (function.Parent is ClassDefinition cd && string.IsNullOrEmpty(function.Name) && function.NameExpression != null && Index > function.NameExpression.StartIndex) { - var loc = function.GetStart(Tree); - ShouldCommitByDefault = false; - result = Analysis.GetOverrideable(loc).Select(o => ToOverrideCompletionItem(o, cd, new string(' ', loc.Column - 1))); - return true; - } - - result = null; - return false; - } - - private CompletionItem ToOverrideCompletionItem(IOverloadResult o, ClassDefinition cd, string indent) { - return new CompletionItem { - label = o.Name, - insertText = MakeOverrideCompletionString(indent, o, cd.Name), - insertTextFormat = InsertTextFormat.PlainText, - kind = CompletionItemKind.Method - }; - } - - private bool NoCompletionsInFunctionDefinition(FunctionDefinition fd) { - // Here we work backwards through the various parts of the definitions. - // When we find that Index is within a part, we return either the available - // completions - if (fd.HeaderIndex > fd.StartIndex && Index > fd.HeaderIndex) { - return false; - } - - if (Index == fd.HeaderIndex) { - return true; - } - - foreach (var p in fd.Parameters.Reverse()) { - if (Index >= p.StartIndex) { - if (p.Annotation != null) { - return Index < p.Annotation.StartIndex; - } - - if (p.DefaultValue != null) { - return Index < p.DefaultValue.StartIndex; - } - } - } - - if (fd.NameExpression != null && fd.NameExpression.StartIndex > fd.KeywordEndIndex && Index >= fd.NameExpression.StartIndex) { - return true; - } - - return Index > fd.KeywordEndIndex; - } - - private bool NoCompletionsInClassDefinition(ClassDefinition cd, out bool addMetadataArg) { - addMetadataArg = false; - - if (cd.HeaderIndex > cd.StartIndex && Index > cd.HeaderIndex) { - return false; - } - - if (Index == cd.HeaderIndex) { - return true; - } - - if (cd.Bases.Length > 0 && Index >= cd.Bases[0].StartIndex) { - foreach (var p in cd.Bases.Reverse()) { - if (Index >= p.StartIndex) { - if (p.Name == null && Tree.LanguageVersion.Is3x() && cd.Bases.All(b => b.Name != "metaclass")) { - addMetadataArg = true; - } - - return false; - } - } - } - - if (cd.NameExpression != null && cd.NameExpression.StartIndex > cd.KeywordEndIndex && Index >= cd.NameExpression.StartIndex) { - return true; - } - - return Index > cd.KeywordEndIndex; - } - - private bool TryGetCompletionsInForStatement(ForStatement forStatement, out IEnumerable result) { - result = null; - - if (forStatement.Left == null) { - return false; - } - - if (forStatement.InIndex > forStatement.StartIndex) { - if (Index > forStatement.InIndex + 2) { - return false; - } - - if (Index >= forStatement.InIndex) { - ApplicableSpan = new SourceSpan(Tree.IndexToLocation(forStatement.InIndex), Tree.IndexToLocation(forStatement.InIndex + 2)); - result = Once(InKeywordCompletion); - return true; - } - } - - if (forStatement.Left.StartIndex > forStatement.StartIndex && forStatement.Left.EndIndex > forStatement.Left.StartIndex && Index > forStatement.Left.EndIndex) { - SetApplicableSpanToLastToken(forStatement); - result = Once(InKeywordCompletion); - return true; - } - - return forStatement.ForIndex >= forStatement.StartIndex && Index > forStatement.ForIndex + 3; - } - - private bool TryGetCompletionsInWithStatement(WithStatement withStatement, out IEnumerable result) { - result = null; - - if (Index > withStatement.HeaderIndex && withStatement.HeaderIndex > withStatement.StartIndex) { - return false; - } - - foreach (var item in withStatement.Items.Reverse().MaybeEnumerate()) { - if (item.AsIndex > item.StartIndex) { - if (Index > item.AsIndex + 2) { - return true; - } - - if (Index >= item.AsIndex) { - ApplicableSpan = new SourceSpan(Tree.IndexToLocation(item.AsIndex), Tree.IndexToLocation(item.AsIndex + 2)); - result = Once(AsKeywordCompletion); - return true; - } - } - - if (item.ContextManager != null && !(item.ContextManager is ErrorExpression)) { - if (Index > item.ContextManager.EndIndex && item.ContextManager.EndIndex > item.ContextManager.StartIndex) { - result = Once(AsKeywordCompletion); - return true; - } - - if (Index >= item.ContextManager.StartIndex) { - return false; - } - } - } - - return false; - } - - private bool TryGetCompletionsInRaiseStatement(RaiseStatement raiseStatement, out IEnumerable result) { - result = null; - - // raise Type, Value, Traceback with Cause - if (raiseStatement.Cause != null && Index >= raiseStatement.CauseFieldStartIndex) { - return false; - } - - if (raiseStatement.Traceback != null && Index >= raiseStatement.TracebackFieldStartIndex) { - return false; - } - - if (raiseStatement.Value != null && Index >= raiseStatement.ValueFieldStartIndex) { - return false; - } - - if (raiseStatement.ExceptType == null) { - return false; - } - - if (Index <= raiseStatement.ExceptType.EndIndex) { - return false; - } - - if (Tree.LanguageVersion.Is3x()) { - SetApplicableSpanToLastToken(raiseStatement); - result = Once(FromKeywordCompletion); - } - - return true; - } - - private bool TryGetCompletionsInExceptStatement(TryStatementHandler tryStatement, out IEnumerable result) { - result = null; - - // except Test as Target - if (tryStatement.Target != null && Index >= tryStatement.Target.StartIndex) { - return true; - } - - if (tryStatement.Test is TupleExpression || tryStatement.Test is null) { - return false; - } - - if (Index <= tryStatement.Test.EndIndex) { - return false; - } - - SetApplicableSpanToLastToken(tryStatement); - result = Once(AsKeywordCompletion); - return true; - } - - private bool IsInsideComment() { - var match = Array.BinarySearch(Tree.CommentLocations, Position); - // If our index = -1, it means we're before the first comment - if (match == -1) { - return false; - } - - if (match < 0) { - // If we couldn't find an exact match for this position, get the nearest - // matching comment before this point - match = ~match - 1; - } - - if (match >= Tree.CommentLocations.Length) { - Debug.Fail("Failed to find nearest preceding comment in AST"); - return false; - } - - if (Tree.CommentLocations[match].Line != Position.Line) { - return false; - } - - if (Tree.CommentLocations[match].Column >= Position.Column) { - return false; - } - - // We are inside a comment - return true; - } - - private IEnumerable GetCompletionsFromError() { - if (!(Node is ErrorExpression)) { - return null; - } - - if (Statement is AssignmentStatement assign && Node == assign.Right) { - return null; - } - - bool ScopeIsClassDefinition(out ClassDefinition classDefinition) { - classDefinition = Scope as ClassDefinition ?? (Scope as FunctionDefinition)?.Parent as ClassDefinition; - return classDefinition != null; - } - - var tokens = Tokens.Reverse().ToArray(); - - string exprString; - SourceLocation loc; - var lastToken = tokens.FirstOrDefault(); - var nextLast = tokens.ElementAtOrDefault(1).Value?.Kind ?? TokenKind.EndOfFile; - switch (lastToken.Value.Kind) { - case TokenKind.Dot: - exprString = ReadExpression(tokens.Skip(1)); - ApplicableSpan = new SourceSpan(Position, Position); - return Analysis.GetMembers(exprString, Position, Options).Select(ToCompletionItem); - - case TokenKind.KeywordDef when lastToken.Key.End < Index && ScopeIsClassDefinition(out var cd): - ApplicableSpan = new SourceSpan(Position, Position); - loc = GetTokenSpan(lastToken.Key).Start; - ShouldCommitByDefault = false; - return Analysis.GetOverrideable(loc).Select(o => ToOverrideCompletionItem(o, cd, new string(' ', loc.Column - 1))); - - case TokenKind.Name when nextLast == TokenKind.Dot: - exprString = ReadExpression(tokens.Skip(2)); - ApplicableSpan = new SourceSpan(GetTokenSpan(lastToken.Key).Start, Position); - return Analysis.GetMembers(exprString, Position, Options).Select(ToCompletionItem); - - case TokenKind.Name when nextLast == TokenKind.KeywordDef && ScopeIsClassDefinition(out var cd): - ApplicableSpan = new SourceSpan(GetTokenSpan(lastToken.Key).Start, Position); - loc = GetTokenSpan(tokens.ElementAt(1).Key).Start; - ShouldCommitByDefault = false; - return Analysis.GetOverrideable(loc).Select(o => ToOverrideCompletionItem(o, cd, new string(' ', loc.Column - 1))); - - case TokenKind.KeywordFor: - case TokenKind.KeywordAs: - return lastToken.Key.Start <= Index && Index <= lastToken.Key.End ? null : Empty; - - default: - Debug.WriteLine($"Unhandled completions from error.\nTokens were: ({lastToken.Value.Image}:{lastToken.Value.Kind}), {string.Join(", ", tokens.AsEnumerable().Take(10).Select(t => $"({t.Value.Image}:{t.Value.Kind})"))}"); - return null; - } - } - - private IEnumerable GetCompletionsFromTopLevel() { - if (Node != null && Node.EndIndex < Index) { - return Empty; - } - - var options = Options | GetMemberOptions.ForEval | GetMemberOptionsForTopLevelCompletions(Statement, Index, out var span); - if (span.HasValue) { - ApplicableSpan = new SourceSpan(Tree.IndexToLocation(span.Value.Start), Tree.IndexToLocation(span.Value.End)); - } - - ShouldAllowSnippets = options.HasFlag(GetMemberOptions.IncludeExpressionKeywords); - - _log.Log(TraceEventType.Verbose, "Completing all names"); - var members = Analysis.GetAllMembers(Position, options); - - var finder = new ExpressionFinder(Tree, new GetExpressionOptions { Calls = true }); - if (finder.GetExpression(Index) is CallExpression callExpr && callExpr.GetArgumentAtIndex(Tree, Index, out _)) { - var argNames = Analysis.GetSignatures(callExpr.Target, Position) - .SelectMany(o => o.Parameters).Select(p => p?.Name) - .Where(n => !string.IsNullOrEmpty(n)) - .Distinct() - .Except(callExpr.Args.MaybeEnumerate().Select(a => a.Name).Where(n => !string.IsNullOrEmpty(n))) - .Select(n => new MemberResult($"{n}=", PythonMemberType.NamedArgument) as IMemberResult) - .ToArray(); - - _log.Log(TraceEventType.Verbose, $"Including {argNames.Length} named arguments"); - members = members.Concat(argNames); - } - - return members - .Where(m => !string.IsNullOrEmpty(m.Completion) || !string.IsNullOrEmpty(m.Name)) - .Select(ToCompletionItem); - } - - private static GetMemberOptions GetMemberOptionsForTopLevelCompletions(Node statement, int index, out IndexSpan? span) { - span = null; - - const GetMemberOptions noKeywords = GetMemberOptions.None; - const GetMemberOptions exceptionsOnly = GetMemberOptions.ExceptionsOnly; - const GetMemberOptions includeExpressionKeywords = GetMemberOptions.IncludeExpressionKeywords; - const GetMemberOptions includeStatementKeywords = GetMemberOptions.IncludeStatementKeywords; - const GetMemberOptions includeAllKeywords = includeExpressionKeywords | includeStatementKeywords; - - switch (statement) { - // Disallow keywords, unless we're between the end of decorators and the - // end of the "[async] def" keyword. - case FunctionDefinition fd when index > fd.KeywordEndIndex || fd.Decorators != null && index < fd.Decorators.EndIndex: - case ClassDefinition cd when index > cd.KeywordEndIndex || cd.Decorators != null && index < cd.Decorators.EndIndex: - return noKeywords; - - case TryStatementHandler tryStatement when tryStatement.Test is TupleExpression || index >= tryStatement.Test.StartIndex: - return exceptionsOnly; - - case null: - return includeAllKeywords; - - // Always allow keywords in non-keyword statements - case ExpressionStatement _: - return includeAllKeywords; - - case ImportStatement _: - case FromImportStatement _: - return includeAllKeywords; - - // Allow keywords at start of assignment, but not in subsequent names - case AssignmentStatement ss: - var firstAssign = ss.Left?.FirstOrDefault(); - return firstAssign == null || index <= firstAssign.EndIndex ? includeAllKeywords : includeExpressionKeywords; - - // Allow keywords when we are in another keyword - case Statement s when index <= s.KeywordEndIndex: - var keywordStart = s.KeywordEndIndex - s.KeywordLength; - if (index >= keywordStart) { - span = new IndexSpan(keywordStart, s.KeywordLength); - } else if ((s as IMaybeAsyncStatement)?.IsAsync == true) { - // Must be in the "async" at the start of the keyword - span = new IndexSpan(s.StartIndex, "async".Length); - } - return includeAllKeywords; - - case RaiseStatement raise when raise.ExceptType != null && index >= raise.ExceptType.StartIndex || index > raise.KeywordEndIndex: - return includeExpressionKeywords | exceptionsOnly; - - // TryStatementHandler is 'except', but not a Statement subclass - case TryStatementHandler except when index <= except.KeywordEndIndex: - var exceptKeywordStart = except.KeywordEndIndex - except.KeywordLength; - if (index >= exceptKeywordStart) { - span = new IndexSpan(exceptKeywordStart, except.KeywordLength); - } - - return includeAllKeywords; - - // Allow keywords in function body (we'd have a different statement if we were deeper) - case FunctionDefinition fd when index >= fd.HeaderIndex: - return includeAllKeywords; - - // Allow keywords within with blocks, but not in their definition - case WithStatement ws: - return index >= ws.HeaderIndex || index <= ws.KeywordEndIndex ? includeAllKeywords : includeExpressionKeywords; - - default: - return includeExpressionKeywords; - } - } - - private static readonly CompletionItem MetadataArgCompletion = ToCompletionItem("metaclass=", PythonMemberType.NamedArgument); - private static readonly CompletionItem AsKeywordCompletion = ToCompletionItem("as", PythonMemberType.Keyword); - private static readonly CompletionItem FromKeywordCompletion = ToCompletionItem("from", PythonMemberType.Keyword); - private static readonly CompletionItem InKeywordCompletion = ToCompletionItem("in", PythonMemberType.Keyword); - private static readonly CompletionItem ImportKeywordCompletion = ToCompletionItem("import", PythonMemberType.Keyword); - private static readonly CompletionItem WithKeywordCompletion = ToCompletionItem("with", PythonMemberType.Keyword); - private static readonly CompletionItem StarCompletion = ToCompletionItem("*", PythonMemberType.Keyword); - - private CompletionItem ToCompletionItem(IMemberResult m) { - var completion = m.Completion; - if (string.IsNullOrEmpty(completion)) { - completion = m.Name; - } - - if (string.IsNullOrEmpty(completion)) { - return default(CompletionItem); - } - - var doc = _textBuilder.GetDocumentation(m.Values, string.Empty); - var kind = ToCompletionItemKind(m.MemberType); - - var res = new CompletionItem { - label = m.Name, - insertText = completion, - insertTextFormat = InsertTextFormat.PlainText, - documentation = string.IsNullOrWhiteSpace(doc) ? null : new MarkupContent { - kind = _textBuilder.DisplayOptions.preferredFormat, - value = doc - }, - // Place regular items first, advanced entries last - sortText = char.IsLetter(completion, 0) ? "1" : "2", - kind = ToCompletionItemKind(m.MemberType), - _kind = m.MemberType.ToString().ToLowerInvariant() - }; - - if (_addBrackets && (kind == CompletionItemKind.Constructor || kind == CompletionItemKind.Function || kind == CompletionItemKind.Method)) { - res.insertText += "($0)"; - res.insertTextFormat = InsertTextFormat.Snippet; - } - - return res; - } - - private static CompletionItem ToCompletionItem(string text, PythonMemberType type, string label = null) { - return new CompletionItem { - label = label ?? text, - insertText = text, - insertTextFormat = InsertTextFormat.PlainText, - // Place regular items first, advanced entries last - sortText = char.IsLetter(text, 0) ? "1" : "2", - kind = ToCompletionItemKind(type), - _kind = type.ToString().ToLowerInvariant() - }; - } - - private static CompletionItemKind ToCompletionItemKind(PythonMemberType memberType) { - switch (memberType) { - case PythonMemberType.Unknown: return CompletionItemKind.None; - case PythonMemberType.Class: return CompletionItemKind.Class; - case PythonMemberType.Instance: return CompletionItemKind.Value; - case PythonMemberType.Enum: return CompletionItemKind.Enum; - case PythonMemberType.EnumInstance: return CompletionItemKind.EnumMember; - case PythonMemberType.Function: return CompletionItemKind.Function; - case PythonMemberType.Method: return CompletionItemKind.Method; - case PythonMemberType.Module: return CompletionItemKind.Module; - case PythonMemberType.Constant: return CompletionItemKind.Constant; - case PythonMemberType.Event: return CompletionItemKind.Event; - case PythonMemberType.Field: return CompletionItemKind.Field; - case PythonMemberType.Property: return CompletionItemKind.Property; - case PythonMemberType.Multiple: return CompletionItemKind.Value; - case PythonMemberType.Keyword: return CompletionItemKind.Keyword; - case PythonMemberType.CodeSnippet: return CompletionItemKind.Snippet; - case PythonMemberType.NamedArgument: return CompletionItemKind.Variable; - default: - return CompletionItemKind.None; - } - } - - private static string MakeOverrideDefParameter(ParameterResult result) { - if (!string.IsNullOrEmpty(result.DefaultValue)) { - return result.Name + "=" + result.DefaultValue; - } - - return result.Name; - } - - private static string MakeOverrideCallParameter(ParameterResult result) { - if (result.Name.StartsWithOrdinal("*")) { - return result.Name; - } - - if (!string.IsNullOrEmpty(result.DefaultValue)) { - return result.Name + "=" + result.Name; - } - - return result.Name; - } - - private string MakeOverrideCompletionString(string indentation, IOverloadResult result, string className) { - var sb = new StringBuilder(); - - ParameterResult first; - ParameterResult[] skipFirstParameters; - ParameterResult[] allParameters; - if (result.FirstParameter != null) { - first = result.FirstParameter; - skipFirstParameters = result.Parameters; - allParameters = new[] {first}.Concat(skipFirstParameters).ToArray(); - } else { - first = result.Parameters.FirstOrDefault(); - skipFirstParameters = result.Parameters.Skip(1).ToArray(); - allParameters = result.Parameters; - } - - sb.AppendLine(result.Name + "(" + string.Join(", ", allParameters.Select(MakeOverrideDefParameter)) + "):"); - sb.Append(indentation); - - if (allParameters.Length > 0) { - var parameterString = string.Join(", ", skipFirstParameters.Select(MakeOverrideCallParameter)); - - if (Tree.LanguageVersion.Is3x()) { - sb.AppendFormat("return super().{0}({1})", - result.Name, - parameterString); - } else if (!string.IsNullOrEmpty(className)) { - sb.AppendFormat("return super({0}, {1}).{2}({3})", - className, - first?.Name ?? string.Empty, - result.Name, - parameterString); - } else { - sb.Append("pass"); - } - } else { - sb.Append("pass"); - } - - return sb.ToString(); - } - - private string ReadExpression(IEnumerable> tokens) { - var expr = ReadExpressionTokens(tokens); - return string.Join("", expr.Select(e => e.VerbatimImage ?? e.Image)); - } - - private IEnumerable ReadExpressionTokens(IEnumerable> tokens) { - int nesting = 0; - var exprTokens = new Stack(); - int currentLine = -1; - - foreach (var t in tokens) { - var p = GetTokenSpan(t.Key).Start; - if (p.Line > currentLine) { - currentLine = p.Line; - } else if (p.Line < currentLine && nesting == 0) { - break; - } - - exprTokens.Push(t.Value); - - switch (t.Value.Kind) { - case TokenKind.RightParenthesis: - case TokenKind.RightBracket: - case TokenKind.RightBrace: - nesting += 1; - break; - case TokenKind.LeftParenthesis: - case TokenKind.LeftBracket: - case TokenKind.LeftBrace: - if (--nesting < 0) { - exprTokens.Pop(); - return exprTokens; - } - - break; - - case TokenKind.Comment: - exprTokens.Pop(); - break; - - case TokenKind.Name: - case TokenKind.Constant: - case TokenKind.Dot: - case TokenKind.Ellipsis: - case TokenKind.MatMultiply: - case TokenKind.KeywordAwait: - break; - - case TokenKind.Assign: - case TokenKind.LeftShiftEqual: - case TokenKind.RightShiftEqual: - case TokenKind.BitwiseAndEqual: - case TokenKind.BitwiseOrEqual: - case TokenKind.ExclusiveOrEqual: - exprTokens.Pop(); - return exprTokens; - - default: - if (t.Value.Kind >= TokenKind.FirstKeyword || nesting == 0) { - exprTokens.Pop(); - return exprTokens; - } - break; - } - } - - return exprTokens; - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/DiagnosticsErrorSink.cs b/src/LanguageServer/Impl/Implementation/DiagnosticsErrorSink.cs deleted file mode 100644 index 2bbad8d11..000000000 --- a/src/LanguageServer/Impl/Implementation/DiagnosticsErrorSink.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.Python.Core; -using Microsoft.Python.Core.Text; -using Microsoft.Python.Parsing; -using Microsoft.PythonTools.Analysis; - -namespace Microsoft.Python.LanguageServer.Implementation { - class DiagnosticsErrorSink : ErrorSink { - private readonly string _source; - private readonly Action _onDiagnostic; - private readonly IReadOnlyList> _taskCommentMap; - - public DiagnosticsErrorSink(string source, Action onDiagnostic, IReadOnlyDictionary taskCommentMap = null) { - _source = source; - _onDiagnostic = onDiagnostic; - _taskCommentMap = taskCommentMap?.ToArray(); - } - - public override void Add(string message, SourceSpan span, int errorCode, Severity severity) { - var d = new Diagnostic { - code = "E{0}".FormatInvariant(errorCode), - message = message, - source = _source, - severity = GetSeverity(severity), - range = span - }; - - _onDiagnostic(d); - } - - public void ProcessTaskComment(object sender, CommentEventArgs e) { - var text = e.Text.TrimStart('#').Trim(); - - var d = new Diagnostic { - message = text, - range = e.Span, - source = _source - }; - - bool found = false; - foreach (var kv in _taskCommentMap.MaybeEnumerate().OrderByDescending(kv => kv.Key.Length)) { - if (text.IndexOfOrdinal(kv.Key, ignoreCase: true) >= 0) { - d.code = kv.Key; - d.severity = kv.Value; - found = true; - break; - } - } - - if (found) { - _onDiagnostic(d); - } - } - - internal static DiagnosticSeverity GetSeverity(Severity severity) { - switch (severity) { - case Severity.Ignore: return DiagnosticSeverity.Unspecified; - case Severity.Information: return DiagnosticSeverity.Information; - case Severity.Warning: return DiagnosticSeverity.Warning; - case Severity.Error: return DiagnosticSeverity.Error; - case Severity.FatalError: return DiagnosticSeverity.Error; - default: return DiagnosticSeverity.Unspecified; - } - } - - internal static Severity GetSeverity(DiagnosticSeverity severity) { - switch (severity) { - case DiagnosticSeverity.Unspecified: return Severity.Ignore; - case DiagnosticSeverity.Information: return Severity.Information; - case DiagnosticSeverity.Warning: return Severity.Warning; - case DiagnosticSeverity.Error: return Severity.Error; - default: return Severity.Ignore; - } - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/DocumentReader.cs b/src/LanguageServer/Impl/Implementation/DocumentReader.cs deleted file mode 100644 index 3027d9e12..000000000 --- a/src/LanguageServer/Impl/Implementation/DocumentReader.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using Microsoft.PythonTools.Analysis; - -namespace Microsoft.Python.LanguageServer.Implementation { - public sealed class DocumentReader : IDocumentReader { - private readonly IDocument _doc; - private readonly int _part; - private string _content; - private bool _read; - - public DocumentReader(IDocument doc, int part) { - _doc = doc; - _part = part; - } - - public string ReadToEnd() => Content; - - public string Read(int start, int count) => Content?.Substring(start, count); - - private string Content { - get { - if (_content == null && !_read) { - var reader = _doc.ReadDocument(_part, out _); - _read = true; - if (reader == null) { - return null; - } - using (reader) { - _content = reader.ReadToEnd(); - } - } - return _content; - } - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/EditorFiles.cs b/src/LanguageServer/Impl/Implementation/EditorFiles.cs deleted file mode 100644 index 89c397477..000000000 --- a/src/LanguageServer/Impl/Implementation/EditorFiles.cs +++ /dev/null @@ -1,227 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Core.Threading; -using Microsoft.PythonTools.Analysis; -using Microsoft.PythonTools.Intellisense; - -namespace Microsoft.Python.LanguageServer.Implementation { - internal sealed class EditorFiles : IDisposable { - private readonly ConcurrentDictionary _files = new ConcurrentDictionary(); - private readonly Server _server; - private readonly SingleThreadSynchronizationContext _syncContext; - - public EditorFiles(Server server) { - _server = server; - _syncContext = new SingleThreadSynchronizationContext(); - } - - public EditorFile GetDocument(Uri uri) => _files.GetOrAdd(uri, _ => new EditorFile(_server, _syncContext)); - public void Remove(Uri uri) => _files.TryRemove(uri, out _); - public void Open(Uri uri) => GetDocument(uri).Open(uri); - public void Close(Uri uri) => GetDocument(uri).Close(uri); - - public void UpdateDiagnostics() { - foreach (var entry in _server.ProjectFiles.OfType()) { - GetDocument(entry.DocumentUri).UpdateAnalysisDiagnostics(entry, -1); - } - } - - public void Dispose() => _syncContext.Dispose(); - } - - internal sealed class EditorFile { - private readonly Server _server; - private readonly SynchronizationContext _syncContext; - private readonly List _pendingChanges = new List(); - private readonly object _lock = new object(); - - private readonly IDictionary _parseBufferDiagnostics = new Dictionary(); - private IEnumerable _lastReportedParseDiagnostics; - private IEnumerable _lastReportedAnalysisDiagnostics; - private bool _ignoreDiagnosticsVersion; - - public EditorFile(Server server, SynchronizationContext syncContext) { - _server = server; - _syncContext = syncContext; - } - - public bool IsOpen { get; private set; } - - public void Open(Uri documentUri) { - IsOpen = true; - // Force update of the diagnostics if reporting of issues in closed files was turned off. - _ignoreDiagnosticsVersion = true; - - if (_lastReportedAnalysisDiagnostics != null) { - PublishDiagnostics(_lastReportedAnalysisDiagnostics ?? _lastReportedParseDiagnostics ?? Array.Empty()); - } - } - - public void Close(Uri documentUri) { - IsOpen = false; - _syncContext.Post(_ => HideDiagnostics(documentUri), null); - } - - public async Task DidChangeTextDocument(DidChangeTextDocumentParams @params, bool enqueueForParsing, CancellationToken cancellationToken) { - var changes = @params.contentChanges; - if (changes == null || changes.Length == 0) { - return; - } - - var uri = @params.textDocument.uri; - var doc = _server.ProjectFiles.GetEntry(uri) as IDocument; - if (doc == null) { - return; - } - - try { - var part = _server.ProjectFiles.GetPart(uri); - _server.TraceMessage($"Received changes for {uri}"); - - var docVersion = Math.Max(doc.GetDocumentVersion(part), 0); - var fromVersion = Math.Max(@params.textDocument._fromVersion ?? docVersion, 0); - - if (fromVersion > docVersion && @params.contentChanges?.Any(c => c.range == null) != true) { - // Expected from version hasn't been seen yet, and there are no resets in this - // change, so enqueue it for later. - _server.TraceMessage($"Deferring changes for {uri} until version {fromVersion} is seen"); - lock (_pendingChanges) { - _pendingChanges.Add(@params); - } - return; - } - - var toVersion = @params.textDocument.version ?? (fromVersion + changes.Length); - - cancellationToken.ThrowIfCancellationRequested(); - doc.UpdateDocument(part, new DocumentChangeSet( - fromVersion, - toVersion, - changes.Select(c => new DocumentChange { - ReplacedSpan = c.range.GetValueOrDefault(), - WholeBuffer = !c.range.HasValue, - InsertedText = c.text - }) - )); - - DidChangeTextDocumentParams? next = null; - lock (_pendingChanges) { - var notExpired = _pendingChanges - .Where(p => p.textDocument.version.GetValueOrDefault() >= toVersion) - .OrderBy(p => p.textDocument.version.GetValueOrDefault()) - .ToArray(); - - _pendingChanges.Clear(); - if (notExpired.Any()) { - next = notExpired.First(); - _pendingChanges.AddRange(notExpired.Skip(1)); - } - } - if (next.HasValue) { - await DidChangeTextDocument(next.Value, false, cancellationToken); - } - } finally { - if (enqueueForParsing) { - _server.TraceMessage($"Applied changes to {uri}"); - await _server.EnqueueItemAsync(doc, enqueueForAnalysis: @params._enqueueForAnalysis ?? true); - } - } - } - - public void UpdateParseDiagnostics(VersionCookie vc, Uri documentUri) { - List diags = null; - - lock (_lock) { - var last = _parseBufferDiagnostics; - - foreach (var kv in vc.GetAllParts(documentUri)) { - var part = _server.ProjectFiles.GetPart(kv.Key); - if (!last.TryGetValue(part, out var lastVersion) || lastVersion.Version < kv.Value.Version || _ignoreDiagnosticsVersion) { - last[part] = kv.Value; - diags = diags ?? new List(); - diags.Add(new PublishDiagnosticsEventArgs { - uri = kv.Key, - diagnostics = kv.Value.Diagnostics, - _version = kv.Value.Version - }); - } - } - _ignoreDiagnosticsVersion = false; - _lastReportedParseDiagnostics = diags ?? _lastReportedParseDiagnostics; - - if (diags != null) { - PublishDiagnostics(diags); - } - } - } - - public void UpdateAnalysisDiagnostics(IPythonProjectEntry projectEntry, int version) { - lock (_lock) { - var diags = _server.Analyzer.GetDiagnostics(projectEntry); - var settings = _server.Settings; - - for (var i = 0; i < diags.Count; i++) { - diags[i].severity = settings.analysis.GetEffectiveSeverity(diags[i].code, diags[i].severity); - } - - if (projectEntry is IDocument) { - if (_lastReportedParseDiagnostics != null) { - diags = diags.Concat(_lastReportedParseDiagnostics.SelectMany(d => d.diagnostics)).ToArray(); - } - } - - var lastPublishedVersion = _lastReportedAnalysisDiagnostics?.FirstOrDefault()?._version; - version = version >= 0 ? version : (lastPublishedVersion.HasValue ? lastPublishedVersion.Value : 0); - - _lastReportedAnalysisDiagnostics = new[] { new PublishDiagnosticsEventArgs { - uri = projectEntry.DocumentUri, - diagnostics = diags, - _version = version - }}; - - PublishDiagnostics(_lastReportedAnalysisDiagnostics); - } - } - - private void PublishDiagnostics(IEnumerable args) { - _syncContext.Post(_ => { - foreach (var a in args) { - if (!ShouldHideDiagnostics) { - _server.PublishDiagnostics(a); - } else { - HideDiagnostics(a.uri); - } - } - }, null); - } - - private void HideDiagnostics(Uri documentUri) { - _server.PublishDiagnostics(new PublishDiagnosticsEventArgs { - uri = documentUri, - diagnostics = Array.Empty() - }); - } - - private bool ShouldHideDiagnostics => !IsOpen && _server.Settings.analysis.openFilesOnly; - } -} diff --git a/src/LanguageServer/Impl/Implementation/ParseQueue.cs b/src/LanguageServer/Impl/Implementation/ParseQueue.cs deleted file mode 100644 index d177c7dea..000000000 --- a/src/LanguageServer/Impl/Implementation/ParseQueue.cs +++ /dev/null @@ -1,262 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Core; -using Microsoft.PythonTools.Analysis; -using Microsoft.PythonTools.Intellisense; -using Microsoft.Python.Parsing; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.LanguageServer.Implementation { - class ParseQueue: IDisposable { - public const string PythonParserSource = "Python"; - private const string TaskCommentSource = "Task comment"; - - private readonly ConcurrentDictionary _parsing; - private readonly VolatileCounter _parsingInProgress; - - public ParseQueue() { - _parsingInProgress = new VolatileCounter(); - _parsing = new ConcurrentDictionary(); - } - - public void Dispose() => _parsing.Clear(); - - public int Count => _parsingInProgress.Count; - - public Task WaitForAllAsync() => _parsingInProgress.WaitForZeroAsync(); - - public Task EnqueueAsync(IDocument doc, PythonLanguageVersion languageVersion) { - if (doc == null) { - throw new ArgumentNullException(nameof(doc)); - } - - var task = new ParseTask(this, doc, languageVersion); - try { - return _parsing.AddOrUpdate(doc.DocumentUri, task, (d, prev) => task.ContinueAfter(prev)).Start(); - } finally { - task.DisposeIfNotStarted(); - } - } - - private IPythonParse ParseWorker(IDocument doc, PythonLanguageVersion languageVersion) { - IPythonParse result = null; - - if (doc is IExternalProjectEntry externalEntry) { - using (var r = doc.ReadDocument(0, out var version)) { - if (r == null) { - throw new FileNotFoundException("failed to parse file", externalEntry.FilePath); - } - result = new StaticPythonParse(null, new VersionCookie(version)); - externalEntry.ParseContent(r, result.Cookie); - } - } else if (doc is IPythonProjectEntry pyEntry) { - var lastParse = pyEntry.GetCurrentParse(); - result = ParsePythonEntry(pyEntry, languageVersion, lastParse?.Cookie as VersionCookie); - } else { - Debug.Fail($"Don't know how to parse {doc.GetType().FullName}"); - } - return result; - } - - private IPythonParse ParsePythonEntry(IPythonProjectEntry entry, PythonLanguageVersion languageVersion, VersionCookie lastParseCookie) { - PythonAst tree; - var doc = (IDocument)entry; - var buffers = new SortedDictionary(); - foreach (var part in doc.DocumentParts.Reverse()) { - using (var r = doc.ReadDocumentBytes(part, out int version)) { - if (r == null) { - continue; - } - - if (version >= 0 && lastParseCookie != null && lastParseCookie.Versions.TryGetValue(part, out var lastParse) && lastParse.Version >= version) { - buffers[part] = lastParse; - continue; - } - - buffers[part] = ParsePython(r, entry, languageVersion, version); - } - } - - if (!buffers.Any()) { - // If the document is a real file, we should have been able to parse. - if (entry.DocumentUri.IsFile) { - throw new FileNotFoundException("failed to parse file {0}".FormatInvariant(entry.DocumentUri.AbsoluteUri), entry.FilePath); - } - // Otherwise, it is likely just empty for now, so no need to cause a fuss - return null; - } - - var cookie = new VersionCookie(buffers); - - if (buffers.Count == 1) { - tree = buffers.First().Value.Ast; - } else { - tree = new PythonAst(buffers.Values.Select(v => v.Ast)); - } - - return new StaticPythonParse(tree, cookie); - } - - public Dictionary TaskCommentMap { get; set; } - - public DiagnosticSeverity InconsistentIndentation { get; set; } - - sealed class ParseTask { - private readonly ParseQueue _queue; - private readonly IDocument _document; - private readonly PythonLanguageVersion _languageVersion; - - private readonly IPythonParse _parse; - - private readonly TaskCompletionSource _tcs; - private Task _previous; - - // State transitions: - // UNSTARTED -> QUEUED when Start() called - // QUEUED -> STARTED when worker starts running on worker thread - // STARTED -> DISPOSED when task completes - // UNSTARTED -> DISPOSED when DisposeIfNotStarted() is called - private const int UNSTARTED = 0; - private const int QUEUED = 1; - private const int STARTED = 2; - private const int DISPOSED = 3; - private int _state = UNSTARTED; - - public ParseTask(ParseQueue queue, IDocument document, PythonLanguageVersion languageVersion) { - _queue = queue; - _document = document; - _languageVersion = languageVersion; - - _queue._parsingInProgress.Increment(); - _parse = (_document as IPythonProjectEntry)?.BeginParse(); - - _tcs = new TaskCompletionSource(); - } - - public Task Task => _tcs.Task; - - public void DisposeIfNotStarted() { - if (Interlocked.CompareExchange(ref _state, DISPOSED, UNSTARTED) == UNSTARTED) { - DisposeWorker(); - } - } - - private void DisposeWorker() { - Debug.Assert(Volatile.Read(ref _state) == DISPOSED); - - _parse?.Dispose(); - _queue._parsingInProgress.Decrement(); - } - - public ParseTask ContinueAfter(ParseTask currentTask) { - // Set our previous task - Volatile.Write(ref _previous, currentTask?.Task); - - return this; - } - - public Task Start() { - int actualState = Interlocked.CompareExchange(ref _state, QUEUED, UNSTARTED); - if (actualState != UNSTARTED) { - if (actualState == DISPOSED) { - throw new ObjectDisposedException(GetType().FullName); - } - throw new InvalidOperationException("cannot start parsing"); - } - - // If we were pending, calling start ensures we are queued - var previous = Interlocked.Exchange(ref _previous, null); - if (previous != null) { - previous.ContinueWith(StartAfterTask); - return Task; - } - - ThreadPool.QueueUserWorkItem(StartWork); - return Task; - } - - private void StartAfterTask(Task previous) { - ThreadPool.QueueUserWorkItem(StartWork); - } - - private void StartWork(object state) { - int actualState = Interlocked.CompareExchange(ref _state, STARTED, QUEUED); - if (actualState != QUEUED) { - // Silently complete if we are not queued. - if (Interlocked.Exchange(ref _state, DISPOSED) != DISPOSED) { - DisposeWorker(); - } - return; - } - - try { - var r = _queue.ParseWorker(_document, _languageVersion); - if (r != null && _parse != null) { - _parse.Tree = r.Tree; - _parse.Cookie = r.Cookie; - _parse.Complete(); - } - _tcs.SetResult(r?.Cookie); - } catch (Exception ex) { - _tcs.SetException(ex); - } finally { - if (Interlocked.CompareExchange(ref _state, DISPOSED, STARTED) == STARTED) { - DisposeWorker(); - } - } - } - } - - - private BufferVersion ParsePython( - Stream stream, - IPythonProjectEntry entry, - PythonLanguageVersion languageVersion, - int version - ) { - var opts = new ParserOptions { - BindReferences = true, - IndentationInconsistencySeverity = DiagnosticsErrorSink.GetSeverity(InconsistentIndentation), - StubFile = entry.DocumentUri.AbsolutePath.EndsWithOrdinal(".pyi", ignoreCase: true) - }; - - List diags = null; - - if (entry.DocumentUri != null) { - diags = new List(); - opts.ErrorSink = new DiagnosticsErrorSink(PythonParserSource, d => { lock (diags) diags.Add(d); }); - var tcm = TaskCommentMap; - if (tcm != null && tcm.Any()) { - opts.ProcessComment += new DiagnosticsErrorSink(TaskCommentSource, d => { lock (diags) diags.Add(d); }, tcm).ProcessTaskComment; - } - } - - var parser = Parser.CreateParser(stream, languageVersion, opts); - var tree = parser.ParseFile(); - - return new BufferVersion(version, tree, diags.MaybeEnumerate()); - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/ProjectFiles.cs b/src/LanguageServer/Impl/Implementation/ProjectFiles.cs deleted file mode 100644 index efc9de332..000000000 --- a/src/LanguageServer/Impl/Implementation/ProjectFiles.cs +++ /dev/null @@ -1,108 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using Microsoft.Python.Core; -using Microsoft.PythonTools.Analysis; -using Microsoft.PythonTools.Intellisense; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.LanguageServer.Implementation { - internal sealed class ProjectFiles : IDisposable, IEnumerable { - private readonly ConcurrentDictionary _projectFiles = new ConcurrentDictionary(); - private bool _disposed; - - public IProjectEntry GetOrAddEntry(Uri documentUri, IProjectEntry entry) { - ThrowIfDisposed(); - return _projectFiles.GetOrAdd(documentUri, entry); - } - - public IProjectEntry RemoveEntry(Uri documentUri) { - ThrowIfDisposed(); - return _projectFiles.TryRemove(documentUri, out var entry) ? entry : null; - } - - public IEnumerable GetLoadedFiles() { - ThrowIfDisposed(); - return _projectFiles.Keys.Select(k => k.AbsoluteUri); - } - - public IProjectEntry GetEntry(Uri documentUri, bool throwIfMissing = true) { - ThrowIfDisposed(); - - IProjectEntry entry = null; - if ((documentUri == null || !_projectFiles.TryGetValue(documentUri, out entry)) && throwIfMissing) { - throw new LanguageServerException(LanguageServerException.UnknownDocument, "unknown document"); - } - return entry; - } - - public void GetEntry(TextDocumentIdentifier document, int? expectedVersion, out ProjectEntry entry, out PythonAst tree) { - ThrowIfDisposed(); - - entry = GetEntry(document.uri) as ProjectEntry; - if (entry == null) { - throw new LanguageServerException(LanguageServerException.UnsupportedDocumentType, "unsupported document"); - } - var parse = entry.GetCurrentParse(); - tree = parse?.Tree; - if (expectedVersion.HasValue && parse?.Cookie is VersionCookie vc) { - if (vc.Versions.TryGetValue(GetPart(document.uri), out var bv)) { - if (bv.Version == expectedVersion.Value) { - tree = bv.Ast; - } - } - } - } - - public void Dispose() { - _disposed = true; - } - - internal int GetPart(Uri documentUri) { - ThrowIfDisposed(); - - var f = documentUri.Fragment; - int i; - if (string.IsNullOrEmpty(f) || - !f.StartsWithOrdinal("#") || - !int.TryParse(f.Substring(1), NumberStyles.Integer, CultureInfo.InvariantCulture, out i)) { - i = 0; - } - return i; - } - private void ThrowIfDisposed() { - if (_disposed) { - throw new ObjectDisposedException(nameof(ProjectFiles)); - } - } - - #region IEnumerable - public IEnumerator GetEnumerator() => GetAll().GetEnumerator(); - IEnumerator IEnumerable.GetEnumerator() => GetAll().GetEnumerator(); - - private ICollection GetAll() { - ThrowIfDisposed(); - return _projectFiles.Values; - } - #endregion - } -} diff --git a/src/LanguageServer/Impl/Implementation/Server.Completion.cs b/src/LanguageServer/Impl/Implementation/Server.Completion.cs deleted file mode 100644 index 7ab08626a..000000000 --- a/src/LanguageServer/Impl/Implementation/Server.Completion.cs +++ /dev/null @@ -1,129 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Core.Text; -using Microsoft.Python.LanguageServer.Extensions; -using Microsoft.Python.Parsing; -using Microsoft.Python.Parsing.Ast; -using Microsoft.PythonTools.Analysis; - -namespace Microsoft.Python.LanguageServer.Implementation { - public sealed partial class Server { - public override async Task Completion(CompletionParams @params, CancellationToken cancellationToken) { - var uri = @params.textDocument.uri; - - ProjectFiles.GetEntry(@params.textDocument, @params._version, out var entry, out var tree); - TraceMessage($"Completions in {uri} at {@params.position}"); - - tree = GetParseTree(entry, uri, cancellationToken, out var version) ?? tree; - var analysis = entry != null ? await entry.GetAnalysisAsync(50, cancellationToken) : null; - if (analysis == null) { - TraceMessage($"No analysis found for {uri}"); - return new CompletionList(); - } - - var opts = GetOptions(@params.context); - var ctxt = new CompletionAnalysis(analysis, tree, @params.position, opts, Settings.completion, _displayTextBuilder, Logger, () => entry.ReadDocument(ProjectFiles.GetPart(uri), out _)); - - var members = string.IsNullOrEmpty(@params._expr) - ? ctxt.GetCompletions() - : ctxt.GetCompletionsFromString(@params._expr); - - if (members == null) { - TraceMessage($"No completions at {@params.position} in {uri}"); - return new CompletionList(); - } - - if (!Settings.completion.showAdvancedMembers) { - members = members.Where(m => !m.label.StartsWith("__")); - } - - var filterKind = @params.context?._filterKind; - if (filterKind.HasValue && filterKind != CompletionItemKind.None) { - TraceMessage($"Only returning {filterKind.Value} items"); - members = members.Where(m => m.kind == filterKind.Value); - } - - var res = new CompletionList { - items = members.ToArray(), - _expr = ctxt.ParentExpression?.ToCodeString(tree, CodeFormattingOptions.Traditional), - _commitByDefault = ctxt.ShouldCommitByDefault, - _allowSnippet = ctxt.ShouldAllowSnippets - }; - - res._applicableSpan = GetApplicableSpan(ctxt, @params, tree); - LogMessage(MessageType.Info, $"Found {res.items.Length} completions for {uri} at {@params.position} after filtering"); - - await InvokeExtensionsAsync((ext, token) - => (ext as ICompletionExtension)?.HandleCompletionAsync(uri, analysis, tree, @params.position, res, cancellationToken), cancellationToken); - - return res; - } - - private SourceSpan? GetApplicableSpan(CompletionAnalysis ca, CompletionParams @params, PythonAst tree) { - if (ca.ApplicableSpan.HasValue) { - return ca.ApplicableSpan; - } - - SourceLocation trigger = @params.position; - if (ca.Node != null) { - var span = ca.Node.GetSpan(tree); - if (@params.context?.triggerKind == CompletionTriggerKind.TriggerCharacter) { - if (span.End > trigger) { - // Span start may be after the trigger if there is bunch of whitespace - // between dot and next token such as in 'sys . version' - span = new SourceSpan(new SourceLocation(span.Start.Line, Math.Min(span.Start.Column, trigger.Column)), span.End); - } - } - if (span.End != span.Start) { - return span; - } - } - - if (@params.context?.triggerKind == CompletionTriggerKind.TriggerCharacter) { - var ch = @params.context?.triggerCharacter.FirstOrDefault() ?? '\0'; - return new SourceSpan( - trigger.Line, - Tokenizer.IsIdentifierStartChar(ch) ? Math.Max(1, trigger.Column - 1) : trigger.Column, - trigger.Line, - trigger.Column - ); - } - - return null; - } - - public override Task CompletionItemResolve(CompletionItem item, CancellationToken token) { - // TODO: Fill out missing values in item - return Task.FromResult(item); - } - - private GetMemberOptions GetOptions(CompletionContext? context) { - var opts = GetMemberOptions.None; - if (context.HasValue) { - var c = context.Value; - if (c._intersection) { - opts |= GetMemberOptions.IntersectMultipleResults; - } - } - return opts; - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/Server.Documents.cs b/src/LanguageServer/Impl/Implementation/Server.Documents.cs new file mode 100644 index 000000000..d84fe1a5b --- /dev/null +++ b/src/LanguageServer/Impl/Implementation/Server.Documents.cs @@ -0,0 +1,78 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; + +namespace Microsoft.Python.LanguageServer.Implementation { + public sealed partial class Server { + public void DidOpenTextDocument(DidOpenTextDocumentParams @params) { + _disposableBag.ThrowIfDisposed(); + _log?.Log(TraceEventType.Verbose, $"Opening document {@params.textDocument.uri}"); + + _rdt.OpenDocument(@params.textDocument.uri, @params.textDocument.text); + } + + public void DidChangeTextDocument(DidChangeTextDocumentParams @params) { + _disposableBag.ThrowIfDisposed(); + var doc = _rdt.GetDocument(@params.textDocument.uri); + if (doc != null) { + var changes = new List(); + foreach (var c in @params.contentChanges) { + Debug.Assert(c.range.HasValue); + var change = new DocumentChange { + InsertedText = c.text, + ReplacedSpan = c.range.Value + }; + changes.Add(change); + } + doc.Update(changes); + } else { + _log?.Log(TraceEventType.Warning, $"Unable to find document for {@params.textDocument.uri}"); + } + } + + public void DidChangeWatchedFiles(DidChangeWatchedFilesParams @params) { + foreach (var c in @params.changes.MaybeEnumerate()) { + _disposableBag.ThrowIfDisposed(); + // TODO: handle? + } + } + + public void DidCloseTextDocument(DidCloseTextDocumentParams @params) { + _disposableBag.ThrowIfDisposed(); + _rdt.CloseDocument(@params.textDocument.uri); + } + + private IDocumentAnalysis GetAnalysis(Uri uri, CancellationToken cancellationToken) { + var document = _rdt.GetDocument(uri); + if (document != null) { + document.GetAnalysisAsync(cancellationToken).Wait(200); + return document.GetAnyAnalysis(); + } + _log?.Log(TraceEventType.Error, $"Unable to find document {uri}"); + return null; + } + } +} diff --git a/src/LanguageServer/Impl/Implementation/Server.Editor.cs b/src/LanguageServer/Impl/Implementation/Server.Editor.cs new file mode 100644 index 000000000..10720c56f --- /dev/null +++ b/src/LanguageServer/Impl/Implementation/Server.Editor.cs @@ -0,0 +1,84 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.LanguageServer.Completion; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.LanguageServer.Sources; + +namespace Microsoft.Python.LanguageServer.Implementation { + public sealed partial class Server { + private CompletionSource _completionSource; + private HoverSource _hoverSource; + private SignatureSource _signatureSource; + + public async Task Completion(CompletionParams @params, CancellationToken cancellationToken) { + var uri = @params.textDocument.uri; + _log?.Log(TraceEventType.Verbose, $"Completions in {uri} at {@params.position}"); + + var res = new CompletionList(); + var analysis = GetAnalysis(uri, cancellationToken); + if(analysis != null) { + var result = await _completionSource.GetCompletionsAsync(analysis, @params.position, cancellationToken); + res.items = result.Completions.ToArray(); + } + + //await InvokeExtensionsAsync((ext, token) + // => (ext as ICompletionExtension)?.HandleCompletionAsync(uri, analysis, tree, @params.position, res, cancellationToken), cancellationToken); + return res; + } + + public Task CompletionItemResolve(CompletionItem item, CancellationToken token) { + // TODO: Fill out missing values in item + return Task.FromResult(item); + } + + public async Task Hover(TextDocumentPositionParams @params, CancellationToken cancellationToken) { + var uri = @params.textDocument.uri; + _log?.Log(TraceEventType.Verbose, $"Hover in {uri} at {@params.position}"); + + var analysis = GetAnalysis(uri, cancellationToken); + if (analysis != null) { + return await _hoverSource.GetHoverAsync(analysis, @params.position, cancellationToken); + } + return null; + } + + public async Task SignatureHelp(TextDocumentPositionParams @params, CancellationToken cancellationToken) { + var uri = @params.textDocument.uri; + _log?.Log(TraceEventType.Verbose, $"Signatures in {uri} at {@params.position}"); + + var analysis = GetAnalysis(uri, cancellationToken); + if (analysis != null) { + return await _signatureSource.GetSignatureAsync(analysis, @params.position, cancellationToken); + } + return null; + } + + public async Task GotoDefinition(TextDocumentPositionParams @params, CancellationToken cancellationToken) { + var uri = @params.textDocument.uri; + _log?.Log(TraceEventType.Verbose, $"Goto Definition in {uri} at {@params.position}"); + + var analysis = GetAnalysis(uri, cancellationToken); + var ds = new DefinitionSource(); + var reference = await ds.FindDefinitionAsync(analysis, @params.position, cancellationToken); + return reference != null ? new[] { reference } : Array.Empty(); + } + } +} diff --git a/src/LanguageServer/Impl/Implementation/Server.Extensions.cs b/src/LanguageServer/Impl/Implementation/Server.Extensions.cs index 086434ad7..a68b60d5c 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Extensions.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Extensions.cs @@ -15,26 +15,33 @@ // permissions and limitations under the License. using System; +using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Core; -using Microsoft.Python.Core.Shell; +using Microsoft.Python.LanguageServer.Extensibility; using Microsoft.Python.LanguageServer.Extensions; +using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer.Implementation { partial class Server { + private readonly ConcurrentDictionary _extensions = new ConcurrentDictionary(); public async Task LoadExtensionAsync(PythonAnalysisExtensionParams extension, IServiceContainer services, CancellationToken cancellationToken) { try { var provider = ActivateObject(extension.assembly, extension.typeName, null); if (provider == null) { - LogMessage(MessageType.Error, $"Extension provider {extension.assembly} {extension.typeName} failed to load"); + _log?.Log(TraceEventType.Error, $"Extension provider {extension.assembly} {extension.typeName} failed to load"); return; } - var ext = await provider.CreateAsync(this, extension.properties ?? new Dictionary(), cancellationToken); + var ext = await provider.CreateAsync(_services, extension.properties ?? new Dictionary(), cancellationToken); if (ext == null) { - LogMessage(MessageType.Error, $"Extension provider {extension.assembly} {extension.typeName} returned null"); + _log?.Log(TraceEventType.Error, $"Extension provider {extension.assembly} {extension.typeName} returned null"); return; } @@ -48,16 +55,16 @@ public async Task LoadExtensionAsync(PythonAnalysisExtensionParams extension, IS if (!string.IsNullOrEmpty(n)) { _extensions.AddOrUpdate(n, ext, (_, previous) => { - (previous as IDisposable)?.Dispose(); + previous?.Dispose(); return ext; }); } } catch (Exception ex) { - LogMessage(MessageType.Error, $"Error loading extension {extension.typeName} from'{extension.assembly}': {ex}"); + _log?.Log(TraceEventType.Error, $"Error loading extension {extension.typeName} from'{extension.assembly}': {ex}"); } } - public async Task ExtensionCommand(ExtensionCommandParams @params, CancellationToken token) { + public async Task ExtensionCommandAsync(ExtensionCommandParams @params, CancellationToken token) { if (string.IsNullOrEmpty(@params.extensionName)) { throw new ArgumentNullException(nameof(@params.extensionName)); } @@ -66,9 +73,12 @@ public async Task ExtensionCommand(ExtensionCommandParam throw new LanguageServerException(LanguageServerException.UnknownExtension, "No extension loaded with name: " + @params.extensionName); } - return new ExtensionCommandResult { - properties = await ext?.ExecuteCommand(@params.command, @params.properties, token) - }; + if (ext != null) { + return new ExtensionCommandResult { + properties = await ext.ExecuteCommand(@params.command, @params.properties, token) + }; + } + return new ExtensionCommandResult(); } private async Task InvokeExtensionsAsync(Func action, CancellationToken cancellationToken) { @@ -79,9 +89,34 @@ private async Task InvokeExtensionsAsync(Func(string assemblyName, string typeName, Dictionary properties) { + if (string.IsNullOrEmpty(assemblyName) || string.IsNullOrEmpty(typeName)) { + return default(T); + } + try { + var assembly = File.Exists(assemblyName) + ? Assembly.LoadFrom(assemblyName) + : Assembly.Load(new AssemblyName(assemblyName)); + + var type = assembly.GetType(typeName, true); + + return (T)Activator.CreateInstance( + type, + BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, + null, + properties == null ? Array.Empty() : new object[] { properties }, + CultureInfo.CurrentCulture + ); + } catch (Exception ex) { + _log?.Log(TraceEventType.Warning, ex.ToString()); + } + + return default; + } } } diff --git a/src/LanguageServer/Impl/Implementation/Server.FindReferences.cs b/src/LanguageServer/Impl/Implementation/Server.FindReferences.cs index e1f5fb2b4..d31d25d38 100644 --- a/src/LanguageServer/Impl/Implementation/Server.FindReferences.cs +++ b/src/LanguageServer/Impl/Implementation/Server.FindReferences.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -15,131 +14,19 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; -using System.Linq; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core.Text; -using Microsoft.Python.Parsing; -using Microsoft.Python.Parsing.Ast; -using Microsoft.PythonTools.Analysis; -using Microsoft.PythonTools.Intellisense; +using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { - public override async Task FindReferences(ReferencesParams @params, CancellationToken cancellationToken) { - await WaitForCompleteAnalysisAsync(cancellationToken); + public async Task FindReferences(ReferencesParams @params, CancellationToken cancellationToken) { var uri = @params.textDocument.uri; - ProjectFiles.GetEntry(@params.textDocument, @params._version, out var entry, out var tree); + _log?.Log(TraceEventType.Verbose, $"References in {uri} at {@params.position}"); - TraceMessage($"References in {uri} at {@params.position}"); - - var analysis = entry != null ? await entry.GetAnalysisAsync(50, cancellationToken) : null; - if (analysis == null) { - TraceMessage($"No analysis found for {uri}"); - return Array.Empty(); - } - - tree = GetParseTree(entry, uri, cancellationToken, out var version); - var modRefs = GetModuleReferences(entry, tree, version, @params); - - IEnumerable result; - if (!string.IsNullOrEmpty(@params._expr)) { - TraceMessage($"Getting references for {@params._expr}"); - result = analysis.GetVariables(@params._expr, @params.position); - } else { - var finder = new ExpressionFinder(tree, GetExpressionOptions.FindDefinition); - if (finder.GetExpression(@params.position) is Expression expr) { - TraceMessage($"Getting references for {expr.ToCodeString(tree, CodeFormattingOptions.Traditional)}"); - result = analysis.GetVariables(expr, @params.position); - } else { - TraceMessage($"No references found in {uri} at {@params.position}"); - result = Enumerable.Empty(); - } - } - - var filtered = result.Where(v => v.Type != VariableType.None); - if (!(@params.context?.includeDeclaration ?? false)) { - filtered = filtered.Where(v => v.Type != VariableType.Definition); - } - if (!(@params.context?._includeValues ?? false)) { - filtered = filtered.Where(v => v.Type != VariableType.Value); - } - - var res = filtered.Select(v => new Reference { - uri = v.Location.DocumentUri, - range = v.Location.Span, - _kind = ToReferenceKind(v.Type), - _version = version?.Version - }) - .Concat(modRefs) - .GroupBy(r => r, ReferenceComparer.Instance) - .Select(g => g.OrderByDescending(r => (SourceLocation)r.range.end).ThenBy(r => (int?)r._kind ?? int.MaxValue).First()) - .ToArray(); - - return res; - } - - private IEnumerable GetModuleReferences(IPythonProjectEntry entry, PythonAst tree, BufferVersion version, ReferencesParams @params) { - if (!@params.context?.includeDeclaration == true) { - return Enumerable.Empty(); - } - - var index = tree.LocationToIndex(@params.position); - var w = new ImportedModuleNameWalker(entry, index, tree); - tree.Walk(w); - - if (w.ImportedType != null) { - @params._expr = w.ImportedType.Name; - return Enumerable.Empty(); - } - - var modulesNamesInRange = w.ImportedModules.Where(m => { - var start = tree.LocationToIndex(m.SourceSpan.Start); - var end = tree.LocationToIndex(m.SourceSpan.End); - return start <= index && index < end; - }).ToArray(); - - if (modulesNamesInRange.Length == 0) { - return Enumerable.Empty(); - } - - var refs = new List(); - foreach (var n in modulesNamesInRange) { - if (Analyzer.Modules.TryGetImportedModule(n.Name, out var modRef) && modRef.AnalysisModule != null) { - // Return a module reference - refs.AddRange(modRef.AnalysisModule.Locations - .Select(l => new Reference { - uri = l.DocumentUri, - range = l.Span, - _version = version?.Version, - _kind = ReferenceKind.Definition - }) - .ToArray()); - } - } - return refs; - } - - private static ReferenceKind ToReferenceKind(VariableType type) { - switch (type) { - case VariableType.None: return ReferenceKind.Value; - case VariableType.Definition: return ReferenceKind.Definition; - case VariableType.Reference: return ReferenceKind.Reference; - case VariableType.Value: return ReferenceKind.Value; - default: return ReferenceKind.Value; - } - } - - private sealed class ReferenceComparer : IEqualityComparer { - public static readonly IEqualityComparer Instance = new ReferenceComparer(); - private ReferenceComparer() { } - public bool Equals(Reference x, Reference y) - => x.uri == y.uri && (SourceLocation)x.range.start == y.range.start; - - public int GetHashCode(Reference obj) - => new { u = obj.uri, l = obj.range.start.line, c = obj.range.start.character }.GetHashCode(); + return Array.Empty(); } } } diff --git a/src/LanguageServer/Impl/Implementation/Server.OnTypeFormatting.cs b/src/LanguageServer/Impl/Implementation/Server.Formatting.cs similarity index 76% rename from src/LanguageServer/Impl/Implementation/Server.OnTypeFormatting.cs rename to src/LanguageServer/Impl/Implementation/Server.Formatting.cs index 8846d3dec..605af2a0d 100644 --- a/src/LanguageServer/Impl/Implementation/Server.OnTypeFormatting.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Formatting.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -15,14 +14,17 @@ // permissions and limitations under the License. using System; +using System.Diagnostics; +using System.IO; using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Core; -using Microsoft.PythonTools.Analysis; +using Microsoft.Python.LanguageServer.Formatting; +using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { - public override async Task DocumentOnTypeFormatting(DocumentOnTypeFormattingParams @params, CancellationToken cancellationToken) { + public async Task DocumentOnTypeFormatting(DocumentOnTypeFormattingParams @params, CancellationToken cancellationToken) { int targetLine; switch (@params.ch) { @@ -38,24 +40,23 @@ public override async Task DocumentOnTypeFormatting(DocumentOnTypeFo } var uri = @params.textDocument.uri; - - if (!(ProjectFiles.GetEntry(uri) is IDocument doc)) { + var doc = _rdt.GetDocument(uri); + if (doc == null) { return Array.Empty(); } - var part = ProjectFiles.GetPart(uri); - using (var reader = doc.ReadDocument(part, out _)) { + using (var reader = new StringReader(doc.Content)) { if (@params.ch == ":") { return await BlockFormatter.ProvideEdits(reader, @params.position, @params.options); } - var lineFormatter = new LineFormatter(reader, Analyzer.LanguageVersion); + var lineFormatter = new LineFormatter(reader, doc.Interpreter.LanguageVersion); var edits = lineFormatter.FormatLine(targetLine); var unmatchedToken = lineFormatter.UnmatchedToken(targetLine); if (unmatchedToken != null) { var message = Resources.LineFormatter_UnmatchedToken.FormatInvariant(unmatchedToken.Value.token, unmatchedToken.Value.line + 1); - LogMessage(MessageType.Warning, message); + _log?.Log(TraceEventType.Warning, message); } return edits; diff --git a/src/LanguageServer/Impl/Implementation/Server.GoToDefinition.cs b/src/LanguageServer/Impl/Implementation/Server.GoToDefinition.cs deleted file mode 100644 index 2e98b7183..000000000 --- a/src/LanguageServer/Impl/Implementation/Server.GoToDefinition.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Linq; -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.Python.LanguageServer.Implementation { - public sealed partial class Server { - public override async Task GotoDefinition(TextDocumentPositionParams @params, CancellationToken cancellationToken) { - var references = await FindReferences(new ReferencesParams { - textDocument = @params.textDocument, - position = @params.position, - context = new ReferenceContext { - includeDeclaration = true, - _includeValues = true - } - }, cancellationToken); - return references.Where(r => r._kind == ReferenceKind.Definition && r.uri != null).ToArray(); - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/Server.Hover.cs b/src/LanguageServer/Impl/Implementation/Server.Hover.cs deleted file mode 100644 index 1e1617b25..000000000 --- a/src/LanguageServer/Impl/Implementation/Server.Hover.cs +++ /dev/null @@ -1,176 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.PythonTools.Analysis; -using Microsoft.PythonTools.Analysis.Documentation; -using Microsoft.PythonTools.Analysis.Values; -using Microsoft.PythonTools.Intellisense; -using Microsoft.Python.Parsing; -using Microsoft.Python.Parsing.Ast; -using Microsoft.Python.Core.Text; - -namespace Microsoft.Python.LanguageServer.Implementation { - public sealed partial class Server { - private static readonly Hover EmptyHover = new Hover { - contents = new MarkupContent { kind = MarkupKind.PlainText, value = string.Empty } - }; - - private DocumentationBuilder _displayTextBuilder; - - public override async Task Hover(TextDocumentPositionParams @params, CancellationToken cancellationToken) { - var uri = @params.textDocument.uri; - ProjectFiles.GetEntry(@params.textDocument, @params._version, out var entry, out var tree); - - TraceMessage($"Hover in {uri} at {@params.position}"); - - var analysis = entry != null ? await entry.GetAnalysisAsync(50, cancellationToken) : null; - if (analysis == null) { - TraceMessage($"No analysis found for {uri}"); - return EmptyHover; - } - - tree = GetParseTree(entry, uri, cancellationToken, out var version) ?? tree; - - Expression expr; - SourceSpan? exprSpan; - - var finder = new ExpressionFinder(tree, GetExpressionOptions.Hover); - expr = finder.GetExpression(@params.position) as Expression; - exprSpan = expr?.GetSpan(tree); - - if (expr == null) { - TraceMessage($"No hover info found in {uri} at {@params.position}"); - return EmptyHover; - } - - TraceMessage($"Getting hover for {expr.ToCodeString(tree, CodeFormattingOptions.Traditional)}"); - - var hover = await GetSelfHoverAsync(expr, analysis, tree, @params.position, cancellationToken); - if (hover != null && hover != EmptyHover) { - return hover; - } - - // First try values from expression. This works for the import statement most of the time. - var values = analysis.GetValues(expr, @params.position, null).ToList(); - if (values.Count == 0) { - values = GetImportHover(entry, analysis, tree, @params.position, out hover).ToList(); - if(hover != null) { - return hover; - } - } - - if (values.Count > 0) { - string originalExpr; - if (expr is ConstantExpression || expr is ErrorExpression) { - originalExpr = null; - } else { - originalExpr = @params._expr?.Trim(); - if (string.IsNullOrEmpty(originalExpr)) { - originalExpr = expr.ToCodeString(tree, CodeFormattingOptions.Traditional); - } - } - - var names = values.Select(GetFullTypeName).Where(n => !string.IsNullOrEmpty(n)).Distinct().ToArray(); - var res = new Hover { - contents = GetMarkupContent( - _displayTextBuilder.GetDocumentation(values, originalExpr), - _clientCaps.textDocument?.hover?.contentFormat), - range = exprSpan, - _version = version?.Version, - _typeNames = names - }; - return res; - } - - return EmptyHover; - } - - private async Task GetSelfHoverAsync(Expression expr, IModuleAnalysis analysis, PythonAst tree, Position position, CancellationToken cancellationToken) { - if(!(expr is NameExpression name) || name.Name != "self") { - return null; - } - var classDef = analysis.GetVariables(expr, position).FirstOrDefault(v => v.Type == VariableType.Definition); - if(classDef == null) { - return null; - } - - var instanceInfo = classDef.Variable.Types.OfType().FirstOrDefault(); - if (instanceInfo == null) { - return null; - } - - var cd = instanceInfo.ClassInfo.ClassDefinition; - var classParams = new TextDocumentPositionParams { - position = cd.NameExpression.GetStart(tree), - textDocument = new TextDocumentIdentifier { uri = classDef.Location.DocumentUri } - }; - return await Hover(classParams, cancellationToken); - } - - private IEnumerable GetImportHover(IPythonProjectEntry entry, IModuleAnalysis analysis, PythonAst tree, Position position, out Hover hover) { - hover = null; - - var index = tree.LocationToIndex(position); - var w = new ImportedModuleNameWalker(entry, index, tree); - tree.Walk(w); - - if (w.ImportedType != null) { - return analysis.GetValues(w.ImportedType.Name, position); - } - - var sb = new StringBuilder(); - var span = SourceSpan.Invalid; - foreach (var n in w.ImportedModules) { - if (Analyzer.Modules.TryGetImportedModule(n.Name, out var modRef) && modRef.AnalysisModule != null) { - if (sb.Length > 0) { - sb.AppendLine(); - sb.AppendLine(); - } - sb.Append(_displayTextBuilder.GetModuleDocumentation(modRef)); - span = span.IsValid ? span.Union(n.SourceSpan) : n.SourceSpan; - } - } - if (sb.Length > 0) { - hover = new Hover { - contents = sb.ToString(), - range = span - }; - } - return Enumerable.Empty(); - } - - private static string GetFullTypeName(AnalysisValue value) { - if (value is IHasQualifiedName qualName) { - return qualName.FullyQualifiedName; - } - - if (value is InstanceInfo ii) { - return GetFullTypeName(ii.ClassInfo); - } - - if (value is BuiltinInstanceInfo bii) { - return GetFullTypeName(bii.ClassInfo); - } - - return value?.Name; - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/Server.PrivateHelpers.cs b/src/LanguageServer/Impl/Implementation/Server.PrivateHelpers.cs deleted file mode 100644 index 40764a13a..000000000 --- a/src/LanguageServer/Impl/Implementation/Server.PrivateHelpers.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using System.Linq; -using Microsoft.PythonTools.Analysis.Documentation; - -namespace Microsoft.Python.LanguageServer.Implementation { - public sealed partial class Server { - private RestTextConverter _restTextConverter = new RestTextConverter(); - - private MarkupContent GetMarkupContent(string plainTextContent, IEnumerable preferences) { - if(string.IsNullOrEmpty(plainTextContent)) { - return string.Empty; - } - switch (SelectBestMarkup(preferences, MarkupKind.Markdown, MarkupKind.PlainText)) { - case MarkupKind.Markdown: - return new MarkupContent { - kind = MarkupKind.Markdown, - value = _restTextConverter.ToMarkdown(plainTextContent) - }; - } - return plainTextContent; - } - - private string SelectBestMarkup(IEnumerable requested, params string[] supported) { - if (requested == null) { - return supported.First(); - } - foreach (var k in requested) { - if (supported.Contains(k)) { - return k; - } - } - return MarkupKind.PlainText; - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/Server.Rename.cs b/src/LanguageServer/Impl/Implementation/Server.Rename.cs index f5e674165..ddc8db6f7 100644 --- a/src/LanguageServer/Impl/Implementation/Server.Rename.cs +++ b/src/LanguageServer/Impl/Implementation/Server.Rename.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -14,115 +13,14 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core; -using Microsoft.Python.Parsing.Ast; -using Microsoft.PythonTools.Analysis; +using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { - public override async Task Rename(RenameParams @params, CancellationToken cancellationToken) { - ProjectFiles.GetEntry(@params.textDocument, @params._version, out var entry, out var tree); - if (entry == null || tree == null) { - throw new EditorOperationException(Resources.RenameVariable_UnableGetExpressionAnalysis); - } - - var references = await FindReferences(new ReferencesParams { - textDocument = new TextDocumentIdentifier { uri = @params.textDocument.uri }, - position = @params.position, - context = new ReferenceContext { includeDeclaration = true } - }, cancellationToken); - - if (references.Any(x => x._isModule)) { - throw new EditorOperationException(Resources.RenameVariable_CannotRenameModuleName); - } - - var definition = references.FirstOrDefault(r => r._kind == ReferenceKind.Definition); - if (definition == null) { - throw new EditorOperationException(Resources.RenameVariable_CannotRename); - } - - var definitionSpan = definition.range.ToIndexSpan(tree); - var reader = new DocumentReader(entry as IDocument, ProjectFiles.GetPart(definition.uri)); - var originalName = reader.Read(definitionSpan.Start, definitionSpan.Length); - if (originalName == null) { - throw new EditorOperationException(Resources.RenameVariable_SelectSymbol); - } - if (!references.Any(r => r._kind == ReferenceKind.Definition || r._kind == ReferenceKind.Reference)) { - throw new EditorOperationException(Resources.RenameVariable_NoInformationAvailableForVariable.FormatUI(originalName)); - } - - // See https://en.wikipedia.org/wiki/Name_mangling, Python section. - var privatePrefix = entry.Analysis.GetPrivatePrefix(definition.range.start); - if (!string.IsNullOrEmpty(privatePrefix) && !string.IsNullOrEmpty(originalName) && originalName.StartsWithOrdinal(privatePrefix)) { - originalName = originalName.Substring(privatePrefix.Length + 1); - } - - // Group by URI for more optimal document reading in FilterPrivatePrefixed - var grouped = references - .GroupBy(x => x.uri) - .ToDictionary(g => g.Key, e => e.ToList()); - - var refs = FilterPrivatePrefixed(grouped, originalName, privatePrefix, @params.newName, @params.textDocument.uri, reader); - // Convert to Dictionary - var changes = refs - .ToDictionary( - kvp => kvp.Key, - kvp => kvp.Value.Select(t => new TextEdit { - range = t.range, - newText = @params.newName - }).ToArray()); - - return new WorkspaceEdit { changes = changes }; - } - - private Dictionary> FilterPrivatePrefixed( - Dictionary> refs, - string originalName, - string privatePrefix, - string newName, - Uri documentReaderUri, - IDocumentReader documentReader) { - // Filter out references depending on the private prefix, if any. - foreach (var kvp in refs) { - var ent = ProjectFiles.GetEntry(kvp.Key); - - var ast = (ent as ProjectEntry).GetCurrentParse().Tree; - var reader = kvp.Key == documentReaderUri - ? documentReader - : new DocumentReader(ent as IDocument, ProjectFiles.GetPart(kvp.Key)); - - if (ast == null || reader == null) { - throw new EditorOperationException(Resources.RenameVariable_NoInformationAvailableForVariable.FormatUI(originalName)); - } - - var fullName = $"{privatePrefix}{originalName}"; - for (var i = 0; i < kvp.Value.Count; i++) { - var reference = kvp.Value[i]; - Debug.Assert(reference.range.start.line == reference.range.end.line); - - var actualName = reader.ReadRange(reference.range, ast); - // If name does not match exactly, so we might be renaming a prefixed name - if (string.IsNullOrEmpty(privatePrefix)) { - // Not a mangled case, if names don't match, do not rename. - if (actualName != fullName) { - kvp.Value.RemoveAt(i); - i--; - } - continue; // All good, rename. - } - // If renaming from private name to private name, rename the non-prefixed portion - if (actualName.StartsWith(privatePrefix) && newName.StartsWith("__")) { - reference.range.start.character = reference.range.start.character + privatePrefix.Length; - } - } - } - return refs; + public async Task Rename(RenameParams @params, CancellationToken cancellationToken) { + return new WorkspaceEdit(); } } } diff --git a/src/LanguageServer/Impl/Implementation/Server.SignatureHelp.cs b/src/LanguageServer/Impl/Implementation/Server.SignatureHelp.cs deleted file mode 100644 index 8dd945927..000000000 --- a/src/LanguageServer/Impl/Implementation/Server.SignatureHelp.cs +++ /dev/null @@ -1,165 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Core; -using Microsoft.PythonTools.Analysis; -using Microsoft.Python.Parsing; -using Microsoft.Python.Parsing.Ast; - -namespace Microsoft.Python.LanguageServer.Implementation { - public sealed partial class Server { - public override async Task SignatureHelp(TextDocumentPositionParams @params, CancellationToken token) { - var uri = @params.textDocument.uri; - - TraceMessage($"Signatures in {uri} at {@params.position}"); - - var analysis = GetEntry(@params.textDocument) is ProjectEntry entry - ? await entry.GetAnalysisAsync(waitingTimeout: 50, cancellationToken: token) - : null; - - if (analysis == null) { - TraceMessage($"No analysis found for {uri}"); - return new SignatureHelp(); - } - - ProjectFiles.GetEntry(@params.textDocument, @params._version, out _, out var tree); - - IEnumerable overloads; - int activeSignature = -1, activeParameter = -1; - if (!string.IsNullOrEmpty(@params._expr)) { - TraceMessage($"Getting signatures for {@params._expr}"); - overloads = analysis.GetSignatures(@params._expr, @params.position); - } else { - var finder = new ExpressionFinder(tree, new GetExpressionOptions { Calls = true }); - var index = tree.LocationToIndex(@params.position); - if (finder.GetExpression(@params.position) is CallExpression callExpr) { - TraceMessage($"Getting signatures for {callExpr.ToCodeString(tree, CodeFormattingOptions.Traditional)}"); - overloads = analysis.GetSignatures(callExpr.Target, @params.position); - activeParameter = -1; - if (callExpr.GetArgumentAtIndex(tree, index, out activeParameter) && activeParameter < 0) { - // Returned 'true' and activeParameter == -1 means that we are after - // the trailing comma, so assume partially typed expression such as 'pow(x, y, |) - activeParameter = callExpr.Args.Count; - } - } else { - TraceMessage($"No signatures found in {uri} at {@params.position}"); - return new SignatureHelp(); - } - } - - var sigs = overloads.Select(ToSignatureInformation).ToArray(); - if (activeParameter >= 0 && activeSignature < 0) { - // TODO: Better selection of active signature - activeSignature = sigs - .Select((s, i) => Tuple.Create(s, i)) - .OrderBy(t => t.Item1.parameters.Length) - .FirstOrDefault(t => t.Item1.parameters.Length > activeParameter) - ?.Item2 ?? -1; - } - - activeSignature = activeSignature >= 0 - ? activeSignature - : (sigs.Length > 0 ? 0 : -1); - - return new SignatureHelp { - signatures = sigs, - activeSignature = activeSignature, - activeParameter = activeParameter - }; - } - - private SignatureInformation ToSignatureInformation(IOverloadResult overload) { - var si = new SignatureInformation { - parameters = overload.Parameters.MaybeEnumerate().Select(p => new ParameterInformation { - label = p.Name, - documentation = string.IsNullOrEmpty(p.Documentation) ? null : p.Documentation, - _type = p.Type, - _defaultValue = p.DefaultValue - }).ToArray(), - - _returnTypes = overload.ReturnType.OrderBy(k => k).ToArray() - }; - - if (_clientCaps?.textDocument?.signatureHelp?.signatureInformation?._shortLabel ?? false) { - si.label = overload.Name; - } else { - var doc = overload.Documentation; - // Some function contain signature in the documentation. Example: print. - // We want to use that signature in VS Code since it contains all arguments. - if (si.parameters.Length == 0 && !string.IsNullOrEmpty(doc) && doc.StartsWithOrdinal($"{overload.Name}(")) { - return GetSignatureFromDoc(doc); - } - si.label = "{0}({1})".FormatInvariant( - overload.Name, - string.Join(", ", overload.Parameters.Select(FormatParameter)) - ); - } - - si.documentation = string.IsNullOrEmpty(overload.Documentation) ? null : overload.Documentation; - var formatSetting = _clientCaps?.textDocument?.signatureHelp?.signatureInformation?.documentationFormat; - si.documentation = GetMarkupContent(si.documentation.value, formatSetting); - foreach (var p in si.parameters) { - p.documentation = GetMarkupContent(p.documentation.value, formatSetting); - } - - return si; - } - - private static SignatureInformation GetSignatureFromDoc(string doc) { - var si = new SignatureInformation(); - var firstLineBreak = doc.IndexOfAny(new[] { '\r', '\n' }); - si.label = firstLineBreak > 0 ? doc.Substring(0, firstLineBreak) : doc; - si.documentation = doc.Substring(si.label.Length).TrimStart(); - si.parameters = GetParametersFromDoc(si.label); - return si; - } - - private static ParameterInformation[] GetParametersFromDoc(string doc) { - var openBrace = doc.IndexOf('('); - var closeBrace = doc.LastIndexOf(')'); - - if (openBrace > 0 && closeBrace > 0) { - var args = doc.Substring(openBrace + 1, closeBrace - openBrace - 1).Split(','); - - return args.Select(a => { - var i = a.IndexOf('='); - return new ParameterInformation { - label = i > 0 ? a.Substring(0, i).Trim() : a.Trim(), - }; - }).ToArray(); - } - return Array.Empty(); - } - private static string FormatParameter(ParameterResult p) { - var res = new StringBuilder(p.Name); - if (!string.IsNullOrEmpty(p.Type)) { - res.Append(": "); - res.Append(p.Type); - } - if (!string.IsNullOrEmpty(p.DefaultValue)) { - res.Append('='); - res.Append(p.DefaultValue); - } - return res.ToString(); - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index 5858ee8b5..b5f658a06 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -15,195 +14,41 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; -using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core; -using Microsoft.Python.Core.Text; -using Microsoft.PythonTools.Analysis; -using Microsoft.PythonTools.Analysis.Values; -using Microsoft.PythonTools.Interpreter; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { private static int _symbolHierarchyDepthLimit = 10; private static int _symbolHierarchyMaxSymbols = 1000; - public override async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { - await WaitForCompleteAnalysisAsync(cancellationToken); - - var members = Enumerable.Empty(); - var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - - foreach (var entry in ProjectFiles) { - members = members.Concat( - await GetModuleVariablesAsync(entry as ProjectEntry, opts, @params.query, 50, cancellationToken) - ); - } - - members = members.GroupBy(mr => mr.Name).Select(g => g.First()); - return members.Select(ToSymbolInformation).ToArray(); - } - - public override async Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - await WaitForCompleteAnalysisAsync(cancellationToken); - var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - var entry = ProjectFiles.GetEntry(@params.textDocument.uri); - - var members = await GetModuleVariablesAsync(entry as ProjectEntry, opts, string.Empty, 50, cancellationToken); - return members - .GroupBy(mr => mr.Name) - .Select(g => g.First()) - .Select(ToSymbolInformation) - .ToArray(); - } - - public override async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - await WaitForCompleteAnalysisAsync(cancellationToken); - var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - var entry = ProjectFiles.GetEntry(@params.textDocument.uri); - - var members = await GetModuleVariablesAsync(entry as ProjectEntry, opts, string.Empty, 50, cancellationToken); - return ToDocumentSymbols(members); - } - - private static async Task> GetModuleVariablesAsync(ProjectEntry entry, GetMemberOptions opts, string prefix, int timeout, CancellationToken token) { - var analysis = entry != null ? await entry.GetAnalysisAsync(timeout, token) : null; - return analysis == null ? new List() : GetModuleVariables(entry, opts, prefix, analysis).ToList(); + public async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { + return Array.Empty< SymbolInformation>(); } - private static IEnumerable GetModuleVariables(ProjectEntry entry, GetMemberOptions opts, string prefix, IModuleAnalysis analysis) { - var breadthFirst = analysis.Scope.TraverseBreadthFirst(s => s.Children); - var all = breadthFirst.SelectMany(c => analysis.GetAllAvailableMembersFromScope(c, opts)); - var result = all - .Where(m => { - if (m.Values.Any(v => v.DeclaringModule == entry || - v.Locations - .MaybeEnumerate() - .ExcludeDefault() - .Any(l => l.DocumentUri == entry.DocumentUri))) { - return string.IsNullOrEmpty(prefix) || m.Name.StartsWithOrdinal(prefix, ignoreCase: true); - } - return false; - }) - .Take(_symbolHierarchyMaxSymbols); - return result; + public async Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { + return Array.Empty(); } - private SymbolInformation ToSymbolInformation(IMemberResult m) { - var res = new SymbolInformation { - name = m.Name, - kind = ToSymbolKind(m.MemberType), - _kind = m.MemberType.ToString().ToLowerInvariant() - }; - - var loc = m.Locations.FirstOrDefault(l => !string.IsNullOrEmpty(l.FilePath)); - - if (loc != null) { - res.location = new Location { - uri = loc.DocumentUri, - range = new SourceSpan( - new SourceLocation(loc.StartLine, loc.StartColumn), - new SourceLocation(loc.EndLine ?? loc.StartLine, loc.EndColumn ?? loc.StartColumn) - ) - }; - } - - return res; + public async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { + return Array.Empty(); } - private DocumentSymbol[] ToDocumentSymbols(List members) { - var topLevel = new List(); - var childMap = new Dictionary>(); - - foreach (var m in members) { - var parent = members.FirstOrDefault(x => x.Scope?.Node == m.Scope?.OuterScope?.Node && x.Name == m.Scope?.Name); - if (parent != null) { - if (!childMap.TryGetValue(parent, out var children)) { - childMap[parent] = children = new List(); - } - children.Add(m); - } else { - topLevel.Add(m); - } - } - - var symbols = topLevel - .GroupBy(mr => mr.Name) - .Select(g => g.First()) - .Select(m => ToDocumentSymbol(m, childMap, 0)) - .ToArray(); - - return symbols; - } - - private DocumentSymbol ToDocumentSymbol(IMemberResult m, Dictionary> childMap, int currentDepth) { - var res = new DocumentSymbol { - name = m.Name, - detail = m.Name, - kind = ToSymbolKind(m.MemberType), - deprecated = false, - _functionKind = GetFunctionKind(m) - }; - - if (childMap.TryGetValue(m, out var children) && currentDepth < _symbolHierarchyDepthLimit) { - res.children = children - .Select(x => ToDocumentSymbol(x, childMap, currentDepth + 1)) - .ToArray(); - } else { - res.children = Array.Empty(); - } - - var loc = m.Locations.FirstOrDefault(l => !string.IsNullOrEmpty(l.FilePath)); - - if (loc != null) { - res.range = new SourceSpan( - new SourceLocation(loc.StartLine, loc.StartColumn), - new SourceLocation(loc.EndLine ?? loc.StartLine, loc.EndColumn ?? loc.StartColumn) - ); - res.selectionRange = res.range; - } - return res; - } - - private static string GetFunctionKind(IMemberResult m) { - if (m.MemberType == PythonMemberType.Function) { - var funcInfo = m.Values.OfType().FirstOrDefault(); - if (funcInfo != null) { - if (funcInfo.IsProperty) { - return "property"; - } - if (funcInfo.IsStatic) { - return "staticmethod"; - } - if (funcInfo.IsClassMethod) { - return "classmethod"; - } - } - return "function"; - } - return m.MemberType == PythonMemberType.Class ? "class" : string.Empty; - } private static SymbolKind ToSymbolKind(PythonMemberType memberType) { switch (memberType) { case PythonMemberType.Unknown: return SymbolKind.None; case PythonMemberType.Class: return SymbolKind.Class; case PythonMemberType.Instance: return SymbolKind.Variable; - case PythonMemberType.Enum: return SymbolKind.Enum; - case PythonMemberType.EnumInstance: return SymbolKind.EnumMember; case PythonMemberType.Function: return SymbolKind.Function; case PythonMemberType.Method: return SymbolKind.Method; case PythonMemberType.Module: return SymbolKind.Module; - case PythonMemberType.Constant: return SymbolKind.Constant; - case PythonMemberType.Event: return SymbolKind.Event; - case PythonMemberType.Field: return SymbolKind.Field; case PythonMemberType.Property: return SymbolKind.Property; - case PythonMemberType.Multiple: return SymbolKind.Object; - case PythonMemberType.Keyword: return SymbolKind.None; - case PythonMemberType.CodeSnippet: return SymbolKind.None; - case PythonMemberType.NamedArgument: return SymbolKind.None; + case PythonMemberType.Union: return SymbolKind.Object; + case PythonMemberType.Variable: return SymbolKind.Variable; + case PythonMemberType.Generic: return SymbolKind.TypeParameter; default: return SymbolKind.None; } } diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index a8f653536..08d90d8fb 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -14,74 +14,35 @@ // permissions and limitations under the License. using System; -using System.Collections.Concurrent; -using System.Collections.Generic; using System.Diagnostics; -using System.Globalization; -using System.IO; using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Core.Interpreter; +using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.IO; +using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Shell; -using Microsoft.Python.LanguageServer.Extensions; -using Microsoft.Python.Parsing.Ast; -using Microsoft.PythonTools.Analysis; -using Microsoft.PythonTools.Analysis.Documentation; -using Microsoft.PythonTools.Intellisense; -using Microsoft.PythonTools.Interpreter; -using Microsoft.PythonTools.Interpreter.Ast; +using Microsoft.Python.LanguageServer.Completion; +using Microsoft.Python.LanguageServer.Diagnostics; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.LanguageServer.Sources; namespace Microsoft.Python.LanguageServer.Implementation { - public sealed partial class Server : ServerBase, IPythonLanguageServer, IDisposable { - /// - /// Implements ability to execute module reload on the analyzer thread - /// - private sealed class ReloadModulesQueueItem : IAnalyzable { - private readonly Server _server; - private TaskCompletionSource _tcs = new TaskCompletionSource(); - public Task Task => _tcs.Task; - - public ReloadModulesQueueItem(Server server) { - _server = server; - } - public void Analyze(CancellationToken cancel) { - if (cancel.IsCancellationRequested) { - return; - } - - var currentTcs = Interlocked.Exchange(ref _tcs, new TaskCompletionSource()); - try { - _server.Analyzer.ReloadModulesAsync(cancel).WaitAndUnwrapExceptions(); - foreach (var entry in _server.Analyzer.ModulesByFilename) { - _server.EnqueueItemAsync(entry.Value.ProjectEntry, AnalysisPriority.Normal, false); - } - currentTcs.TrySetResult(true); - } catch (OperationCanceledException oce) { - currentTcs.TrySetCanceled(oce.CancellationToken); - } catch (Exception ex) { - currentTcs.TrySetException(ex); - } - } - } - - private readonly Dictionary _pendingParse = new Dictionary(); - private readonly VolatileCounter _pendingAnalysisEnqueue = new VolatileCounter(); - private readonly ConcurrentDictionary _extensions = new ConcurrentDictionary(); + public sealed partial class Server : IDisposable { private readonly DisposableBag _disposableBag = DisposableBag.Create(); private readonly CancellationTokenSource _shutdownCts = new CancellationTokenSource(); + private readonly IServiceManager _services; - private readonly EditorFiles _editorFiles; + private IPythonInterpreter _interpreter; + private IRunningDocumentTable _rdt; private ClientCapabilities _clientCaps; - private bool _traceLogging; - private bool _analysisUpdates; - private ReloadModulesQueueItem _reloadModulesQueueItem; - // If null, all files must be added manually - private string _rootDir; + private ILogger _log; public static InformationDisplayOptions DisplayOptions { get; private set; } = new InformationDisplayOptions { preferredFormat = MarkupKind.PlainText, @@ -92,13 +53,8 @@ public void Analyze(CancellationToken cancel) { maxDocumentationLines = 100 }; - public Server(IServiceContainer services = null): base(services) { - - AnalysisQueue = new AnalysisQueue(); - AnalysisQueue.UnhandledException += Analysis_UnhandledException; - - ParseQueue = new ParseQueue(); - _editorFiles = new EditorFiles(this); + public Server(IServiceManager services) { + _services = services; _disposableBag .Add(() => { @@ -106,37 +62,16 @@ public Server(IServiceContainer services = null): base(services) { ext.Dispose(); } }) - .Add(ProjectFiles) - .Add(() => Analyzer?.Dispose()) - .Add(AnalysisQueue) - .Add(_editorFiles) .Add(() => _shutdownCts.Cancel()); } - private void Analysis_UnhandledException(object sender, UnhandledExceptionEventArgs e) { - Debug.Fail(e.ExceptionObject.ToString()); - LogMessage(MessageType.Error, e.ExceptionObject.ToString()); - } - - internal AnalysisQueue AnalysisQueue { get; } - internal ParseQueue ParseQueue { get; } - - internal PythonAnalyzer Analyzer { get; private set; } internal ServerSettings Settings { get; private set; } = new ServerSettings(); - internal ProjectFiles ProjectFiles { get; } = new ProjectFiles(); - - public void Dispose() => _disposableBag.TryDispose(); - #region ILogger - public void TraceMessage(IFormattable message) { - if (_traceLogging) { - LogMessage(MessageType.Log, message.ToString()); - } - } - #endregion + public IServiceContainer Services => _services; + public void Dispose() => _disposableBag.TryDispose(); #region Client message handling - internal InitializeResult GetInitializeResult() => new InitializeResult { + private InitializeResult GetInitializeResult() => new InitializeResult { capabilities = new ServerCapabilities { textDocumentSync = new TextDocumentSyncOptions { openClose = true, @@ -159,508 +94,69 @@ public void TraceMessage(IFormattable message) { } }; - public override async Task Initialize(InitializeParams @params, CancellationToken cancellationToken) { + public async Task InitializeAsync(InitializeParams @params, CancellationToken cancellationToken) { _disposableBag.ThrowIfDisposed(); - await DoInitializeAsync(@params, cancellationToken); - return GetInitializeResult(); - } + _clientCaps = @params.capabilities; + _log = _services.GetService(); - public override Task Shutdown() { - _disposableBag.ThrowIfDisposed(); - return Task.CompletedTask; - } + if (@params.initializationOptions.displayOptions != null) { + DisplayOptions = @params.initializationOptions.displayOptions; + } - public override async Task DidOpenTextDocument(DidOpenTextDocumentParams @params, CancellationToken token) { - _disposableBag.ThrowIfDisposed(); - TraceMessage($"Opening document {@params.textDocument.uri}"); + _services.AddService(new DiagnosticsService(_services)); - _editorFiles.Open(@params.textDocument.uri); - var entry = ProjectFiles.GetEntry(@params.textDocument.uri, throwIfMissing: false); - var doc = entry as IDocument; - if (doc != null) { - if (@params.textDocument.text != null) { - doc.ResetDocument(@params.textDocument.version, @params.textDocument.text); - } - await EnqueueItemAsync(doc); - } else if (entry == null) { - IAnalysisCookie cookie = null; - if (@params.textDocument.text != null) { - cookie = new InitialContentCookie { - Content = @params.textDocument.text, - Version = @params.textDocument.version - }; - } - entry = await AddFileAsync(@params.textDocument.uri, cookie); - } - } + var analyzer = new PythonAnalyzer(_services, @params.rootPath); + _services.AddService(analyzer); - public override Task DidChangeTextDocument(DidChangeTextDocumentParams @params, CancellationToken cancellationToken) { - _disposableBag.ThrowIfDisposed(); - var openedFile = _editorFiles.GetDocument(@params.textDocument.uri); - return openedFile.DidChangeTextDocument(@params, true, cancellationToken); - } + _services.AddService(new RunningDocumentTable(@params.rootPath, _services)); + _rdt = _services.GetService(); - public override async Task DidChangeWatchedFiles(DidChangeWatchedFilesParams @params, CancellationToken token) { - foreach (var c in @params.changes.MaybeEnumerate()) { - _disposableBag.ThrowIfDisposed(); - IProjectEntry entry; - switch (c.type) { - case FileChangeType.Created: - entry = await LoadFileAsync(c.uri); - if (entry != null) { - TraceMessage($"Saw {c.uri} created and loaded new entry"); - } else { - LogMessage(MessageType.Warning, $"Failed to load {c.uri}"); - } - break; - case FileChangeType.Deleted: - await UnloadFileAsync(c.uri); - break; - case FileChangeType.Changed: - if ((entry = ProjectFiles.GetEntry(c.uri, false)) is IDocument doc) { - // If document version is >=0, it is loaded in memory. - if (doc.GetDocumentVersion(0) < 0) { - await EnqueueItemAsync(doc, AnalysisPriority.Low); - } - } - break; - } - } - } + // TODO: multi-root workspaces. + var rootDir = @params.rootUri != null ? @params.rootUri.ToAbsolutePath() : PathUtils.NormalizePath(@params.rootPath); + var configuration = InterpreterConfiguration.FromDictionary(@params.initializationOptions.interpreter.properties); + configuration.SearchPaths = @params.initializationOptions.searchPaths; + configuration.TypeshedPath = @params.initializationOptions.typeStubSearchPaths.FirstOrDefault(); - public override async Task DidCloseTextDocument(DidCloseTextDocumentParams @params, CancellationToken token) { - _disposableBag.ThrowIfDisposed(); - _editorFiles.Close(@params.textDocument.uri); + _interpreter = await PythonInterpreter.CreateAsync(configuration, rootDir, _services, cancellationToken); + _services.AddService(_interpreter); - if (ProjectFiles.GetEntry(@params.textDocument.uri) is IDocument doc) { - // No need to keep in-memory buffers now - doc.ResetDocument(-1, null); - // Pick up any changes on disk that we didn't know about - await EnqueueItemAsync(doc, AnalysisPriority.Low); - } + DisplayStartupInfo(); + + var ds = new PlainTextDocumentationSource(); + _completionSource = new CompletionSource(ds, Settings.completion); + _hoverSource = new HoverSource(ds); + _signatureSource = new SignatureSource(ds); + + return GetInitializeResult(); } - public override async Task DidChangeConfiguration(DidChangeConfigurationParams @params, CancellationToken cancellationToken) { + public Task Shutdown() { _disposableBag.ThrowIfDisposed(); + return Task.CompletedTask; + } - if (Analyzer == null) { - LogMessage(MessageType.Error, "Change configuration notification sent to uninitialized server"); - return; - } + public async Task DidChangeConfiguration(DidChangeConfigurationParams @params, CancellationToken cancellationToken) { + _disposableBag.ThrowIfDisposed(); var reanalyze = true; if (@params.settings != null) { if (@params.settings is ServerSettings settings) { reanalyze = HandleConfigurationChanges(settings); } else { - LogMessage(MessageType.Error, "change configuration notification sent unsupported settings"); + _log?.Log(TraceEventType.Error, "change configuration notification sent unsupported settings"); return; } } - - if (reanalyze) { - await ReloadModulesAsync(cancellationToken); - } } - - public async Task ReloadModulesAsync(CancellationToken token) { - LogMessage(MessageType._General, Resources.ReloadingModules); - - // Make sure reload modules is executed on the analyzer thread. - var task = _reloadModulesQueueItem.Task; - AnalysisQueue.Enqueue(_reloadModulesQueueItem, AnalysisPriority.Normal); - await task; - - // re-analyze all of the modules when we get a new set of modules loaded... - foreach (var entry in Analyzer.ModulesByFilename) { - AnalysisQueue.Enqueue(entry.Value.ProjectEntry, AnalysisPriority.Normal); - } - LogMessage(MessageType._General, Resources.Done); - } - - public override Task ExecuteCommand(ExecuteCommandParams @params, CancellationToken token) { - _disposableBag.ThrowIfDisposed(); - Command(new CommandEventArgs { - command = @params.command, - arguments = @params.arguments - }); - return Task.FromResult((object)null); - } - #endregion - - #region Non-LSP public API - public IProjectEntry GetEntry(TextDocumentIdentifier document) => ProjectFiles.GetEntry(document.uri); - public IProjectEntry GetEntry(Uri documentUri, bool throwIfMissing = true) => ProjectFiles.GetEntry(documentUri, throwIfMissing); - - public int GetPart(TextDocumentIdentifier document) => ProjectFiles.GetPart(document.uri); - public IEnumerable GetLoadedFiles() => ProjectFiles.GetLoadedFiles(); - - public Task LoadFileAsync(Uri documentUri) => AddFileAsync(documentUri); - - public Task UnloadFileAsync(Uri documentUri) { - var entry = RemoveEntry(documentUri); - if (entry != null) { - Analyzer.RemoveModule(entry, e => EnqueueItemAsync(e as IDocument, AnalysisPriority.Normal, parse: false).DoNotWait()); - return Task.FromResult(true); - } - - return Task.FromResult(false); - } - - public int EstimateRemainingWork() { - return ParseQueue.Count + AnalysisQueue.Count; - } - - public event EventHandler OnParseComplete; - private void ParseComplete(Uri uri, int version) { - TraceMessage($"Parse complete for {uri} at version {version}"); - OnParseComplete?.Invoke(this, new ParseCompleteEventArgs { uri = uri, version = version }); - } - - public event EventHandler OnAnalysisComplete; - private void AnalysisComplete(Uri uri, int version) { - if (_analysisUpdates) { - TraceMessage($"Analysis complete for {uri} at version {version}"); - OnAnalysisComplete?.Invoke(this, new AnalysisCompleteEventArgs { uri = uri, version = version }); - } - } - - public event EventHandler OnAnalysisQueued; - private void AnalysisQueued(Uri uri) { - if (_analysisUpdates) { - TraceMessage($"Analysis queued for {uri}"); - OnAnalysisQueued?.Invoke(this, new AnalysisQueuedEventArgs { uri = uri }); - } - } - - #endregion - - #region IPythonLanguageServer - public PythonAst GetCurrentAst(Uri documentUri) { - ProjectFiles.GetEntry(documentUri, null, out var entry, out var tree); - return entry.GetCurrentParse()?.Tree; - } - - public Task GetAstAsync(Uri documentUri, CancellationToken token) { - ProjectFiles.GetEntry(documentUri, null, out var entry, out var tree); - entry.WaitForCurrentParse(Timeout.Infinite, token); - return Task.FromResult(entry.GetCurrentParse()?.Tree); - } - - public Task GetAnalysisAsync(Uri documentUri, CancellationToken token) { - ProjectFiles.GetEntry(documentUri, null, out var entry, out var tree); - return entry.GetAnalysisAsync(Timeout.Infinite, token); - } - - public IProjectEntry GetProjectEntry(Uri documentUri) => ProjectFiles.GetEntry(documentUri); #endregion #region Private Helpers - - private IProjectEntry RemoveEntry(Uri documentUri) { - var entry = ProjectFiles.RemoveEntry(documentUri); - _editorFiles.Remove(documentUri); - return entry; - } - - private async Task DoInitializeAsync(InitializeParams @params, CancellationToken token) { - _disposableBag.ThrowIfDisposed(); - - Analyzer = await AnalysisQueue.ExecuteInQueueAsync(ct => CreateAnalyzer(@params.initializationOptions.interpreter, token), AnalysisPriority.High); - - _disposableBag.ThrowIfDisposed(); - _clientCaps = @params.capabilities; - _traceLogging = @params.initializationOptions.traceLogging; - _analysisUpdates = @params.initializationOptions.analysisUpdates; - - Analyzer.EnableDiagnostics = _clientCaps?.python?.liveLinting ?? true; - _reloadModulesQueueItem = new ReloadModulesQueueItem(this); - - if (@params.initializationOptions.displayOptions != null) { - DisplayOptions = @params.initializationOptions.displayOptions; - } - _displayTextBuilder = DocumentationBuilder.Create(DisplayOptions); - - DisplayStartupInfo(); - - if (@params.rootUri != null) { - _rootDir = @params.rootUri.ToAbsolutePath(); - } else if (!string.IsNullOrEmpty(@params.rootPath)) { - _rootDir = PathUtils.NormalizePath(@params.rootPath); - } - - Analyzer.SetRoot(_rootDir); - Analyzer.SetSearchPaths(@params.initializationOptions.searchPaths.MaybeEnumerate()); - Analyzer.SetTypeStubPaths(@params.initializationOptions.typeStubSearchPaths.MaybeEnumerate()); - - Analyzer.Interpreter.ModuleNamesChanged += Interpreter_ModuleNamesChanged; - } - - private void Interpreter_ModuleNamesChanged(object sender, EventArgs e) { - Analyzer.Modules.Reload(); - foreach (var entry in Analyzer.ModulesByFilename) { - AnalysisQueue.Enqueue(entry.Value.ProjectEntry, AnalysisPriority.Normal); - } - } - private void DisplayStartupInfo() { - LogMessage(MessageType._General, Resources.LanguageServerVersion.FormatInvariant(Assembly.GetExecutingAssembly().GetName().Version)); - LogMessage(MessageType._General, - string.IsNullOrEmpty(Analyzer.InterpreterFactory?.Configuration?.InterpreterPath) + _log?.Log(TraceEventType.Information, Resources.LanguageServerVersion.FormatInvariant(Assembly.GetExecutingAssembly().GetName().Version)); + _log?.Log(TraceEventType.Information, + string.IsNullOrEmpty(_interpreter.Configuration.InterpreterPath) ? Resources.InitializingForGenericInterpreter - : Resources.InitializingForPythonInterpreter.FormatInvariant(Analyzer.InterpreterFactory.Configuration.InterpreterPath)); - } - - private T ActivateObject(string assemblyName, string typeName, Dictionary properties) { - if (string.IsNullOrEmpty(assemblyName) || string.IsNullOrEmpty(typeName)) { - return default(T); - } - try { - var assembly = File.Exists(assemblyName) - ? Assembly.LoadFrom(assemblyName) - : Assembly.Load(new AssemblyName(assemblyName)); - - var type = assembly.GetType(typeName, true); - - return (T)Activator.CreateInstance( - type, - BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Instance, - null, - properties == null ? Array.Empty() : new object[] { properties }, - CultureInfo.CurrentCulture - ); - } catch (Exception ex) { - LogMessage(MessageType.Warning, ex.ToString()); - } - - return default(T); - } - - private async Task CreateAnalyzer(PythonInitializationOptions.Interpreter interpreter, CancellationToken token) { - var factory = ActivateObject(interpreter.assembly, interpreter.typeName, interpreter.properties) - ?? new AstPythonInterpreterFactory(interpreter.properties); - - var analyzer = await PythonAnalyzer.CreateAsync(factory, token); - LogMessage(MessageType.Info, $"Created {analyzer.Interpreter.GetType().FullName} instance from {factory.GetType().FullName}"); - return analyzer; - } - - private IEnumerable GetImportNames(Uri document) { - var filePath = GetLocalPath(document); - - if (!string.IsNullOrEmpty(filePath)) { - if (!string.IsNullOrEmpty(_rootDir) && ModulePath.FromBasePathAndFile_NoThrow(_rootDir, filePath, out var mp)) { - yield return mp; - } - - foreach (var sp in Analyzer.GetSearchPaths()) { - if (ModulePath.FromBasePathAndFile_NoThrow(sp, filePath, out mp)) { - yield return mp; - } - } - } - - if (document.Scheme == "python") { - var path = Path.Combine(document.Host, document.AbsolutePath.Replace(Path.AltDirectorySeparatorChar, Path.DirectorySeparatorChar).Trim(Path.DirectorySeparatorChar)); - if (ModulePath.FromBasePathAndFile_NoThrow("", path, true, out var mp, out _, out _)) { - yield return mp; - } - } - } - - private string GetLocalPath(Uri uri) { - if (uri == null) { - return null; - } - if (uri.IsFile) { - return uri.ToAbsolutePath(); - } - var scheme = uri.Scheme; - var path = uri.GetComponents(UriComponents.Path, UriFormat.SafeUnescaped).Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); - var bits = new List(path.Length + 2); - bits.Add("_:"); // Non-existent root - bits.Add(scheme.ToUpperInvariant()); - bits.AddRange(path); - return string.Join(Path.DirectorySeparatorChar.ToString(), bits); - } - - private async Task AddFileAsync(Uri documentUri, IAnalysisCookie cookie = null) { - var item = ProjectFiles.GetEntry(documentUri, throwIfMissing: false); - if (item != null) { - return item; - } - - var path = GetLocalPath(documentUri); - var aliases = GetImportNames(documentUri).Select(mp => mp.ModuleName).ToArray(); - if (aliases.Length == 0) { - aliases = new[] { Path.GetFileNameWithoutExtension(path) }; - } - - var first = aliases.First(); - - var pyItem = Analyzer.AddModule(first, path, documentUri, cookie); - item = pyItem; - - var actualItem = ProjectFiles.GetOrAddEntry(documentUri, item); - if (actualItem != item) { - return actualItem; - } - - var reanalyzeEntries = Analyzer.GetEntriesThatImportModule(pyItem.ModuleName, true).ToArray(); - pyItem.NewAnalysis += OnProjectEntryNewAnalysis; - - if (item is IDocument doc) { - await EnqueueItemAsync(doc); - } - - foreach (var entryRef in reanalyzeEntries) { - await EnqueueItemAsync(entryRef as IDocument, AnalysisPriority.Low, parse: false); - } - - return item; - } - - private void OnProjectEntryNewAnalysis(object sender, EventArgs e) { - // Events are fired from a background thread and may come - // after items were actually already disposed. - if (_disposableBag.IsDisposed) { - return; - } - - var pythonProjectEntry = (IPythonProjectEntry)sender; - TraceMessage($"Received new analysis for {pythonProjectEntry.DocumentUri}"); - - var version = 0; - var parse = pythonProjectEntry.GetCurrentParse(); - if (_analysisUpdates) { - if (parse?.Cookie is VersionCookie vc && vc.Versions.Count > 0) { - foreach (var kv in vc.GetAllParts(pythonProjectEntry.DocumentUri)) { - AnalysisComplete(kv.Key, kv.Value.Version); - if (kv.Value.Version > version) { - version = kv.Value.Version; - } - } - } else { - AnalysisComplete(pythonProjectEntry.DocumentUri, 0); - } - } - - _editorFiles.GetDocument(pythonProjectEntry.DocumentUri).UpdateAnalysisDiagnostics(pythonProjectEntry, version); - } - - private void RemoveDocumentParseCounter(Task t, IDocument doc, VolatileCounter counter) { - if (t.IsCompleted) { - lock (_pendingParse) { - if (counter.IsZero) { - if (_pendingParse.TryGetValue(doc, out var existing) && existing == counter) { - _pendingParse.Remove(doc); - } - return; - } - } - } - counter.WaitForChangeToZeroAsync().ContinueWith(t2 => RemoveDocumentParseCounter(t, doc, counter)); - } - - private IDisposable GetDocumentParseCounter(IDocument doc, out int count) { - VolatileCounter counter; - lock (_pendingParse) { - if (!_pendingParse.TryGetValue(doc, out counter)) { - _pendingParse[doc] = counter = new VolatileCounter(); - // Automatically remove counter from the dictionary when it reaches zero. - counter.WaitForChangeToZeroAsync().ContinueWith(t => RemoveDocumentParseCounter(t, doc, counter)); - } - var res = counter.Incremented(); - count = counter.Count; - return res; - } - } - - internal Task EnqueueItemAsync(IDocument doc, AnalysisPriority priority = AnalysisPriority.Normal, bool parse = true, bool enqueueForAnalysis = true) { - _disposableBag.ThrowIfDisposed(); - var pending = _pendingAnalysisEnqueue.Incremented(); - try { - var entry = doc as ProjectEntry; - entry?.ResetCompleteAnalysis(); - - // If we don't need to parse, use null cookie - var cookieTask = Task.FromResult(null); - if (parse) { - using (GetDocumentParseCounter(doc, out var count)) { - if (count > 3) { - // Rough check to prevent unbounded queueing. If we have - // multiple parses in queue, we will get the latest doc - // version in one of the ones to come. - return Task.CompletedTask; - } - TraceMessage($"Parsing document {doc.DocumentUri}"); - cookieTask = ParseQueue.EnqueueAsync(doc, Analyzer.LanguageVersion); - } - } - - // The call must be fire and forget, but should not be yielding. - // It is called from DidChangeTextDocument which must fully finish - // since otherwise Complete() may come before the change is enqueued - // for processing and the completion list will be driven off the stale data. - return cookieTask.ContinueWith(t => { - if (t.IsFaulted) { - // Happens when file got deleted before processing - pending.Dispose(); - LogMessage(MessageType.Error, t.Exception.Message); - return; - } - OnDocumentChangeProcessingComplete(doc, t.Result as VersionCookie, enqueueForAnalysis, priority, pending); - }); - } catch { - pending?.Dispose(); - throw; - } - } - - private void OnDocumentChangeProcessingComplete(IDocument doc, VersionCookie vc, bool enqueueForAnalysis, AnalysisPriority priority, IDisposable disposeWhenEnqueued) { - try { - _shutdownCts.Token.ThrowIfCancellationRequested(); - if (vc != null) { - foreach (var kv in vc.GetAllParts(doc.DocumentUri)) { - ParseComplete(kv.Key, kv.Value.Version); - } - } else { - ParseComplete(doc.DocumentUri, 0); - } - - if (doc is IAnalyzable analyzable && enqueueForAnalysis && !_shutdownCts.Token.IsCancellationRequested) { - AnalysisQueued(doc.DocumentUri); - AnalysisQueue.Enqueue(analyzable, priority); - } - - disposeWhenEnqueued?.Dispose(); - disposeWhenEnqueued = null; - if (vc != null) { - _editorFiles.GetDocument(doc.DocumentUri).UpdateParseDiagnostics(vc, doc.DocumentUri); - } - } catch (OperationCanceledException ex) { - LogMessage(MessageType.Warning, $"Parsing {doc.DocumentUri} cancelled"); - TraceMessage($"{ex}"); - } catch (Exception ex) { - LogMessage(MessageType.Error, ex.ToString()); - } finally { - disposeWhenEnqueued?.Dispose(); - } - } - - private PythonAst GetParseTree(IPythonProjectEntry entry, Uri documentUri, CancellationToken token, out BufferVersion bufferVersion) { - PythonAst tree = null; - bufferVersion = null; - var parse = entry.WaitForCurrentParse(Timeout.Infinite, token); - if (parse != null) { - tree = parse.Tree; - if (parse.Cookie is VersionCookie vc) { - if (vc.Versions.TryGetValue(ProjectFiles.GetPart(documentUri), out bufferVersion)) { - tree = bufferVersion.Ast ?? tree; - } - } - } - return tree; + : Resources.InitializingForPythonInterpreter.FormatInvariant(_interpreter.Configuration.InterpreterPath)); } private bool HandleConfigurationChanges(ServerSettings newSettings) { @@ -674,38 +170,15 @@ private bool HandleConfigurationChanges(ServerSettings newSettings) { return true; } - if (newSettings.analysis.openFilesOnly != oldSettings.analysis.openFilesOnly) { - _editorFiles.UpdateDiagnostics(); - return false; - } - if (!newSettings.analysis.errors.SetEquals(oldSettings.analysis.errors) || !newSettings.analysis.warnings.SetEquals(oldSettings.analysis.warnings) || !newSettings.analysis.information.SetEquals(oldSettings.analysis.information) || !newSettings.analysis.disabled.SetEquals(oldSettings.analysis.disabled)) { - _editorFiles.UpdateDiagnostics(); + return true; } return false; } #endregion - - internal Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken) - => Task.WhenAny(WaitForCompleteAnalysisWorker(cancellationToken), Task.Delay(Timeout.Infinite, cancellationToken)).Unwrap(); - - private async Task WaitForCompleteAnalysisWorker(CancellationToken cancellationToken) { - // Wait for all current parsing to complete - TraceMessage($"Waiting for parsing to complete."); - await ParseQueue.WaitForAllAsync(); - TraceMessage($"Parsing complete. Waiting for analysis entries to enqueue."); - await _pendingAnalysisEnqueue.WaitForZeroAsync(); - TraceMessage($"Enqueue complete. Waiting for analysis to complete."); - await AnalysisQueue.WaitForCompleteAsync(); - foreach (var pf in ProjectFiles) { - TraceMessage($" Waiting for analysis of {pf.DocumentUri} to complete."); - await GetAnalysisAsync(pf.DocumentUri, cancellationToken); - } - TraceMessage($"Analysis complete."); - } } } diff --git a/src/LanguageServer/Impl/Implementation/ServerBase.cs b/src/LanguageServer/Impl/Implementation/ServerBase.cs deleted file mode 100644 index 5d9bfddd9..000000000 --- a/src/LanguageServer/Impl/Implementation/ServerBase.cs +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Python.Core; -using Microsoft.Python.Core.Logging; - -namespace Microsoft.Python.LanguageServer.Implementation { - public abstract class ServerBase { - protected IServiceContainer Services { get; private set; } - public ILogger Logger { get; private set; } - - protected ServerBase(IServiceContainer services) { - SetServices(services); - } - - /// - /// For tests - /// - internal void SetServices(IServiceContainer services) { - Services = services; - Logger = services?.GetService(); - } - - #region Client Requests - - public abstract Task Initialize(InitializeParams @params, CancellationToken cancellationToken); - - public virtual Task Initialized(InitializedParams @params, CancellationToken cancellationToken) => Task.CompletedTask; - - public virtual Task Shutdown() => Task.CompletedTask; - - public virtual Task Exit() => Task.CompletedTask; - - public virtual void CancelRequest() { } // Does nothing - - public abstract Task DidChangeConfiguration(DidChangeConfigurationParams @params, CancellationToken cancellationToken); - - public virtual Task DidChangeWatchedFiles(DidChangeWatchedFilesParams @params, CancellationToken cancellationToken) - => Task.CompletedTask; - - public virtual Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) - => Task.FromResult(Array.Empty()); - - public virtual Task ExecuteCommand(ExecuteCommandParams @params, CancellationToken cancellationToken) - => Task.FromResult((object)null); - - public abstract Task DidOpenTextDocument(DidOpenTextDocumentParams @params, CancellationToken cancellationToken); - - public abstract Task DidChangeTextDocument(DidChangeTextDocumentParams @params, CancellationToken cancellationToken); - - public virtual Task WillSaveTextDocument(WillSaveTextDocumentParams @params, CancellationToken cancellationToken) - => Task.CompletedTask; - - public virtual Task WillSaveWaitUntilTextDocument(WillSaveTextDocumentParams @params, CancellationToken cancellationToken) - => Task.FromResult(Array.Empty()); - - public virtual Task DidSaveTextDocument(DidSaveTextDocumentParams @params, CancellationToken cancellationToken) - => Task.CompletedTask; - - public abstract Task DidCloseTextDocument(DidCloseTextDocumentParams @params, CancellationToken cancellationToken); - - public virtual Task Completion(CompletionParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task CompletionItemResolve(CompletionItem item, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task Hover(TextDocumentPositionParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task SignatureHelp(TextDocumentPositionParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task GotoDefinition(TextDocumentPositionParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task FindReferences(ReferencesParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task DocumentHighlight(TextDocumentPositionParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) - => Task.FromResult(Array.Empty()); - - public virtual Task CodeAction(CodeActionParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task CodeLens(TextDocumentPositionParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task CodeLensResolve(CodeLens item, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task DocumentLink(DocumentLinkParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task DocumentLinkResolve(DocumentLink item, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task DocumentFormatting(DocumentFormattingParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task DocumentRangeFormatting(DocumentRangeFormattingParams @params, CancellationToken cancellationToken) => throw new NotImplementedException(); - - public virtual Task DocumentOnTypeFormatting(DocumentOnTypeFormattingParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - - public virtual Task Rename(RenameParams @params, CancellationToken cancellationToken) - => throw new NotImplementedException(); - #endregion - - #region Server Requests - public void LogMessage(MessageType mt, string message) => Logger.Log(mt.ToTraceEventType(), message); - - public event EventHandler OnCommand; - public void Command(CommandEventArgs e) => OnCommand?.Invoke(this, e); - - public event EventHandler OnRegisterCapability; - public Task RegisterCapability(RegistrationParams @params) { - var evt = OnRegisterCapability; - if (evt == null) { - return Task.CompletedTask; - } - var tcs = new TaskCompletionSource(); - var e = new RegisterCapabilityEventArgs(tcs) { @params = @params }; - evt(this, e); - return tcs.Task; - } - - - public event EventHandler OnUnregisterCapability; - public Task UnregisterCapability(UnregistrationParams @params) { - var evt = OnUnregisterCapability; - if (evt == null) { - return Task.CompletedTask; - } - var tcs = new TaskCompletionSource(); - var e = new UnregisterCapabilityEventArgs(tcs) { @params = @params }; - evt(this, e); - return tcs.Task; - } - - public event EventHandler OnApplyWorkspaceEdit; - public Task ApplyWorkspaceEdit(ApplyWorkspaceEditParams @params) { - var evt = OnApplyWorkspaceEdit; - if (evt == null) { - return Task.FromResult((ApplyWorkspaceEditResponse)null); - } - var tcs = new TaskCompletionSource(); - var e = new ApplyWorkspaceEditEventArgs(tcs) { @params = @params }; - evt(this, e); - return tcs.Task; - } - - public event EventHandler OnPublishDiagnostics; - public void PublishDiagnostics(PublishDiagnosticsEventArgs e) => OnPublishDiagnostics?.Invoke(this, e); - - #endregion - - /// - /// Represents a disposable that does nothing on disposal. - /// - private sealed class EmptyDisposable : IDisposable { - /// - /// Singleton default disposable. - /// - public static EmptyDisposable Instance { get; } = new EmptyDisposable(); - - private EmptyDisposable() { } - - /// - /// Does nothing. - /// - public void Dispose() { } - } - } -} diff --git a/src/LanguageServer/Impl/Implementation/VolatileCounter.cs b/src/LanguageServer/Impl/Implementation/VolatileCounter.cs deleted file mode 100644 index a5c2cf581..000000000 --- a/src/LanguageServer/Impl/Implementation/VolatileCounter.cs +++ /dev/null @@ -1,106 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Diagnostics; -using System.Threading; -using System.Threading.Tasks; - -namespace Microsoft.Python.LanguageServer.Implementation { - sealed class VolatileCounter { - private int _count; - private TaskCompletionSource _waitForChange; - private TaskCompletionSource _waitForZero; - - public int Count => Volatile.Read(ref _count); - - public bool IsZero => Count == 0; - - private sealed class DecrementOnDispose : IDisposable { - public VolatileCounter _self; - public void Dispose() => Interlocked.Exchange(ref _self, null)?.Decrement(); - } - - public IDisposable Incremented() { - Increment(); - return new DecrementOnDispose { _self = this }; - } - - /// - /// Increments the counter and returns true if it changes to non-zero. - /// - public bool Increment() { - int newCount = Interlocked.Increment(ref _count); - Interlocked.Exchange(ref _waitForChange, null)?.SetResult(newCount); - return newCount == 1; - } - - /// - /// Decrements the counter and returns true if it changes to zero. - /// - public bool Decrement() { - int origCount, newCount; - do { - origCount = Volatile.Read(ref _count); - newCount = origCount - 1; - if (newCount < 0) { - Debug.Fail("mismatched decrement"); - return false; - } - } while (Interlocked.CompareExchange(ref _count, newCount, origCount) != origCount); - - Interlocked.Exchange(ref _waitForChange, null)?.SetResult(newCount); - if (newCount > 0) { - return false; - } - Interlocked.Exchange(ref _waitForZero, null)?.SetResult(null); - return true; - } - - public Task WaitForChangeAsync() { - var tcs = Volatile.Read(ref _waitForChange); - if (tcs == null) { - tcs = new TaskCompletionSource(); - tcs = Interlocked.CompareExchange(ref _waitForChange, tcs, null) ?? tcs; - } - return tcs.Task; - } - - public Task WaitForZeroAsync() { - if (IsZero) { - return Task.CompletedTask; - } - var tcs = Volatile.Read(ref _waitForZero); - if (tcs == null) { - tcs = new TaskCompletionSource(); - tcs = Interlocked.CompareExchange(ref _waitForZero, tcs, null) ?? tcs; - } - if (IsZero) { - tcs.TrySetResult(null); - } - return tcs.Task; - } - - public async Task WaitForChangeToZeroAsync() { - var tcs = Volatile.Read(ref _waitForZero); - if (tcs == null) { - tcs = new TaskCompletionSource(); - tcs = Interlocked.CompareExchange(ref _waitForZero, tcs, null) ?? tcs; - } - await tcs.Task; - } - } -} diff --git a/src/LanguageServer/Impl/LanguageServer.Lifetime.cs b/src/LanguageServer/Impl/LanguageServer.Lifetime.cs index dd47fca46..5eae32578 100644 --- a/src/LanguageServer/Impl/LanguageServer.Lifetime.cs +++ b/src/LanguageServer/Impl/LanguageServer.Lifetime.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -16,15 +15,10 @@ using System; using System.Diagnostics; -using System.IO; -using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.FileSystemGlobbing; -using Microsoft.Extensions.FileSystemGlobbing.Abstractions; -using Microsoft.Python.Analysis.Core.Interpreter; using Microsoft.Python.Core; -using Microsoft.Python.Core.IO; +using Microsoft.Python.LanguageServer.Protocol; using Newtonsoft.Json.Linq; using StreamJsonRpc; @@ -38,25 +32,16 @@ public partial class LanguageServer { public async Task Initialize(JToken token, CancellationToken cancellationToken) { _initParams = token.ToObject(); MonitorParentProcess(_initParams); - var priorityToken = await _prioritizer.InitializePriorityAsync(cancellationToken); - if (_initParams.initializationOptions.asyncStartup) { - _server.Initialize(_initParams, cancellationToken) - .ContinueWith(DisposeStateContinuation, priorityToken, CancellationToken.None, TaskContinuationOptions.ExecuteSynchronously, TaskScheduler.Default) - .DoNotWait(); - return _server.GetInitializeResult(); - } - - try { - return await _server.Initialize(_initParams, cancellationToken); - } finally { - priorityToken.Dispose(); + using (await _prioritizer.InitializePriorityAsync(cancellationToken)) { + return await _server.InitializeAsync(_initParams, cancellationToken); } } [JsonRpcMethod("initialized")] - public async Task Initialized(JToken token, CancellationToken cancellationToken) { - await _server.Initialized(ToObject(token), cancellationToken); + public Task Initialized(JToken token, CancellationToken cancellationToken) { + //await _server.Initialized(ToObject(token), cancellationToken); _rpc.NotifyAsync("python/languageServerStarted").DoNotWait(); + return Task.CompletedTask; } [JsonRpcMethod("shutdown")] @@ -68,56 +53,13 @@ public async Task Shutdown() { } [JsonRpcMethod("exit")] - public async Task Exit() { - await _server.Exit(); + public void Exit() { + _server.Dispose(); _sessionTokenSource.Cancel(); // Per https://microsoft.github.io/language-server-protocol/specification#exit Environment.Exit(_shutdown ? 0 : 1); } - private Task LoadDirectoryFiles() { - string rootDir = null; - if (_initParams.rootUri != null) { - rootDir = _initParams.rootUri.ToAbsolutePath(); - } else if (!string.IsNullOrEmpty(_initParams.rootPath)) { - rootDir = PathUtils.NormalizePath(_initParams.rootPath); - } - - if (string.IsNullOrEmpty(rootDir)) { - return Task.CompletedTask; - } - - var matcher = new Matcher(); - var included = _initParams.initializationOptions.includeFiles; - matcher.AddIncludePatterns(included != null && included.Length > 0 ? included : new[] { "**/*" }); - matcher.AddExcludePatterns(_initParams.initializationOptions.excludeFiles ?? Enumerable.Empty()); - - var dib = new DirectoryInfoWrapper(new DirectoryInfo(rootDir)); - var matchResult = matcher.Execute(dib); - - _server.LogMessage(MessageType.Log, $"Loading files from {rootDir}"); - return LoadFromDirectoryAsync(rootDir, matchResult); - } - - private async Task LoadFromDirectoryAsync(string rootDir, PatternMatchingResult matchResult) { - using (_server.AnalysisQueue.Pause()) { - foreach (var file in matchResult.Files) { - if (_sessionTokenSource.IsCancellationRequested) { - break; - } - - var path = Path.Combine(rootDir, PathUtils.NormalizePath(file.Path)); - if (!ModulePath.IsPythonSourceFile(path)) { - if (ModulePath.IsPythonFile(path, true, true, true)) { - // TODO: Deal with scrapable files (if we need to do anything?) - } - continue; - } - await _server.LoadFileAsync(new Uri(path)); - } - } - } - private void MonitorParentProcess(InitializeParams p) { // Monitor parent process Process parentProcess = null; @@ -143,11 +85,5 @@ private void MonitorParentProcess(InitializeParams p) { }).DoNotWait(); } } - - private async Task IfTestWaitForAnalysisCompleteAsync() { - if (_initParams.initializationOptions.testEnvironment) { - await _server.WaitForCompleteAnalysisAsync(_shutdownCts.Token); - } - } } } diff --git a/src/LanguageServer/Impl/LanguageServer.cs b/src/LanguageServer/Impl/LanguageServer.cs index 716ea25d8..d856906c4 100644 --- a/src/LanguageServer/Impl/LanguageServer.cs +++ b/src/LanguageServer/Impl/LanguageServer.cs @@ -19,6 +19,8 @@ using System.Diagnostics; using System.Threading; using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Idle; @@ -26,7 +28,8 @@ using Microsoft.Python.Core.Shell; using Microsoft.Python.Core.Text; using Microsoft.Python.Core.Threading; -using Microsoft.Python.LanguageServer.Diagnostics; +using Microsoft.Python.LanguageServer.Extensibility; +using Microsoft.Python.LanguageServer.Protocol; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using StreamJsonRpc; @@ -40,7 +43,6 @@ namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class LanguageServer : IDisposable { private readonly DisposableBag _disposables = new DisposableBag(nameof(LanguageServer)); private readonly CancellationTokenSource _sessionTokenSource = new CancellationTokenSource(); - private readonly object _lock = new object(); private readonly Prioritizer _prioritizer = new Prioritizer(); private readonly CancellationTokenSource _shutdownCts = new CancellationTokenSource(); @@ -51,15 +53,13 @@ public sealed partial class LanguageServer : IDisposable { private JsonRpc _rpc; private JsonSerializer _jsonSerializer; - private bool _filesLoaded; private PathsWatcher _pathsWatcher; private IIdleTimeTracker _idleTimeTracker; - private DiagnosticsPublisher _diagnosticsPublisher; private bool _watchSearchPaths; private string[] _searchPaths = Array.Empty(); - public CancellationToken Start(IServiceContainer services, JsonRpc rpc) { + public CancellationToken Start(IServiceManager services, JsonRpc rpc) { _server = new Server(services); _services = services; _rpc = rpc; @@ -72,22 +72,11 @@ public CancellationToken Start(IServiceContainer services, JsonRpc rpc) { var rpcTraceListener = new TelemetryRpcTraceListener(_telemetry); _rpc.TraceSource.Listeners.Add(rpcTraceListener); - _diagnosticsPublisher = new DiagnosticsPublisher(_server, services); - _server.OnApplyWorkspaceEdit += OnApplyWorkspaceEdit; - _server.OnRegisterCapability += OnRegisterCapability; - _server.OnUnregisterCapability += OnUnregisterCapability; - _server.AnalysisQueue.UnhandledException += OnAnalysisQueueUnhandledException; - _disposables - .Add(() => _server.OnApplyWorkspaceEdit -= OnApplyWorkspaceEdit) - .Add(() => _server.OnRegisterCapability -= OnRegisterCapability) - .Add(() => _server.OnUnregisterCapability -= OnUnregisterCapability) - .Add(() => _server.AnalysisQueue.UnhandledException -= OnAnalysisQueueUnhandledException) .Add(() => _shutdownCts.Cancel()) .Add(_prioritizer) .Add(() => _pathsWatcher?.Dispose()) - .Add(() => _rpc.TraceSource.Listeners.Remove(rpcTraceListener)) - .Add(_diagnosticsPublisher); + .Add(() => _rpc.TraceSource.Listeners.Remove(rpcTraceListener)); return _sessionTokenSource.Token; } @@ -97,14 +86,8 @@ public void Dispose() { _server.Dispose(); } - #region Events private void OnApplyWorkspaceEdit(object sender, ApplyWorkspaceEditEventArgs e) => _rpc.NotifyWithParameterObjectAsync("workspace/applyEdit", e.@params).DoNotWait(); - private void OnRegisterCapability(object sender, RegisterCapabilityEventArgs e) - => _rpc.NotifyWithParameterObjectAsync("client/registerCapability", e.@params).DoNotWait(); - private void OnUnregisterCapability(object sender, UnregisterCapabilityEventArgs e) - => _rpc.NotifyWithParameterObjectAsync("client/unregisterCapability", e.@params).DoNotWait(); - #endregion #region Workspace [JsonRpcMethod("workspace/didChangeConfiguration")] @@ -123,15 +106,16 @@ public async Task DidChangeConfiguration(JToken token, CancellationToken cancell settings.completion.addBrackets = GetSetting(autoComplete, "addBrackets", false); var analysis = pythonSection["analysis"]; - settings.analysis.openFilesOnly = GetSetting(analysis, "openFilesOnly", false); settings.diagnosticPublishDelay = GetSetting(analysis, "diagnosticPublishDelay", 1000); settings.symbolsHierarchyDepthLimit = GetSetting(analysis, "symbolsHierarchyDepthLimit", 10); settings.symbolsHierarchyMaxSymbols = GetSetting(analysis, "symbolsHierarchyMaxSymbols", 1000); _logger.LogLevel = GetLogLevel(analysis).ToTraceEventType(); - _diagnosticsPublisher.PublishingDelay = settings.diagnosticPublishDelay; - HandlePathWatchChange(token); + var ds = _services.GetService(); + ds.PublishingDelay = settings.diagnosticPublishDelay; + + HandlePathWatchChange(token, cancellationToken); var errors = GetSetting(analysis, "errors", Array.Empty()); var warnings = GetSetting(analysis, "warnings", Array.Empty()); @@ -140,18 +124,13 @@ public async Task DidChangeConfiguration(JToken token, CancellationToken cancell settings.analysis.SetErrorSeverityOptions(errors, warnings, information, disabled); await _server.DidChangeConfiguration(new DidChangeConfigurationParams { settings = settings }, cancellationToken); - - if (!_filesLoaded) { - await LoadDirectoryFiles(); - } - _filesLoaded = true; } } [JsonRpcMethod("workspace/didChangeWatchedFiles")] public async Task DidChangeWatchedFiles(JToken token, CancellationToken cancellationToken) { using (await _prioritizer.DocumentChangePriorityAsync(cancellationToken)) { - await _server.DidChangeWatchedFiles(ToObject(token), cancellationToken); + _server.DidChangeWatchedFiles(ToObject(token)); } } @@ -164,9 +143,9 @@ public async Task WorkspaceSymbols(JToken token, Cancellati #endregion #region Commands - [JsonRpcMethod("workspace/executeCommand")] - public Task ExecuteCommand(JToken token, CancellationToken cancellationToken) - => _server.ExecuteCommand(ToObject(token), cancellationToken); + //[JsonRpcMethod("workspace/executeCommand")] + //public Task ExecuteCommand(JToken token, CancellationToken cancellationToken) + // => _server.ExecuteCommandAsync(ToObject(token), cancellationToken); #endregion #region TextDocument @@ -174,27 +153,27 @@ public Task ExecuteCommand(JToken token, CancellationToken cancellationT public async Task DidOpenTextDocument(JToken token, CancellationToken cancellationToken) { _idleTimeTracker?.NotifyUserActivity(); using (await _prioritizer.DocumentChangePriorityAsync(cancellationToken)) { - await _server.DidOpenTextDocument(ToObject(token), cancellationToken); + _server.DidOpenTextDocument(ToObject(token)); } } [JsonRpcMethod("textDocument/didChange")] public async Task DidChangeTextDocument(JToken token, CancellationToken cancellationToken) { _idleTimeTracker?.NotifyUserActivity(); - using (await _prioritizer.DocumentChangePriorityAsync()) { + using (await _prioritizer.DocumentChangePriorityAsync(cancellationToken)) { var @params = ToObject(token); var version = @params.textDocument.version; if (version == null || @params.contentChanges.IsNullOrEmpty()) { - await _server.DidChangeTextDocument(@params, cancellationToken); + _server.DidChangeTextDocument(@params); return; } - // _server.DidChangeTextDocument can handle change buckets with decreasing version and without overlaping + // _server.DidChangeTextDocument can handle change buckets with decreasing version and without overlapping // Split change into buckets that will be properly handled var changes = SplitDidChangeTextDocumentParams(@params, version.Value); foreach (var change in changes) { - await _server.DidChangeTextDocument(change, cancellationToken); + _server.DidChangeTextDocument(change); } } } @@ -235,25 +214,19 @@ private static DidChangeTextDocumentParams CreateDidChangeTextDocumentParams(Did }; [JsonRpcMethod("textDocument/willSave")] - public Task WillSaveTextDocument(JToken token, CancellationToken cancellationToken) - => _server.WillSaveTextDocument(ToObject(token), cancellationToken); + public void WillSaveTextDocument(JToken token) { } - public Task WillSaveWaitUntilTextDocument(JToken token, CancellationToken cancellationToken) - => _server.WillSaveWaitUntilTextDocument(ToObject(token), cancellationToken); + [JsonRpcMethod("textDocument/willSaveWaitUntilTextDocument")] + public TextEdit[] WillSaveWaitUntilTextDocument(JToken token) => Array.Empty(); [JsonRpcMethod("textDocument/didSave")] - public async Task DidSaveTextDocument(JToken token, CancellationToken cancellationToken) { - _idleTimeTracker?.NotifyUserActivity(); - using (await _prioritizer.DocumentChangePriorityAsync(cancellationToken)) { - await _server.DidSaveTextDocument(ToObject(token), cancellationToken); - } - } + public void DidSaveTextDocument(JToken token) => _idleTimeTracker?.NotifyUserActivity(); [JsonRpcMethod("textDocument/didClose")] public async Task DidCloseTextDocument(JToken token, CancellationToken cancellationToken) { _idleTimeTracker?.NotifyUserActivity(); using (await _prioritizer.DocumentChangePriorityAsync(cancellationToken)) { - await _server.DidCloseTextDocument(ToObject(token), cancellationToken); + _server.DidCloseTextDocument(ToObject(token)); } } #endregion @@ -262,98 +235,98 @@ public async Task DidCloseTextDocument(JToken token, CancellationToken cancellat [JsonRpcMethod("textDocument/completion")] public async Task Completion(JToken token, CancellationToken cancellationToken) { await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.Completion(ToObject(token), cancellationToken); + return await _server.Completion(ToObject(token), GetToken(cancellationToken)); } [JsonRpcMethod("completionItem/resolve")] public Task CompletionItemResolve(JToken token, CancellationToken cancellationToken) - => _server.CompletionItemResolve(ToObject(token), cancellationToken); + => _server.CompletionItemResolve(ToObject(token), GetToken(cancellationToken)); [JsonRpcMethod("textDocument/hover")] public async Task Hover(JToken token, CancellationToken cancellationToken) { await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.Hover(ToObject(token), cancellationToken); + return await _server.Hover(ToObject(token), GetToken(cancellationToken)); } [JsonRpcMethod("textDocument/signatureHelp")] public async Task SignatureHelp(JToken token, CancellationToken cancellationToken) { await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.SignatureHelp(ToObject(token), cancellationToken); + return await _server.SignatureHelp(ToObject(token), GetToken(cancellationToken)); } [JsonRpcMethod("textDocument/definition")] public async Task GotoDefinition(JToken token, CancellationToken cancellationToken) { await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.GotoDefinition(ToObject(token), cancellationToken); + return await _server.GotoDefinition(ToObject(token), GetToken(cancellationToken)); } [JsonRpcMethod("textDocument/references")] public async Task FindReferences(JToken token, CancellationToken cancellationToken) { await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.FindReferences(ToObject(token), cancellationToken); + return await _server.FindReferences(ToObject(token), GetToken(cancellationToken)); } - [JsonRpcMethod("textDocument/documentHighlight")] - public async Task DocumentHighlight(JToken token, CancellationToken cancellationToken) { - await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.DocumentHighlight(ToObject(token), cancellationToken); - } + //[JsonRpcMethod("textDocument/documentHighlight")] + //public async Task DocumentHighlight(JToken token, CancellationToken cancellationToken) { + // await _prioritizer.DefaultPriorityAsync(cancellationToken); + // return await _server.DocumentHighlight(ToObject(token), cancellationToken); + //} [JsonRpcMethod("textDocument/documentSymbol")] public async Task DocumentSymbol(JToken token, CancellationToken cancellationToken) { await _prioritizer.DefaultPriorityAsync(cancellationToken); // This call is also used by VSC document outline and it needs correct information - return await _server.HierarchicalDocumentSymbol(ToObject(token), cancellationToken); - } - - [JsonRpcMethod("textDocument/codeAction")] - public async Task CodeAction(JToken token, CancellationToken cancellationToken) { - await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.CodeAction(ToObject(token), cancellationToken); - } - - [JsonRpcMethod("textDocument/codeLens")] - public async Task CodeLens(JToken token, CancellationToken cancellationToken) { - await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.CodeLens(ToObject(token), cancellationToken); - } - - [JsonRpcMethod("codeLens/resolve")] - public Task CodeLensResolve(JToken token, CancellationToken cancellationToken) - => _server.CodeLensResolve(ToObject(token), cancellationToken); - - [JsonRpcMethod("textDocument/documentLink")] - public async Task DocumentLink(JToken token, CancellationToken cancellationToken) { - await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.DocumentLink(ToObject(token), cancellationToken); - } - - [JsonRpcMethod("documentLink/resolve")] - public Task DocumentLinkResolve(JToken token, CancellationToken cancellationToken) - => _server.DocumentLinkResolve(ToObject(token), cancellationToken); - - [JsonRpcMethod("textDocument/formatting")] - public async Task DocumentFormatting(JToken token, CancellationToken cancellationToken) { - await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.DocumentFormatting(ToObject(token), cancellationToken); + return await _server.HierarchicalDocumentSymbol(ToObject(token), GetToken(cancellationToken)); } - [JsonRpcMethod("textDocument/rangeFormatting")] - public async Task DocumentRangeFormatting(JToken token, CancellationToken cancellationToken) { - await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.DocumentRangeFormatting(ToObject(token), cancellationToken); - } + //[JsonRpcMethod("textDocument/codeAction")] + //public async Task CodeAction(JToken token, CancellationToken cancellationToken) { + // await _prioritizer.DefaultPriorityAsync(cancellationToken); + // return await _server.CodeAction(ToObject(token), cancellationToken); + //} + + //[JsonRpcMethod("textDocument/codeLens")] + //public async Task CodeLens(JToken token, CancellationToken cancellationToken) { + // await _prioritizer.DefaultPriorityAsync(cancellationToken); + // return await _server.CodeLens(ToObject(token), cancellationToken); + //} + + //[JsonRpcMethod("codeLens/resolve")] + //public Task CodeLensResolve(JToken token, CancellationToken cancellationToken) + // => _server.CodeLensResolve(ToObject(token), cancellationToken); + + //[JsonRpcMethod("textDocument/documentLink")] + //public async Task DocumentLink(JToken token, CancellationToken cancellationToken) { + // await _prioritizer.DefaultPriorityAsync(cancellationToken); + // return await _server.DocumentLink(ToObject(token), cancellationToken); + //} + + //[JsonRpcMethod("documentLink/resolve")] + //public Task DocumentLinkResolve(JToken token, CancellationToken cancellationToken) + // => _server.DocumentLinkResolve(ToObject(token), cancellationToken); + + //[JsonRpcMethod("textDocument/formatting")] + //public async Task DocumentFormatting(JToken token, CancellationToken cancellationToken) { + // await _prioritizer.DefaultPriorityAsync(cancellationToken); + // return await _server.DocumentFormatting(ToObject(token), cancellationToken); + //} + + //[JsonRpcMethod("textDocument/rangeFormatting")] + //public async Task DocumentRangeFormatting(JToken token, CancellationToken cancellationToken) { + // await _prioritizer.DefaultPriorityAsync(cancellationToken); + // return await _server.DocumentRangeFormatting(ToObject(token), cancellationToken); + //} [JsonRpcMethod("textDocument/onTypeFormatting")] public async Task DocumentOnTypeFormatting(JToken token, CancellationToken cancellationToken) { await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.DocumentOnTypeFormatting(ToObject(token), cancellationToken); + return await _server.DocumentOnTypeFormatting(ToObject(token), GetToken(cancellationToken)); } [JsonRpcMethod("textDocument/rename")] public async Task Rename(JToken token, CancellationToken cancellationToken) { await _prioritizer.DefaultPriorityAsync(cancellationToken); - return await _server.Rename(ToObject(token), cancellationToken); + return await _server.Rename(ToObject(token), GetToken(cancellationToken)); } #endregion @@ -362,6 +335,9 @@ public async Task Rename(JToken token, CancellationToken cancella public Task LoadExtension(JToken token, CancellationToken cancellationToken) => _server.LoadExtensionAsync(ToObject(token), _services, cancellationToken); + [JsonRpcMethod("python/extensionCommand")] + public Task ExtensionCommand(JToken token, CancellationToken cancellationToken) + => _server.ExtensionCommandAsync(ToObject(token), cancellationToken); #endregion private T ToObject(JToken token) => token.ToObject(_jsonSerializer); @@ -371,7 +347,7 @@ private T GetSetting(JToken section, string settingName, T defaultValue) { try { return value != null ? value.ToObject() : defaultValue; } catch (JsonException ex) { - _server.LogMessage(MessageType.Warning, $"Exception retrieving setting '{settingName}': {ex.Message}"); + _logger?.Log(TraceEventType.Warning, $"Exception retrieving setting '{settingName}': {ex.Message}"); } return defaultValue; } @@ -390,7 +366,7 @@ private MessageType GetLogLevel(JToken analysisKey) { return MessageType.Error; } - private void HandlePathWatchChange(JToken section) { + private void HandlePathWatchChange(JToken section, CancellationToken cancellationToken) { var watchSearchPaths = GetSetting(section, "watchSearchPaths", true); if (!watchSearchPaths) { // No longer watching. @@ -404,26 +380,20 @@ private void HandlePathWatchChange(JToken section) { if (!_watchSearchPaths || (_watchSearchPaths && _searchPaths.SetEquals(_initParams.initializationOptions.searchPaths))) { // Were not watching OR were watching but paths have changed. Recreate the watcher. _pathsWatcher?.Dispose(); + var interpreter = _services.GetService(); _pathsWatcher = new PathsWatcher( _initParams.initializationOptions.searchPaths, - () => _server.ReloadModulesAsync(CancellationToken.None).DoNotWait(), + () => interpreter.ModuleResolution.ReloadAsync(cancellationToken).DoNotWait(), _services.GetService() ); } - _watchSearchPaths = watchSearchPaths; + _watchSearchPaths = true; _searchPaths = _initParams.initializationOptions.searchPaths; } - private void OnAnalysisQueueUnhandledException(object sender, UnhandledExceptionEventArgs e) { - if (!(e.ExceptionObject is Exception ex)) { - Debug.Fail($"ExceptionObject was {e.ExceptionObject.GetType()}, not Exception"); - return; - } - - var te = Telemetry.CreateEventWithException("analysis_queue.unhandled_exception", ex); - _telemetry.SendTelemetryAsync(te).DoNotWait(); - } + private static CancellationToken GetToken(CancellationToken original) + => Debugger.IsAttached ? CancellationToken.None : original; private class Prioritizer : IDisposable { private const int InitializePriority = 0; diff --git a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj index de6dba4a4..a462754b9 100644 --- a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj +++ b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj @@ -10,7 +10,9 @@ Exe portable true - + 1701;1702;$(NoWarn) + 7.2 + ..\..\PLS.ruleset @@ -31,9 +33,6 @@ - - - $(AnalysisReference)/Microsoft.Python.Analysis.Engine.dll @@ -43,6 +42,7 @@ + diff --git a/src/LanguageServer/Impl/Properties/AssemblyInfo.cs b/src/LanguageServer/Impl/Properties/AssemblyInfo.cs index d876237fb..fbfb7c32c 100644 --- a/src/LanguageServer/Impl/Properties/AssemblyInfo.cs +++ b/src/LanguageServer/Impl/Properties/AssemblyInfo.cs @@ -16,12 +16,4 @@ using System.Runtime.CompilerServices; -[assembly: InternalsVisibleTo("Microsoft.PythonTools.TestAdapter, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.TestAdapter.Analysis, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.PythonTools.TestAdapter.Executor, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("AnalysisTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("PythonToolsTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestUtilities, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestUtilities.Python, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("TestUtilities.Python.Analysis, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] -[assembly: InternalsVisibleTo("Microsoft.Python.Analysis.Engine.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] +[assembly: InternalsVisibleTo("Microsoft.Python.LanguageServer.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/LanguageServer/Impl/Definitions/Structures.cs b/src/LanguageServer/Impl/Protocol/Classes.cs similarity index 71% rename from src/LanguageServer/Impl/Definitions/Structures.cs rename to src/LanguageServer/Impl/Protocol/Classes.cs index 635b28082..a10b50ba7 100644 --- a/src/LanguageServer/Impl/Definitions/Structures.cs +++ b/src/LanguageServer/Impl/Protocol/Classes.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -16,30 +15,25 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using Microsoft.Python.Core.Text; -using Microsoft.PythonTools.Analysis.Documentation; +using Newtonsoft.Json; -namespace Microsoft.Python.LanguageServer { +namespace Microsoft.Python.LanguageServer.Protocol { [Serializable] - public struct ResponseError { + public sealed class ResponseError { public int code; public string message; } - public struct ResponseError { - public int code; - public string message; - public T data; - } - [Serializable] - public struct Location { + public sealed class Location { public Uri uri; public Range range; } [Serializable] - public struct Command { + public sealed class Command { /// /// Title of the command, like `save`. /// @@ -57,7 +51,7 @@ public struct Command { } [Serializable] - public struct TextEdit { + public sealed class TextEdit { /// /// The range of the text document to be manipulated. To insert /// text into a document create a range where start === end. @@ -79,26 +73,26 @@ public struct TextEdit { } [Serializable] - public struct TextDocumentEdit { + public sealed class TextDocumentEdit { public VersionedTextDocumentIdentifier textDocument; public TextEdit[] edits; } [Serializable] - public struct WorkspaceEdit { + public sealed class WorkspaceEdit { public Dictionary changes; public TextDocumentEdit[] documentChanges; } [Serializable] - public struct TextDocumentIdentifier { + public sealed class TextDocumentIdentifier { public Uri uri; public static implicit operator TextDocumentIdentifier(Uri uri) => new TextDocumentIdentifier { uri = uri }; } [Serializable] - public struct TextDocumentItem { + public sealed class TextDocumentItem { public Uri uri; public string languageId; public int version; @@ -106,14 +100,13 @@ public struct TextDocumentItem { } [Serializable] - public struct VersionedTextDocumentIdentifier { + public sealed class VersionedTextDocumentIdentifier { public Uri uri; public int? version; - public int? _fromVersion; } [Serializable] - public struct DocumentFilter { + public sealed class DocumentFilter { /// /// A language id, like `typescript`. /// @@ -134,7 +127,7 @@ public struct DocumentFilter { /// Required layout for the initializationOptions member of initializeParams /// [Serializable] - public class PythonInitializationOptions { + public sealed class PythonInitializationOptions { [Serializable] public struct Interpreter { /// @@ -158,20 +151,10 @@ public struct Interpreter { public string[] searchPaths = Array.Empty(); /// - /// Secondary paths to search when resolving modules. Not supported by all - /// factories. In generaly, only source files will be discovered, and their - /// contents will be merged with the initial module. + /// Paths to search for module stubs. /// public string[] typeStubSearchPaths = Array.Empty(); - /// - /// Indicates that analysis engine is running in a test environment. - /// Causes initialization and analysis sequences to fully - /// complete before information requests such as hover or - /// completion can be processed. - /// - public bool testEnvironment; - /// /// Controls tooltip display appearance. Different between VS and VS Code. /// @@ -189,26 +172,15 @@ public struct Interpreter { /// public string[] includeFiles = Array.Empty(); - /// - /// Client expects analysis progress updates, including notifications - /// when analysis is complete for a particular document version. - /// - public bool analysisUpdates; - /// /// Enables an even higher level of logging via the logMessage event. /// This will likely have a performance impact. /// public bool traceLogging; - - /// - /// If true, analyzer will be created asynchronously. Used in VS Code. - /// - public bool asyncStartup; } [Serializable] - public class WorkspaceClientCapabilities { + public sealed class WorkspaceClientCapabilities { public bool applyEdit; public struct WorkspaceEditCapabilities { public bool documentChanges; } @@ -264,11 +236,11 @@ public struct SynchronizationCapabilities { public SynchronizationCapabilities? synchronization; [Serializable] - public struct CompletionCapabilities { + public sealed class CompletionCapabilities { public bool dynamicRegistration; [Serializable] - public struct CompletionItemCapabilities { + public sealed class CompletionItemCapabilities { /// /// Client supports snippets as insert text. /// @@ -283,10 +255,10 @@ public struct CompletionItemCapabilities { public string[] documentationFormat; } - public CompletionItemCapabilities? completionItem; + public CompletionItemCapabilities completionItem; [Serializable] - public struct CompletionItemKindCapabilities { + public sealed class CompletionItemKindCapabilities { /// /// The completion item kind values the client supports. When this /// property exists the client also guarantees that it will @@ -299,7 +271,7 @@ public struct CompletionItemKindCapabilities { /// public SymbolKind[] valueSet; } - public CompletionItemKindCapabilities? completionItemKind; + public CompletionItemKindCapabilities completionItemKind; /// /// The client supports to send additional context information for a @@ -307,10 +279,10 @@ public struct CompletionItemKindCapabilities { /// public bool contextSupport; } - public CompletionCapabilities? completion; + public CompletionCapabilities completion; [Serializable] - public struct HoverCapabilities { + public sealed class HoverCapabilities { public bool dynamicRegistration; /// /// Client supports the follow content formats for the content @@ -318,10 +290,10 @@ public struct HoverCapabilities { /// public string[] contentFormat; } - public HoverCapabilities? hover; + public HoverCapabilities hover; [Serializable] - public struct SignatureHelpCapabilities { + public sealed class SignatureHelpCapabilities { public bool dynamicRegistration; public struct SignatureInformationCapabilities { @@ -340,20 +312,20 @@ public struct SignatureInformationCapabilities { } public SignatureInformationCapabilities? signatureInformation; } - public SignatureHelpCapabilities? signatureHelp; + public SignatureHelpCapabilities signatureHelp; [Serializable] - public struct ReferencesCapabilities { public bool dynamicRegistration; } - public ReferencesCapabilities? references; + public sealed class ReferencesCapabilities { public bool dynamicRegistration; } + public ReferencesCapabilities references; [Serializable] - public struct DocumentHighlightCapabilities { public bool dynamicRegistration; } - public DocumentHighlightCapabilities? documentHighlight; + public sealed class DocumentHighlightCapabilities { public bool dynamicRegistration; } + public DocumentHighlightCapabilities documentHighlight; [Serializable] - public struct DocumentSymbolCapabilities { + public sealed class DocumentSymbolCapabilities { public bool dynamicRegistration; - public struct SymbolKindCapabilities { + public sealed class SymbolKindCapabilities { /// /// The symbol kind values the client supports. When this /// property exists the client also guarantees that it will @@ -366,74 +338,55 @@ public struct SymbolKindCapabilities { /// public SymbolKind[] valueSet; } - public SymbolKindCapabilities? symbolKind; + public SymbolKindCapabilities symbolKind; /// /// The client support hierarchical document symbols. /// public bool? hierarchicalDocumentSymbolSupport; } - public DocumentSymbolCapabilities? documentSymbol; + public DocumentSymbolCapabilities documentSymbol; [Serializable] - public struct FormattingCapabilities { public bool dynamicRegistration; } - public FormattingCapabilities? formatting; + public sealed class FormattingCapabilities { public bool dynamicRegistration; } + public FormattingCapabilities formatting; [Serializable] - public struct RangeFormattingCapabilities { public bool dynamicRegistration; } - public RangeFormattingCapabilities? rangeFormatting; + public sealed class RangeFormattingCapabilities { public bool dynamicRegistration; } + public RangeFormattingCapabilities rangeFormatting; [Serializable] - public struct OnTypeFormattingCapabilities { public bool dynamicRegistration; } - public OnTypeFormattingCapabilities? onTypeFormatting; + public sealed class OnTypeFormattingCapabilities { public bool dynamicRegistration; } + public OnTypeFormattingCapabilities onTypeFormatting; - public struct DefinitionCapabilities { public bool dynamicRegistration; } - public DefinitionCapabilities? definition; + public sealed class DefinitionCapabilities { public bool dynamicRegistration; } + public DefinitionCapabilities definition; [Serializable] - public struct CodeActionCapabilities { public bool dynamicRegistration; } - public CodeActionCapabilities? codeAction; + public sealed class CodeActionCapabilities { public bool dynamicRegistration; } + public CodeActionCapabilities codeAction; [Serializable] - public struct CodeLensCapabilities { public bool dynamicRegistration; } - public CodeLensCapabilities? codeLens; + public sealed class CodeLensCapabilities { public bool dynamicRegistration; } + public CodeLensCapabilities codeLens; [Serializable] - public struct DocumentLinkCapabilities { public bool dynamicRegistration; } - public DocumentLinkCapabilities? documentLink; + public sealed class DocumentLinkCapabilities { public bool dynamicRegistration; } + public DocumentLinkCapabilities documentLink; [Serializable] - public struct RenameCapabilities { public bool dynamicRegistration; } - public RenameCapabilities? rename; + public sealed class RenameCapabilities { public bool dynamicRegistration; } + public RenameCapabilities rename; } - /// - /// This struct is for Python-specific extensions. It is included with - /// client capabilities following the specification for extra settings. - /// [Serializable] - public class PythonClientCapabilities { - /// - /// Disables automatic analysis of all files under the root URI. - /// - public bool? manualFileLoad; - - /// - /// Enables rich diagnostics from code analysis. - /// - public bool? liveLinting; - } - - [Serializable] - public class ClientCapabilities { + public sealed class ClientCapabilities { public WorkspaceClientCapabilities workspace; public TextDocumentClientCapabilities textDocument; - public object experimental; - public PythonClientCapabilities python; } [Serializable] - public struct CompletionOptions { + public sealed class CompletionOptions { /// /// The server provides support to resolve additional /// information for a completion item. @@ -446,7 +399,7 @@ public struct CompletionOptions { } [Serializable] - public struct SignatureHelpOptions { + public sealed class SignatureHelpOptions { /// /// The characters that trigger signature help /// automatically. @@ -455,33 +408,33 @@ public struct SignatureHelpOptions { } [Serializable] - public struct CodeLensOptions { + public sealed class CodeLensOptions { public bool resolveProvider; } [Serializable] - public struct DocumentOnTypeFormattingOptions { + public sealed class DocumentOnTypeFormattingOptions { public string firstTriggerCharacter; public string[] moreTriggerCharacter; } [Serializable] - public struct DocumentLinkOptions { + public sealed class DocumentLinkOptions { public bool resolveProvider; } [Serializable] - public struct ExecuteCommandOptions { + public sealed class ExecuteCommandOptions { public string[] commands; } [Serializable] - public class SaveOptions { + public sealed class SaveOptions { public bool includeText; } [Serializable] - public class TextDocumentSyncOptions { + public sealed class TextDocumentSyncOptions { /// /// Open and close notifications are sent to the server. /// @@ -493,34 +446,34 @@ public class TextDocumentSyncOptions { } [Serializable] - public struct ServerCapabilities { + public sealed class ServerCapabilities { public TextDocumentSyncOptions textDocumentSync; public bool hoverProvider; - public CompletionOptions? completionProvider; - public SignatureHelpOptions? signatureHelpProvider; + public CompletionOptions completionProvider; + public SignatureHelpOptions signatureHelpProvider; public bool definitionProvider; public bool referencesProvider; public bool documentHighlightProvider; public bool documentSymbolProvider; public bool workspaceSymbolProvider; public bool codeActionProvider; - public CodeLensOptions? codeLensProvider; + public CodeLensOptions codeLensProvider; public bool documentFormattingProvider; public bool documentRangeFormattingProvider; - public DocumentOnTypeFormattingOptions? documentOnTypeFormattingProvider; + public DocumentOnTypeFormattingOptions documentOnTypeFormattingProvider; public bool renameProvider; - public DocumentLinkOptions? documentLinkProvider; - public ExecuteCommandOptions? executeCommandProvider; + public DocumentLinkOptions documentLinkProvider; + public ExecuteCommandOptions executeCommandProvider; public object experimental; } [Serializable] - public struct MessageActionItem { + public sealed class MessageActionItem { public string title; } [Serializable] - public struct Registration { + public sealed class Registration { public string id; public string method; public IRegistrationOptions registerOptions; @@ -529,8 +482,8 @@ public struct Registration { public interface IRegistrationOptions { } [Serializable] - public struct TextDocumentRegistrationOptions : IRegistrationOptions { - public DocumentFilter? documentSelector; + public sealed class TextDocumentRegistrationOptions : IRegistrationOptions { + public DocumentFilter documentSelector; } [Serializable] @@ -540,42 +493,42 @@ public struct Unregistration { } [Serializable] - public struct FileEvent { + public sealed class FileEvent { public Uri uri; public FileChangeType type; } [Serializable] - public struct DidChangeWatchedFilesRegistrationOptions : IRegistrationOptions { + public sealed class DidChangeWatchedFilesRegistrationOptions : IRegistrationOptions { public FileSystemWatcher[] watchers; } [Serializable] - public struct FileSystemWatcher { + public sealed class FileSystemWatcher { public string globPattern; public WatchKind? type; } [Serializable] - public struct ExecuteCommandRegistrationOptions : IRegistrationOptions { + public sealed class ExecuteCommandRegistrationOptions : IRegistrationOptions { public string[] commands; } [Serializable] - public struct TextDocumentContentChangedEvent { + public sealed class TextDocumentContentChangedEvent { public Range? range; public int? rangeLength; public string text; } - public struct TextDocumentChangeRegistrationOptions : IRegistrationOptions { - public DocumentFilter? documentSelector; + public sealed class TextDocumentChangeRegistrationOptions : IRegistrationOptions { + public DocumentFilter documentSelector; public TextDocumentSyncKind syncKind; } [Serializable] public struct TextDocumentSaveRegistrationOptions : IRegistrationOptions { - public DocumentFilter? documentSelector; + public DocumentFilter documentSelector; public bool includeText; } @@ -601,14 +554,10 @@ public class CompletionList { /// The expression that members are being displayed for. /// public string _expr; - /// - /// When true, completions should commit by default. When false, completions - /// should not commit. If unspecified the client may decide. - /// - public bool? _commitByDefault; } [Serializable] + [DebuggerDisplay("{label}")] public class CompletionItem { public string label; public CompletionItemKind kind; @@ -619,14 +568,11 @@ public class CompletionItem { public bool? preselect; // VS Code 1.25+ public string insertText; public InsertTextFormat insertTextFormat; - public TextEdit? textEdit; + public TextEdit textEdit; public TextEdit[] additionalTextEdits; public string[] commitCharacters; - public Command? command; + public Command command; public object data; - - public string _kind; - public CompletionItemValue[] _values; } // Not in LSP spec @@ -638,14 +584,14 @@ public struct CompletionItemValue { } [Serializable] - public struct CompletionRegistrationOptions : IRegistrationOptions { - public DocumentFilter? documentSelector; + public sealed class CompletionRegistrationOptions : IRegistrationOptions { + public DocumentFilter documentSelector; public string[] triggerCharacters; public bool resolveProvider; } [Serializable] - public class Hover { + public sealed class Hover { public MarkupContent contents; public Range? range; @@ -660,14 +606,14 @@ public class Hover { } [Serializable] - public class SignatureHelp { + public sealed class SignatureHelp { public SignatureInformation[] signatures; public int activeSignature; public int activeParameter; } [Serializable] - public class SignatureInformation { + public sealed class SignatureInformation { public string label; public MarkupContent documentation; public ParameterInformation[] parameters; @@ -676,16 +622,9 @@ public class SignatureInformation { } [Serializable] - public class ParameterInformation { + public sealed class ParameterInformation { public string label; public MarkupContent documentation; - - [NonSerialized] - public string _type; - [NonSerialized] - public string _defaultValue; - [NonSerialized] - public bool? _isOptional; } /// @@ -693,7 +632,7 @@ public class ParameterInformation { /// the kind. /// [Serializable] - public class Reference { + public sealed class Reference { public Uri uri; public Range range; @@ -701,15 +640,10 @@ public class Reference { /// The kind of reference /// public ReferenceKind? _kind; - /// - /// The document version that range applies to - /// - public int? _version; - public bool _isModule; } [Serializable] - public struct DocumentHighlight { + public sealed class DocumentHighlight { public Range range; public DocumentHighlightKind kind; @@ -720,7 +654,7 @@ public struct DocumentHighlight { } [Serializable] - public struct DocumentSymbol { + public sealed class DocumentSymbol { /// /// The name of this symbol. /// @@ -759,16 +693,10 @@ public struct DocumentSymbol { /// Children of this symbol, e.g. properties of a class. /// public DocumentSymbol[] children; - - /// - /// Custom field provides more information on the function or method such as - /// 'classmethod' or 'property' that are not part of the . - /// - public string _functionKind; } [Serializable] - public struct SymbolInformation { + public sealed class SymbolInformation { public string name; public SymbolKind kind; public Location location; @@ -779,54 +707,46 @@ public struct SymbolInformation { /// symbols. /// public string containerName; - - /// - /// The document version that location applies to - /// - public int? _version; - public string _kind; } [Serializable] - public struct CodeLens { + public sealed class CodeLens { public Range range; - public Command? command; + public Command command; public object data; - - /// - /// The document version that range applies to - /// - public int? _version; } [Serializable] - public struct DocumentLink { + public sealed class DocumentLink { public Range range; public Uri target; - - /// - /// The document version tha range applies to - /// - public int? _version; } [Serializable] public struct DocumentLinkRegistrationOptions : IRegistrationOptions { - public DocumentFilter? documentSelector; + public DocumentFilter documentSelector; public bool resolveProvider; } [Serializable] - public struct FormattingOptions { + public sealed class FormattingOptions { public int tabSize; public bool insertSpaces; } [Serializable] - public struct DocumentOnTypeFormattingRegistrationOptions : IRegistrationOptions { - public DocumentFilter? documentSelector; + public sealed class DocumentOnTypeFormattingRegistrationOptions : IRegistrationOptions { + public DocumentFilter documentSelector; public string firstTriggerCharacter; public string[] moreTriggerCharacters; } + + [JsonObject] + public sealed class PublishDiagnosticsParams { + [JsonProperty] + public Uri uri; + [JsonProperty] + public Diagnostic[] diagnostics; + } } diff --git a/src/LanguageServer/Impl/Protocol/Diagnostic.cs b/src/LanguageServer/Impl/Protocol/Diagnostic.cs new file mode 100644 index 000000000..68dcac78b --- /dev/null +++ b/src/LanguageServer/Impl/Protocol/Diagnostic.cs @@ -0,0 +1,54 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Core.Text; + +namespace Microsoft.Python.LanguageServer.Protocol { + public class Diagnostic { + /// + /// The range at which the message applies. + /// + public Range range; + + /// + /// The diagnostics severity. + /// + public DiagnosticSeverity severity; + + /// + /// The diagnostics code (string, such as 'unresolved-import'). + /// + public string code; + + /// + /// A human-readable string describing the source of this + /// diagnostic, e.g. 'typescript' or 'super lint'. + /// + public string source; + + /// + /// The diagnostics message. + /// + public string message; + } + + public enum DiagnosticSeverity : int { + Unspecified = 0, + Error = 1, + Warning = 2, + Information = 3, + Hint = 4 + } +} diff --git a/src/LanguageServer/Impl/Definitions/Enums.cs b/src/LanguageServer/Impl/Protocol/Enums.cs similarity index 88% rename from src/LanguageServer/Impl/Definitions/Enums.cs rename to src/LanguageServer/Impl/Protocol/Enums.cs index ca7cf1917..e332a9fa6 100644 --- a/src/LanguageServer/Impl/Definitions/Enums.cs +++ b/src/LanguageServer/Impl/Protocol/Enums.cs @@ -1,5 +1,4 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation +// Copyright(c) Microsoft Corporation // All rights reserved. // // Licensed under the Apache License, Version 2.0 (the License); you may not use @@ -17,7 +16,7 @@ using System; using System.Diagnostics; -namespace Microsoft.Python.LanguageServer { +namespace Microsoft.Python.LanguageServer.Protocol { public sealed class SerializeAsAttribute : Attribute { public object Value { get; } @@ -65,13 +64,13 @@ public enum SymbolKind { TypeParameter = 26 } - public enum TextDocumentSyncKind : int { + public enum TextDocumentSyncKind { None = 0, Full = 1, Incremental = 2 } - public enum MessageType : int { + public enum MessageType { /// /// General language server output relevant to the user /// such as information on Python interpreter type. @@ -118,35 +117,35 @@ public static TraceEventType ToTraceEventType(this MessageType mt) { } } - public enum FileChangeType : int { + public enum FileChangeType { Created = 1, Changed = 2, Deleted = 3 } - public enum WatchKind : int { + public enum WatchKind { Create = 1, Change = 2, Delete = 4 } - public enum TextDocumentSaveReason : int { + public enum TextDocumentSaveReason { Manual = 1, AfterDelay = 2, FocusOut = 3 } - public enum CompletionTriggerKind : int { + public enum CompletionTriggerKind { Invoked = 1, TriggerCharacter = 2 } - public enum InsertTextFormat : int { + public enum InsertTextFormat { PlainText = 1, Snippet = 2 } - public enum CompletionItemKind : int { + public enum CompletionItemKind { None = 0, Text = 1, Method = 2, @@ -175,14 +174,14 @@ public enum CompletionItemKind : int { TypeParameter = 25 } - public enum DocumentHighlightKind : int { + public enum DocumentHighlightKind { Text = 1, Read = 2, Write = 3 } // Not in the LSP spec. - public enum ReferenceKind : int { + public enum ReferenceKind { Definition = 1, Reference = 2, Value = 3 diff --git a/src/LanguageServer/Impl/Protocol/InformationDisplayOptions.cs b/src/LanguageServer/Impl/Protocol/InformationDisplayOptions.cs new file mode 100644 index 000000000..f5d6314ab --- /dev/null +++ b/src/LanguageServer/Impl/Protocol/InformationDisplayOptions.cs @@ -0,0 +1,25 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.LanguageServer.Protocol { + public sealed class InformationDisplayOptions { + public string preferredFormat; + public bool trimDocumentationLines; + public int maxDocumentationLineLength; + public bool trimDocumentationText; + public int maxDocumentationTextLength; + public int maxDocumentationLines; + } +} diff --git a/src/LanguageServer/Impl/Protocol/LanguageServerException.cs b/src/LanguageServer/Impl/Protocol/LanguageServerException.cs new file mode 100644 index 000000000..0ce665668 --- /dev/null +++ b/src/LanguageServer/Impl/Protocol/LanguageServerException.cs @@ -0,0 +1,39 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections; + +namespace Microsoft.Python.LanguageServer.Protocol { + [Serializable] + public sealed class LanguageServerException : Exception { + public const int UnknownDocument = 1; + public const int UnsupportedDocumentType = 2; + public const int MismatchedVersion = 3; + public const int UnknownExtension = 4; + + public int Code => (int)Data["Code"]; + + public sealed override IDictionary Data => base.Data; + + public LanguageServerException(int code, string message) : base(message) { + Data["Code"] = code; + } + + public LanguageServerException(int code, string message, Exception innerException) : base(message, innerException) { + Data["Code"] = code; + } + } +} diff --git a/src/LanguageServer/Impl/Protocol/MarkupContent.cs b/src/LanguageServer/Impl/Protocol/MarkupContent.cs new file mode 100644 index 000000000..43b27ae9e --- /dev/null +++ b/src/LanguageServer/Impl/Protocol/MarkupContent.cs @@ -0,0 +1,26 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; + +namespace Microsoft.Python.LanguageServer.Protocol { + [Serializable] + public sealed class MarkupContent { + public string kind; + public string value; + + public static implicit operator MarkupContent(string text) => new MarkupContent { kind = MarkupKind.PlainText, value = text }; + } +} diff --git a/src/LanguageServer/Impl/Protocol/MarkupKind.cs b/src/LanguageServer/Impl/Protocol/MarkupKind.cs new file mode 100644 index 000000000..512fe1316 --- /dev/null +++ b/src/LanguageServer/Impl/Protocol/MarkupKind.cs @@ -0,0 +1,21 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.LanguageServer.Protocol { + public static class MarkupKind { + public const string PlainText = "plaintext"; + public const string Markdown = "markdown"; + } +} diff --git a/src/LanguageServer/Impl/Protocol/Messages.cs b/src/LanguageServer/Impl/Protocol/Messages.cs new file mode 100644 index 000000000..8c8a103dd --- /dev/null +++ b/src/LanguageServer/Impl/Protocol/Messages.cs @@ -0,0 +1,241 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Microsoft.Python.Core.Text; + +namespace Microsoft.Python.LanguageServer.Protocol { + [Serializable] + public sealed class InitializeParams { + public int? processId; + public string rootPath; + public Uri rootUri; + public PythonInitializationOptions initializationOptions; + public ClientCapabilities capabilities; + public TraceLevel trace; + } + + [Serializable] + public struct InitializeResult { + public ServerCapabilities capabilities; + } + + [Serializable] + public struct InitializedParams { } + + public sealed class ShowMessageEventArgs : EventArgs { + public MessageType type { get; set; } + public string message { get; set; } + } + + [Serializable] + public class ShowMessageRequestParams { + public MessageType type; + public string message; + public MessageActionItem[] actions; + } + + public sealed class CommandEventArgs : EventArgs { + public string command; + public object[] arguments; + } + + [Serializable] + public sealed class RegistrationParams { + public Registration[] registrations; + } + + [ComVisible(false)] + public sealed class RegisterCapabilityEventArgs : CallbackEventArgs { + internal RegisterCapabilityEventArgs(TaskCompletionSource task) : base(task) { } + } + + [Serializable] + public sealed class UnregistrationParams { + public Unregistration[] unregistrations; + } + + [ComVisible(false)] + public sealed class UnregisterCapabilityEventArgs : CallbackEventArgs { + internal UnregisterCapabilityEventArgs(TaskCompletionSource task) : base(task) { } + } + + [Serializable] + public sealed class DidChangeConfigurationParams { + public object settings; + } + + [Serializable] + public sealed class DidChangeWatchedFilesParams { + public FileEvent[] changes; + } + + [Serializable] + public sealed class WorkspaceSymbolParams { + public string query; + } + + [Serializable] + public sealed class ExecuteCommandParams { + public string command; + public object[] arguments; + } + + [Serializable] + public sealed class ApplyWorkspaceEditParams { + /// + /// An optional label of the workspace edit.This label is + /// presented in the user interface for example on an undo + /// stack to undo the workspace edit. + /// + public string label; + public WorkspaceEdit edit; + } + + [Serializable] + public sealed class ApplyWorkspaceEditResponse { + public bool applied; + } + + [ComVisible(false)] + public sealed class ApplyWorkspaceEditEventArgs : CallbackEventArgs { + internal ApplyWorkspaceEditEventArgs(TaskCompletionSource task) : base(task) { } + } + + [Serializable] + public sealed class DidOpenTextDocumentParams { + public TextDocumentItem textDocument; + } + + [Serializable] + public sealed class DidChangeTextDocumentParams { + public VersionedTextDocumentIdentifier textDocument; + public TextDocumentContentChangedEvent[] contentChanges; + + // Defaults to true, but can be set to false to suppress analysis + public bool? _enqueueForAnalysis; + } + + [Serializable] + public sealed class WillSaveTextDocumentParams { + public TextDocumentIdentifier textDocument; + public TextDocumentSaveReason reason; + } + + [Serializable] + public sealed class DidSaveTextDocumentParams { + public TextDocumentIdentifier textDocument; + public string content; + } + + [Serializable] + public sealed class DidCloseTextDocumentParams { + public TextDocumentIdentifier textDocument; + } + + [Serializable] + public sealed class TextDocumentPositionParams { + public TextDocumentIdentifier textDocument; + public Position position; + } + + [Serializable] + public sealed class CompletionParams { + public TextDocumentIdentifier textDocument; + public Position position; + public CompletionContext context; + } + + [Serializable] + public sealed class CompletionContext { + public CompletionTriggerKind triggerKind; + public string triggerCharacter; + } + + [Serializable] + public sealed class ReferencesParams { + public TextDocumentIdentifier textDocument; + public Position position; + public ReferenceContext context; + } + + public sealed class ReferenceContext { + public bool includeDeclaration; + } + + [Serializable] + public sealed class DocumentSymbolParams { + public TextDocumentIdentifier textDocument; + } + + [Serializable] + public struct CodeActionParams { + public TextDocumentIdentifier textDocument; + public Range range; + public CodeActionContext context; + } + + [Serializable] + public sealed class CodeActionContext { + public Diagnostic[] diagnostics; + + /// + /// The intended version that diagnostic locations apply to. The request may + /// fail if the server cannot map correctly. + /// + public int? _version; + } + + [Serializable] + public sealed class DocumentLinkParams { + public TextDocumentIdentifier textDocument; + } + + [Serializable] + public sealed class DocumentFormattingParams { + public TextDocumentIdentifier textDocument; + public FormattingOptions options; + } + + [Serializable] + public sealed class DocumentRangeFormattingParams { + public TextDocumentIdentifier textDocument; + public Range range; + public FormattingOptions options; + } + + [Serializable] + public sealed class DocumentOnTypeFormattingParams { + public TextDocumentIdentifier textDocument; + public Position position; + public string ch; + public FormattingOptions options; + } + + [Serializable] + public sealed class RenameParams { + public TextDocumentIdentifier textDocument; + public Position position; + public string newName; + } + + [Serializable] + public sealed class LogMessageParams { + public MessageType type; + public string message; + } +} diff --git a/src/LanguageServer/Impl/Resources.Designer.cs b/src/LanguageServer/Impl/Resources.Designer.cs index 33c50439f..47aad9931 100644 --- a/src/LanguageServer/Impl/Resources.Designer.cs +++ b/src/LanguageServer/Impl/Resources.Designer.cs @@ -60,6 +60,15 @@ internal Resources() { } } + /// + /// Looks up a localized string similar to Code analysis is in progress.... + /// + internal static string AnalysisIsInProgressHover { + get { + return ResourceManager.GetString("AnalysisIsInProgressHover", resourceCulture); + } + } + /// /// Looks up a localized string similar to done.. /// diff --git a/src/LanguageServer/Impl/Resources.resx b/src/LanguageServer/Impl/Resources.resx index 630472020..4054f5a97 100644 --- a/src/LanguageServer/Impl/Resources.resx +++ b/src/LanguageServer/Impl/Resources.resx @@ -117,6 +117,9 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + Code analysis is in progress... + done. diff --git a/src/LanguageServer/Impl/Services/Logger.cs b/src/LanguageServer/Impl/Services/Logger.cs index e9cff7c3d..2a01cf385 100644 --- a/src/LanguageServer/Impl/Services/Logger.cs +++ b/src/LanguageServer/Impl/Services/Logger.cs @@ -19,6 +19,7 @@ using System.Threading.Tasks; using Microsoft.Python.Core; using Microsoft.Python.Core.Logging; +using Microsoft.Python.LanguageServer.Protocol; using StreamJsonRpc; namespace Microsoft.Python.LanguageServer.Services { @@ -55,11 +56,5 @@ public Task LogMessageAsync(string message, TraceEventType eventType) { }; return _rpc.NotifyWithParameterObjectAsync("window/logMessage", parameters); } - - [Serializable] - private class LogMessageParams { - public MessageType type; - public string message; - } } } diff --git a/src/LanguageServer/Impl/Services/TraceEventTypeExtensions.cs b/src/LanguageServer/Impl/Services/TraceEventTypeExtensions.cs index da9abd9a2..ec0d75373 100644 --- a/src/LanguageServer/Impl/Services/TraceEventTypeExtensions.cs +++ b/src/LanguageServer/Impl/Services/TraceEventTypeExtensions.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System.Diagnostics; +using Microsoft.Python.LanguageServer.Protocol; namespace Microsoft.Python.LanguageServer.Services { internal static class TraceEventTypeExtensions { diff --git a/src/LanguageServer/Impl/Services/UIService.cs b/src/LanguageServer/Impl/Services/UIService.cs index 4b45dda49..e2199869b 100644 --- a/src/LanguageServer/Impl/Services/UIService.cs +++ b/src/LanguageServer/Impl/Services/UIService.cs @@ -17,6 +17,7 @@ using System.Linq; using System.Threading.Tasks; using Microsoft.Python.Core.Shell; +using Microsoft.Python.LanguageServer.Protocol; using StreamJsonRpc; namespace Microsoft.Python.LanguageServer.Services { @@ -41,7 +42,7 @@ public async Task ShowMessageAsync(string message, string[] actions, Tra message = message, actions = actions.Select(a => new MessageActionItem { title = a }).ToArray() }; - var result = await _rpc.InvokeWithParameterObjectAsync("window/showMessageRequest", parameters); + var result = await _rpc.InvokeWithParameterObjectAsync("window/showMessageRequest", parameters); return result?.title; } diff --git a/src/LanguageServer/Impl/Sources/DefinitionSource.cs b/src/LanguageServer/Impl/Sources/DefinitionSource.cs new file mode 100644 index 000000000..24815fec9 --- /dev/null +++ b/src/LanguageServer/Impl/Sources/DefinitionSource.cs @@ -0,0 +1,125 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Net.WebSockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Analyzer.Expressions; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Completion; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Sources { + internal sealed class DefinitionSource { + public async Task FindDefinitionAsync(IDocumentAnalysis analysis, SourceLocation location, CancellationToken cancellationToken = default) { + ExpressionLocator.FindExpression(analysis.Ast, location, + FindExpressionOptions.Hover, out var exprNode, out var statement, out var exprScope); + + if (exprNode is ConstantExpression) { + return null; // No hover for literals. + } + if (!(exprNode is Expression expr)) { + return null; + } + + var eval = analysis.ExpressionEvaluator; + using (eval.OpenScope(analysis.Document, exprScope)) { + var value = await eval.GetValueFromExpressionAsync(expr, cancellationToken); + return await FromMemberAsync(value, expr, eval, cancellationToken); + } + } + + private async Task FromMemberAsync(IMember value, Expression expr, IExpressionEvaluator eval, CancellationToken cancellationToken) { + cancellationToken.ThrowIfCancellationRequested(); + + Node node = null; + IPythonModule module = null; + + switch (value) { + case IPythonClassType cls: + node = cls.ClassDefinition; + module = cls.DeclaringModule; + break; + case IPythonFunctionType fn: + node = fn.FunctionDefinition; + module = fn.DeclaringModule; + break; + case IPythonPropertyType prop: + node = prop.FunctionDefinition; + module = prop.DeclaringModule; + break; + case IPythonModule mod: { + var member = eval.LookupNameInScopes(mod.Name, out var scope); + if (member != null && scope != null) { + var v = scope.Variables[mod.Name]; + if (v != null) { + return new Reference { range = v.Location.Span, uri = v.Location.DocumentUri }; + } + } + break; + } + case IPythonInstance instance when instance.Type is IPythonFunctionType ft: { + node = ft.FunctionDefinition; + module = ft.DeclaringModule; + break; + } + case IPythonInstance _ when expr is NameExpression nex: { + var member = eval.LookupNameInScopes(nex.Name, out var scope); + if (member != null && scope != null) { + var v = scope.Variables[nex.Name]; + if (v != null) { + return new Reference { range = v.Location.Span, uri = v.Location.DocumentUri }; + } + } + break; + } + case IPythonInstance _ when expr is MemberExpression mex: { + var target = await eval.GetValueFromExpressionAsync(mex.Target, cancellationToken); + var type = target?.GetPythonType(); + var member = type?.GetMember(mex.Name); + if (member is IPythonInstance v) { + return new Reference { range = v.Location.Span, uri = v.Location.DocumentUri }; + } + return await FromMemberAsync(member, null, eval, cancellationToken); + } + } + + if (node != null && CanNavigateToModule(module) && module is IDocument doc) { + return new Reference { + range = node.GetSpan(doc.GetAnyAst()), uri = doc.Uri + }; + } + + return null; + } + + private static bool CanNavigateToModule(IPythonModule m) { +#if DEBUG + // Allow navigation anywhere in debug. + return m.ModuleType != ModuleType.Specialized && m.ModuleType != ModuleType.Unresolved; +#else + return m.ModuleType == ModuleType.User || m.ModuleType == ModuleType.Package || m.ModuleType == ModuleType.Library; +#endif + } + } +} diff --git a/src/LanguageServer/Impl/Sources/HoverSource.cs b/src/LanguageServer/Impl/Sources/HoverSource.cs new file mode 100644 index 000000000..57c3603ce --- /dev/null +++ b/src/LanguageServer/Impl/Sources/HoverSource.cs @@ -0,0 +1,124 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Analyzer.Expressions; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Completion; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Sources { + internal sealed class HoverSource { + private readonly IDocumentationSource _docSource; + + public HoverSource(IDocumentationSource docSource) { + _docSource = docSource; + } + + public async Task GetHoverAsync(IDocumentAnalysis analysis, SourceLocation location, CancellationToken cancellationToken = default) { + if (analysis is EmptyAnalysis) { + return new Hover { contents = Resources.AnalysisIsInProgressHover }; + } + + ExpressionLocator.FindExpression(analysis.Ast, location, + FindExpressionOptions.Hover, out var node, out var statement, out var scope); + + if (node is ConstantExpression || !(node is Expression expr)) { + return null; // No hover for literals. + } + + var range = new Range { + start = expr.GetStart(analysis.Ast), + end = expr.GetEnd(analysis.Ast), + }; + + var eval = analysis.ExpressionEvaluator; + switch (statement) { + case FromImportStatement fi when node is NameExpression nex: { + // In 'from A import B as C' B is not declared as a variable + // so we have to fetch the type manually. + var index = fi.Names.IndexOf(nex); + if (index >= 0) { + using (eval.OpenScope(analysis.Document, scope)) { + var variable = eval.LookupNameInScopes(fi.Root.MakeString(), out _); + if (variable.GetPythonType() is IPythonModule mod) { + var v = mod.GetMember(nex.Name)?.GetPythonType(); + return new Hover { + contents = _docSource.GetHover(mod.Name, v), + range = range + }; + } + } + } + + break; + } + case ImportStatement imp: { + // In 'import A as B' 'A' is not declared as a variable + // so we have to fetch the type manually. + var index = location.ToIndex(analysis.Ast); + var dottedName = imp.Names.FirstOrDefault(n => n.StartIndex <= index && index < n.EndIndex); + if (dottedName != null) { + var mod = analysis.Document.Interpreter.ModuleResolution.GetImportedModule(dottedName.MakeString()); + if (mod != null) { + return new Hover { + contents = _docSource.GetHover(null, mod), + range = range + }; + } + } + break; + } + } + + IMember value; + IPythonType type; + using (eval.OpenScope(analysis.Document, scope)) { + value = await analysis.ExpressionEvaluator.GetValueFromExpressionAsync(expr, cancellationToken); + type = value?.GetPythonType(); + if (type == null) { + return null; + } + } + + // Figure out name, if any + var name = (expr as MemberExpression)?.Name; + name = name ?? (node as NameExpression)?.Name; + + // Special case hovering over self or cls + if ((name.EqualsOrdinal("self") || name.EqualsOrdinal("cls")) && type is IPythonClassType) { + return new Hover { + contents = _docSource.GetHover(null, type), + range = range + }; + } + + name = name == null && statement is ClassDefinition cd ? cd.Name : name; + name = name == null && statement is FunctionDefinition fd ? fd.Name : name; + + return new Hover { + contents = _docSource.GetHover(name, value), + range = range + }; + } + } +} diff --git a/src/LanguageServer/Impl/Sources/PlainTextDocumentationSource.cs b/src/LanguageServer/Impl/Sources/PlainTextDocumentationSource.cs new file mode 100644 index 000000000..e510f10d0 --- /dev/null +++ b/src/LanguageServer/Impl/Sources/PlainTextDocumentationSource.cs @@ -0,0 +1,118 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.LanguageServer.Protocol; + +namespace Microsoft.Python.LanguageServer.Sources { + internal sealed class PlainTextDocumentationSource : IDocumentationSource { + public InsertTextFormat DocumentationFormat => InsertTextFormat.PlainText; + + public MarkupContent GetHover(string name, IMember member) { + // We need to tell between instance and type. + var type = member.GetPythonType(); + if (type.IsUnknown()) { + return new MarkupContent { kind = MarkupKind.PlainText, value = name }; + } + + string text; + if (member is IPythonInstance) { + if (!(type is IPythonFunctionType)) { + text = !string.IsNullOrEmpty(name) ? $"{name}: {type.Name}" : $"{type.Name}"; + return new MarkupContent { kind = MarkupKind.PlainText, value = text }; + } + } + + var typeDoc = !string.IsNullOrEmpty(type.Documentation) ? $"\n\n{type.Documentation}" : string.Empty; + switch (type) { + case IPythonPropertyType prop: + text = GetPropertyHoverString(prop); + break; + + case IPythonFunctionType ft: + text = GetFunctionHoverString(ft); + break; + + case IPythonClassType cls: + var clsDoc = !string.IsNullOrEmpty(cls.Documentation) ? $"\n\n{cls.Documentation}" : string.Empty; + text = $"class {cls.Name}{clsDoc}"; + break; + + case IPythonModule mod: + text = !string.IsNullOrEmpty(mod.Name) ? $"module {mod.Name}{typeDoc}" : $"module{typeDoc}"; + break; + + default: + text = !string.IsNullOrEmpty(name) ? $"type {name}: {type.Name}{typeDoc}" : $"{type.Name}{typeDoc}"; + break; + } + + return new MarkupContent { + kind = MarkupKind.PlainText, value = text + }; + } + + public MarkupContent FormatDocumentation(string documentation) { + return new MarkupContent { kind = MarkupKind.PlainText, value = documentation }; + } + + public string GetSignatureString(IPythonFunctionType ft, int overloadIndex = 0) { + var o = ft.Overloads[overloadIndex]; + + var parms = GetFunctionParameters(ft); + var parmString = string.Join(", ", parms); + var annString = string.IsNullOrEmpty(o.ReturnDocumentation) ? string.Empty : $" -> {o.ReturnDocumentation}"; + + return $"{ft.Name}({parmString}){annString}"; + } + + public MarkupContent FormatParameterDocumentation(IParameterInfo parameter) { + if (!string.IsNullOrEmpty(parameter.Documentation)) { + return FormatDocumentation(parameter.Documentation); + } + // TODO: show fully qualified type? + var text = parameter.Type.IsUnknown() ? parameter.Name : $"{parameter.Name}: {parameter.Type.Name}"; + return new MarkupContent { kind = MarkupKind.PlainText, value = text }; + } + + private string GetPropertyHoverString(IPythonPropertyType prop, int overloadIndex = 0) { + var decTypeString = prop.DeclaringType != null ? $"{prop.DeclaringType.Name}." : string.Empty; + var propDoc = !string.IsNullOrEmpty(prop.Documentation) ? $"\n\n{prop.Documentation}" : string.Empty; + return $"{decTypeString}{propDoc}"; + } + + private string GetFunctionHoverString(IPythonFunctionType ft, int overloadIndex = 0) { + var sigString = GetSignatureString(ft, overloadIndex); + var decTypeString = ft.DeclaringType != null ? $"{ft.DeclaringType.Name}." : string.Empty; + var funcDoc = !string.IsNullOrEmpty(ft.Documentation) ? $"\n\n{ft.Documentation}" : string.Empty; + return $"{decTypeString}{sigString}{funcDoc}"; + } + + private IEnumerable GetFunctionParameters(IPythonFunctionType ft, int overloadIndex = 0) { + var o = ft.Overloads[overloadIndex]; // TODO: display all? + var skip = ft.IsStatic || ft.IsUnbound ? 0 : 1; + return o.Parameters.Skip(skip).Select(p => { + if (!string.IsNullOrEmpty(p.DefaultValueString)) { + return $"{p.Name}={p.DefaultValueString}"; + } + return p.Type.IsUnknown() ? p.Name : $"{p.Name}: {p.Type.Name}"; + }); + } + } +} diff --git a/src/LanguageServer/Impl/Sources/SignatureSource.cs b/src/LanguageServer/Impl/Sources/SignatureSource.cs new file mode 100644 index 000000000..39b6e8579 --- /dev/null +++ b/src/LanguageServer/Impl/Sources/SignatureSource.cs @@ -0,0 +1,100 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Analyzer.Expressions; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Completion; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.LanguageServer.Sources { + internal sealed class SignatureSource { + private readonly IDocumentationSource _docSource; + + public SignatureSource(IDocumentationSource docSource) { + _docSource = docSource; + } + + public async Task GetSignatureAsync(IDocumentAnalysis analysis, SourceLocation location, CancellationToken cancellationToken = default) { + if (analysis is EmptyAnalysis) { + return null; + } + + ExpressionLocator.FindExpression(analysis.Ast, location, + FindExpressionOptions.Hover, out var node, out var statement, out var scope); + + IMember value = null; + var call = node as CallExpression; + if (call != null) { + using (analysis.ExpressionEvaluator.OpenScope(analysis.Document, scope)) { + value = await analysis.ExpressionEvaluator.GetValueFromExpressionAsync(call.Target, cancellationToken); + } + } + + var ft = value?.GetPythonType(); + if (ft == null) { + return null; + } + + var skip = ft.IsStatic || ft.IsUnbound ? 0 : 1; + + var signatures = new SignatureInformation[ft.Overloads.Count]; + for (var i = 0; i < ft.Overloads.Count; i++) { + var o = ft.Overloads[i]; + + var parameters = o.Parameters.Skip(skip).Select(p => new ParameterInformation { + label = p.Name, + documentation = _docSource.FormatParameterDocumentation(p) + }).ToArray(); + + signatures[i] = new SignatureInformation { + label = _docSource.GetSignatureString(ft, i), + documentation = _docSource.FormatDocumentation(ft.Documentation), + parameters = parameters + }; + } + + var index = location.ToIndex(analysis.Ast); + if (call.GetArgumentAtIndex(analysis.Ast, index, out var activeParameter) && activeParameter < 0) { + // Returned 'true' and activeParameter == -1 means that we are after + // the trailing comma, so assume partially typed expression such as 'pow(x, y, |) + activeParameter = call.Args.Count; + } + + var activeSignature = -1; + if (activeParameter >= 0) { + // TODO: Better selection of active signature by argument set + activeSignature = signatures + .Select((s, i) => Tuple.Create(s, i)) + .OrderBy(t => t.Item1.parameters.Length) + .FirstOrDefault(t => t.Item1.parameters.Length > activeParameter) + ?.Item2 ?? -1; + } + + return new SignatureHelp { + signatures = signatures.ToArray(), + activeSignature = activeSignature, + activeParameter = activeParameter + }; + } + } +} diff --git a/src/LanguageServer/Impl/Telemetry.cs b/src/LanguageServer/Impl/Telemetry.cs index 95c5ac89e..14eb3378a 100644 --- a/src/LanguageServer/Impl/Telemetry.cs +++ b/src/LanguageServer/Impl/Telemetry.cs @@ -19,6 +19,7 @@ using System.Reflection; using Microsoft.Python.Core; using Microsoft.Python.Core.Shell; +using Microsoft.Python.LanguageServer.Protocol; using StreamJsonRpc; namespace Microsoft.Python.LanguageServer.Implementation { diff --git a/src/LanguageServer/Test/AssemblySetup.cs b/src/LanguageServer/Test/AssemblySetup.cs new file mode 100644 index 000000000..0263bbff5 --- /dev/null +++ b/src/LanguageServer/Test/AssemblySetup.cs @@ -0,0 +1,34 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Core.Testing; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public sealed class AssemblySetup { + [AssemblyInitialize] + public static void Initialize(TestContext testContext) => AnalysisTestEnvironment.Initialize(); + + private class AnalysisTestEnvironment : TestEnvironmentImpl, ITestEnvironment { + public static void Initialize() { + var instance = new AnalysisTestEnvironment(); + Instance = instance; + TestEnvironment.Current = instance; + } + } + } +} diff --git a/src/LanguageServer/Test/BlockFormatterTests.cs b/src/LanguageServer/Test/BlockFormatterTests.cs new file mode 100644 index 000000000..915b998ea --- /dev/null +++ b/src/LanguageServer/Test/BlockFormatterTests.cs @@ -0,0 +1,304 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Formatting; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.LanguageServer.Tests.FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class BlockFormatterTests { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + } + + [TestCleanup] + public void TestCleanup() { + TestEnvironmentImpl.TestCleanup(); + } + + [TestMethod, Priority(0)] + public void NullReader() { + Func> func = () => BlockFormatter.ProvideEdits(null, new Position(), new FormattingOptions()); + func.Should().Throw().And.ParamName.Should().Be("reader"); + } + + [TestMethod, Priority(0)] + public async Task FirstLine() { + using (var reader = new StringReader(string.Empty)) { + var edits = await BlockFormatter.ProvideEdits(reader, new Position { line = 0, character = 4 }, new FormattingOptions()); + edits.Should().BeEmpty(); + } + } + + [TestMethod, Priority(0)] + public void TooShort() { + using (var reader = new StringReader("a + b")) { + Func> func = () => BlockFormatter.ProvideEdits(reader, new Position { line = 1, character = 4 }, new FormattingOptions()); + func.Should().Throw().And.ParamName.Should().Be("position"); + } + } + + [TestMethod, Priority(0)] + public async Task NoMatch() { + var code = @"d = { + 'a': a, + 'b': +"; + using (var reader = new StringReader(code)) { + var edits = await BlockFormatter.ProvideEdits(reader, new Position { line = 2, character = 7 }, new FormattingOptions()); + edits.Should().BeEmpty(); + } + } + + [DataRow("elseBlocksFirstLine2.py", 3, 7, true, 2, 0, 2)] + [DataRow("elseBlocksFirstLine4.py", 3, 9, true, 4, 0, 4)] + [DataRow("elseBlocksFirstLineTab.py", 3, 6, false, 4, 0, 1)] + [DataTestMethod, Priority(0)] + public async Task ElseBlock(string filename, int line, int col, bool insertSpaces, int tabSize, int startCharacter, int endCharacter) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = insertSpaces, tabSize = tabSize }; + + var src = TestData.GetPath("TestData", "Formatting", filename); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().OnlyHaveTextEdit(string.Empty, (line, startCharacter, line, endCharacter)); + } + } + + [DataRow(6, 22, 0, 2)] + [DataRow(35, 13, 0, 2)] + [DataRow(54, 19, 0, 2)] + [DataRow(76, 9, 0, 2)] + [DataRow(143, 22, 0, 2)] + [DataRow(172, 11, 0, 2)] + [DataRow(195, 12, 0, 2)] + [DataTestMethod, Priority(0)] + public async Task TryBlockTwoSpace(int line, int col, int startCharacter, int endCharacter) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = true, tabSize = 2 }; + + var src = TestData.GetPath("TestData", "Formatting", "tryBlocks2.py"); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().OnlyHaveTextEdit(string.Empty, (line, startCharacter, line, endCharacter)); + } + } + + [DataRow(15, 21)] + [DataRow(47, 12)] + [DataRow(157, 25)] + [DataTestMethod, Priority(0)] + public async Task TryBlockTwoSpaceNoEdits(int line, int col) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = true, tabSize = 2 }; + + var src = TestData.GetPath("TestData", "Formatting", "tryBlocks2.py"); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().BeEmpty(); + } + } + + [DataRow(6, 22, 0, 4)] + [DataRow(35, 13, 0, 4)] + [DataRow(54, 19, 0, 4)] + [DataRow(76, 9, 0, 4)] + [DataRow(143, 22, 0, 4)] + [DataRow(172, 11, 0, 4)] + [DataRow(195, 12, 0, 4)] + [DataTestMethod, Priority(0)] + public async Task TryBlockFourSpace(int line, int col, int startCharacter, int endCharacter) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = true, tabSize = 4 }; + + var src = TestData.GetPath("TestData", "Formatting", "tryBlocks4.py"); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().OnlyHaveTextEdit(string.Empty, (line, startCharacter, line, endCharacter)); + } + } + + [DataRow(15, 21)] + [DataRow(47, 12)] + [DataRow(157, 25)] + [DataTestMethod, Priority(0)] + public async Task TryBlockFourSpaceNoEdits(int line, int col) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = true, tabSize = 4 }; + + var src = TestData.GetPath("TestData", "Formatting", "tryBlocks4.py"); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().BeEmpty(); + } + } + + [DataRow(6, 22, 0, 2)] + [DataRow(35, 13, 0, 2)] + [DataRow(54, 19, 0, 2)] + [DataRow(76, 9, 0, 2)] + [DataRow(143, 22, 0, 2)] + [DataRow(172, 11, 0, 3)] + [DataRow(195, 12, 0, 2)] + [DataTestMethod, Priority(0)] + public async Task TryBlockTab(int line, int col, int startCharacter, int endCharacter) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = false, tabSize = 4 }; + + var src = TestData.GetPath("TestData", "Formatting", "tryBlocksTab.py"); + var newText = new string('\t', endCharacter - 1); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().OnlyHaveTextEdit(newText, (line, startCharacter, line, endCharacter)); + } + } + + [DataRow(4, 18, 0, 2)] + [DataRow(7, 18, 0, 2)] + [DataRow(21, 18, 0, 2)] + [DataRow(38, 7, 0, 2)] + [DataRow(47, 13, 0, 2)] + [DataRow(57, 9, 0, 2)] + [DataRow(66, 20, 0, 2)] + [DataRow(69, 20, 0, 2)] + [DataRow(83, 20, 0, 2)] + [DataRow(109, 15, 0, 2)] + [DataRow(119, 11, 0, 2)] + [DataRow(134, 9, 0, 2)] + [DataTestMethod, Priority(0)] + public async Task ElseBlockTwoSpace(int line, int col, int startCharacter, int endCharacter) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = true, tabSize = 2 }; + + var src = TestData.GetPath("TestData", "Formatting", "elseBlocks2.py"); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().OnlyHaveTextEdit(string.Empty, (line, startCharacter, line, endCharacter)); + } + } + + [DataRow(345, 18)] + [DataRow(359, 18)] + [DataTestMethod, Priority(0)] + public async Task ElseBlockTwoSpaceNoEdits(int line, int col) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = true, tabSize = 2 }; + + var src = TestData.GetPath("TestData", "Formatting", "elseBlocks2.py"); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().BeEmpty(); + } + } + + [DataRow(4, 18, 0, 4)] + [DataRow(7, 18, 0, 4)] + [DataRow(21, 18, 0, 4)] + [DataRow(38, 7, 0, 4)] + [DataRow(47, 13, 0, 4)] + [DataRow(57, 9, 0, 4)] + [DataRow(66, 20, 0, 4)] + [DataRow(69, 20, 0, 4)] + [DataRow(83, 20, 0, 4)] + [DataRow(109, 15, 0, 4)] + [DataRow(119, 11, 0, 4)] + [DataRow(134, 9, 0, 4)] + [DataTestMethod, Priority(0)] + public async Task ElseBlockFourSpace(int line, int col, int startCharacter, int endCharacter) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = true, tabSize = 4 }; + + var src = TestData.GetPath("TestData", "Formatting", "elseBlocks4.py"); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().OnlyHaveTextEdit(string.Empty, (line, startCharacter, line, endCharacter)); + } + } + + [DataRow(345, 18)] + [DataTestMethod, Priority(0)] + public async Task ElseBlockFourSpaceNoEdits(int line, int col) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = true, tabSize = 4 }; + + var src = TestData.GetPath("TestData", "Formatting", "elseBlocks4.py"); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().BeEmpty(); + } + } + + [DataRow(4, 18, 0, 1)] + [DataRow(7, 18, 0, 1)] + [DataRow(21, 18, 0, 1)] + [DataRow(38, 7, 0, 1)] + [DataRow(47, 13, 0, 1)] + [DataRow(57, 9, 0, 1)] + [DataRow(66, 20, 0, 1)] + [DataRow(69, 20, 0, 1)] + [DataRow(83, 20, 0, 1)] + [DataRow(109, 15, 0, 1)] + [DataRow(119, 11, 0, 1)] + [DataRow(134, 9, 0, 1)] + [DataTestMethod, Priority(0)] + public async Task ElseBlockTab(int line, int col, int startCharacter, int endCharacter) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = true, tabSize = 2 }; + + var src = TestData.GetPath("TestData", "Formatting", "elseBlocksTab.py"); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().OnlyHaveTextEdit(string.Empty, (line, startCharacter, line, endCharacter)); + } + } + + [DataRow(345, 18)] + [DataTestMethod, Priority(0)] + public async Task ElseBlockTabNoEdits(int line, int col) { + var position = new Position { line = line, character = col }; + var options = new FormattingOptions { insertSpaces = true, tabSize = 2 }; + + var src = TestData.GetPath("TestData", "Formatting", "elseBlocksTab.py"); + + using (var reader = new StreamReader(src)) { + var edits = await BlockFormatter.ProvideEdits(reader, position, options); + edits.Should().BeEmpty(); + } + } + } +} diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs new file mode 100644 index 000000000..f5a83b82a --- /dev/null +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -0,0 +1,809 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Completion; +using Microsoft.Python.LanguageServer.Protocol; +using Microsoft.Python.LanguageServer.Sources; +using Microsoft.Python.LanguageServer.Tests.FluentAssertions; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class CompletionTests : LanguageServerTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task TopLevelVariables() { + const string code = @" +x = 'str' +y = 1 + +class C: + def method(self): + return 1.0 + +"; + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var comps = (await cs.GetCompletionsAsync(analysis, new SourceLocation(8, 1))).Completions.ToArray(); + comps.Select(c => c.label).Should().Contain("C", "x", "y", "while", "for", "yield"); + } + + [TestMethod, Priority(0)] + public async Task StringMembers() { + const string code = @" +x = 'str' +x. +"; + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var comps = (await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 3))).Completions.ToArray(); + comps.Select(c => c.label).Should().Contain(new[] { @"isupper", @"capitalize", @"split" }); + } + + [TestMethod, Priority(0)] + public async Task ModuleMembers() { + const string code = @" +import datetime +datetime.datetime. +"; + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var comps = (await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 19))).Completions.ToArray(); + comps.Select(c => c.label).Should().Contain(new[] { "now", @"tzinfo", @"ctime" }); + } + + [TestMethod, Priority(0)] + public async Task MembersIncomplete() { + const string code = @" +class ABCDE: + def method1(self): pass + +ABC +ABCDE.me +"; + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var comps = (await cs.GetCompletionsAsync(analysis, new SourceLocation(5, 4))).Completions.ToArray(); + comps.Select(c => c.label).Should().Contain(@"ABCDE"); + + comps = (await cs.GetCompletionsAsync(analysis, new SourceLocation(6, 9))).Completions.ToArray(); + comps.Select(c => c.label).Should().Contain("method1"); + } + + [DataRow(PythonLanguageVersion.V36, "value")] + [DataRow(PythonLanguageVersion.V37, "object")] + [DataTestMethod] + public async Task OverrideCompletions3X(PythonLanguageVersion version, string parameterName) { + const string code = @" +class oar(list): + def + pass +"; + var analysis = await GetAnalysisAsync(code, version); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 9)); + + result.Should().HaveItem("append") + .Which.Should().HaveInsertText($"append(self, {parameterName}):{Environment.NewLine} return super().append({parameterName})") + .And.HaveInsertTextFormat(InsertTextFormat.PlainText); + } + + [TestMethod, Priority(0)] + public async Task TypeAtEndOfMethod() { + const string code = @" +class Fob(object): + def oar(self, a): + pass + + + def fob(self): + pass + +x = Fob() +x.oar(100) +"; + + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(4, 8)); + result.Should().HaveItem("a"); + } + + [TestMethod, Priority(0)] + public async Task TypeAtEndOfIncompleteMethod() { + var code = @" +class Fob(object): + def oar(self, a): + + +x = Fob() +x.oar(100) +"; + + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(4, 8)); + result.Should().HaveItem("a"); + } + + [TestMethod, Priority(0)] + public async Task TypeIntersectionUserDefinedTypes() { + const string code = @" +class C1(object): + def fob(self): pass + +class C2(object): + def oar(self): pass + +c = C1() +c.fob() +c = C2() +c. +"; + + + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(11, 3)); + result.Should().NotContainLabels("fob"); + result.Should().HaveLabels("oar"); + } + + [DataRow(false, "B, self")] + [DataRow(true, "")] + [DataTestMethod, Priority(0)] + public async Task ForOverrideArgs(bool is3x, string superArgs) { + const string code = @" +class A(object): + def foo(self, a, b=None, *args, **kwargs): + pass + +class B(A): + def f"; + + var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(7, 10)); + result.Should() + .HaveInsertTexts($"foo(self, a, b=None, *args, **kwargs):{Environment.NewLine} return super({superArgs}).foo(a, b=b, *args, **kwargs)") + .And.NotContainInsertTexts($"foo(self, a, b = None, *args, **kwargs):{Environment.NewLine} return super({superArgs}).foo(a, b = b, *args, **kwargs)"); + } + + [DataRow(false)] + [DataRow(true)] + [DataTestMethod, Priority(0)] + public async Task InRaise(bool is3X) { + var version = is3X ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X; + + var analysis = await GetAnalysisAsync("raise ", version); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 7)); + result.Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); + + if (is3X) { + analysis = await GetAnalysisAsync("raise Exception from ", PythonVersions.LatestAvailable3X); + cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 7)); + result.Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 17)); + result.Should().HaveInsertTexts("from").And.NotContainInsertTexts("Exception", "def", "abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 22)); + result.Should().HaveAnyCompletions(); + + analysis = await GetAnalysisAsync("raise Exception fr ", PythonVersions.LatestAvailable3X); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 19)); + result.Should().HaveInsertTexts("from") + .And.NotContainInsertTexts("Exception", "def", "abs") + .And.Subject.ApplicableSpan.Should().Be(1, 17, 1, 19); + } + + analysis = await GetAnalysisAsync("raise Exception, x, y", version); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 17)); + result.Should().HaveAnyCompletions(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 20)); + result.Should().HaveAnyCompletions(); + } + + [TestMethod, Priority(0)] + public async Task InExcept() { + var analysis = await GetAnalysisAsync("try:\n pass\nexcept "); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 8)); + result.Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); + + analysis = await GetAnalysisAsync("try:\n pass\nexcept ("); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 9)); + result.Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); + + analysis = await GetAnalysisAsync("try:\n pass\nexcept Exception as "); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 8)); + result.Should().HaveInsertTexts("Exception", "ValueError").And.NotContainInsertTexts("def", "abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 18)); + result.Should().HaveInsertTexts("as").And.NotContainInsertTexts("def", "abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 22)); + result.Should().HaveNoCompletion(); + + analysis = await GetAnalysisAsync("try:\n pass\nexc"); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 18)); + result.Should().HaveInsertTexts("except", "def", "abs"); + + analysis = await GetAnalysisAsync("try:\n pass\nexcept Exception a"); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 19)); + result.Should().HaveInsertTexts("as") + .And.NotContainInsertTexts("Exception", "def", "abs") + .And.Subject.ApplicableSpan.Should().Be(3, 18, 3, 19); + } + + [TestMethod, Priority(0)] + public async Task AfterDot() { + const string code = @" +x = 1 +x. +x.( ) +x(x. ) +x. +x +"; + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 3)); + result.Should().HaveLabels("real", @"imag").And.NotContainLabels("abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 4)); + result.Should().HaveLabels("real", @"imag").And.NotContainLabels("abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 5)); + result.Should().HaveLabels("real", @"imag").And.NotContainLabels("abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(4, 3)); + result.Should().HaveLabels("real", @"imag").And.NotContainLabels("abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(5, 5)); + result.Should().HaveLabels("real", @"imag").And.NotContainLabels("abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(6, 4)); + result.Should().HaveLabels("real", @"imag").And.NotContainLabels("abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(7, 2)); + result.Should().HaveLabels("abs").And.NotContainLabels("real", @"imag"); + } + + [TestMethod, Priority(0)] + public async Task AfterAssign() { + var analysis = await GetAnalysisAsync("x = x\ny = "); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 4)); + result.Should().HaveLabels("x", "abs"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 5)); + result.Should().HaveLabels("x", "abs"); + } + + [TestMethod, Priority(0)] + public async Task MethodFromBaseClass2X() { + const string code = @" +import unittest +class Simple(unittest.TestCase): + def test_exception(self): + self.assertRaises(TypeError). +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable2X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(5, 38)); + result.Should().HaveInsertTexts("exception"); + } + + [TestMethod, Priority(0)] + public async Task MethodFromBaseClass3X() { + const string code = @" +import unittest +class Simple(unittest.TestCase): + def test_exception(self): + self.assertRaises(TypeError). +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(5, 38)); + result.Should().HaveInsertTexts("exception"); + } + + [TestMethod, Priority(0)] + public async Task WithWhitespaceAroundDot() { + const string code = @"import sys +sys . version +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 7)); + result.Should().HaveLabels("argv"); + } + + [TestMethod, Priority(0)] + public async Task MarkupKindValid() { + var analysis = await GetAnalysisAsync("import sys\nsys.\n"); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 5)); + result.Completions?.Select(i => i.documentation.kind) + .Should().NotBeEmpty().And.BeSubsetOf(new[] { MarkupKind.PlainText, MarkupKind.Markdown }); + } + + [TestMethod, Priority(0)] + public async Task NewType() { + const string code = @" +from typing import NewType + +Foo = NewType('Foo', dict) +foo: Foo = Foo({ }) +foo. +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(6, 5)); + result.Should().HaveLabels("clear", "copy", "items", "keys", "update", "values"); + } + + [TestMethod, Priority(0)] + public async Task GenericListBase() { + const string code = @" +from typing import List + +def func(a: List[str]): + a. + a[0]. + pass +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(5, 7)); + result.Should().HaveLabels("clear", "copy", "count", "index", "remove", "reverse"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(6, 10)); + result.Should().HaveLabels("capitalize"); + } + + [TestMethod, Priority(0)] + public async Task GenericDictBase() { + const string code = @" +from typing import Dict + +def func(a: Dict[int, str]): + a. + a[0]. + pass +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(5, 7)); + result.Should().HaveLabels("keys", "values"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(6, 10)); + result.Should().HaveLabels("capitalize"); + } + + [TestMethod, Priority(0)] + public async Task ForwardRef() { + const string code = @" +class D(object): + def oar(self, x): + abc = C() + abc.fob(2) + a = abc.fob(2.0) + a.oar(('a', 'b', 'c', 'd')) + +class C(object): + def fob(self, x): + D().oar('abc') + D().oar(['a', 'b', 'c']) + return D() + def baz(self): pass +"; + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var completionInD = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 5)); + var completionInOar = await cs.GetCompletionsAsync(analysis, new SourceLocation(5, 9)); + var completionForAbc = await cs.GetCompletionsAsync(analysis, new SourceLocation(5, 13)); + + completionInD.Should().HaveLabels("C", "D", "oar") + .And.NotContainLabels("a", "abc", "self", "x", "fob", "baz"); + + completionInOar.Should().HaveLabels("C", "D", "a", "oar", "abc", "self", "x") + .And.NotContainLabels("fob", "baz"); + + completionForAbc.Should().HaveLabels("baz", "fob"); + } + + [TestMethod, Priority(0)] + public async Task SimpleGlobals() { + const string code = @" +class x(object): + def abc(self): + pass + +a = x() +x.abc() +"; + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var objectMemberNames = analysis.Document.Interpreter.GetBuiltinType(BuiltinTypeId.Object).GetMemberNames(); + + var completion = await cs.GetCompletionsAsync(analysis, new SourceLocation(7, 1)); + var completionX = await cs.GetCompletionsAsync(analysis, new SourceLocation(7, 3)); + + completion.Should().HaveLabels("a", "x").And.NotContainLabels("abc", "self"); + completionX.Should().HaveLabels(objectMemberNames).And.HaveLabels("abc"); + } + + [DataRow(true)] + [DataRow(false)] + [DataTestMethod, Priority(0)] + public async Task InFunctionDefinition(bool is3X) { + var version = is3X ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X; + + var analysis = await GetAnalysisAsync("def f(a, b:int, c=2, d:float=None): pass", version); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 5)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 7)); + result.Should().HaveNoCompletion(); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 8)); + result.Should().HaveNoCompletion(); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 10)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 14)); + result.Should().HaveLabels("int"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 17)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 29)); + result.Should().HaveLabels("float"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 34)); + result.Should().HaveLabels("NotImplemented"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 35)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 36)); + result.Should().HaveLabels("any"); + } + + [TestMethod, Priority(0)] + public async Task InFunctionDefinition_2X() { + var analysis = await GetAnalysisAsync("@dec" + Environment.NewLine + "def f(): pass", PythonVersions.LatestAvailable2X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 1)); + result.Should().HaveLabels("any"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 2)); + result.Should().HaveLabels("abs").And.NotContainLabels("def"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 1)); + result.Should().HaveLabels("def"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 2)); + result.Should().HaveLabels("def"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 5)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 6)); + result.Should().HaveNoCompletion(); + } + + [TestMethod, Priority(0)] + public async Task InFunctionDefinition_3X() { + var analysis = await GetAnalysisAsync("@dec" + Environment.NewLine + "async def f(): pass", PythonVersions.LatestAvailable3X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 1)); + result.Should().HaveLabels("any"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 2)); + result.Should().HaveLabels("abs").And.NotContainLabels("def"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 1)); + result.Should().HaveLabels("def"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 12)); + result.Should().HaveLabels("def"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 13)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 14)); + result.Should().HaveNoCompletion(); + } + + [DataRow(false)] + [DataRow(true)] + [TestMethod, Priority(0)] + public async Task InClassDefinition(bool is3x) { + var version = is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X; + + var analysis = await GetAnalysisAsync("class C(object, parameter=MC): pass", version); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 8)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 9)); + if (is3x) { + result.Should().HaveLabels(@"metaclass=", "object"); + } else { + result.Should().HaveLabels("object").And.NotContainLabels(@"metaclass="); + } + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 15)); + result.Should().HaveLabels("any"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 17)); + result.Should().HaveLabels("any"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 29)); + result.Should().HaveLabels("object"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 30)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 31)); + result.Should().HaveLabels("any"); + + analysis = await GetAnalysisAsync("class D(o", version); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 8)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 9)); + result.Should().HaveLabels("any"); + + analysis = await GetAnalysisAsync(@"class E(metaclass=MC,o): pass", version); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 22)); + result.Should().HaveLabels("object").And.NotContainLabels(@"metaclass="); + } + + [TestMethod, Priority(0)] + public async Task InWithStatement() { + var analysis = await GetAnalysisAsync("with x as y, z as w: pass"); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 6)); + result.Should().HaveAnyCompletions(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 8)); + result.Should().HaveInsertTexts("as").And.NotContainInsertTexts("abs", "dir"); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 11)); + result.Should().HaveNoCompletion(); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 14)); + result.Should().HaveAnyCompletions(); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 17)); + result.Should().HaveInsertTexts("as").And.NotContainInsertTexts("abs", "dir"); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 21)); + result.Should().HaveAnyCompletions(); + + + analysis = await GetAnalysisAsync("with "); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 6)); + result.Should().HaveAnyCompletions(); + + analysis = await GetAnalysisAsync("with x "); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 6)); + result.Should().HaveAnyCompletions(); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 8)); + result.Should().HaveInsertTexts("as").And.NotContainInsertTexts("abs", "dir"); + + analysis = await GetAnalysisAsync("with x as "); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 6)); + result.Should().HaveAnyCompletions(); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 8)); + result.Should().HaveInsertTexts("as").And.NotContainInsertTexts("abs", "dir"); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 11)); + result.Should().HaveNoCompletion(); + } + + [TestMethod, Priority(0)] + public async Task InImport() { + var code = @" +import unittest.case as C, unittest +from unittest.case import TestCase as TC, TestCase +"; + + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 7)); + result.Should().HaveLabels("from", "import", "abs", "dir").And.NotContainLabels("abc"); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 8)); + result.Should().HaveLabels("abc", @"unittest").And.NotContainLabels("abs", "dir"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 17)); + result.Should().HaveLabels("case").And.NotContainLabels("abc", @"unittest", "abs", "dir"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 23)); + result.Should().HaveLabels("as").And.NotContainLabels("abc", @"unittest", "abs", "dir"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 25)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 28)); + result.Should().HaveLabels("abc", @"unittest").And.NotContainLabels("abs", "dir"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 5)); + result.Should().HaveLabels("from", "import", "abs", "dir").And.NotContainLabels("abc"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 6)); + result.Should().HaveLabels("abc", @"unittest").And.NotContainLabels("abs", "dir"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 15)); + result.Should().HaveLabels("case").And.NotContainLabels("abc", @"unittest", "abs", "dir"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 20)); + result.Should().HaveLabels("import").And.NotContainLabels("abc", @"unittest", "abs", "dir"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 22)); + result.Should().HaveLabels("import") + .And.NotContainLabels("abc", @"unittest", "abs", "dir") + .And.Subject.ApplicableSpan.Should().Be(3, 20, 3, 26); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 27)); + result.Should().HaveLabels("TestCase").And.NotContainLabels("abs", "dir", "case"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 36)); + result.Should().HaveLabels("as").And.NotContainLabels("abc", @"unittest", "abs", "dir"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 39)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 44)); + result.Should().HaveLabels("TestCase").And.NotContainLabels("abs", "dir", "case"); + + code = @" +from unittest.case imp +pass +"; + analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 22)); + result.Should().HaveLabels("import") + .And.NotContainLabels("abc", @"unittest", "abs", "dir") + .And.Subject.ApplicableSpan.Should().Be(2, 20, 2, 23); + + code = @" +import unittest.case a +pass"; + analysis = await GetAnalysisAsync(code); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 23)); + result.Should().HaveLabels("as") + .And.NotContainLabels("abc", @"unittest", "abs", "dir") + .And.Subject.ApplicableSpan.Should().Be(2, 22, 2, 23); + + code = @" +from unittest.case import TestCase a +pass"; + analysis = await GetAnalysisAsync(code); + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(2, 37)); + result.Should().HaveLabels("as") + .And.NotContainLabels("abc", @"unittest", "abs", "dir") + .And.Subject.ApplicableSpan.Should().Be(2, 36, 2, 37); + } + + [TestMethod, Priority(0)] + public async Task ForOverride() { + const string code = @" +class A(object): + def i(): pass + def +pass"; + var analysis = await GetAnalysisAsync(code); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 9)); + result.Should().HaveNoCompletion(); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(4, 8)); + result.Should().HaveInsertTexts("def").And.NotContainInsertTexts("__init__"); + + result = await cs.GetCompletionsAsync(analysis, new SourceLocation(4, 9)); + result.Should().HaveLabels("__init__").And.NotContainLabels("def"); + } + + [TestMethod, Priority(0)] + public async Task NoCompletionInString() { + + var analysis = await GetAnalysisAsync("\"str.\""); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 6)); + result.Should().HaveNoCompletion(); + } + + [TestMethod, Priority(0)] + public async Task NoCompletionInComment() { + + var analysis = await GetAnalysisAsync("x = 1 #str. more text"); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(1, 12)); + result.Should().HaveNoCompletion(); + } + + [DataRow(false)] + [DataRow(true)] + [DataTestMethod, Priority(0)] + public async Task OsMembers(bool is3x) { + const string code = @" +import os +os. +"; + var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 4)); + result.Should().HaveLabels("path", @"devnull", "SEEK_SET", @"curdir"); + } + + [DataRow(false), Ignore("https://github.com/Microsoft/python-language-server/issues/574")] + [DataRow(true)] + [DataTestMethod, Priority(0)] + public async Task OsPathMembers(bool is3x) { + const string code = @" +import os +os.path. +"; + var analysis = await GetAnalysisAsync(code, is3x ? PythonVersions.LatestAvailable3X : PythonVersions.LatestAvailable2X); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + + var result = await cs.GetCompletionsAsync(analysis, new SourceLocation(3, 9)); + result.Should().HaveLabels("split", @"getsize", @"islink", @"abspath"); + } + } +} diff --git a/src/LanguageServer/Test/FluentAssertions/AssertionsFactory.cs b/src/LanguageServer/Test/FluentAssertions/AssertionsFactory.cs new file mode 100644 index 000000000..50c29fb39 --- /dev/null +++ b/src/LanguageServer/Test/FluentAssertions/AssertionsFactory.cs @@ -0,0 +1,47 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Completion; +using Microsoft.Python.LanguageServer.Protocol; + +namespace Microsoft.Python.LanguageServer.Tests.FluentAssertions { + [ExcludeFromCodeCoverage] + internal static class AssertionsFactory { + public static CompletionItemAssertions Should(this CompletionItem completionItem) + => new CompletionItemAssertions(completionItem); + + public static CompletionResultAssertions Should(this CompletionResult completionResult) + => new CompletionResultAssertions(completionResult); + + public static RangeAssertions Should(this Range range) => new RangeAssertions(range); + public static RangeAssertions Should(this Range? range) => new RangeAssertions(range.Value); + + public static SignatureHelpAssertions Should(this SignatureHelp signatureHelp) + => new SignatureHelpAssertions(signatureHelp); + + public static SignatureInformationAssertions Should(this SignatureInformation signatureInformation) + => new SignatureInformationAssertions(signatureInformation); + + public static SourceSpanAssertions Should(this SourceSpan span) => new SourceSpanAssertions(span); + public static SourceSpanAssertions Should(this SourceSpan? span) => new SourceSpanAssertions(span.Value); + + public static TextEditCollectionAssertions Should(this IEnumerable textEdits) + => new TextEditCollectionAssertions(textEdits); + } +} diff --git a/src/LanguageServer/Test/FluentAssertions/CompletionItemAssertions.cs b/src/LanguageServer/Test/FluentAssertions/CompletionItemAssertions.cs new file mode 100644 index 000000000..bab24c22b --- /dev/null +++ b/src/LanguageServer/Test/FluentAssertions/CompletionItemAssertions.cs @@ -0,0 +1,62 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Diagnostics.CodeAnalysis; +using FluentAssertions; +using FluentAssertions.Execution; +using FluentAssertions.Primitives; +using Microsoft.Python.LanguageServer.Protocol; +using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; + +namespace Microsoft.Python.LanguageServer.Tests.FluentAssertions { + [ExcludeFromCodeCoverage] + internal sealed class CompletionItemAssertions : ReferenceTypeAssertions { + public CompletionItemAssertions(CompletionItem subject) { + Subject = subject; + } + + protected override string Identifier => nameof(CompletionItem); + + [CustomAssertion] + public AndConstraint HaveInsertText(string insertText, string because = "", params object[] reasonArgs) { + Execute.Assertion.ForCondition(string.Equals(Subject.insertText, insertText, StringComparison.Ordinal)) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected '{Subject.label}' completion to have insert text '{DoubleEscape(insertText)}'{{reason}}, but it has '{DoubleEscape(Subject.insertText)}'"); + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndConstraint HaveInsertTextFormat(InsertTextFormat insertTextFormat, string because = "", params object[] reasonArgs) { + Execute.Assertion.ForCondition(Subject.insertTextFormat == insertTextFormat) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected '{Subject.label}' completion to have insert text format '{insertTextFormat}'{{reason}}, but it has '{Subject.insertTextFormat}'"); + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndConstraint HaveDocumentation(string documentation, string because = "", params object[] reasonArgs) { + Execute.Assertion.BecauseOf(because, reasonArgs) + .AssertIsNotNull(Subject.documentation, $"'{Subject.label}' completion", "documentation", "\'CompletionItem.documentation\'") + .Then + .ForCondition(string.Equals(Subject.documentation.value, documentation, StringComparison.Ordinal)) + .FailWith($"Expected '{Subject.label}' completion to have documentation '{documentation}'{{reason}}, but it has '{Subject.documentation.value}'"); + + return new AndConstraint(this); + } + } +} diff --git a/src/LanguageServer/Test/FluentAssertions/CompletionResultAssertions.cs b/src/LanguageServer/Test/FluentAssertions/CompletionResultAssertions.cs new file mode 100644 index 000000000..c68a1ceb9 --- /dev/null +++ b/src/LanguageServer/Test/FluentAssertions/CompletionResultAssertions.cs @@ -0,0 +1,154 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using FluentAssertions; +using FluentAssertions.Execution; +using FluentAssertions.Primitives; +using Microsoft.Python.LanguageServer.Completion; +using Microsoft.Python.LanguageServer.Protocol; +using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; + +namespace Microsoft.Python.LanguageServer.Tests.FluentAssertions { + [ExcludeFromCodeCoverage] + internal sealed class CompletionResultAssertions : ReferenceTypeAssertions { + public CompletionResultAssertions(CompletionResult subject) { + Subject = subject; + } + + protected override string Identifier => nameof(CompletionResult); + + [CustomAssertion] + public AndConstraint OnlyHaveLabels(params string[] labels) + => OnlyHaveLabels(labels, string.Empty); + + [CustomAssertion] + public AndConstraint HaveAnyCompletions(string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + + var errorMessage = Subject.Completions != null + ? Subject.Completions.Count > 0 ? null : $"Expected {GetName()} to have completion items{{reason}}, but CompletionList.items collection is empty." + : $"Expected {GetName()} to have completion items{{reason}}, but CompletionList.items collection is null."; + + Execute.Assertion.ForCondition(errorMessage == null) + .BecauseOf(because, reasonArgs) + .FailWith(errorMessage); + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndConstraint HaveNoCompletion(string because = "", params object[] reasonArgs) + => OnlyHaveLabels(Array.Empty(), because, reasonArgs); + + [CustomAssertion] + public AndConstraint OnlyHaveLabels(IEnumerable labels, string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + + var actual = Subject.Completions?.Select(i => i.label).ToArray() ?? Array.Empty(); + var expected = labels.ToArray(); + + var errorMessage = GetAssertCollectionOnlyContainsMessage(actual, expected, GetName(), "label", "labels"); + + Execute.Assertion.ForCondition(errorMessage == null) + .BecauseOf(because, reasonArgs) + .FailWith(errorMessage); + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndWhichConstraint HaveItem(string label, string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + + var actual = Subject.Completions?.Where(i => string.Equals(i.label, label, StringComparison.Ordinal)).ToArray() ?? Array.Empty(); + var errorMessage = GetAssertCollectionContainsMessage(actual.Select(i => i.label).ToArray(), new [] { label }, GetName(), "label", "labels"); + + Execute.Assertion.ForCondition(errorMessage == null) + .BecauseOf(because, reasonArgs) + .FailWith(errorMessage); + + return new AndWhichConstraint(this, actual[0]); + } + + [CustomAssertion] + public AndConstraint HaveLabels(params string[] labels) + => HaveLabels(labels, string.Empty); + + [CustomAssertion] + public AndConstraint HaveLabels(IEnumerable labels, string because = "", params object[] reasonArgs) + => HaveAttribute(labels, i => i.label, "label", "labels", because, reasonArgs); + + [CustomAssertion] + public AndConstraint HaveInsertTexts(params string[] insertTexts) + => HaveInsertTexts(insertTexts, string.Empty); + + [CustomAssertion] + public AndConstraint HaveInsertTexts(IEnumerable insertTexts, string because = "", params object[] reasonArgs) + => HaveAttribute(insertTexts, i => i.insertText, "insert text", "insert texts", because, reasonArgs); + + private AndConstraint HaveAttribute(IEnumerable attributes, Func attributeSelector, string itemNameSingle, string itemNamePlural, string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + + var actual = Subject.Completions?.Select(attributeSelector).ToArray() ?? Array.Empty(); + var expected = attributes.ToArray(); + + var errorMessage = GetAssertCollectionContainsMessage(actual, expected, GetName(), itemNameSingle, itemNamePlural); + + Execute.Assertion.ForCondition(errorMessage == null) + .BecauseOf(because, reasonArgs) + .FailWith(errorMessage); + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndConstraint NotContainLabels(params string[] labels) + => NotContainLabels(labels, string.Empty); + + [CustomAssertion] + public AndConstraint NotContainLabels(IEnumerable labels, string because = "", params object[] reasonArgs) + => NotContainAttributes(labels, i => i.label, "label", "labels", because, reasonArgs); + + [CustomAssertion] + public AndConstraint NotContainInsertTexts(params string[] insertTexts) + => NotContainInsertTexts(insertTexts, string.Empty); + + [CustomAssertion] + public AndConstraint NotContainInsertTexts(IEnumerable insertTexts, string because = "", params object[] reasonArgs) + => NotContainAttributes(insertTexts, i => i.insertText, "insert text", "insert texts", because, reasonArgs); + + public AndConstraint NotContainAttributes(IEnumerable attributes, Func attributeSelector, string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + + var actual = Subject.Completions?.Select(attributeSelector).ToArray() ?? Array.Empty(); + var expected = attributes.ToArray(); + + var errorMessage = GetAssertCollectionNotContainMessage(actual, expected, GetName(), "label", "labels"); + + Execute.Assertion.ForCondition(errorMessage == null) + .BecauseOf(because, reasonArgs) + .FailWith(errorMessage); + + return new AndConstraint(this); + } + + [CustomAssertion] + private static string GetName() => CallerIdentifier.DetermineCallerIdentity() ?? "completion list items"; + } +} diff --git a/src/LanguageServer/Test/FluentAssertions/SignatureHelpAssertions.cs b/src/LanguageServer/Test/FluentAssertions/SignatureHelpAssertions.cs new file mode 100644 index 000000000..d1a1bb5d7 --- /dev/null +++ b/src/LanguageServer/Test/FluentAssertions/SignatureHelpAssertions.cs @@ -0,0 +1,77 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using FluentAssertions; +using FluentAssertions.Execution; +using FluentAssertions.Primitives; +using Microsoft.Python.LanguageServer.Protocol; +using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; + +namespace Microsoft.Python.LanguageServer.Tests.FluentAssertions { + [ExcludeFromCodeCoverage] + internal sealed class SignatureHelpAssertions : ReferenceTypeAssertions { + public SignatureHelpAssertions(SignatureHelp subject) { + Subject = subject; + } + + protected override string Identifier => nameof(SignatureHelp); + + public AndWhichConstraint OnlyHaveSignature(string signature, string because = "", params object[] reasonArgs) { + var constraint = HaveSingleSignature(); + var actual = constraint.Which.label; + + Execute.Assertion.ForCondition(string.Equals(actual, signature, StringComparison.Ordinal)) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected SignatureHelp to have single signature '{signature}'{{reason}}, but it has '{actual}'."); + + return constraint; + } + + public AndConstraint OnlyHaveSignatures(string[] signatures) + => OnlyHaveSignatures(signatures, string.Empty); + + public AndConstraint OnlyHaveSignatures(IEnumerable signatures, string because = "", params object[] reasonArgs) { + var expected = signatures.ToArray(); + var actual = Subject.signatures.Select(s => s.label).ToArray(); + var errorMessage = GetAssertCollectionOnlyContainsMessage(actual, expected, "SignatureHelp", "signature", "signatures"); + + Execute.Assertion.ForCondition(errorMessage == null) + .BecauseOf(because, reasonArgs) + .FailWith(errorMessage); + + return new AndConstraint(this); + } + + public AndWhichConstraint HaveSingleSignature(string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + + var signature = Subject.signatures != null && Subject.signatures.Length > 0 ? Subject.signatures[0] : null; + + Execute.Assertion.ForCondition(signature != null) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected SignatureHelp to have single signature{{reason}}, but it has none."); + + Execute.Assertion.ForCondition(Subject.signatures.Length == 1) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected SignatureHelp to have single signature{{reason}}, but it has {Subject.signatures.Length}."); + + return new AndWhichConstraint(this, signature); + } + } +} diff --git a/src/LanguageServer/Test/FluentAssertions/SignatureInformationAssertions.cs b/src/LanguageServer/Test/FluentAssertions/SignatureInformationAssertions.cs new file mode 100644 index 000000000..67bb30752 --- /dev/null +++ b/src/LanguageServer/Test/FluentAssertions/SignatureInformationAssertions.cs @@ -0,0 +1,81 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using FluentAssertions; +using FluentAssertions.Execution; +using FluentAssertions.Primitives; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.LanguageServer.Protocol; + +namespace Microsoft.Python.LanguageServer.Tests.FluentAssertions { + [ExcludeFromCodeCoverage] + internal sealed class SignatureInformationAssertions : ReferenceTypeAssertions { + public SignatureInformationAssertions(SignatureInformation subject) { + Subject = subject; + } + + protected override string Identifier => nameof(SignatureInformation); + + public AndConstraint HaveNoParameters(string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + + var count = Subject.parameters?.Length ?? 0; + Execute.Assertion.ForCondition(count == 0) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected signature '{Subject.label}' to have no parameters{{reason}}, but it has {count} instead."); + + return new AndConstraint(this); + } + + public AndConstraint OnlyHaveParameterLabels(params string[] labels) + => OnlyHaveParameterLabels(labels, string.Empty); + + public AndConstraint OnlyHaveParameterLabels(IEnumerable labels, string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + + var actual = Subject.parameters?.Select(i => i.label).ToArray() ?? new string[0]; + var expected = labels.ToArray(); + + var errorMessage = AssertionsUtilities.GetAssertCollectionOnlyContainsMessage(actual, expected, $"signature '{Subject.label}'", "parameter label", "parameter labels"); + + Execute.Assertion.ForCondition(errorMessage == null) + .BecauseOf(because, reasonArgs) + .FailWith(errorMessage); + + return new AndConstraint(this); + } + + public AndConstraint HaveMarkdownDocumentation(string documentation, string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + + var errorMessage = Subject.documentation == null + ? $"Expected signature '{Subject.label}' to have markdown documentation {documentation}{{reason}}, but it has no documentation" + : Subject.documentation.kind != MarkupKind.Markdown + ? $"Expected signature '{Subject.label}' to have markdown documentation '{documentation}'{{reason}}, but it has {Subject.documentation.kind} documentation" + : !string.Equals(Subject.documentation.value, documentation) + ? $"Expected signature '{Subject.label}' to have markdown documentation '{documentation}'{{reason}}, but it has '{Subject.documentation.value}'" + : null; + + Execute.Assertion.ForCondition(errorMessage == null) + .BecauseOf(because, reasonArgs) + .FailWith(errorMessage); + + return new AndConstraint(this); + } + } +} diff --git a/src/LanguageServer/Test/FluentAssertions/SourceSpanAssertions.cs b/src/LanguageServer/Test/FluentAssertions/SourceSpanAssertions.cs new file mode 100644 index 000000000..390cd8d51 --- /dev/null +++ b/src/LanguageServer/Test/FluentAssertions/SourceSpanAssertions.cs @@ -0,0 +1,47 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using FluentAssertions; +using FluentAssertions.Execution; +using Microsoft.Python.Core.Text; +using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; + +namespace Microsoft.Python.LanguageServer.Tests.FluentAssertions { + internal sealed class SourceSpanAssertions { + public SourceSpan? Subject { get; } + + public SourceSpanAssertions(SourceSpan? span) { + Subject = span; + } + + public AndConstraint Be(int startLine, int startCharacter, int endLine, int endCharacter, string because = "", params object[] becauseArgs) { + var span = new SourceSpan( + new SourceLocation(startLine, startCharacter), + new SourceLocation(endLine, endCharacter) + ); + return Be(span, because, becauseArgs); + } + + public AndConstraint Be(SourceSpan span, string because = "", params object[] becauseArgs) { + Execute.Assertion.ForCondition(Subject.HasValue && RangeEquals(Subject.Value, span)) + .BecauseOf(because, becauseArgs) + .FailWith($"Expected range to be {span.ToString()}{{reason}}, but found {SubjectString}."); + + return new AndConstraint(this); + } + + private string SubjectString => Subject.HasValue ? Subject.Value.ToString() : "none"; + } +} diff --git a/src/LanguageServer/Test/FluentAssertions/TextEditCollectionAssertions.cs b/src/LanguageServer/Test/FluentAssertions/TextEditCollectionAssertions.cs new file mode 100644 index 000000000..d3e8534ad --- /dev/null +++ b/src/LanguageServer/Test/FluentAssertions/TextEditCollectionAssertions.cs @@ -0,0 +1,113 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using FluentAssertions; +using FluentAssertions.Collections; +using FluentAssertions.Execution; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Protocol; +using static Microsoft.Python.Analysis.Tests.FluentAssertions.AssertionsUtilities; + +namespace Microsoft.Python.LanguageServer.Tests.FluentAssertions { + [ExcludeFromCodeCoverage] + internal sealed class TextEditCollectionAssertions : SelfReferencingCollectionAssertions { + public TextEditCollectionAssertions(IEnumerable references) : base(references) { } + + protected override string Identifier => nameof(TextEdit) + "Collection"; + + + [CustomAssertion] + public AndConstraint OnlyHaveTextEdit(string expectedText, (int startLine, int startCharacter, int endLine, int endCharacter) expectedRange, string because = "", params object[] reasonArgs) + => OnlyHaveTextEdits(new[] {(expectedText, expectedRange)}, because, reasonArgs); + + [CustomAssertion] + public AndConstraint OnlyHaveTextEdits(params (string expectedText, (int startLine, int startCharacter, int endLine, int endCharacter) expectedRange)[] textEdits) + => OnlyHaveTextEdits(textEdits, string.Empty); + + [CustomAssertion] + public AndConstraint OnlyHaveTextEdits(IEnumerable<(string expectedText, (int startLine, int startCharacter, int endLine, int endCharacter) expectedRange)> textEdits, string because = "", params object[] reasonArgs) { + var expected = textEdits.ToArray(); + foreach (var (expectedText, (startLine, startCharacter, endLine, endCharacter)) in expected) { + HaveTextEditAt(expectedText, (startLine, startCharacter, endLine, endCharacter), because, reasonArgs); + } + + var excess = Subject.Select(r => (r.newText, (r.range.start.line, r.range.start.character, r.range.end.line, r.range.end.character))) + .Except(expected) + .ToArray(); + + if (excess.Length > 0) { + var excessString = string.Join(", ", excess.Select(((string text, (int, int, int, int) range) te) => $"({te.text}, {te.range.ToString()})")); + var errorMessage = expected.Length > 1 + ? $"Expected {GetSubjectName()} to have only {expected.Length} textEdits{{reason}}, but it also has textEdits: {excessString}." + : expected.Length > 0 + ? $"Expected {GetSubjectName()} to have only one reference{{reason}}, but it also has textEdits: {excessString}." + : $"Expected {GetSubjectName()} to have no textEdits{{reason}}, but it has textEdits: {excessString}."; + + Execute.Assertion.BecauseOf(because, reasonArgs).FailWith(errorMessage); + } + + return new AndConstraint(this); + } + + [CustomAssertion] + public AndConstraint HaveTextEditAt(string expectedText, (int startLine, int startCharacter, int endLine, int endCharacter) expectedRange, string because = "", params object[] reasonArgs) { + var range = new Range { + start = new Position { line = expectedRange.startLine, character = expectedRange.startCharacter }, + end = new Position { line = expectedRange.endLine, character = expectedRange.endCharacter } + }; + + var errorMessage = GetHaveTextEditErrorMessage(expectedText, range); + if (errorMessage != string.Empty) { + var assertion = Execute.Assertion.BecauseOf(because, reasonArgs); + assertion.AddNonReportable("expectedText", expectedText); + assertion.AddNonReportable("expectedRange", range); + assertion.AddNonReportable("currentTexts", GetQuotedNames(Subject.Select(te => te.newText))); + assertion.FailWith(errorMessage); + } + + return new AndConstraint(this); + } + + [CustomAssertion] + private string GetHaveTextEditErrorMessage(string expectedText, Range expectedRange) { + var candidates = Subject.Where(av => string.Equals(av.newText, expectedText, StringComparison.Ordinal)).ToArray(); + if (candidates.Length == 0) { + return "Expected {context:subject} to have text edit with newText \'{expectedText}\'{reason}, but " + + (Subject.Any() ? "it has {currentTexts}" : "it is empty"); + } + + var candidatesWithRange = candidates.Where(c => RangeEquals(c.range, expectedRange)).ToArray(); + if (candidatesWithRange.Length > 1) { + return $"Expected {{context:subject}} to have only one text edit with newText '{{expectedText}}' and range {{expectedRange}}{{reason}}, but there are {candidatesWithRange.Length}"; + } + + if (candidatesWithRange.Length == 0) { + return "Expected {context:subject} to have text edit with newText \'{expectedText}\' in range {expectedRange}{reason}, but " + + (candidates.Length == 1 + ? $"it has range {candidates[0].range.ToString()}" + : $"they are in ranges {string.Join(", ", candidates.Select(te => te.range.ToString()))}"); + } + + return string.Empty; + } + + [CustomAssertion] + private static string GetSubjectName() => CallerIdentifier.DetermineCallerIdentity() ?? "collection"; + } +} diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs new file mode 100644 index 000000000..47efefa1f --- /dev/null +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -0,0 +1,90 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Sources; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; +using Microsoft.Python.LanguageServer.Tests.FluentAssertions; + +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class GoToDefinitionTests : LanguageServerTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task BasicDefinitions() { + const string code = @" +import os + +x = os.path + +class C: + z: int + def method(self, a, b): + self.z = 1 + return 1.0 + +def func(a, b): + a = 3 + b = a + return 1 + +y = func(1, 2) +x = 1 +c = C() +c.method(1, 2) +"; + var analysis = await GetAnalysisAsync(code); + var ds = new DefinitionSource(); + + var reference = await ds.FindDefinitionAsync(analysis, new SourceLocation(4, 5)); + reference.range.Should().Be(1, 7, 1, 9); + + reference = await ds.FindDefinitionAsync(analysis, new SourceLocation(9, 9)); + reference.range.Should().Be(7, 15, 7, 19); + + reference = await ds.FindDefinitionAsync(analysis, new SourceLocation(9, 14)); + reference.range.Should().Be(6, 4, 6, 5); + + reference = await ds.FindDefinitionAsync(analysis, new SourceLocation(13, 5)); + reference.range.Should().Be(11, 9, 11, 10); + + reference = await ds.FindDefinitionAsync(analysis, new SourceLocation(14, 9)); + reference.range.Should().Be(11, 9, 11, 10); + + reference = await ds.FindDefinitionAsync(analysis, new SourceLocation(17, 5)); + reference.range.Should().Be(11, 0, 14, 12); + + reference = await ds.FindDefinitionAsync(analysis, new SourceLocation(18, 1)); + reference.range.Should().Be(17, 0, 17, 1); // TODO: store all locations + + reference = await ds.FindDefinitionAsync(analysis, new SourceLocation(19, 5)); + reference.range.Should().Be(5, 0, 9, 18); + + reference = await ds.FindDefinitionAsync(analysis, new SourceLocation(20, 5)); + reference.range.Should().Be(7, 4, 9, 18); + } + } +} diff --git a/src/LanguageServer/Test/HoverTests.cs b/src/LanguageServer/Test/HoverTests.cs new file mode 100644 index 000000000..caf48b76f --- /dev/null +++ b/src/LanguageServer/Test/HoverTests.cs @@ -0,0 +1,158 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis; +using Microsoft.Python.Core.Text; +using Microsoft.Python.LanguageServer.Sources; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class HoverTests : LanguageServerTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task BasicTypes() { + const string code = @" +x = 'str' + +class C: + '''Class C is awesome''' + def method(self, a:int, b) -> float: + '''Returns a float!!!''' + return 1.0 + +def func(a, b): + '''Does nothing useful''' + return 1 + +y = func(1, 2) +string = str +"; + var analysis = await GetAnalysisAsync(code); + var hs = new HoverSource(new PlainTextDocumentationSource()); + + var hover = await hs.GetHoverAsync(analysis, new SourceLocation(2, 2)); + hover.contents.value.Should().Be("x: str"); + + hover = await hs.GetHoverAsync(analysis, new SourceLocation(2, 7)); + hover.Should().BeNull(); + + hover = await hs.GetHoverAsync(analysis, new SourceLocation(4, 7)); + hover.contents.value.Should().Be("class C\n\nClass C is awesome"); + + hover = await hs.GetHoverAsync(analysis, new SourceLocation(6, 9)); + hover.contents.value.Should().Be("C.method(a: int, b) -> float\n\nReturns a float!!!"); + + hover = await hs.GetHoverAsync(analysis, new SourceLocation(6, 22)); + hover.contents.value.Should().Be("a: int"); + + hover = await hs.GetHoverAsync(analysis, new SourceLocation(10, 7)); + hover.contents.value.Should().Be("func(a, b)\n\nDoes nothing useful"); + + hover = await hs.GetHoverAsync(analysis, new SourceLocation(14, 2)); + hover.contents.value.Should().Be("y: int"); + + hover = await hs.GetHoverAsync(analysis, new SourceLocation(15, 2)); + hover.contents.value.Should().StartWith("class str\n\nstr(object='') -> str"); + } + + [DataRow(false)] + [DataRow(true)] + [DataTestMethod] + public async Task HoverSpanCheck(bool is3x) { + const string code = @" +import datetime +datetime.datetime.now().day +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var hs = new HoverSource(new PlainTextDocumentationSource()); + + await AssertHover(hs, analysis, new SourceLocation(3, 2), "module datetime*", new SourceSpan(3, 1, 3, 9)); + await AssertHover(hs, analysis, new SourceLocation(3, 11), "class datetime*", new SourceSpan(3, 1, 3, 18)); + await AssertHover(hs, analysis, new SourceLocation(3, 20), "datetime.now(tz: Optional[tzinfo]) -> datetime*", new SourceSpan(3, 1, 3, 22)); + } + + [TestMethod, Priority(0)] + public async Task FromImportHover() { + const string code = @" +from os import path as p +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var hs = new HoverSource(new PlainTextDocumentationSource()); + + await AssertHover(hs, analysis, new SourceLocation(2, 6), "module os*", new SourceSpan(2, 6, 2, 8)); + await AssertHover(hs, analysis, new SourceLocation(2, 16), "module*", new SourceSpan(2, 16, 2, 20)); + await AssertHover(hs, analysis, new SourceLocation(2, 24), "module*", new SourceSpan(2, 24, 2, 25)); + } + + [TestMethod, Priority(0)] + public async Task ImportAsNameHover() { + const string code = @" +import datetime as d123 +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var hs = new HoverSource(new PlainTextDocumentationSource()); + + await AssertHover(hs, analysis, new SourceLocation(2, 11), "module datetime*", new SourceSpan(2, 8, 2, 16)); + await AssertHover(hs, analysis, new SourceLocation(2, 21), "module datetime*", new SourceSpan(2, 20, 2, 24)); + } + + [TestMethod, Priority(0)] + public async Task SelfHover() { + const string code = @" +class Base(object): + def fob_base(self): + pass + +class Derived(Base): + def fob_derived(self): + self.fob_base() + pass +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + var hs = new HoverSource(new PlainTextDocumentationSource()); + await AssertHover(hs, analysis, new SourceLocation(3, 19), "class Base*", new SourceSpan(3, 18, 3, 22)); + await AssertHover(hs, analysis, new SourceLocation(8, 8), "class Derived*", new SourceSpan(8, 8, 8, 12)); + } + + private static async Task AssertHover(HoverSource hs, IDocumentAnalysis analysis, SourceLocation position, string hoverText, SourceSpan? span = null) { + var hover = await hs.GetHoverAsync(analysis, position); + + if (hoverText.EndsWith("*")) { + // Check prefix first, but then show usual message for mismatched value + if (!hover.contents.value.StartsWith(hoverText.Remove(hoverText.Length - 1))) { + Assert.AreEqual(hoverText, hover.contents.value); + } + } else { + Assert.AreEqual(hoverText, hover.contents.value); + } + if (span.HasValue) { + hover.range.Should().Be((Range)span.Value); + } + } + } +} diff --git a/src/LanguageServer/Test/LanguageServerTestBase.cs b/src/LanguageServer/Test/LanguageServerTestBase.cs new file mode 100644 index 000000000..8f32a3a54 --- /dev/null +++ b/src/LanguageServer/Test/LanguageServerTestBase.cs @@ -0,0 +1,22 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Tests; + +namespace Microsoft.Python.LanguageServer.Tests { + public abstract class LanguageServerTestBase : AnalysisTestBase { + protected static readonly ServerSettings ServerSettings = new ServerSettings(); + } +} diff --git a/src/LanguageServer/Test/LineFormatterTests.cs b/src/LanguageServer/Test/LineFormatterTests.cs new file mode 100644 index 000000000..aff05c289 --- /dev/null +++ b/src/LanguageServer/Test/LineFormatterTests.cs @@ -0,0 +1,482 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.IO; +using FluentAssertions; +using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.LanguageServer.Formatting; +using Microsoft.Python.LanguageServer.Tests.FluentAssertions; +using Microsoft.Python.Parsing; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.LanguageServer.Tests { + [TestClass] + public class LineFormatterTests { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + } + + [TestCleanup] + public void TestCleanup() { + TestEnvironmentImpl.TestCleanup(); + } + + [TestMethod, Priority(0)] + public void LineOutOfBounds() { + AssertNoEdits("a+b", line: -1); + AssertNoEdits("a+b", line: 1); + } + + [DataRow("")] + [DataRow(" ")] + [DataRow("\t")] + [DataTestMethod, Priority(0)] + public void FormatEmpty(string code) { + AssertNoEdits(code); + } + + [TestMethod, Priority(0)] + public void OperatorSpacing() { + AssertSingleLineFormat("( x +1 )*y/ 3", "(x + 1) * y / 3"); + } + + [TestMethod, Priority(0)] + public void TupleComma() { + AssertSingleLineFormat("foo =(0 ,)", "foo = (0,)"); + } + + [TestMethod, Priority(0)] + public void ColonRegular() { + AssertSingleLineFormat("if x == 4 : print x,y; x,y= y, x", "if x == 4: print x, y; x, y = y, x", languageVersion: PythonLanguageVersion.V27); + } + + [TestMethod, Priority(0)] + public void ColonSlices() { + AssertSingleLineFormat("x[1: 30]", "x[1:30]"); + } + + [TestMethod, Priority(0)] + public void ColonSlicesInArguments() { + AssertSingleLineFormat("spam ( ham[ 1 :3], {eggs : 2})", "spam(ham[1:3], {eggs: 2})"); + } + + [TestMethod, Priority(0)] + public void ColonSlicesWithDoubleColon() { + AssertSingleLineFormat("ham [1:9 ], ham[ 1: 9: 3], ham[: 9 :3], ham[1: :3], ham [ 1: 9:]", "ham[1:9], ham[1:9:3], ham[:9:3], ham[1::3], ham[1:9:]"); + } + + [TestMethod, Priority(0)] + public void ColonSlicesWithOperators() { + AssertSingleLineFormat("ham [lower+ offset :upper+offset]", "ham[lower + offset : upper + offset]"); + } + + [TestMethod, Priority(0)] + public void ColonSlicesWithFunctions() { + AssertSingleLineFormat("ham[ : upper_fn ( x) : step_fn(x )], ham[ :: step_fn(x)]", "ham[: upper_fn(x) : step_fn(x)], ham[:: step_fn(x)]"); + } + + [TestMethod, Priority(0)] + public void ColonInForLoop() { + AssertSingleLineFormat("for index in range( len(fruits) ): ", "for index in range(len(fruits)):"); + } + + [TestMethod, Priority(0)] + public void TrailingComment() { + AssertSingleLineFormat("x=1 # comment", "x = 1 # comment"); + } + + [TestMethod, Priority(0)] + public void SingleComment() { + AssertSingleLineFormat("# comment", "# comment"); + } + + [TestMethod, Priority(0)] + public void CommentWithLeadingWhitespace() { + AssertSingleLineFormat(" # comment", "# comment", editStart: 3); + } + + [TestMethod, Priority(0)] + public void AsterisksArgsKwargs() { + AssertSingleLineFormat("foo( *a, ** b)", "foo(*a, **b)"); + } + + [DataRow("for x in(1,2,3)", "for x in (1, 2, 3)")] + [DataRow("assert(1,2,3)", "assert (1, 2, 3)")] + [DataRow("if (True|False)and(False/True)and not ( x )", "if (True | False) and (False / True) and not (x)")] + [DataRow("while (True|False)", "while (True | False)")] + [DataRow("yield(a%b)", "yield (a % b)")] + [DataTestMethod, Priority(0)] + public void BraceAfterKeyword(string code, string expected) { + AssertSingleLineFormat(code, expected); + } + + [DataRow("x.y", "x.y")] + [DataRow("x. y", "x.y")] + [DataRow("5 .y", "5 .y")] + [DataTestMethod, Priority(0)] + public void DotOperator(string code, string expected) { + AssertSingleLineFormat(code, expected); + } + + [TestMethod, Priority(0)] + public void DoubleAsterisk() { + AssertSingleLineFormat("foo(a**2, **k)", "foo(a ** 2, **k)"); + } + + [TestMethod, Priority(0)] + public void Lambda() { + AssertSingleLineFormat("lambda * args, :0", "lambda *args,: 0"); + } + + [TestMethod, Priority(0)] + public void CommaExpression() { + AssertSingleLineFormat("x=1,2,3", "x = 1, 2, 3"); + } + + [TestMethod, Priority(0)] + public void IsExpression() { + AssertSingleLineFormat("a( (False is 2) is 3)", "a((False is 2) is 3)"); + } + + [TestMethod, Priority(0)] + public void FunctionReturningTuple() { + AssertSingleLineFormat("x,y=f(a)", "x, y = f(a)"); + } + + [TestMethod, Priority(0)] + public void FromDotImport() { + AssertSingleLineFormat("from. import A", "from . import A"); + } + + [TestMethod, Priority(0)] + public void FromDotDotImport() { + AssertSingleLineFormat("from ..import A", "from .. import A"); + } + + [TestMethod, Priority(0)] + public void FromDotDotXImport() { + AssertSingleLineFormat("from..x import A", "from ..x import A"); + } + + [DataRow("z=r\"\"", "z = r\"\"")] + [DataRow("z=rf\"\"", "z = rf\"\"")] + [DataRow("z=R\"\"", "z = R\"\"")] + [DataRow("z=RF\"\"", "z = RF\"\"")] + [DataTestMethod, Priority(0)] + public void RawStrings(string code, string expected) { + AssertSingleLineFormat(code, expected); + } + + [DataRow("x = - y", "x = -y")] + [DataRow("x = + y", "x = +y")] + [DataRow("x = ~ y", "x = ~y")] + [DataRow("x =-1", "x = -1")] + [DataRow("x = +1", "x = +1")] + [DataRow("x = ~1", "x = ~1")] + [DataRow("x = (-y)", "x = (-y)")] + [DataRow("x = (+ y)", "x = (+y)")] + [DataRow("x = (~ y)", "x = (~y)")] + [DataRow("x =(-1)", "x = (-1)")] + [DataRow("x = (+ 1)", "x = (+1)")] + [DataRow("x = ( ~1)", "x = (~1)")] + [DataRow("foo(-3.14, +1, ~0xDEADBEEF)", "foo(-3.14, +1, ~0xDEADBEEF)")] + [DataRow("foo(a=-3.14, b=+1, c=~0xDEADBEEF)", "foo(a=-3.14, b=+1, c=~0xDEADBEEF)")] + [DataTestMethod, Priority(0)] + public void UnaryOperators(string code, string expected) { + AssertSingleLineFormat(code, expected); + } + + [DataRow("def foo(x:int=3,x=100.)", "def foo(x: int = 3, x=100.)")] + [DataRow("def foo(x:Union[int,str]=3,x=100.)", "def foo(x: Union[int, str] = 3, x=100.)")] + [DataTestMethod, Priority(0)] + public void EqualsWithTypeHints(string code, string expected) { + AssertSingleLineFormat(code, expected); + } + + [TestMethod, Priority(0)] + public void TrailingCommaAssignment() { + AssertSingleLineFormat("a, =[1]", "a, = [1]"); + } + + [TestMethod, Priority(0)] + public void IfTrue() { + AssertSingleLineFormat("if(True) :", "if (True):"); + } + + [TestMethod, Priority(0)] + public void LambdaArguments() { + AssertSingleLineFormat("l4= lambda x =lambda y =lambda z= 1: z: y(): x()", "l4 = lambda x=lambda y=lambda z=1: z: y(): x()"); + } + + [DataRow("x = foo(\n * param1,\n * param2\n)", "*param1,", 1, 2)] + [DataRow("x = foo(\n * param1,\n * param2\n)", "*param2", 2, 2)] + [DataTestMethod, Priority(0)] + public void StarInMultilineArguments(string code, string expected, int line, int editStart) { + AssertSingleLineFormat(code, expected, line: line, editStart: editStart); + } + + [TestMethod, Priority(0)] + public void Arrow() { + AssertSingleLineFormat("def f(a, \n ** k: 11) -> 12: pass", "**k: 11) -> 12: pass", line: 1, editStart: 4); + } + + [DataRow("def foo(x = 1)", "def foo(x=1)", 0, 0)] + [DataRow("def foo(a\n, x = 1)", ", x=1)", 1, 0)] + [DataRow("foo(a ,b,\n x = 1)", "x=1)", 1, 2)] + [DataRow("if True:\n if False:\n foo(a , bar(\n x = 1)", "x=1)", 3, 6)] + [DataRow("z=foo (0 , x= 1, (3+7) , y , z )", "z = foo(0, x=1, (3 + 7), y, z)", 0, 0)] + [DataRow("foo (0,\n x= 1,", "x=1,", 1, 1)] + [DataRow(@"async def fetch(): + async with aiohttp.ClientSession() as session: + async with session.ws_connect( + ""http://127.0.0.1:8000/"", headers = cookie) as ws: # add unwanted spaces", @"""http://127.0.0.1:8000/"", headers=cookie) as ws: # add unwanted spaces", 3, 8)] + [DataRow("def pos0key1(*, key): return key\npos0key1(key= 100)", "pos0key1(key=100)", 1, 0)] + [DataRow("def test_string_literals(self):\n x= 1; y =2; self.assertTrue(len(x) == 0 and x == y)", "x = 1; y = 2; self.assertTrue(len(x) == 0 and x == y)", 1, 2)] + [DataTestMethod, Priority(0)] + public void MultilineFunctionCall(string code, string expected, int line, int editStart) { + AssertSingleLineFormat(code, expected, line: line, editStart: editStart); + } + + [TestMethod, Priority(0)] + public void RemoveTrailingSpace() { + AssertSingleLineFormat("a+b ", "a + b"); + } + + // https://github.com/Microsoft/vscode-python/issues/1783 + [DataRow("*a, b, c = 1, 2, 3")] + [DataRow("a, *b, c = 1, 2, 3")] + [DataRow("a, b, *c = 1, 2, 3")] + [DataRow("a, *b, = 1, 2, 3")] + [DataTestMethod, Priority(0)] + public void IterableUnpacking(string code) { + AssertSingleLineFormat(code, code); + } + + // https://github.com/Microsoft/vscode-python/issues/1792 + // https://www.python.org/dev/peps/pep-0008/#pet-peeves + [DataRow("ham[lower+offset : upper+offset]", "ham[lower + offset : upper + offset]")] + [DataRow("ham[: upper_fn(x) : step_fn(x)], ham[:: step_fn(x)]", "ham[: upper_fn(x) : step_fn(x)], ham[:: step_fn(x)]")] + [DataRow("ham[lower + offset : upper + offset]", "ham[lower + offset : upper + offset]")] + [DataRow("ham[1: 9], ham[1 : 9], ham[1 :9 :3]", "ham[1:9], ham[1:9], ham[1:9:3]")] + [DataRow("ham[lower : : upper]", "ham[lower::upper]")] + [DataRow("ham[ : upper]", "ham[:upper]")] + [DataRow("foo[-5:]", "foo[-5:]")] + [DataRow("foo[:-5]", "foo[:-5]")] + [DataRow("foo[+5:]", "foo[+5:]")] + [DataRow("foo[:+5]", "foo[:+5]")] + [DataRow("foo[~5:]", "foo[~5:]")] + [DataRow("foo[:~5]", "foo[:~5]")] + [DataRow("foo[-a:]", "foo[-a:]")] + [DataTestMethod, Priority(0)] + public void SlicingPetPeeves(string code, string expected) { + AssertSingleLineFormat(code, expected); + } + + [TestMethod, Priority(0)] + public void SlicingMultilineNonSimple() { + AssertSingleLineFormat("arr[:foo\n\n\n\n.bar]", "arr[: foo"); + } + + // https://github.com/Microsoft/vscode-python/issues/1784 + [TestMethod, Priority(0)] + public void LiteralFunctionCall() { + AssertSingleLineFormat("5 .bit_length()", "5 .bit_length()"); + } + + // https://github.com/Microsoft/vscode-python/issues/2323 + [TestMethod, Priority(0)] + public void MultilineFString() { + AssertNoEdits(@"f"""""" +select* from { table} +where { condition} +order by { order_columns} +limit { limit_num}; """"""", line: 5); + } + + [TestMethod, Priority(0)] + public void Ellipsis() { + AssertSingleLineFormat("x=...", "x = ..."); + } + + [DataRow("print(*[1], *[2], 3)")] + [DataRow("dict(**{'x': 1}, y=2, **{'z': 3})")] + [DataRow("*range(4), 4")] + [DataRow("[*range(4), 4]")] + [DataRow("{*range(4), 4}")] + [DataRow("{'x': 1, **{'y': 2}}")] + [DataRow("{'x': 1, **{'x': 2}}")] + [DataRow("{**{'x': 2}, 'x': 1}")] + [DataTestMethod, Priority(0)] + public void PEP448(string code) { + AssertSingleLineFormat(code, code); + } + + [TestMethod, Priority(0)] + public void MultilineStringAssignment() { + AssertSingleLineFormat("x='''abc\ntest'''abc", "x = '''abc"); + } + + [TestMethod, Priority(0)] + public void MultilineDefaultArg() { + AssertSingleLineFormat("def foo(x='''abc\ntest''')", "def foo(x='''abc"); + } + + [TestMethod, Priority(0)] + public void LineContinuation() { + AssertSingleLineFormat("a+b+ \\\n", "a + b + \\"); + } + + [DataRow("foo.a() \\\n .b() \\\n .c()", "foo.a() \\", 0, 0, 9)] + [DataRow("foo.a() \\\r\n .b() \\\r\n .c()", "foo.a() \\", 0, 0, 9)] + [DataRow("foo.a() \\\n .b() \\\n .c()", ".b() \\", 1, 3, 9)] + [DataRow("foo.a() \\\r\n .b() \\\r\n .c()", ".b() \\", 1, 3, 9)] + [DataRow("foo.a() \\\n .b() \\\n .c()", ".c()", 2, 3, 7)] + [DataRow("foo.a() \\\r\n .b() \\\r\n .c()", ".c()", 2, 3, 7)] + [DataTestMethod, Priority(0)] + public void MultilineChainedCall(string code, string expected, int line, int characterStart, int characterEnd) { + var edits = new LineFormatter(new StringReader(code), PythonLanguageVersion.V36).FormatLine(line); + edits.Should().OnlyHaveTextEdit(expected, (line, characterStart, line, characterEnd)); + } + + [DataRow("a[:, :, :, 1]")] + [DataRow("a[x:y, x + 1 :y, :, 1]")] + [DataRow("a[:, 1:3]")] + [DataRow("a[:, :3, :]")] + [DataRow("a[:, 3:, :]")] + [DataTestMethod, Priority(0)] + public void BracketCommas(string code) { + AssertSingleLineFormat(code, code); + } + + [TestMethod, Priority(0)] + public void MultilineStringTrailingComment() { + AssertSingleLineFormat("'''\nfoo\n''' # comment", " # comment", line: 2, editStart: 3); + } + + [DataRow("`a`")] + [DataRow("foo(`a`)")] + [DataRow("`a` if a else 'oops'")] + [DataTestMethod, Priority(0)] + public void Backtick(string code) { + AssertSingleLineFormat(code, code, languageVersion: PythonLanguageVersion.V27); + } + + [DataRow("exec code", PythonLanguageVersion.V27)] + [DataRow("exec (code)", PythonLanguageVersion.V27)] + [DataRow("exec(code)", PythonLanguageVersion.V37)] + [DataTestMethod, Priority(0)] + public void ExecStatement(string code, PythonLanguageVersion version) { + AssertSingleLineFormat(code, code, languageVersion: version); + } + + [TestMethod, Priority(0)] + public void CommentAfterOperator() { + AssertSingleLineFormat("a+# comment\nb", "a + # comment"); + } + + [DataRow("def foo()):\n a+b", "a + b", 1, 4, ")", 0)] + [DataRow("x = [1, 2]\nx += [3]]\na+b", "a + b", 2, 0, "]", 1)] + [DataRow("x = { foo: bar } } }\na+b", "a + b", 1, 0, "}", 0)] + [DataTestMethod, Priority(0)] + public void UnmatchedBracket(string code, string expected, int line, int editStart, string unmatched, int unmatchedLine) { + AssertSingleLineFormat(code, expected, line: line, editStart: editStart, unmatched: (unmatched, unmatchedLine)); + } + + [DataRow("'a''b'", "'a' 'b'")] + [DataRow("'a' 'b'", "'a' 'b'")] + [DataRow("'''a''''''b'''", "'''a''' '''b'''")] + [DataRow("'''a'''r'''b'''", "'''a''' r'''b'''")] + [DataRow("\"a\"\"b\"", "\"a\" \"b\"")] + [DataRow("\"a\" \"b\"", "\"a\" \"b\"")] + [DataRow("\"\"\"a\"\"\"\"\"\"b\"\"\"", "\"\"\"a\"\"\" \"\"\"b\"\"\"")] + [DataRow("\"\"\"a\"\"\"r\"\"\"b\"\"\"", "\"\"\"a\"\"\" r\"\"\"b\"\"\"")] + [DataTestMethod, Priority(0)] + public void StringConcat(string code, string expected) { + AssertSingleLineFormat(code, expected); + } + + [TestMethod, Priority(0)] + public void GrammarFile() { + var src = TestData.GetPath("TestData", "Formatting", "pythonGrammar.py"); + + string fileContents; + using (var reader = new StreamReader(src, true)) { + fileContents = reader.ReadToEnd(); + } + + var lines = fileContents.Split(new[] { "\r\n", "\r", "\n" }, StringSplitOptions.None); + + using (var reader = new StringReader(fileContents)) { + var lineFormatter = new LineFormatter(reader, PythonLanguageVersion.V37); + + for (var i = 0; i < lines.Length; i++) { + var edits = lineFormatter.FormatLine(i); + edits.Should().NotBeNull().And.HaveCountLessOrEqualTo(1); + + if (edits.Length == 0) { + continue; + } + + var edit = edits[0]; + var start = edit.range.start; + var end = edit.range.end; + + start.line.Should().Be(i); + end.line.Should().Be(i); + + var lineText = lines[i]; + edit.newText.Should().Be(lineText.Substring(start.character, end.character - start.character), $"because line {i} should be unchanged"); + } + } + } + + /// + /// Checks that a single line of input text is formatted as expected. + /// + /// Input code to format. + /// The expected result from the formatter. + /// The line number to request to be formatted. + /// Python language version to format. + /// Where the edit should begin (i.e. when whitespace or a multi-line string begins a line). + /// A nullable tuple to check against the line formatter's UnmatchedToken. + public static void AssertSingleLineFormat(string text, string expected, int line = 0, PythonLanguageVersion languageVersion = PythonLanguageVersion.V37, int editStart = 0, (string, int)? unmatched = null) { + Check.ArgumentNotNull(nameof(text), text); + Check.ArgumentNotNull(nameof(expected), expected); + + using (var reader = new StringReader(text)) { + var lineFormatter = new LineFormatter(reader, languageVersion); + var edits = lineFormatter.FormatLine(line); + + edits.Should().OnlyHaveTextEdit(expected, (line, editStart, line, text.Split('\n')[line].Length)); + lineFormatter.UnmatchedToken(line).Should().Be(unmatched); + } + } + + public static void AssertNoEdits(string text, int line = 0, PythonLanguageVersion languageVersion = PythonLanguageVersion.V37) { + Check.ArgumentNotNull(nameof(text), text); + + using (var reader = new StringReader(text)) { + var edits = new LineFormatter(reader, languageVersion).FormatLine(line); + edits.Should().BeEmpty(); + } + } + } +} diff --git a/src/LanguageServer/Test/Microsoft.Python.LanguageServer.Tests.csproj b/src/LanguageServer/Test/Microsoft.Python.LanguageServer.Tests.csproj new file mode 100644 index 000000000..f1a3b09f0 --- /dev/null +++ b/src/LanguageServer/Test/Microsoft.Python.LanguageServer.Tests.csproj @@ -0,0 +1,46 @@ + + + netcoreapp2.1 + Microsoft.Python.LanguageServer.Tests + Microsoft.Python.LanguageServer.Tests + + + + 1701;1702$(NoWarn) + 7.2 + + + ..\..\PLS.ruleset + + + ..\..\PLS.ruleset + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + + + + + + + + + + diff --git a/src/PLS.sln b/src/PLS.sln index 630b965cb..d19e7a2d0 100644 --- a/src/PLS.sln +++ b/src/PLS.sln @@ -11,10 +11,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{80AA38A1 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Product", "Product", "{C465393D-145E-4695-A7DB-AF55951BD533}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.Engine", "Analysis\Engine\Impl\Microsoft.Python.Analysis.Engine.csproj", "{55679AE9-8C3D-4F26-A0B5-10A5B2F1789F}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.Engine.Tests", "Analysis\Engine\Test\Microsoft.Python.Analysis.Engine.Tests.csproj", "{1CFA416B-6932-432F-8C75-34B5615D7664}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Core", "Core\Impl\Microsoft.Python.Core.csproj", "{84EB780C-55D0-4DEF-B5F7-D63696E11764}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Core.Tests", "Core\Test\Microsoft.Python.Core.Tests.csproj", "{AFEA5563-CED6-4932-8BB7-2A83D797FB36}" @@ -29,6 +25,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.C EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.Analysis.Tests", "Analysis\Ast\Test\Microsoft.Python.Analysis.Tests.csproj", "{D8D85896-5DB0-4FA6-B744-910A272C39F9}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Python.LanguageServer.Tests", "LanguageServer\Test\Microsoft.Python.LanguageServer.Tests.csproj", "{3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -43,14 +41,6 @@ Global {B1F6F2EA-6465-4D63-9457-D856F17EDF38}.Debug|Any CPU.Build.0 = Debug|Any CPU {B1F6F2EA-6465-4D63-9457-D856F17EDF38}.Release|Any CPU.ActiveCfg = Release|Any CPU {B1F6F2EA-6465-4D63-9457-D856F17EDF38}.Release|Any CPU.Build.0 = Release|Any CPU - {55679AE9-8C3D-4F26-A0B5-10A5B2F1789F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {55679AE9-8C3D-4F26-A0B5-10A5B2F1789F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {55679AE9-8C3D-4F26-A0B5-10A5B2F1789F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {55679AE9-8C3D-4F26-A0B5-10A5B2F1789F}.Release|Any CPU.Build.0 = Release|Any CPU - {1CFA416B-6932-432F-8C75-34B5615D7664}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1CFA416B-6932-432F-8C75-34B5615D7664}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1CFA416B-6932-432F-8C75-34B5615D7664}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1CFA416B-6932-432F-8C75-34B5615D7664}.Release|Any CPU.Build.0 = Release|Any CPU {84EB780C-55D0-4DEF-B5F7-D63696E11764}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {84EB780C-55D0-4DEF-B5F7-D63696E11764}.Debug|Any CPU.Build.0 = Debug|Any CPU {84EB780C-55D0-4DEF-B5F7-D63696E11764}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -79,6 +69,10 @@ Global {D8D85896-5DB0-4FA6-B744-910A272C39F9}.Debug|Any CPU.Build.0 = Debug|Any CPU {D8D85896-5DB0-4FA6-B744-910A272C39F9}.Release|Any CPU.ActiveCfg = Release|Any CPU {D8D85896-5DB0-4FA6-B744-910A272C39F9}.Release|Any CPU.Build.0 = Release|Any CPU + {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -86,8 +80,6 @@ Global GlobalSection(NestedProjects) = preSolution {4E648207-0C9F-45DB-AA30-84BCDAFE698D} = {C465393D-145E-4695-A7DB-AF55951BD533} {B1F6F2EA-6465-4D63-9457-D856F17EDF38} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} - {55679AE9-8C3D-4F26-A0B5-10A5B2F1789F} = {C465393D-145E-4695-A7DB-AF55951BD533} - {1CFA416B-6932-432F-8C75-34B5615D7664} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} {84EB780C-55D0-4DEF-B5F7-D63696E11764} = {C465393D-145E-4695-A7DB-AF55951BD533} {AFEA5563-CED6-4932-8BB7-2A83D797FB36} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} {C59C4212-95B8-43FA-B909-60652FA5E8E0} = {C465393D-145E-4695-A7DB-AF55951BD533} @@ -95,6 +87,7 @@ Global {615CC909-CDDD-49B1-87B8-CE8A637613E9} = {C465393D-145E-4695-A7DB-AF55951BD533} {2C8DE250-41F4-4FC5-A661-76E2A4172891} = {C465393D-145E-4695-A7DB-AF55951BD533} {D8D85896-5DB0-4FA6-B744-910A272C39F9} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} + {3BAB87E1-79FD-45D1-8564-CAF87D4D16CA} = {80AA38A1-3E82-4B87-BB21-FDEDD2CC87E6} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {ABC12ED7-0EC8-4219-8A14-A058F7942D92} diff --git a/src/Parsing/Impl/Ast/DecoratorStatement.cs b/src/Parsing/Impl/Ast/DecoratorStatement.cs index f0443a7eb..23ade24de 100644 --- a/src/Parsing/Impl/Ast/DecoratorStatement.cs +++ b/src/Parsing/Impl/Ast/DecoratorStatement.cs @@ -38,8 +38,8 @@ public override void Walk(PythonWalker walker) { public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken cancellationToken = default) { if (await walker.WalkAsync(this, cancellationToken)) { - foreach (var decorator in Decorators.MaybeEnumerate()) { - await decorator?.WalkAsync(walker, cancellationToken); + foreach (var decorator in Decorators.MaybeEnumerate().ExcludeDefault()) { + await decorator.WalkAsync(walker, cancellationToken); } } await walker.PostWalkAsync(this, cancellationToken); diff --git a/src/Parsing/Impl/Ast/FunctionDefinition.cs b/src/Parsing/Impl/Ast/FunctionDefinition.cs index 7e7f81b08..86ca85838 100644 --- a/src/Parsing/Impl/Ast/FunctionDefinition.cs +++ b/src/Parsing/Impl/Ast/FunctionDefinition.cs @@ -201,7 +201,7 @@ public override void Walk(PythonWalker walker) { public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken cancellationToken = default) { if (await walker.WalkAsync(this, cancellationToken)) { if (NameExpression != null) { - await NameExpression?.WalkAsync(walker, cancellationToken); + await NameExpression.WalkAsync(walker, cancellationToken); } foreach (var p in _parameters.MaybeEnumerate()) { diff --git a/src/Parsing/Impl/Ast/PythonNameBinder.cs b/src/Parsing/Impl/Ast/PythonNameBinder.cs index 0b3ec0178..41dde3ef7 100644 --- a/src/Parsing/Impl/Ast/PythonNameBinder.cs +++ b/src/Parsing/Impl/Ast/PythonNameBinder.cs @@ -188,7 +188,7 @@ internal PythonVariable DefineDeleted(string/*!*/ name) { internal void ReportSyntaxWarning(string message, Node node) => _errorSink.Add(message, _ast.NewLineLocations, node.StartIndex, node.EndIndex, ErrorCodes.SyntaxError, Severity.Warning); - internal void ReportSyntaxError(string message, Node node) => _errorSink.Add(message, _ast.NewLineLocations, node.StartIndex, node.EndIndex, ErrorCodes.SyntaxError, Severity.FatalError); + internal void ReportSyntaxError(string message, Node node) => _errorSink.Add(message, _ast.NewLineLocations, node.StartIndex, node.EndIndex, ErrorCodes.SyntaxError, Severity.Error); #region AstBinder Overrides diff --git a/src/Parsing/Impl/Ast/TypeAnnotation.cs b/src/Parsing/Impl/Ast/TypeAnnotation.cs index 2f6aa7e35..eb2c367dd 100644 --- a/src/Parsing/Impl/Ast/TypeAnnotation.cs +++ b/src/Parsing/Impl/Ast/TypeAnnotation.cs @@ -27,7 +27,7 @@ public TypeAnnotation(PythonLanguageVersion version, Expression expr) { Expression = expr ?? throw new ArgumentNullException(nameof(expr)); } - public static TypeAnnotation FromType(TypeAnnotationConverter converter, T type) where T : class + public static TypeAnnotation FromType(TypeAnnotationConverter converter, T type) where T : class => throw new NotImplementedException(); public PythonLanguageVersion LanguageVersion { get; } @@ -81,13 +81,18 @@ public T GetResult(TypeAnnotationConverter converter) where T : class { } public override bool Walk(ConstantExpression node) { - if (node.Value is string s) { - _parse(s)?.Walk(this); - } else if (node.Value is AsciiString a) { - _parse(a.String)?.Walk(this); - } else if (node.Value == null) { - _ops.Add(new NameOp { Name = "None" }); + switch (node.Value) { + case string s: + _parse(s)?.Walk(this); + break; + case AsciiString a: + _parse(a.String)?.Walk(this); + break; + case null: + _ops.Add(new NameOp { Name = "None" }); + break; } + return false; } @@ -348,7 +353,7 @@ public abstract class TypeAnnotationConverter where T : class { /// union type, return null. /// public virtual IReadOnlyList GetUnionTypes(T unionType) => null; - + /// /// Returns True if the provided type is not fully defined and diff --git a/src/Parsing/Impl/CollectingErrorSink.cs b/src/Parsing/Impl/CollectingErrorSink.cs index 44098fb19..4ac93da99 100644 --- a/src/Parsing/Impl/CollectingErrorSink.cs +++ b/src/Parsing/Impl/CollectingErrorSink.cs @@ -20,7 +20,7 @@ namespace Microsoft.Python.Parsing { public class CollectingErrorSink : ErrorSink { public override void Add(string message, SourceSpan span, int errorCode, Severity severity) { - if (severity == Severity.Error || severity == Severity.FatalError) { + if (severity == Severity.Error) { Errors.Add(new ErrorResult(message, span)); } else if (severity == Severity.Warning) { Warnings.Add(new ErrorResult(message, span)); diff --git a/src/Parsing/Impl/Parser.cs b/src/Parsing/Impl/Parser.cs index e30e9da07..1ada9a6ae 100644 --- a/src/Parsing/Impl/Parser.cs +++ b/src/Parsing/Impl/Parser.cs @@ -303,7 +303,7 @@ internal void ReportSyntaxError(int start, int end, string message, int errorCod _tokenizer.GetLineLocations(), start, end, errorCode, - Severity.FatalError); + Severity.Error); } #endregion @@ -2667,9 +2667,6 @@ private Statement ParseSuite() { } while (true) { - var s = ParseStmt(); - - l.Add(s); if (MaybeEat(TokenKind.Dedent)) { // dedent white space belongs to the statement which follows the suite if (_verbatim) { @@ -2677,6 +2674,10 @@ private Statement ParseSuite() { } break; } + + var s = ParseStmt(); + l.Add(s); + if (PeekToken().Kind == TokenKind.EndOfFile) { ReportSyntaxError("unexpected end of file"); break; // error handling @@ -4790,7 +4791,7 @@ private Token NextToken() { private Token PeekToken2() { if (_lookahead2.Token == null) { _lookahead2 = new TokenWithSpan(_tokenizer.GetNextToken(), _tokenizer.TokenSpan); - _lookahead2WhiteSpace = _tokenizer.PreceedingWhiteSpace; + _lookahead2WhiteSpace = _tokenizer.PrecedingWhiteSpace; } return _lookahead2.Token; } @@ -4803,7 +4804,7 @@ private void FetchLookahead() { _lookahead2WhiteSpace = null; } else { _lookahead = new TokenWithSpan(_tokenizer.GetNextToken(), _tokenizer.TokenSpan); - _lookaheadWhiteSpace = _tokenizer.PreceedingWhiteSpace; + _lookaheadWhiteSpace = _tokenizer.PrecedingWhiteSpace; } } @@ -4859,7 +4860,7 @@ public TokenizerErrorSink(Parser parser) { } public override void Add(string message, SourceSpan span, int errorCode, Severity severity) { - if (_parser._errorCode == 0 && (severity == Severity.Error || severity == Severity.FatalError)) { + if (_parser._errorCode == 0 && severity == Severity.Error) { _parser._errorCode = errorCode; } @@ -4972,7 +4973,7 @@ public static void GetEncodingFromMagicDesignator(string text, out Encoding enco encodingIndex, encodingIndex + encodingName.Length, ErrorCodes.SyntaxError, - Severity.FatalError + Severity.Error ); encoding = Encoding.UTF8; } else if (isUtf8) { @@ -5001,7 +5002,7 @@ public static void GetEncodingFromMagicDesignator(string text, out Encoding enco ex.Index, ex.Index + 1, ErrorCodes.SyntaxError, - Severity.FatalError + Severity.Error ); return new StreamReader(new PartiallyReadStream(readBytes, stream), encoding); } diff --git a/src/Parsing/Impl/Severity.cs b/src/Parsing/Impl/Severity.cs index cbd401399..ef01dacdf 100644 --- a/src/Parsing/Impl/Severity.cs +++ b/src/Parsing/Impl/Severity.cs @@ -16,10 +16,9 @@ namespace Microsoft.Python.Parsing { public enum Severity { - Ignore, - Warning, Error, - FatalError, - Information + Warning, + Information, + Hint } } diff --git a/src/Parsing/Impl/Tokenizer.cs b/src/Parsing/Impl/Tokenizer.cs index 2582b6fc3..8289016ac 100644 --- a/src/Parsing/Impl/Tokenizer.cs +++ b/src/Parsing/Impl/Tokenizer.cs @@ -33,13 +33,11 @@ namespace Microsoft.Python.Parsing { /// IronPython tokenizer /// [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1704:IdentifiersShouldBeSpelledCorrectly", MessageId = "Tokenizer")] - public sealed partial class Tokenizer { - private readonly PythonLanguageVersion _langVersion; + public sealed class Tokenizer { + private readonly bool _disableLineFeedLineSeparator = false; + private readonly SourceCodeKind _kind = SourceCodeKind.AutoDetect; private State _state; - private bool _disableLineFeedLineSeparator = false; - private SourceCodeKind _kind = SourceCodeKind.AutoDetect; private ErrorSink _errors; - private Severity _indentationInconsistencySeverity; private List _newLineLocations; private List _commentLocations; private SourceLocation _initialLocation; @@ -56,7 +54,7 @@ public sealed partial class Tokenizer { public const int DefaultBufferCapacity = 1024; private readonly Dictionary _names; - private static object _nameFromBuffer = new object(); + private static readonly object _nameFromBuffer = new object(); // pre-calculated strings for space indentation strings so we usually don't allocate. private static readonly string[] SpaceIndentation, TabIndentation; @@ -68,20 +66,20 @@ public Tokenizer(PythonLanguageVersion version, ErrorSink errorSink = null, Toke public Tokenizer(PythonLanguageVersion version, ErrorSink errorSink, TokenizerOptions options, Action commentProcessor) { _errors = errorSink ?? ErrorSink.Null; _commentProcessor = commentProcessor; - _state = new State(options); + _state = new State(options, MaxIndent); PrintFunction = false; UnicodeLiterals = false; _names = new Dictionary(new TokenEqualityComparer(this)); - _langVersion = version; + LanguageVersion = version; _options = options; } static Tokenizer() { - SpaceIndentation = new String[80]; + SpaceIndentation = new string[80]; for (var i = 0; i < 80; i++) { SpaceIndentation[i] = new string(' ', i + 1); } - TabIndentation = new String[10]; + TabIndentation = new string[10]; for (var i = 0; i < 10; i++) { TabIndentation[i] = new string('\t', i + 1); } @@ -89,7 +87,7 @@ static Tokenizer() { public bool Verbatim => (_options & TokenizerOptions.Verbatim) != 0; - public PythonLanguageVersion LanguageVersion => _langVersion; + public PythonLanguageVersion LanguageVersion { get; } private bool StubFile => _options.HasFlag(TokenizerOptions.StubFile); @@ -120,7 +118,6 @@ public List ReadTokens(int characterCount) { return tokens; } - public object CurrentState => _state; public int CurrentLine => _newLineLocations.Count; public SourceLocation CurrentPosition => IndexToLocation(CurrentIndex); @@ -147,15 +144,7 @@ internal ErrorSink ErrorSink { } } - internal Severity IndentationInconsistencySeverity { - get => _indentationInconsistencySeverity; - set { - _indentationInconsistencySeverity = value; - if (value != Severity.Ignore && _state.IndentFormat == null) { - _state.IndentFormat = new string[MaxIndent]; - } - } - } + internal Severity IndentationInconsistencySeverity { get; set; } = Severity.Warning; public bool IsEndOfFile => Peek() == EOF; @@ -177,9 +166,9 @@ public void Initialize(object state, TextReader reader, SourceLocation initialLo throw new ArgumentException("bad state provided"); } - _state = new State((State)state, Verbatim); + _state = new State((State)state, Verbatim, MaxIndent); } else { - _state = new State(_options); + _state = new State(_options, MaxIndent); } Debug.Assert(_reader == null, "Must uninitialize tokenizer before reinitializing"); @@ -371,17 +360,11 @@ internal bool TryGetTokenString(int len, out string tokenString) { internal bool UnicodeLiterals { get; set; } /// - /// Return the white space proceeding the last fetched token. Returns an empty string if - /// the tokenizer was not created in verbatim mode. + /// Return the white space preceding the last fetched token. + /// Returns an empty string if the tokenizer was not created + /// in verbatim mode. /// - public string PreceedingWhiteSpace { - get { - if (!Verbatim) { - return ""; - } - return _state.CurWhiteSpace.ToString(); - } - } + public string PrecedingWhiteSpace => !Verbatim ? string.Empty : _state.CurWhiteSpace.ToString(); public Token GetNextToken() { if (Verbatim) { @@ -503,7 +486,7 @@ private Token Next() { case 'U': _state.LastNewLine = false; // The u prefix was reintroduced to Python 3.3 in PEP 414 - if (_langVersion.Is2x() || _langVersion >= PythonLanguageVersion.V33) { + if (LanguageVersion.Is2x() || LanguageVersion >= PythonLanguageVersion.V33) { return ReadNameOrUnicodeString(); } return ReadName(); @@ -514,14 +497,14 @@ private Token Next() { case 'b': case 'B': _state.LastNewLine = false; - if (_langVersion >= PythonLanguageVersion.V26) { + if (LanguageVersion >= PythonLanguageVersion.V26) { return ReadNameOrBytes(); } return ReadName(); case 'f': case 'F': _state.LastNewLine = false; - if (_langVersion >= PythonLanguageVersion.V36) { + if (LanguageVersion >= PythonLanguageVersion.V36) { return ReadNameOrFormattedString(); } return ReadName(); @@ -534,7 +517,7 @@ private Token Next() { ch = Peek(); if (ch >= '0' && ch <= '9') { return ReadFraction(); - } else if (ch == '.' && (StubFile || _langVersion.Is3x())) { + } else if (ch == '.' && (StubFile || LanguageVersion.Is3x())) { NextChar(); if (Peek() == '.') { NextChar(); @@ -701,10 +684,10 @@ private Token ReadNameOrBytes() { private Token ReadNameOrRawString() { bool isBytes = false, isFormatted = false; - if (_langVersion >= PythonLanguageVersion.V33) { + if (LanguageVersion >= PythonLanguageVersion.V33) { isBytes = NextChar('b') || NextChar('B'); } - if (_langVersion >= PythonLanguageVersion.V36 && !isBytes) { + if (LanguageVersion >= PythonLanguageVersion.V36 && !isBytes) { isFormatted = NextChar('f') || NextChar('F'); } if (NextChar('\"')) { @@ -1018,7 +1001,7 @@ private Token MakeStringToken(char quote, bool isRaw, bool isUnicode, bool isByt } else if (isBytes) { makeUnicode = false; } else { - makeUnicode = _langVersion.Is3x() || UnicodeLiterals || StubFile; + makeUnicode = LanguageVersion.Is3x() || UnicodeLiterals || StubFile; } if (makeUnicode) { @@ -1067,7 +1050,7 @@ private Token ReadNumber(int start) { if (start == '0') { if (NextChar('x') || NextChar('X')) { return ReadHexNumber(); - } else if (_langVersion >= PythonLanguageVersion.V26) { + } else if (LanguageVersion >= PythonLanguageVersion.V26) { if ((NextChar('b') || NextChar('B'))) { return ReadBinaryNumber(); } else if (NextChar('o') || NextChar('O')) { @@ -1116,7 +1099,7 @@ private Token ReadNumber(int start) { } case '_': - if (_langVersion < PythonLanguageVersion.V36) { + if (LanguageVersion < PythonLanguageVersion.V36) { goto default; } break; @@ -1139,7 +1122,7 @@ private Token ReadNumber(int start) { var image = GetTokenString(); var val = ParseInteger(GetTokenString(), b); - if (b == 8 && _langVersion.Is3x() && (!(val is int) || !((int)val == 0))) { + if (b == 8 && LanguageVersion.Is3x() && (!(val is int) || !((int)val == 0))) { ReportSyntaxError(BufferTokenSpan, "invalid token", ErrorCodes.SyntaxError); } @@ -1183,7 +1166,7 @@ private Token ReadBinaryNumber() { case 'L': MarkTokenEnd(); - if (_langVersion.Is3x()) { + if (LanguageVersion.Is3x()) { ReportSyntaxError(new IndexSpan(_tokenEndIndex - 1, 1), "invalid token", ErrorCodes.SyntaxError); } @@ -1193,7 +1176,7 @@ private Token ReadBinaryNumber() { return new ConstantValueToken(useBigInt ? bigInt : (BigInteger)iVal); case '_': - if (_langVersion < PythonLanguageVersion.V36) { + if (LanguageVersion < PythonLanguageVersion.V36) { goto default; } break; @@ -1235,7 +1218,7 @@ private Token ReadOctalNumber() { return new ConstantValueToken(ParseBigInteger(GetTokenSubstring(2), 8)); case '_': - if (_langVersion < PythonLanguageVersion.V36) { + if (LanguageVersion < PythonLanguageVersion.V36) { goto default; } break; @@ -1293,7 +1276,7 @@ private Token ReadHexNumber() { return new ConstantValueToken(ParseBigInteger(tokenStr.Substring(2), 16)); case '_': - if (_langVersion < PythonLanguageVersion.V36) { + if (LanguageVersion < PythonLanguageVersion.V36) { goto default; } break; @@ -1396,7 +1379,7 @@ private Token ReadExponent(bool leftIsFloat = false) { return new ConstantValueToken(ParseComplex(tokenStr)); case '_': - if (_langVersion < PythonLanguageVersion.V36) { + if (LanguageVersion < PythonLanguageVersion.V36) { goto default; } ch = NextChar(); @@ -1435,7 +1418,7 @@ private Token ReadExponent(bool leftIsFloat = false) { } private bool ReportInvalidNumericLiteral(string tokenStr, bool eIsForExponent = false, bool allowLeadingUnderscore = false) { - if (_langVersion >= PythonLanguageVersion.V36 && tokenStr.Contains("_")) { + if (LanguageVersion >= PythonLanguageVersion.V36 && tokenStr.Contains("_")) { if (tokenStr.Contains("__") || (!allowLeadingUnderscore && tokenStr.StartsWithOrdinal("_")) || tokenStr.EndsWithOrdinal("_") || tokenStr.Contains("._") || tokenStr.Contains("_.")) { ReportSyntaxError(TokenSpan, "invalid token", ErrorCodes.SyntaxError); @@ -1447,7 +1430,7 @@ private bool ReportInvalidNumericLiteral(string tokenStr, bool eIsForExponent = return true; } } - if (_langVersion.Is3x() && tokenStr.EndsWithOrdinal("l", ignoreCase: true)) { + if (LanguageVersion.Is3x() && tokenStr.EndsWithOrdinal("l", ignoreCase: true)) { ReportSyntaxError(new IndexSpan(_tokenEndIndex - 1, 1), "invalid token", ErrorCodes.SyntaxError); return true; } @@ -1489,7 +1472,7 @@ private Token ReadName() { } else if (ch == 'w') { ch = NextChar(); if (ch == 'i') { - if ((_langVersion >= PythonLanguageVersion.V26 || WithStatement) && NextChar() == 't' && NextChar() == 'h' && !IsNamePart(Peek())) { + if ((LanguageVersion >= PythonLanguageVersion.V26 || WithStatement) && NextChar() == 't' && NextChar() == 'h' && !IsNamePart(Peek())) { // with is a keyword in 2.6 and up return TransformStatementToken(Tokens.KeywordWithToken); } @@ -1521,7 +1504,7 @@ private Token ReadName() { } } else if (ch == 'r') { if (NextChar() == 'i' && NextChar() == 'n' && NextChar() == 't' && !IsNamePart(Peek())) { - if (!PrintFunction && !_langVersion.Is3x() && !StubFile) { + if (!PrintFunction && !LanguageVersion.Is3x() && !StubFile) { return TransformStatementToken(Tokens.KeywordPrintToken); } } @@ -1553,7 +1536,7 @@ private Token ReadName() { ch = NextChar(); if (ch == 'e') { if (NextChar() == 'c' && !IsNamePart(Peek())) { - if (_langVersion.Is2x()) { + if (LanguageVersion.Is2x()) { return TransformStatementToken(Tokens.KeywordExecToken); } } @@ -1612,7 +1595,7 @@ private Token ReadName() { return Tokens.KeywordAndToken; } } else if (ch == 's') { - if ((_langVersion >= PythonLanguageVersion.V26 || WithStatement) && !IsNamePart(Peek())) { + if ((LanguageVersion >= PythonLanguageVersion.V26 || WithStatement) && !IsNamePart(Peek())) { // as is a keyword in 2.6 and up or when from __future__ import with_statement is used MarkTokenEnd(); return Tokens.KeywordAsToken; @@ -1623,13 +1606,13 @@ private Token ReadName() { return TransformStatementToken(Tokens.KeywordAssertToken); } } else if (ch == 'y') { - if (_langVersion >= PythonLanguageVersion.V35 && NextChar() == 'n' && NextChar() == 'c' && !IsNamePart(Peek())) { + if (LanguageVersion >= PythonLanguageVersion.V35 && NextChar() == 'n' && NextChar() == 'c' && !IsNamePart(Peek())) { MarkTokenEnd(); return Tokens.KeywordAsyncToken; } } } else if (ch == 'w') { - if (_langVersion >= PythonLanguageVersion.V35 && NextChar() == 'a' && NextChar() == 'i' && NextChar() == 't' && !IsNamePart(Peek())) { + if (LanguageVersion >= PythonLanguageVersion.V35 && NextChar() == 'a' && NextChar() == 'i' && NextChar() == 't' && !IsNamePart(Peek())) { MarkTokenEnd(); return Tokens.KeywordAwaitToken; } @@ -1650,7 +1633,7 @@ private Token ReadName() { if (ch == 't' && !IsNamePart(Peek())) { MarkTokenEnd(); return Tokens.KeywordNotToken; - } else if (_langVersion.Is3x() && ch == 'n' && NextChar() == 'l' && NextChar() == 'o' && NextChar() == 'c' && NextChar() == 'a' && NextChar() == 'l' && !IsNamePart(Peek())) { + } else if (LanguageVersion.Is3x() && ch == 'n' && NextChar() == 'l' && NextChar() == 'o' && NextChar() == 'c' && NextChar() == 'a' && NextChar() == 'l' && !IsNamePart(Peek())) { return TransformStatementToken(Tokens.KeywordNonlocalToken); } } @@ -1664,12 +1647,12 @@ private Token ReadName() { MarkTokenEnd(); return Tokens.KeywordLambdaToken; } - } else if ((_langVersion.Is3x() || StubFile) && ch == 'T') { + } else if ((LanguageVersion.Is3x() || StubFile) && ch == 'T') { if (NextChar() == 'r' && NextChar() == 'u' && NextChar() == 'e' && !IsNamePart(Peek())) { MarkTokenEnd(); return Tokens.KeywordTrueToken; } - } else if ((_langVersion.Is3x() || StubFile) && ch == 'F') { + } else if ((LanguageVersion.Is3x() || StubFile) && ch == 'F') { if (NextChar() == 'a' && NextChar() == 'l' && NextChar() == 's' && NextChar() == 'e' && !IsNamePart(Peek())) { MarkTokenEnd(); return Tokens.KeywordFalseToken; @@ -1744,7 +1727,7 @@ private Token NextOperator(int ch) { } return Tokens.ModToken; case '<': - if (_langVersion.Is2x() && NextChar('>')) { + if (LanguageVersion.Is2x() && NextChar('>')) { return Tokens.LessThanGreaterThanToken; } if (NextChar('=')) { @@ -1822,7 +1805,7 @@ private Token NextOperator(int ch) { case ':': return Tokens.ColonToken; case '`': - if (_langVersion.Is2x()) { + if (LanguageVersion.Is2x()) { return Tokens.BackQuoteToken; } break; @@ -2047,15 +2030,13 @@ private bool ReadIndentationAfterNewLine(NewLineKind startingKind) { MarkTokenEnd(); if (_tokenEndIndex != _tokenStartIndex) { - // We've captured a line of significant identation + // We've captured a line of significant indentation // (i.e. not pure whitespace or comment). Check that // any of this indentation that's in common with the // current indent level is constructed in exactly // the same way (i.e. has the same mix of spaces and // tabs etc.). - if (IndentationInconsistencySeverity != Severity.Ignore) { - CheckIndent(sb, noAllocWhiteSpace, _tokenStartIndex + startingKind.GetSize()); - } + CheckIndent(sb, noAllocWhiteSpace, _tokenStartIndex + startingKind.GetSize()); } // if there's a blank line then we don't want to mess w/ the @@ -2109,7 +2090,7 @@ private void CheckIndent(StringBuilder sb, string noAllocWhiteSpace, int indentS _newLineLocations.ToArray(), indentStart, _tokenEndIndex, - ErrorCodes.TabError, _indentationInconsistencySeverity + ErrorCodes.TabError, IndentationInconsistencySeverity ); break; } @@ -2121,7 +2102,9 @@ private void SetIndent(int spaces, StringBuilder chars, string noAllocWhiteSpace var current = _state.Indent[_state.IndentLevel]; if (spaces == current) { return; - } else if (spaces > current) { + } + + if (spaces > current) { _state.Indent[++_state.IndentLevel] = spaces; if (_state.IndentFormat != null) { if (chars != null) { @@ -2132,14 +2115,14 @@ private void SetIndent(int spaces, StringBuilder chars, string noAllocWhiteSpace } _state.PendingDedents = -1; return; - } else { - current = DoDedent(spaces, current); + } - if (spaces != current && indentStart != -1) { - ReportSyntaxError( - new IndexSpan(indentStart, spaces), - "unindent does not match any outer indentation level", ErrorCodes.IndentationError); - } + current = DoDedent(spaces, current); + + if (spaces != current && indentStart != -1) { + ReportSyntaxError( + new IndexSpan(indentStart, spaces), + "unindent does not match any outer indentation level", ErrorCodes.IndentationError); } } @@ -2205,7 +2188,7 @@ private object ParseComplex(string s) { } private void ReportSyntaxError(IndexSpan span, string message, int errorCode) - => _errors.Add(message, _newLineLocations.ToArray(), span.Start, span.End, errorCode, Severity.FatalError); + => _errors.Add(message, _newLineLocations.ToArray(), span.Start, span.End, errorCode, Severity.Error); [Conditional("DUMP_TOKENS")] private static void DumpToken(Token token) @@ -2239,8 +2222,8 @@ public override int GetHashCode() => (IsRaw ? 0x01 : 0) | (IsFormatted ? 0x10 : 0); public static bool operator ==(IncompleteString left, IncompleteString right) { - if ((object)left == null) { - return (object)right == null; + if (left is null) { + return right is null; } return left.Equals(right); } @@ -2286,7 +2269,7 @@ struct State : IEquatable { public StringBuilder NextWhiteSpace; public GroupingRecovery GroupingRecovery; - public State(State state, bool verbatim) { + public State(State state, bool verbatim, int maxIndent) { Indent = (int[])state.Indent.Clone(); LastNewLine = state.LastNewLine; BracketLevel = state.BraceLevel; @@ -2294,7 +2277,7 @@ public State(State state, bool verbatim) { BraceLevel = state.BraceLevel; PendingDedents = state.PendingDedents; IndentLevel = state.IndentLevel; - IndentFormat = (state.IndentFormat != null) ? (string[])state.IndentFormat.Clone() : null; + IndentFormat = (string[])state.IndentFormat?.Clone(); IncompleteString = state.IncompleteString; if (verbatim) { CurWhiteSpace = new StringBuilder(state.CurWhiteSpace.ToString()); @@ -2304,9 +2287,10 @@ public State(State state, bool verbatim) { NextWhiteSpace = null; } GroupingRecovery = null; + IndentFormat = new string[maxIndent]; } - public State(TokenizerOptions options) { + public State(TokenizerOptions options, int maxIndent) { Indent = new int[MaxIndent]; // TODO LastNewLine = true; BracketLevel = ParenLevel = BraceLevel = PendingDedents = IndentLevel = 0; @@ -2320,6 +2304,7 @@ public State(TokenizerOptions options) { NextWhiteSpace = null; } GroupingRecovery = null; + IndentFormat = new string[maxIndent]; } public override bool Equals(object obj) { diff --git a/src/Parsing/Test/ParserTests.cs b/src/Parsing/Test/ParserTests.cs index e6116044f..b4eb6f68d 100644 --- a/src/Parsing/Test/ParserTests.cs +++ b/src/Parsing/Test/ParserTests.cs @@ -3038,7 +3038,7 @@ public ErrorInfo(string msg, int startIndex, int startLine, int startCol, int en } private void ParseErrors(string filename, PythonLanguageVersion version, params ErrorInfo[] errors) { - ParseErrors(filename, version, Severity.Ignore, errors); + ParseErrors(filename, version, Severity.Hint, errors); } private static string FormatError(ErrorResult r) { @@ -3061,7 +3061,7 @@ private void ParseErrors(string filename, PythonLanguageVersion version, Severit for (var i = 0; i < sink.Errors.Count; i++) { foundErrors.AppendFormat("{0}{1}{2}", FormatError(sink.Errors[i]), - i == sink.Errors.Count - 1 ? "" : ",", + i == sink.Errors.Count - 1 ? string.Empty : ",", Environment.NewLine ); } @@ -3085,7 +3085,7 @@ private void ParseErrors(string filename, PythonLanguageVersion version, Severit } } - private static PythonAst ParseFileNoErrors(string filename, PythonLanguageVersion version, Severity indentationInconsistencySeverity = Severity.Ignore) { + private static PythonAst ParseFileNoErrors(string filename, PythonLanguageVersion version, Severity indentationInconsistencySeverity = Severity.Hint) { var errorSink = new CollectingErrorSink(); var ast = ParseFile(filename, errorSink, version, indentationInconsistencySeverity); foreach (var warn in errorSink.Warnings) { @@ -3098,7 +3098,7 @@ private static PythonAst ParseFileNoErrors(string filename, PythonLanguageVersio return ast; } - private static PythonAst ParseFile(string filename, ErrorSink errorSink, PythonLanguageVersion version, Severity indentationInconsistencySeverity = Severity.Ignore) { + private static PythonAst ParseFile(string filename, ErrorSink errorSink, PythonLanguageVersion version, Severity indentationInconsistencySeverity = Severity.Hint) { var src = TestData.GetPath("TestData", "Grammar", filename); using (var reader = new StreamReader(src, true)) { var parser = Parser.CreateParser(reader, version, new ParserOptions() { ErrorSink = errorSink, IndentationInconsistencySeverity = indentationInconsistencySeverity }); @@ -3106,7 +3106,7 @@ private static PythonAst ParseFile(string filename, ErrorSink errorSink, PythonL } } - private static PythonAst ParseString(string content, ErrorSink errorSink, PythonLanguageVersion version, Severity indentationInconsistencySeverity = Severity.Ignore) { + private static PythonAst ParseString(string content, ErrorSink errorSink, PythonLanguageVersion version, Severity indentationInconsistencySeverity = Severity.Hint) { using (var reader = new StringReader(content)) { var parser = Parser.CreateParser(reader, version, new ParserOptions() { ErrorSink = errorSink, IndentationInconsistencySeverity = indentationInconsistencySeverity }); return parser.ParseFile(); diff --git a/src/Parsing/Test/TokenizerRoundTripTest.cs b/src/Parsing/Test/TokenizerRoundTripTest.cs index 22a45441d..75730ff17 100644 --- a/src/Parsing/Test/TokenizerRoundTripTest.cs +++ b/src/Parsing/Test/TokenizerRoundTripTest.cs @@ -179,7 +179,7 @@ private static List TestOneString(PythonLanguageVersion version, while ((token = tokenizer.GetNextToken()) != Tokens.EndOfFileToken) { tokens.Add(new TokenWithSpan(token, tokenizer.TokenSpan)); - output.Append(tokenizer.PreceedingWhiteSpace); + output.Append(tokenizer.PrecedingWhiteSpace); output.Append(token.VerbatimImage); const int contextSize = 50; @@ -211,7 +211,7 @@ private static List TestOneString(PythonLanguageVersion version, prevOffset = output.Length; } - output.Append(tokenizer.PreceedingWhiteSpace); + output.Append(tokenizer.PrecedingWhiteSpace); Assert.AreEqual(originalText.Length, output.Length); return tokens;