From 6095ae56576cea9b31aa57997c40e239865d6cd5 Mon Sep 17 00:00:00 2001 From: Justin Chen Date: Thu, 30 Jul 2020 11:47:38 -0700 Subject: [PATCH] Added Handler for Semantic Tokenization (#1328) * added basic semantic token support * removed unnecessary imports * removed unnecessary field * minor refactoring changes * minor refactoring changes * change tokenize to non async * rename handler * refactoring + copyright * renamed handler file * Delete log20200713.txt * moved/refactored handler * added e2e tets * updated test * remove pragma * removed extra spacing * added testing for converting from PS token to semantic tokens * refactored the functions related to converting between tokens * refactored ConvertSemanticToken * fixed tests * added more test cases * fixed spacing * added enum test * fixed spacing issues * fixed spacing, added note about token array representation * changed name to PsesSemanticTokensHandler * reformatted fields * renamed file * used Assert.Collection instead of Assert.Single * modified yml file to fix build * undo changes in yml file * addressed issues in PR * added basic semantic token support * removed unnecessary imports * removed unnecessary field * minor refactoring changes * minor refactoring changes * change tokenize to non async * rename handler * refactoring + copyright * renamed handler file * moved/refactored handler * added e2e tets * Delete log20200713.txt * updated test * remove pragma * removed extra spacing * added testing for converting from PS token to semantic tokens * refactored the functions related to converting between tokens * refactored ConvertSemanticToken * fixed tests * added more test cases * fixed spacing * added enum test * fixed spacing issues * fixed spacing, added note about token array representation * changed name to PsesSemanticTokensHandler * reformatted fields * renamed file * used Assert.Collection instead of Assert.Single * addressed issues in PR * remove unused using * Delete Untitled-1.json Co-authored-by: Justin Chen --- .../Server/PsesLanguageServer.cs | 1 + .../Handlers/PsesSemanticTokensHandler.cs | 168 ++++++++++++++++ .../Services/TextDocument/SemanticToken.cs | 27 +++ .../LanguageServerProtocolMessageTests.cs | 31 +++ .../Language/SemanticTokenTest.cs | 188 ++++++++++++++++++ 5 files changed, 415 insertions(+) create mode 100644 src/PowerShellEditorServices/Services/TextDocument/Handlers/PsesSemanticTokensHandler.cs create mode 100644 src/PowerShellEditorServices/Services/TextDocument/SemanticToken.cs create mode 100644 test/PowerShellEditorServices.Test/Language/SemanticTokenTest.cs diff --git a/src/PowerShellEditorServices/Server/PsesLanguageServer.cs b/src/PowerShellEditorServices/Server/PsesLanguageServer.cs index 82abc2b72..52eb2dec4 100644 --- a/src/PowerShellEditorServices/Server/PsesLanguageServer.cs +++ b/src/PowerShellEditorServices/Server/PsesLanguageServer.cs @@ -92,6 +92,7 @@ public async Task StartAsync() .WithHandler() .WithHandler() .WithHandler() + .WithHandler() .OnInitialize( async (languageServer, request, cancellationToken) => { diff --git a/src/PowerShellEditorServices/Services/TextDocument/Handlers/PsesSemanticTokensHandler.cs b/src/PowerShellEditorServices/Services/TextDocument/Handlers/PsesSemanticTokensHandler.cs new file mode 100644 index 000000000..a5a520783 --- /dev/null +++ b/src/PowerShellEditorServices/Services/TextDocument/Handlers/PsesSemanticTokensHandler.cs @@ -0,0 +1,168 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Collections.Generic; +using System.Management.Automation.Language; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.PowerShell.EditorServices.Services; +using Microsoft.PowerShell.EditorServices.Services.TextDocument; +using Microsoft.PowerShell.EditorServices.Utility; +using OmniSharp.Extensions.LanguageServer.Protocol; +using OmniSharp.Extensions.LanguageServer.Protocol.Document.Proposals; +using OmniSharp.Extensions.LanguageServer.Protocol.Models; +using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals; + +namespace Microsoft.PowerShell.EditorServices.Handlers +{ + internal class PsesSemanticTokensHandler : SemanticTokensHandler + { + private static readonly SemanticTokensRegistrationOptions s_registrationOptions = new SemanticTokensRegistrationOptions + { + DocumentSelector = LspUtils.PowerShellDocumentSelector, + Legend = new SemanticTokensLegend(), + DocumentProvider = new Supports( + isSupported: true, + new SemanticTokensDocumentProviderOptions + { + Edits = true + }), + RangeProvider = true + }; + + private readonly ILogger _logger; + private readonly WorkspaceService _workspaceService; + + public PsesSemanticTokensHandler(ILogger logger, WorkspaceService workspaceService) + : base(s_registrationOptions) + { + _logger = logger; + _workspaceService = workspaceService; + } + + protected override Task Tokenize(SemanticTokensBuilder builder, ITextDocumentIdentifierParams identifier, + CancellationToken cancellationToken) + { + ScriptFile file = _workspaceService.GetFile(identifier.TextDocument.Uri); + foreach (Token token in file.ScriptTokens) + { + PushToken(token, builder); + } + return Task.CompletedTask; + } + + private static void PushToken(Token token, SemanticTokensBuilder builder) + { + foreach (SemanticToken sToken in ConvertToSemanticTokens(token)) + { + builder.Push( + sToken.Line, + sToken.Column, + length: sToken.Text.Length, + sToken.Type, + tokenModifiers: sToken.TokenModifiers); + } + } + + internal static IEnumerable ConvertToSemanticTokens(Token token) + { + if (token is StringExpandableToken stringExpandableToken) + { + // Try parsing tokens within the string + if (stringExpandableToken.NestedTokens != null) + { + foreach (Token t in stringExpandableToken.NestedTokens) + { + foreach (SemanticToken subToken in ConvertToSemanticTokens(t)) + yield return subToken; + } + yield break; + } + } + + SemanticTokenType mappedType = MapSemanticTokenType(token); + if (mappedType == null) + { + yield break; + } + + //Note that both column and line numbers are 0-based + yield return new SemanticToken( + token.Text, + mappedType, + line: token.Extent.StartLineNumber - 1, + column: token.Extent.StartColumnNumber - 1, + tokenModifiers: Array.Empty()); + } + + private static SemanticTokenType MapSemanticTokenType(Token token) + { + // First check token flags + if ((token.TokenFlags & TokenFlags.Keyword) != 0) + { + return SemanticTokenType.Keyword; + } + + if ((token.TokenFlags & TokenFlags.CommandName) != 0) + { + return SemanticTokenType.Function; + } + + if (token.Kind != TokenKind.Generic && (token.TokenFlags & + (TokenFlags.BinaryOperator | TokenFlags.UnaryOperator | TokenFlags.AssignmentOperator)) != 0) + { + return SemanticTokenType.Operator; + } + + if ((token.TokenFlags & TokenFlags.TypeName) != 0) + { + return SemanticTokenType.Type; + } + + if ((token.TokenFlags & TokenFlags.MemberName) != 0) + { + return SemanticTokenType.Member; + } + + // Only check token kind after checking flags + switch (token.Kind) + { + case TokenKind.Comment: + return SemanticTokenType.Comment; + + case TokenKind.Parameter: + case TokenKind.Generic when token is StringLiteralToken slt && slt.Text.StartsWith("--"): + return SemanticTokenType.Parameter; + + case TokenKind.Variable: + case TokenKind.SplattedVariable: + return SemanticTokenType.Variable; + + case TokenKind.StringExpandable: + case TokenKind.StringLiteral: + case TokenKind.HereStringExpandable: + case TokenKind.HereStringLiteral: + return SemanticTokenType.String; + + case TokenKind.Number: + return SemanticTokenType.Number; + + case TokenKind.Generic: + return SemanticTokenType.Function; + } + + return null; + } + + protected override Task GetSemanticTokensDocument( + ITextDocumentIdentifierParams @params, + CancellationToken cancellationToken) + { + return Task.FromResult(new SemanticTokensDocument(GetRegistrationOptions().Legend)); + } + } +} diff --git a/src/PowerShellEditorServices/Services/TextDocument/SemanticToken.cs b/src/PowerShellEditorServices/Services/TextDocument/SemanticToken.cs new file mode 100644 index 000000000..a687690b2 --- /dev/null +++ b/src/PowerShellEditorServices/Services/TextDocument/SemanticToken.cs @@ -0,0 +1,27 @@ +using System.Collections.Generic; +using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals; + +namespace Microsoft.PowerShell.EditorServices.Services.TextDocument +{ + internal class SemanticToken + { + public SemanticToken(string text, SemanticTokenType type, int line, int column, IEnumerable tokenModifiers) + { + Line = line; + Text = text; + Column = column; + Type = type; + TokenModifiers = tokenModifiers; + } + + public string Text { get; set ;} + + public int Line { get; set; } + + public int Column { get; set; } + + public SemanticTokenType Type { get; set; } + + public IEnumerable TokenModifiers { get; set; } + } +} diff --git a/test/PowerShellEditorServices.Test.E2E/LanguageServerProtocolMessageTests.cs b/test/PowerShellEditorServices.Test.E2E/LanguageServerProtocolMessageTests.cs index 05826f514..a6047542f 100644 --- a/test/PowerShellEditorServices.Test.E2E/LanguageServerProtocolMessageTests.cs +++ b/test/PowerShellEditorServices.Test.E2E/LanguageServerProtocolMessageTests.cs @@ -19,6 +19,7 @@ using OmniSharp.Extensions.LanguageServer.Protocol.Document; using OmniSharp.Extensions.LanguageServer.Protocol.Models; using OmniSharp.Extensions.LanguageServer.Protocol.Workspace; +using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals; using Xunit; using Xunit.Abstractions; using Range = OmniSharp.Extensions.LanguageServer.Protocol.Models.Range; @@ -1136,5 +1137,35 @@ await PsesLanguageClient Assert.Equal("Get-ChildItem", expandAliasResult.Text); } + + [Fact] + public async Task CanSendSemanticTokenRequest() + { + string scriptContent = "function"; + string scriptPath = NewTestFile(scriptContent); + + SemanticTokens result = + await PsesLanguageClient + .SendRequest( + "textDocument/semanticTokens", + new SemanticTokensParams + { + TextDocument = new TextDocumentIdentifier + { + Uri = new Uri(scriptPath) + } + }) + .Returning(CancellationToken.None); + + // More information about how this data is generated can be found at + // https://github.com/microsoft/vscode-extension-samples/blob/5ae1f7787122812dcc84e37427ca90af5ee09f14/semantic-tokens-sample/vscode.proposed.d.ts#L71 + var expectedArr = new int[5] + { + // line, index, token length, token type, token modifiers + 0, 0, scriptContent.Length, 2, 0 //function token: line 0, index 0, length, type 2 = keyword, no modifiers + }; + + Assert.Equal(expectedArr, result.Data.ToArray()); + } } } diff --git a/test/PowerShellEditorServices.Test/Language/SemanticTokenTest.cs b/test/PowerShellEditorServices.Test/Language/SemanticTokenTest.cs new file mode 100644 index 000000000..3dbd66e7d --- /dev/null +++ b/test/PowerShellEditorServices.Test/Language/SemanticTokenTest.cs @@ -0,0 +1,188 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using System; +using System.Collections.Generic; +using System.IO; +using System.Management.Automation.Language; +using System.Threading.Tasks; +using Microsoft.PowerShell.EditorServices.Services.TextDocument; +using Microsoft.PowerShell.EditorServices.Handlers; +using OmniSharp.Extensions.LanguageServer.Protocol; +using OmniSharp.Extensions.LanguageServer.Protocol.Models.Proposals; +using Xunit; + +namespace Microsoft.PowerShell.EditorServices.Test.Language +{ + public class SemanticTokenTest + { + [Fact] + public async Task TokenizesFunctionElements() + { + string text = @" +function Get-Sum { + param( [int]$a, [int]$b ) + return $a + $b +} +"; + ScriptFile scriptFile = new ScriptFile( + // Use any absolute path. Even if it doesn't exist. + DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")), + text, + Version.Parse("5.0")); + + foreach (Token t in scriptFile.ScriptTokens) + { + List mappedTokens = new List(PsesSemanticTokensHandler.ConvertToSemanticTokens(t)); + switch (t.Text) + { + case "function": + case "param": + case "return": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Keyword == sToken.Type); + break; + case "Get-Sum": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Function == sToken.Type); + break; + case "$a": + case "$b": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Variable == sToken.Type); + break; + case "[int]": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Type == sToken.Type); + break; + case "+": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Operator == sToken.Type); + break; + } + } + } + + [Fact] + public async Task TokenizesStringExpansion() + { + string text = "Write-Host \"$(Test-Property Get-Whatever) $(Get-Whatever)\""; + ScriptFile scriptFile = new ScriptFile( + // Use any absolute path. Even if it doesn't exist. + DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")), + text, + Version.Parse("5.0")); + + Token commandToken = scriptFile.ScriptTokens[0]; + List mappedTokens = new List(PsesSemanticTokensHandler.ConvertToSemanticTokens(commandToken)); + Assert.Single(mappedTokens, sToken => SemanticTokenType.Function == sToken.Type); + + Token stringExpandableToken = scriptFile.ScriptTokens[1]; + mappedTokens = new List(PsesSemanticTokensHandler.ConvertToSemanticTokens(stringExpandableToken)); + Assert.Collection(mappedTokens, + sToken => Assert.Equal(SemanticTokenType.Function, sToken.Type), + sToken => Assert.Equal(SemanticTokenType.Function, sToken.Type), + sToken => Assert.Equal(SemanticTokenType.Function, sToken.Type) + ); + } + + [Fact] + public async Task RecognizesTokensWithAsterisk() + { + string text = @" +function Get-A*A { +} +Get-A*A +"; + ScriptFile scriptFile = new ScriptFile( + // Use any absolute path. Even if it doesn't exist. + DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")), + text, + Version.Parse("5.0")); + + foreach (Token t in scriptFile.ScriptTokens) + { + List mappedTokens = new List(PsesSemanticTokensHandler.ConvertToSemanticTokens(t)); + switch (t.Text) + { + case "function": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Keyword == sToken.Type); + break; + case "Get-A*A": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Function == sToken.Type); + break; + } + } + } + + [Fact] + public async Task RecognizesArrayMemberInExpandableString() + { + string text = "\"$(@($Array).Count) OtherText\""; + ScriptFile scriptFile = new ScriptFile( + // Use any absolute path. Even if it doesn't exist. + DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")), + text, + Version.Parse("5.0")); + + foreach (Token t in scriptFile.ScriptTokens) + { + List mappedTokens = new List(PsesSemanticTokensHandler.ConvertToSemanticTokens(t)); + switch (t.Text) + { + case "$Array": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Variable == sToken.Type); + break; + case "Count": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Member == sToken.Type); + break; + } + } + } + + [Fact] + public async Task RecognizesCurlyQuotedString() + { + string text = "“^[-'a-z]*”"; + ScriptFile scriptFile = new ScriptFile( + // Use any absolute path. Even if it doesn't exist. + DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")), + text, + Version.Parse("5.0")); + + List mappedTokens = new List(PsesSemanticTokensHandler.ConvertToSemanticTokens(scriptFile.ScriptTokens[0])); + Assert.Single(mappedTokens, sToken => SemanticTokenType.String == sToken.Type); + } + + [Fact] + public async Task RecognizeEnum() + { + string text = @" +enum MyEnum{ + one + two + three +} +"; + ScriptFile scriptFile = new ScriptFile( + // Use any absolute path. Even if it doesn't exist. + DocumentUri.FromFileSystemPath(Path.Combine(Path.GetTempPath(), "TestFile.ps1")), + text, + Version.Parse("5.0")); + + foreach (Token t in scriptFile.ScriptTokens) + { + List mappedTokens = new List(PsesSemanticTokensHandler.ConvertToSemanticTokens(t)); + switch (t.Text) + { + case "enum": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Keyword == sToken.Type); + break; + case "MyEnum": + case "one": + case "two": + case "three": + Assert.Single(mappedTokens, sToken => SemanticTokenType.Member == sToken.Type); + break; + } + } + } + } +}