Fix #300 and #301 (#302)

* Fix #300 & #301

apps/erlangbridge/src/gen_lsp_doc_server.erl

@@ -8,8 +8,10 @@
 -export([get_syntax_tree/1, get_dodged_syntax_tree/1, get_references/1, get_inlayhints/1]).
 -export([root_available/0, config_change/0, project_modules/0, get_module_file/1, get_module_files/1, get_build_dir/0, find_source_file/1]).
 -export([init/1,handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]).
+-include("./lsp_log.hrl").
 
 -define(SERVER, ?MODULE).
+-define(IIF(Cond, Then, Else), if Cond -> Then; true -> Else end).
 
 -record(state,
         {root_available = false :: boolean(),
@@ -187,10 +189,10 @@ handle_cast({project_file_deleted, File}, State) ->
     {noreply, delete_project_files([File], State)}.
 
 handle_info({worker_result, File, _Result}, State) ->
-    gen_lsp_server:lsp_log("Parsed in background: ~s~n", [File]),
+    ?LOG("Parsed in background: ~s~n", [unicode:characters_to_binary(File)]),
     {noreply, parse_next_file_in_background(State)};
 handle_info({worker_error, File, _Exception}, State) ->
-    gen_lsp_server:lsp_log("Parse in background failed: ~s~n", [File]),
+    ?LOG("Parse in background failed: ~s~n", [unicode:characters_to_binary(File)]),
     {noreply, parse_next_file_in_background(State)};
 handle_info(_Info, State) ->
     {noreply, State}.
@@ -342,39 +344,47 @@ scan_project_files(State = #state{project_modules = OldProjectModules}) ->
     %% Find all source (*.erl) files not excluded by any filter
     CollectProjSrcFilesFun =
         fun(File, AccProjectModules = #{}) ->
-            case lsp_utils:is_path_excluded(File, SearchExclude) of
+            % File can contain unicode characters (e.g. smiley in the path, like in otp repo)            
+            case lsp_utils:is_path_excluded(unicode:characters_to_binary(File), SearchExclude) of
                 true  -> AccProjectModules;
                 false -> do_add_project_file(File, AccProjectModules, BuildDir)
             end
         end,
     AllProjectModules = filelib:fold_files(gen_lsp_config_server:root(), "\\.erl$",
                                            true, CollectProjSrcFilesFun, #{}),
-    gen_lsp_server:lsp_log(
+    NbFiles = maps:size(AllProjectModules),
+    LogAllProjectModules = ?IIF(NbFiles > 200, #{}, AllProjectModules),
+    ?LOG(
         "~p: Project modules (~p) with all source files:~n  ~p~n",
-        [?MODULE, maps:size(AllProjectModules), AllProjectModules]),
+        [?MODULE, NbFiles, LogAllProjectModules]),
     %% Filter out source files sym-linked to the original ones by rebar3
     %% from '_build' directory
     ProjectModules =
         maps:map(
             fun(_Module, Files) -> module_files_to_parse(Files, BuildDir) end,
             AllProjectModules),
-    gen_lsp_server:lsp_log(
+
+    NbPrjFiles = maps:size(ProjectModules),
+    LogProjectModules = ?IIF(NbPrjFiles > 200, #{}, ProjectModules),
+    ?LOG(
         "~p: Project modules (~p) without rebar3 duplicated source files:~n  ~p~n",
-        [?MODULE, maps:size(ProjectModules), ProjectModules]),
+        [?MODULE, maps:size(ProjectModules), LogProjectModules]),
     %% Get the complete list of source files (already scanned and newly found)
     OldProjFiles = lists:usort(lists:append(maps:values(OldProjectModules))),
     ProjFiles = lists:usort(lists:append(maps:values(ProjectModules))),
     %% Drop unwanted files, e.g. newly excluded by a filter change
     FilesToDrop = OldProjFiles -- ProjFiles,
-    gen_lsp_server:lsp_log(
+    ?LOG(
         "~p: Files to drop: (~p)~n  ~p~n",
         [?MODULE, length(FilesToDrop), lists:sort(FilesToDrop)]),
     State2 = delete_project_files(FilesToDrop, State),
     %% Get files not parsed yet
     FilesToParse = ProjFiles -- OldProjFiles,
-    gen_lsp_server:lsp_log(
+    NbFileToParse = length(FilesToParse),
+    LogFilesToParse = ?IIF(NbFileToParse > 200, lists:sublist(FilesToParse,200), FilesToParse),
+    ?LOG(
         "~p: Files to parse: (~p)~n  ~p~n",
-        [?MODULE, length(FilesToParse), lists:sort(FilesToParse)]),
+        [?MODULE, length(FilesToParse), lists:sort(LogFilesToParse)]),
 
     %% Load the 1st source file (and continue later one-by-one ...)
     NewState = State2#state{project_modules = ProjectModules,
@@ -418,7 +428,7 @@ parse_and_store(File, ContentsFile) ->
             lsp_navigation:fold_references(fun (Reference, Line, Column, End, _) ->
                 ets:insert(references, {File, Reference, Line, Column, End})
             end, undefined, File, SyntaxTree),
-            ets:insert(document_inlayhints, {File, lsp_navigation:full_inlayhints_info(File,SyntaxTree)})
+            ets:insert(document_inlayhints, {File, lsp_navigation:full_inlayhints_info(File,SyntaxTree, DodgedSyntaxTree)})
     end,
     case DodgedSyntaxTree of
         undefined -> ok;

apps/erlangbridge/src/lsp_fun_utils.erl

@@ -0,0 +1,64 @@
+-module(lsp_fun_utils).
+
+
+-export([get_function_range/1, get_type_range/1]).
+
+-include("lsp_log.hrl").
+
+
+get_function_range({function, {L, C}, _FnName, _FnArity, Body}) ->
+    LastClause = lists:last(Body),
+    case get_latest_lc(LastClause) of
+        {-1,-1} -> {L,C,L,C};
+        {L1,C1} -> {L,C,L1+1,C1}
+    end.
+
+get_type_range({attribute, {L, C}, type, {_Type, TypDef,_}}) ->
+    case get_latest_lc(TypDef) of
+        {-1,-1} -> {L,1,L,1};
+        {L1,C1} -> {L,C,L1+1,C1}
+    end.
+
+% basic filter to get the latest line/column
+get_latest_lc({clause, {L, C}, _, _, Body}) ->
+    case Body of
+        [] -> {L,C};
+        _ -> get_latest_lc(lists:last(Body))
+    end;
+get_latest_lc({T, {L, C}, _, Args}) when T =:= call orelse T =:= 'case' orelse T =:= record ->
+    case Args of
+        [] -> {L,C};
+        _ -> get_latest_lc(lists:last(Args))
+    end;
+get_latest_lc({M, {_, _}, _, Args}) when M =:= match orelse M =:= cons orelse M =:= map_field_assoc 
+                                    orelse M =:= record_field ->
+    get_latest_lc(Args);
+get_latest_lc({'try', {_, _}, A1, A2, A3, A4}) ->
+    List = A1 ++ A2 ++ A3 ++ A4,
+    get_latest_lc(lists:last(List));
+get_latest_lc({T, {L, C}, Clauses}=_Other) when T =:= tuple orelse T =:= 'if' orelse T =:= map ->
+    case Clauses of
+        [] -> {L,C};
+        _ -> get_latest_lc(lists:last(Clauses))
+    end;
+get_latest_lc({type, {L, C}, _Type, TypeDefList}) when is_list(TypeDefList) ->
+    case TypeDefList of
+        [] -> {L,C};
+        _ -> get_latest_lc(lists:last(TypeDefList))
+    end;
+get_latest_lc({_, {L, C}}=_Other) ->
+    %?LOG("get_latest_lc: 0 token: ~p", [_Other]),
+    {L,C};
+get_latest_lc({_, {L, C}, _}=_Other) ->
+    %?LOG("get_latest_lc: 1 token: ~p", [_Other]),
+    {L,C};
+get_latest_lc({_, {L, C}, _, _}=_Other) ->
+    %?LOG("get_latest_lc: 2 token: ~p", [_Other]),
+    {L,C};
+get_latest_lc({_, {L, C}, _, _, _}=_Other) ->
+    %?LOG("get_latest_lc: 3 token: ~p", [_Other]),
+    {L,C};
+get_latest_lc(_Other) ->
+    ?LOG("get_latest_lc: unknown token: ~p", [_Other]),
+    {-1,-1}.
+

apps/erlangbridge/src/lsp_handlers.erl

@@ -262,13 +262,13 @@ textDocument_inlayHints(_Socket, Params) ->
 
 textDocument_documentSymbol(_Socket, Params) ->
     Uri = mapmapget(textDocument, uri, Params),
-    lists:map(fun ({Name, Kind, Line}) ->
+    lists:map(fun ({Name, Kind, {L, C, L1, C1}}) ->
         #{
             name => Name,
             kind => Kind, 
             location => #{ 
                 uri => Uri, 
-                range => lsp_utils:client_range(Line, 1, 1)
+                range => lsp_utils:client_range(L, C, L1, C1)
             }
         }
     end, lsp_navigation:symbol_info(lsp_utils:file_uri_to_file(Uri))).

apps/erlangbridge/src/lsp_inlayhints.erl

@@ -1,10 +1,29 @@
 -module(lsp_inlayhints).
 
--export([inlayhint_analyze/3, generate_inlayhints/2]).
+-export([inlayhint_analyze/3, generate_inlayhints/3]).
 
 -include("lsp_log.hrl").
 
-inlayhint_analyze(SyntaxTree, _CurrentFile, #{defs := Defs, calls := Calls} = Dict) ->
+-ifdef(OTP_RELEASE).
+inlayhint_analyze(SyntaxTree, _CurrentFile, Dict) ->
+    %do not crash on error, just log, in order to avoid exception while parsing file
+    try
+        internal_inlayhint_analyze(SyntaxTree, _CurrentFile, Dict)
+    catch Error:Exception:StackStrace ->
+        ?LOG("inlayhint_analyze error ~p:~p, stacktrace:~p", [Error, Exception, StackStrace]),
+        Dict
+    end.
+-else.
+inlayhint_analyze(SyntaxTree, _CurrentFile, Dict) ->
+    try
+        internal_inlayhint_analyze(SyntaxTree, _CurrentFile, Dict)
+    catch Error:Exception ->
+        ?LOG("inlayhint_analyze error ~p:~p, stacktrace:~p", [Error, Exception, erlang:get_stacktrace()]),
+        Dict
+    end.
+-endif.
+
+internal_inlayhint_analyze(SyntaxTree, _CurrentFile, #{defs := Defs, calls := Calls} = Dict) ->
     case SyntaxTree of
         %TODO, get args from spec if exists
         {function, _Location, FuncName, Arity, Content} when Arity > 0 ->
@@ -106,39 +125,56 @@ choose_better_arg(VarName, Arg1, Arg2) ->
     end.
 
 
-generate_inlayhints([], _Defs) ->
+generate_inlayhints([], _Defs, _Macros) ->
     [];
-generate_inlayhints([#{args := Args, func_name := FName} | RestCalls], Defs) ->
+generate_inlayhints([#{args := Args, func_name := FName} | RestCalls], Defs, Macros) ->
     L = length(Args),
     %% filter calls and definitions by arity and name
     case
         lists:filter(
             fun(#{arity := Arity, func_name := Dfn}) -> Arity =:= L andalso FName =:= Dfn end, Defs
         )
     of
-        [#{args := DArgs}] -> filter_and_map_args(Args, DArgs);
+        [#{args := DArgs}] -> filter_and_map_args(Args, DArgs, Macros);
         _ -> []
     end ++
-        generate_inlayhints(RestCalls, Defs).
+        generate_inlayhints(RestCalls, Defs, Macros).
 
-filter_and_map_args([], _Defs) ->
+filter_and_map_args([], _Defs, _Macros) ->
     [];
-filter_and_map_args([{Index, Call} | RestCalls], Defs) ->
+filter_and_map_args([{Index, Call} | RestCalls], Defs, Macros) ->
     %get corresponding argument in definition
     D = lists:nth(Index + 1, Defs),
     %?LOG("try_match:~p, ~p", [Call, D]),
-    try_match_parameter(Call, D) ++ filter_and_map_args(RestCalls, Defs).
+    try_match_parameter(Call, D, Macros) ++ filter_and_map_args(RestCalls, Defs, Macros).
 
-try_match_parameter({var, _, VarName}, {var, _, DefVarName}) when VarName =:= DefVarName ->
+try_match_parameter({var, _, VarName}, {var, _, DefVarName}, _Macros) when VarName =:= DefVarName ->
     % call var name and definition var name are equal, no inlay for this argument
     [];
-try_match_parameter({_, Position, _}, DefArg) ->
-   new_inlay(Position, DefArg);
-try_match_parameter({call, Position, _,_}, DefArg)  ->
-   new_inlay(Position, DefArg);
-try_match_parameter(_, _)  ->
+try_match_parameter({_P, Position, _}, DefArg, Macros) ->
+    NewPosition = update_position(Position, Macros),
+    new_inlay(NewPosition, DefArg);
+try_match_parameter({call, Position, _,_}, DefArg, _Macros)  ->
+    new_inlay(Position, DefArg);
+try_match_parameter(_, _, _)  ->
     [].
 
+update_position(LC, undefined) ->
+    LC;
+update_position(LC, []) ->
+    LC;
+update_position({Line, Column}, Macros) ->
+    %% check if the position is inside a macro
+    case lists:filter(fun
+        ({{L,C}, _, Length}) when L =:= Line, C =< Column, C + Length >= Column -> true;
+        (_) -> false 
+        end, 
+    Macros) of
+        [] -> {Line, Column};
+        [{{NewLine, NewColumn}, _, _}] -> {NewLine, NewColumn};
+        _ -> {Line, Column}
+    end.
+
 new_inlay(Position, DefArg) ->
     Label = case DefArg of
         {match, _, _,_} ->  "match: ";

apps/erlangbridge/src/lsp_navigation.erl

@@ -2,7 +2,7 @@
 
 -export([definition/3, hover_info/3, function_description/2, function_description/3, references/3, function_clauses/3]).
 -export([codelens_info/1, symbol_info/1, record_fields/2, find_function_with_line/2, fold_references/4]).
--export([inlayhints_info/3, full_inlayhints_info/2, functions/2]).
+-export([inlayhints_info/3, full_inlayhints_info/3, functions/2]).
 -export([inlinevalues_info/2, local_function_references/3]).
 -import(lsp_syntax,[fold_in_syntax_tree/4, find_in_syntax_tree/2, fold_in_syntax_tree/3]).
 
@@ -78,16 +78,19 @@ inlayhints_info(File, {LS, _CS}, {LE,_CE}) ->
     %?LOG("inlayhint_info result:~p, filtered:~p", [length(Res), length(FilteredRes)]),
     FilteredRes.
 
-full_inlayhints_info(File, SyntaxTree) ->
+full_inlayhints_info(File, SyntaxTree, DodgedSyntaxTree) ->
     % return empty if not enabled -> optimize background parsing
     case gen_lsp_config_server:inlayHintsEnabled() of
     false -> [];
     _ ->
+        % should be considered to be optimized using only the dodged syntax tree
+        % in order to get inlay hints for the whole file
+        Macros = lsp_syntax:get_macros(DodgedSyntaxTree),
         #{defs := Defs, calls := Calls} = fold_in_syntax_tree(fun lsp_inlayhints:inlayhint_analyze/3,
                     #{defs => [], calls => []},
                     File, SyntaxTree),
         %?LOG("full_inlayhints_info result:~p, ~p", [Defs, Calls]),
-        NewInlays = [X || X <- lsp_inlayhints:generate_inlayhints(Calls, Defs)],
+        NewInlays = [X || X <- lsp_inlayhints:generate_inlayhints(Calls, Defs, Macros)],
         NewInlays
     end.
 
@@ -177,17 +180,21 @@ expressions_2(_, S) ->
 
 
 symbol_info(File) ->
-    lists:reverse(fold_in_syntax_tree(fun
+    % documentation : https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#symbolKind
+    List = lists:reverse(fold_in_syntax_tree(fun
         (_SyntaxTree, CurrentFile, Acc) when CurrentFile =/= File ->
             Acc;
-        ({function, {L, _}, Function, Arity, _}, _CurrentFile, Acc) ->
+        ({function, {_, _}, Function, Arity, _}=F, _CurrentFile, Acc) ->
             FullName = iolist_to_binary(io_lib:format("~p/~p", [Function, Arity])),
-            [{FullName, 12, L} | Acc];
+            [{FullName, 12, lsp_fun_utils:get_function_range(F)} | Acc];
         ({attribute, {L, _}, record, {Record, _}}, _CurrentFile, Acc) ->
-            [{Record, 23, L} | Acc];
+            [{Record, 23, {L, 1, L, 1}} | Acc];
+        ({attribute, {_, _}, type, {Type, _, _}}=T, _CurrentFile, Acc) ->
+            [{Type, 5, lsp_fun_utils:get_type_range(T)} | Acc];
         (_SyntaxTree, _CurrentFile, Acc) ->
             Acc
-    end, [], File)).
+    end, [], File)),
+    List.
 
 %return list of function for specified file
 functions(File, FunName) ->
@@ -660,13 +667,13 @@ find_definition_in_file(_Type, _Name, _File, []) ->
 find_definition_in_file(macro, MacroName, File,
                         [{tree, attribute, _,
                           {attribute, {atom, _, define},
-                           [{_, Line, MacroName}, _]}}
+                           [{_, {Line,_C}, MacroName}, _]}}
                          | _Forms]) ->
     {File, Line, 1, 1};
 find_definition_in_file(macro, MacroName, File,
                         [{tree, attribute, _,
                           {attribute, {atom, _, define},
-                           [{_, _, _, {_, {_, Line, MacroName}, _}}, _]}}
+                           [{_, _, _, {_, {_, {Line,_C}, MacroName}, _}}, _]}}
                          | _Forms]) ->
     {File, Line, 1, 1};
 find_definition_in_file(record, RecordName, File,
@@ -694,14 +701,14 @@ find_field_definition_in_file(_Name, _File, []) ->
     undefined;
 find_field_definition_in_file(Name, File,
                               [{tree, record_field, _,
-                                {record_field, {atom, Line, Name}, _}}
+                                {record_field, {atom, {Line,_C}, Name}, _}}
                                | _Forms]) ->
     {File, Line, 1, 1};
 find_field_definition_in_file(Name, File,
                               [{tree,typed_record_field, _,
                                 {typed_record_field,
                                  {tree, record_field, _,
-                                  {record_field, {atom, Line, Name}, _}}, _}}
+                                  {record_field, {atom, {Line,_C}, Name}, _}}, _}}
                                | _Forms]) ->
     {File, Line, 1, 1};
 find_field_definition_in_file(Name, File, [_ | Forms]) ->

apps/erlangbridge/src/lsp_parse.erl

@@ -18,7 +18,7 @@ parse_source_file(File, ContentsFile) ->
     case epp_parse_file(ContentsFile, get_include_path(File), get_define_from_rebar_config(File)) of
         {ok, FileSyntaxTree} ->
             UpdatedSyntaxTree = update_file_in_forms(File, ContentsFile, FileSyntaxTree),
-            case epp_dodger:parse_file(ContentsFile) of
+            case epp_dodger_parse_file(ContentsFile) of
                 {ok, Forms} ->
                     {UpdatedSyntaxTree, Forms};
                 _ ->
@@ -30,6 +30,16 @@ parse_source_file(File, ContentsFile) ->
             {undefined, undefined}
     end.
 
+%% in order to have LMine and column information in syntax tree, we need to parse file with epp_dodger
+epp_dodger_parse_file(File) ->
+    case file:open(File, [read]) of
+        {ok, Handle} ->
+            R = epp_dodger:parse(Handle, {1,1}),
+            file:close(Handle),            
+            R;
+        Error -> Error
+    end.
+
 parse_config_file(File, ContentsFile) ->
     case file:path_consult(filename:dirname(ContentsFile), ContentsFile) of
       {ok,_, _} -> #{parse_result => true};
@@ -46,7 +56,8 @@ get_include_path(File) ->
         get_include_paths_from_rebar_config(File) ++
         get_standard_include_paths(),
     Paths = lists:filter(fun filelib:is_dir/1, Candidates),
-    gen_lsp_server:lsp_log("get_include_path: ~p", [Paths]),
+    % activate it only for debugging, on big projects it can generate a lot of logs
+    %gen_lsp_server:lsp_log("get_include_path: ~p", [Paths]),
     Paths.
 
 get_standard_include_paths() ->