Skip to content

Commit

Permalink
Issue 89: Adding new sync_method - fsevents
Browse files Browse the repository at this point in the history 
Issue has been detailed at: #89

Currently sync polls directories and files for any changes and recompile the changed files and loads them. When sync starts monitoring more projects under one umbrella CPU can get overwhelmed with this polling. it will be good if sync looks for file system changes notifications and act upon them.

There is a project synrc/fs which provides filesystem change events. This supports Mac, Windows and Linux platforms.

This commit integrates sync with fs. Here is what it does:

1. Added a new app evnironment variable, sync_method. User can specifiy this to scanner or fsevents. By default it will be scanner, which would retain current behavior. If user specifies fsevents, new logic kicks in.
2. discover_src_dirs, will discover all source directories as requested by user. and starts fs process for monited directories.
3. When a source file or a header file changes, sync gets {fs,file_event} event. And it will handle the changed files a second later so that it can suppress/collect all events on a file.
4. Make sure no timers are active in stable state.

Signed-off-by: Vasu Dasari <[email protected]>
  • Loading branch information
@vasu-dasari
vasu-dasari committed Jul 13, 2020
1 parent 9106be7 commit 3969698
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 16 deletions.
4 changes: 4 additions & 0 deletions rebar.config
View file
Original file line number Diff line number Diff line change
@@ -1 +1,5 @@
{erl_opts, [debug_info]}.

{deps, [
{fs, "6.1.1"}
]}.
5 changes: 5 additions & 0 deletions src/sync.app.src
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@
{maintainers, ["Jesse Gumm", "Heinz N. Gies"]},
{licenses, ["MIT"]},
{links, [{"Github", "https://github.com/rustyio/sync"}]},
{applications, [
kernel,
stdlib,
fs
]},
{env, [
{discover_modules_interval, 10000},
{discover_src_dirs_interval, 10000},
Expand Down
165 changes: 149 additions & 16 deletions src/sync_scanner.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

-module(sync_scanner).
-behaviour(gen_server).
-include_lib("kernel/include/file.hrl").

-compile([export_all, nowarn_export_all]).

%% API
Expand Down Expand Up @@ -46,21 +48,31 @@
hrl_file_lastmod = [] :: [{file:filename(), timestamp()}],
timers = [],
patching = false,
paused = false
paused = false,
sync_method = scanner,
modified_files = [] :: [file:filename()],
fsevents_pids = []
}).

start_link() ->
gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).


rescan() ->
io:format("Scanning source files...~n"),
gen_server:cast(?SERVER, discover_modules),
gen_server:cast(?SERVER, discover_src_dirs),
gen_server:cast(?SERVER, discover_src_files),
gen_server:cast(?SERVER, compare_src_files),
gen_server:cast(?SERVER, compare_beams),
gen_server:cast(?SERVER, compare_hrl_files),
ok.
case sync_utils:get_env(sync_method, scanner) of
scanner ->
io:format("Scanning source files...~n"),
gen_server:cast(?SERVER, compare_src_files),
gen_server:cast(?SERVER, compare_beams),
gen_server:cast(?SERVER, compare_hrl_files),
ok;
_ ->
io:format("Listening on fsevents...~n"),
ok
end.

unpause() ->
gen_server:cast(?SERVER, unpause),
Expand Down Expand Up @@ -118,7 +130,9 @@ init([]) ->
%% Display startup message...
sync_notify:startup(get_growl()),

{ok, #state{}}.
{ok, #state{
sync_method = sync_utils:get_env(sync_method, scanner)
}}.

handle_call(_Request, _From, State) ->
Reply = ok,
Expand All @@ -139,7 +153,10 @@ handle_cast(discover_modules, State) ->
FilteredModules = filter_modules_to_scan(Modules),

%% Schedule the next interval...
NewTimers = schedule_cast(discover_modules, 30000, State#state.timers),
NewTimers = case State#state.sync_method of
scanner -> schedule_cast(discover_modules, 30000, State#state.timers);
_ -> proplists:delete(discover_modules, State#state.timers)
end,

%% Return with updated modules...
NewState = State#state { modules=FilteredModules, timers=NewTimers },
Expand Down Expand Up @@ -169,7 +186,10 @@ handle_cast(discover_src_files, State) ->
HrlFiles = lists:usort(lists:foldl(Fhrl, [], State#state.hrl_dirs)),

%% Schedule the next interval...
NewTimers = schedule_cast(discover_src_files, 5000, State#state.timers),
NewTimers = case State#state.sync_method of
scanner -> schedule_cast(discover_src_files, 5000, State#state.timers);
_ -> proplists:delete(discover_src_files, State#state.timers)
end,

%% Return with updated files...
NewState = State#state { src_files=ErlFiles, hrl_files=HrlFiles, timers=NewTimers },
Expand Down Expand Up @@ -198,7 +218,10 @@ handle_cast(compare_beams, State) ->
process_beam_lastmod(State#state.beam_lastmod, NewBeamLastMod, State#state.patching),

%% Schedule the next interval...
NewTimers = schedule_cast(compare_beams, 2000, State#state.timers),
NewTimers = case State#state.sync_method of
scanner -> schedule_cast(compare_beams, 2000, State#state.timers);
_ -> proplists:delete(compare_beams, State#state.timers)
end,

%% Return with updated beam lastmod...
NewState = State#state { beam_lastmod=NewBeamLastMod, timers=NewTimers },
Expand Down Expand Up @@ -240,6 +263,44 @@ handle_cast(compare_hrl_files, State) ->
NewState = State#state { hrl_file_lastmod=NewHrlFileLastMod, timers=NewTimers },
{noreply, NewState};

handle_cast(fsevents_modified_files,
#state{
modified_files = Files, patching = Patching, src_files = SrcFiles, modules = OldModules
} = State) when Files /= [] ->
Recompile = fun(F, P) ->
recompile_src_file(F, P),
{_, M} = determine_compile_fun_and_module_name(F),
M
end,

NewModules1 = lists:foldl(fun
(FileName, Acc) ->
case filename:extension(FileName) of
Ext when Ext == ".erl"; Ext == ".dtl"; Ext == ".lfe"; Ext == ".ex" ->
M = Recompile(FileName, Patching),
[M | Acc];
".hrl" ->
WhoInclude = who_include(FileName, SrcFiles),
[Recompile(SrcFile, Patching) || SrcFile <- WhoInclude] ++ Acc;
_ ->
Acc
end
end, [], Files),

%% It's possible that this module is not known to sync, if yes, add it to modules list
Filter = fun(M) ->
lists:member(M, OldModules) /= true
end,
NewModules = case lists:filter(Filter, NewModules1) of
[] -> OldModules;
R -> lists:usort(OldModules ++ R)
end,

{noreply, State#state{
modified_files = [],
modules = NewModules
}};

handle_cast(info, State) ->
io:format("Modules: ~p~n", [State#state.modules]),
io:format("Source Dirs: ~p~n", [State#state.src_dirs]),
Expand All @@ -256,7 +317,7 @@ handle_cast(_Msg, State) ->
dirs(DirsAndOpts) ->
[begin
sync_options:set_options(Dir, Opts),

%% ensure module out path exists & in our code list
case proplists:get_value(outdir, Opts) of
undefined ->
Expand All @@ -268,6 +329,16 @@ dirs(DirsAndOpts) ->
Dir
end || {Dir, Opts} <- DirsAndOpts].

handle_info({_Pid, {fs,file_event}, {FileName, _Events}},
#state{modified_files = OldModFiles} = State) ->

%% Process the modified files event about a second later. Just
%% to thaw all events happening on a file
{noreply, State#state{
modified_files = lists:usort([FileName | OldModFiles]),
timers = schedule_cast(fsevents_modified_files, 1000, State#state.timers)
}};

handle_info(_Info, State) ->
{noreply, State}.

Expand Down Expand Up @@ -519,7 +590,8 @@ reload_if_necessary(CompileFun, SrcFile, Module, _OldBinary, _Binary, Options, W
%% Module is not yet loaded, load it.
case code:load_file(Module) of
{module, Module} -> ok;
{error, nofile} -> error_no_file(Module)
{error, nofile} ->
error_no_file(Module)
end
end,
gen_server:cast(?SERVER, compare_beams),
Expand Down Expand Up @@ -780,11 +852,72 @@ discover_source_dirs(State, ExtraDirs, ReplaceDirs) ->
%% InitialDirs = sync_utils:initial_src_dirs(),

%% Schedule the next interval...
NewTimers = schedule_cast(discover_src_dirs, 30000, State#state.timers),
case State#state.sync_method of
scanner ->
NewTimers = schedule_cast(discover_src_dirs, 30000, State#state.timers),

%% Return with updated dirs...
NewState = State#state { src_dirs=USortedSrcDirs, hrl_dirs=USortedHrlDirs, timers=NewTimers },
{noreply, NewState};
fsevents ->
%% Stop old processes
start_fsevents(SrcDirs++HrlDirs, ReplaceDirs, State#state{
src_dirs = USortedSrcDirs,
hrl_dirs = USortedHrlDirs
})
end.

%% Return with updated dirs...
NewState = State#state { src_dirs=USortedSrcDirs, hrl_dirs=USortedHrlDirs, timers=NewTimers },
{noreply, NewState}.
start_fsevents(MonitorDirs, ReplaceDirs, State) ->
%% Stop existing fs processes if any
[erlang:exit(Pid, normal) || Pid <- State#state.fsevents_pids],

%% Start new fs processes based on the user inputs
NewPids = lists:foldl(fun
(Dir, Acc) ->
Name = erlang:list_to_atom(Dir),
case file:read_link_info(Dir) of
{ok, #file_info{type = symlink}} ->
Acc;
{ok, _} ->
case fs:start_link(Name, Dir) of
{ok, Pid} ->
fs:subscribe(Name),
[Pid | Acc];
_ ->
Acc
end;
_ ->
Acc
end
end, [], MonitorDirs),

%% It is possible that not all modules are discovered
%% by the time this process is ran. So, handle that
DirsNotDiscovered = lists:foldl(fun
(_, []) ->
[];
(Dir, Acc) ->
lists:filter(fun
(Match) ->
string:split(Dir, Match) == [Dir]
end, Acc)
end, ReplaceDirs, MonitorDirs),

%% Restart the discovery process if needed ...
NewTimers = case DirsNotDiscovered == [] of
true ->
proplists:delete(discover_src_dirs, State#state.timers);
_ ->
T1 = schedule_cast(discover_modules, 3000, State#state.timers),
T2 = schedule_cast(discover_src_dirs, 4000, T1),
schedule_cast(discover_src_files, 5000, T2)
end,

gen_server:cast(?SERVER, compare_beams),
{noreply, State#state{
fsevents_pids = NewPids,
timers = NewTimers
}}.

is_replace_dir(_, []) ->
true;
Expand Down

0 comments on commit 3969698

Please sign in to comment.