Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve driver experience #1262

Merged
merged 6 commits into from
Dec 13, 2024
Merged

Improve driver experience #1262

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e1bd734
Driver: improve output
panglesd Dec 12, 2024
5300a3b
Driver: fix counting of generated units
panglesd Dec 13, 2024
2ac3b0e
Driver: only generate json if requested
panglesd Dec 13, 2024
4a799f1
Driver: remove progress bars at the end
panglesd Dec 13, 2024
082f3c1
Driver: Use Log.app instead of eprintf to display progress
panglesd Dec 13, 2024
ea933c6
Driver: fix double new lines
panglesd Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/driver/common_args.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@ let index_grep =
let doc = "Show compile-index commands containing the string" in
Arg.(value & opt (some string) None & info [ "index-grep" ] ~doc)

let generate_json =
let doc = "Also generate json output" in
Arg.(value & flag & info [ "json-output" ] ~doc)

type t = {
verbose : bool;
odoc_dir : Fpath.t;
Expand All @@ -75,6 +79,7 @@ type t = {
generate_grep : string option;
remap : bool;
index_grep : string option;
generate_json : bool;
}

let term =
Expand All @@ -91,6 +96,7 @@ let term =
and+ nb_workers = nb_workers
and+ odoc_bin = odoc_bin
and+ compile_grep = compile_grep
and+ generate_json = generate_json
and+ link_grep = link_grep
and+ generate_grep = generate_grep
and+ index_grep = index_grep
Expand All @@ -110,4 +116,5 @@ let term =
generate_grep;
remap;
index_grep;
generate_json;
}
78 changes: 41 additions & 37 deletions src/driver/compile.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ let sherlodoc_index_one ~output_dir (index : Odoc_unit.index) =
Sherlodoc.index ~format:`js ~inputs ~dst ();
rel_path

let html_generate ~occurrence_file ~remaps output_dir linked =
let html_generate ~occurrence_file ~remaps ~generate_json output_dir linked =
let tbl = Hashtbl.create 10 in
let _ = OS.Dir.create output_dir |> Result.get_ok in
Sherlodoc.js Fpath.(output_dir // Sherlodoc.js_file);
Expand Down Expand Up @@ -296,42 +296,46 @@ let html_generate ~occurrence_file ~remaps output_dir linked =
in
let html_generate : Fpath.t option -> linked -> unit =
fun remap_file l ->
(if l.to_output then
let output_dir = Fpath.to_string output_dir in
let input_file = l.odocl_file in
match l.kind with
| `Intf { hidden = true; _ } -> ()
| `Impl { src_path; _ } ->
let search_uris, sidebar =
match l.index with
| None -> (None, None)
| Some index ->
let db_path, sidebar = compile_index index in
let search_uris = [ db_path; Sherlodoc.js_file ] in
(Some search_uris, sidebar)
in
Odoc.html_generate_source ?search_uris ?sidebar ~output_dir
~input_file ~source:src_path ();
Odoc.html_generate_source ?search_uris ?sidebar ~output_dir
~input_file ~source:src_path ~as_json:true ();
Atomic.incr Stats.stats.generated_units
| `Asset ->
Odoc.html_generate_asset ~output_dir ~input_file:l.odoc_file
~asset_path:l.input_file ()
| _ ->
let search_uris, sidebar =
match l.index with
| None -> (None, None)
| Some index ->
let db_path, sidebar = compile_index index in
let search_uris = [ db_path; Sherlodoc.js_file ] in
(Some search_uris, sidebar)
in
Odoc.html_generate ?search_uris ?sidebar ?remap:remap_file
~output_dir ~input_file ();
Odoc.html_generate ?search_uris ?sidebar ~output_dir ~input_file
~as_json:true ());
Atomic.incr Stats.stats.generated_units
if l.to_output then
let output_dir = Fpath.to_string output_dir in
let input_file = l.odocl_file in
match l.kind with
| `Intf { hidden = true; _ } -> ()
| `Impl { src_path; _ } ->
let search_uris, sidebar =
match l.index with
| None -> (None, None)
| Some index ->
let db_path, sidebar = compile_index index in
let search_uris = [ db_path; Sherlodoc.js_file ] in
(Some search_uris, sidebar)
in
Odoc.html_generate_source ?search_uris ?sidebar ~output_dir
~input_file ~source:src_path ();
Atomic.incr Stats.stats.generated_units;
if generate_json then (
Odoc.html_generate_source ?search_uris ?sidebar ~output_dir
~input_file ~source:src_path ~as_json:true ();
Atomic.incr Stats.stats.generated_units)
| `Asset ->
Odoc.html_generate_asset ~output_dir ~input_file:l.odoc_file
~asset_path:l.input_file ()
| _ ->
let search_uris, sidebar =
match l.index with
| None -> (None, None)
| Some index ->
let db_path, sidebar = compile_index index in
let search_uris = [ db_path; Sherlodoc.js_file ] in
(Some search_uris, sidebar)
in
Odoc.html_generate ?search_uris ?sidebar ?remap:remap_file ~output_dir
~input_file ();
Atomic.incr Stats.stats.generated_units;
if generate_json then (
Odoc.html_generate ?search_uris ?sidebar ~output_dir ~input_file
~as_json:true ();
Atomic.incr Stats.stats.generated_units)
in
if List.length remaps = 0 then Fiber.List.iter (html_generate None) linked
else
Expand Down
1 change: 1 addition & 0 deletions src/driver/compile.mli
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ val link : compiled list -> linked list
val html_generate :
occurrence_file:Fpath.t ->
remaps:(string * string) list ->
generate_json:bool ->
Fpath.t ->
linked list ->
unit
28 changes: 18 additions & 10 deletions src/driver/odoc_driver.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
(* Odoc driver *)

let render_stats env nprocs =
let render_stats env ~generate_json nprocs =
let if_app f =
match Logs.level () with Some (App | Warning) | None -> f () | _ -> ()
in
Expand All @@ -15,11 +15,11 @@ let render_stats env nprocs =
let total_indexes = Atomic.get Stats.stats.total_indexes in
let bar message total =
let open Progress.Line in
list [ lpad 16 (const message); bar total; count_to total ]
list [ lpad 16 (const message); bar total; rpad 10 (count_to total) ]
in
let procs total =
let open Progress.Line in
list [ lpad 16 (const "Processes"); bar total; count_to total ]
list [ lpad 16 (const "Processes"); bar total; rpad 10 (count_to total) ]
in
let description =
let open Progress.Line in
Expand All @@ -30,7 +30,12 @@ let render_stats env nprocs =
let non_hidden = Atomic.get Stats.stats.non_hidden_units in

let dline x y = Multi.line (bar x y) in
with_reporters
let config = Progress.Config.v ~persistent:false () in
let total_generate =
let units = total_impls + non_hidden + total_mlds in
if generate_json then 2 * units else units
in
with_reporters ~config
Multi.(
dline "Compiling" total
++ dline "Compiling impls" total_impls
Expand All @@ -40,7 +45,7 @@ let render_stats env nprocs =
++ dline "Linking impls" total_impls
++ dline "Linking mlds" total_mlds
++ dline "Indexes" total_indexes
++ dline "HTML" (total_impls + non_hidden + total_mlds)
++ dline "HTML" total_generate
++ line (procs nprocs)
++ descriptions)
(fun comp compimpl compmld compassets link linkimpl linkmld indexes html
Expand Down Expand Up @@ -70,7 +75,7 @@ let render_stats env nprocs =
indexes (i' - i);
html (g' - g);
procs (h' - h);
if g' < non_hidden + total_impls + total_mlds then
if g' < total_generate then
inner (a', b', c', j', d', e', f', i', g', h')
in
inner (0, 0, 0, 0, 0, 0, 0, 0, 0, 0))
Expand Down Expand Up @@ -135,6 +140,7 @@ let run mode
generate_grep;
remap;
index_grep;
generate_json;
} =
Option.iter (fun odoc_bin -> Odoc.odoc := Bos.Cmd.v odoc_bin) odoc_bin;
let _ = Voodoo.find_universe_and_version "foo" in
Expand Down Expand Up @@ -205,6 +211,7 @@ let run mode
else []
in
Logs.debug (fun m -> m "XXXX Remaps length: %d" (List.length remaps));
Format.eprintf "Starting the compilation process... \n%!";
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't this be Logs.info ?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point. However, shouldn't this be Logs.app?

Currently, the Info level is a bit too verbose. Eg:

odoc_driver.exe: [INFO] Unable to determine library of archive ocamlmiddleend: Ignoring.
odoc_driver.exe: [INFO] No files for module: Tophooks
odoc_driver.exe: [INFO] Unable to determine library of archive odoc_info: Ignoring.
odoc_driver.exe: [INFO] No files for module: Findlib
odoc_driver.exe: [INFO] No files for module: Findlib_config
odoc_driver.exe: [INFO] No files for module: Fl_args
odoc_driver.exe: [INFO] No files for module: Fl_lint
odoc_driver.exe: [INFO] No files for module: Fl_meta
odoc_driver.exe: [INFO] No files for module: Fl_metascanner
odoc_driver.exe: [INFO] No files for module: Fl_metatoken
odoc_driver.exe: [INFO] No files for module: Fl_package_base
odoc_driver.exe: [INFO] No files for module: Fl_split
odoc_driver.exe: [INFO] No files for module: Fl_topo
odoc_driver.exe: [INFO] No files for module: Ocaml_args
odoc_driver.exe: [INFO] No files for module: Fl_dynload
odoc_driver.exe: [INFO] No files for module: Topfind

will be displayed for any invokation of odoc_driver. I believe this should be fixed in another PR!

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that's Logs.app !

let () =
Eio.Fiber.both
(fun () ->
Expand Down Expand Up @@ -240,13 +247,15 @@ let run mode
output
in
let () =
Compile.html_generate ~occurrence_file ~remaps html_dir linked
Compile.html_generate ~occurrence_file ~remaps ~generate_json
html_dir linked
in
let _ = Odoc.support_files html_dir in
())
(fun () -> render_stats env nb_workers)
(fun () -> render_stats env ~generate_json nb_workers)
in

Format.eprintf "Collected logs... \n%!";
let grep_log ty s =
let open Astring in
let do_ affix =
Expand Down Expand Up @@ -284,8 +293,7 @@ let run mode
| _ -> ())
!Cmd_outputs.outputs;

Format.eprintf "Final stats: %a@.%!" Stats.pp_stats Stats.stats;
Format.eprintf "Total time: %f@.%!" (Stats.total_time ());
Format.eprintf "Benchmarking... \n%!";
if stats then Stats.bench_results html_dir

open Cmdliner
Expand Down
2 changes: 2 additions & 0 deletions src/driver/packages.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,7 @@ let of_libs ~packages_dir libs =
fix_missing_deps packages

let of_packages ~packages_dir packages =
Format.eprintf "Computing deps... \n%!";
let deps =
if packages = [] then Opam.all_opam_packages () else Opam.deps packages
in
Expand All @@ -441,6 +442,7 @@ let of_packages ~packages_dir packages =

let all = orig @ ps in

Format.eprintf "Analyzing packages needed to be built... \n%!";
let packages =
List.fold_left
(fun acc (pkg, files) ->
Expand Down
Loading