From 1e9155208d5852b9f5023e116963cc3f09c08308 Mon Sep 17 00:00:00 2001 From: Sabine Schmaltz Date: Fri, 14 Jul 2023 11:13:19 +0200 Subject: [PATCH 1/5] fix Cmi_format error in voodoo-do, by checking in prep that the magic numbers match --- src/voodoo-prep/prep.ml | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/voodoo-prep/prep.ml b/src/voodoo-prep/prep.ml index 9f96d87d..6289f61b 100644 --- a/src/voodoo-prep/prep.ml +++ b/src/voodoo-prep/prep.ml @@ -6,10 +6,32 @@ let process_package : Fpath.t -> Package.t -> Fpath.t list -> unit = fun root package files -> let dest = Package.prep_path package in + let format_matches_compiler_version path = + let open Unix in + let cmd = "ocamlobjinfo " ^ Fpath.to_string path in + let ic, oc = open_process cmd in + let rec matches_version ic = + try + let line = input_line ic in + let prefix = "Wrong magic number" in + let str_len = String.length line in + if str_len >= 18 && String.sub line 0 18 = prefix then ( + Printf.eprintf "ERROR: Ocamlobjinfo on %s: %s\n" + (Fpath.to_string path) line; + false) + else if str_len > 0 then matches_version ic + else true + with End_of_file -> true + in + let matches_version = matches_version ic in + let _ = close_process (ic, oc) in + matches_version + in + (* Some packages produce ocaml artefacts that can't be processed with the switch's ocaml compiler - most notably the secondary compiler! This switch is intended to be used to ignore those files *) - let process_ocaml_artefacts = + let is_not_blacklisted = let package_blacklist = [ "ocaml-secondary-compiler" ] in not (List.mem package.name package_blacklist) in @@ -27,16 +49,17 @@ let process_package : Fpath.t -> Package.t -> Fpath.t list -> unit = let no_ext = Fpath.rem_ext filename in let has_hyphen = String.contains (Fpath.to_string filename) '-' in let is_module = - process_ocaml_artefacts + is_not_blacklisted && List.mem ext [ ".cmt"; ".cmti"; ".cmi" ] - && not has_hyphen + && (not has_hyphen) + && format_matches_compiler_version Fpath.(root // fpath) in let do_copy = (not in_build_dir) && (is_in_doc_dir || is_module || List.mem no_ext (List.map Fpath.v [ "META"; "dune-package" ])) in - let is_cma = process_ocaml_artefacts && List.mem ext [ ".cma"; ".cmxa" ] in + let is_cma = is_not_blacklisted && List.mem ext [ ".cma"; ".cmxa" ] in let copy = if do_copy then Fpath.(root // fpath, dest // fpath) :: acc.copy else acc.copy From 8333d326e75ccfb02974ec34c071d21c9e122db8 Mon Sep 17 00:00:00 2001 From: Sabine Schmaltz Date: Tue, 25 Jul 2023 09:18:35 +0200 Subject: [PATCH 2/5] try/with around call to ocamlobjinfo, is_blacklisted --- src/voodoo-prep/prep.ml | 45 +++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/src/voodoo-prep/prep.ml b/src/voodoo-prep/prep.ml index 6289f61b..858b19b9 100644 --- a/src/voodoo-prep/prep.ml +++ b/src/voodoo-prep/prep.ml @@ -9,29 +9,34 @@ let process_package : Fpath.t -> Package.t -> Fpath.t list -> unit = let format_matches_compiler_version path = let open Unix in let cmd = "ocamlobjinfo " ^ Fpath.to_string path in - let ic, oc = open_process cmd in - let rec matches_version ic = - try - let line = input_line ic in - let prefix = "Wrong magic number" in - let str_len = String.length line in - if str_len >= 18 && String.sub line 0 18 = prefix then ( - Printf.eprintf "ERROR: Ocamlobjinfo on %s: %s\n" - (Fpath.to_string path) line; - false) - else if str_len > 0 then matches_version ic - else true - with End_of_file -> true - in - let matches_version = matches_version ic in - let _ = close_process (ic, oc) in - matches_version + try + let ic, oc = open_process cmd in + let rec matches_version ic = + try + let line = input_line ic in + let prefix = "Wrong magic number" in + let str_len = String.length line in + if str_len >= 18 && String.sub line 0 18 = prefix then ( + Printf.eprintf "ERROR: ocamlobjinfo on %s: %s\n" + (Fpath.to_string path) line; + false) + else if str_len > 0 then matches_version ic + else true + with End_of_file -> true + in + let matches_version = matches_version ic in + let _ = close_process (ic, oc) in + matches_version + with _ -> + Printf.eprintf "ERROR: Failed to run ocamlobjinfo on %s\n" + (Fpath.to_string path); + false in (* Some packages produce ocaml artefacts that can't be processed with the switch's ocaml compiler - most notably the secondary compiler! This switch is intended to be used to ignore those files *) - let is_not_blacklisted = + let is_blacklisted = let package_blacklist = [ "ocaml-secondary-compiler" ] in not (List.mem package.name package_blacklist) in @@ -49,7 +54,7 @@ let process_package : Fpath.t -> Package.t -> Fpath.t list -> unit = let no_ext = Fpath.rem_ext filename in let has_hyphen = String.contains (Fpath.to_string filename) '-' in let is_module = - is_not_blacklisted + (not is_blacklisted) && List.mem ext [ ".cmt"; ".cmti"; ".cmi" ] && (not has_hyphen) && format_matches_compiler_version Fpath.(root // fpath) @@ -59,7 +64,7 @@ let process_package : Fpath.t -> Package.t -> Fpath.t list -> unit = && (is_in_doc_dir || is_module || List.mem no_ext (List.map Fpath.v [ "META"; "dune-package" ])) in - let is_cma = is_not_blacklisted && List.mem ext [ ".cma"; ".cmxa" ] in + let is_cma = (not is_blacklisted) && List.mem ext [ ".cma"; ".cmxa" ] in let copy = if do_copy then Fpath.(root // fpath, dest // fpath) :: acc.copy else acc.copy From 2f7b1f4e2bb81ac462e92486298ae45ae2dca32d Mon Sep 17 00:00:00 2001 From: sabine Date: Tue, 25 Jul 2023 16:37:13 +0200 Subject: [PATCH 3/5] Update src/voodoo-prep/prep.ml Co-authored-by: Guillaume Petiot --- src/voodoo-prep/prep.ml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/voodoo-prep/prep.ml b/src/voodoo-prep/prep.ml index 858b19b9..34254e25 100644 --- a/src/voodoo-prep/prep.ml +++ b/src/voodoo-prep/prep.ml @@ -38,7 +38,7 @@ let process_package : Fpath.t -> Package.t -> Fpath.t list -> unit = be used to ignore those files *) let is_blacklisted = let package_blacklist = [ "ocaml-secondary-compiler" ] in - not (List.mem package.name package_blacklist) + List.mem package.name package_blacklist in let foldfn fpath acc = From 0e6ac4d2c9f9e3ae8a2da6c854557c65503537ac Mon Sep 17 00:00:00 2001 From: Guillaume Petiot Date: Wed, 6 Sep 2023 13:14:10 +0100 Subject: [PATCH 4/5] Use read_cmt/cmi functions from compiler-libs (#1) * Use read_cmt/cmi functions from compiler-libs Co-authored-by: Guillaume Petiot --- src/voodoo-prep/dune | 2 +- src/voodoo-prep/prep.ml | 28 +++++++--------------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/src/voodoo-prep/dune b/src/voodoo-prep/dune index c55d26f5..81897c5c 100644 --- a/src/voodoo-prep/dune +++ b/src/voodoo-prep/dune @@ -2,4 +2,4 @@ (name main) (public_name voodoo-prep) (package voodoo-prep) - (libraries cmdliner fpath bos opam-format)) + (libraries cmdliner fpath bos compiler-libs.common opam-format)) diff --git a/src/voodoo-prep/prep.ml b/src/voodoo-prep/prep.ml index 34254e25..39ede92b 100644 --- a/src/voodoo-prep/prep.ml +++ b/src/voodoo-prep/prep.ml @@ -7,29 +7,15 @@ let process_package : Fpath.t -> Package.t -> Fpath.t list -> unit = let dest = Package.prep_path package in let format_matches_compiler_version path = - let open Unix in - let cmd = "ocamlobjinfo " ^ Fpath.to_string path in + let filename = Fpath.to_string path in try - let ic, oc = open_process cmd in - let rec matches_version ic = - try - let line = input_line ic in - let prefix = "Wrong magic number" in - let str_len = String.length line in - if str_len >= 18 && String.sub line 0 18 = prefix then ( - Printf.eprintf "ERROR: ocamlobjinfo on %s: %s\n" - (Fpath.to_string path) line; - false) - else if str_len > 0 then matches_version ic - else true - with End_of_file -> true - in - let matches_version = matches_version ic in - let _ = close_process (ic, oc) in - matches_version + (match Fpath.get_ext path with + | ".cmt" | ".cmti" -> ignore @@ Cmt_format.read_cmt filename + | ".cmi" -> ignore @@ Cmi_format.read_cmi filename + | _ -> assert false); + true with _ -> - Printf.eprintf "ERROR: Failed to run ocamlobjinfo on %s\n" - (Fpath.to_string path); + Format.eprintf "[WARNING] %a ignored: wrong magic number\n%!" Fpath.pp path; false in From 5ba1fe1a12694756c0116b95fd4be2ed52d667d4 Mon Sep 17 00:00:00 2001 From: Sabine Schmaltz Date: Wed, 6 Sep 2023 14:31:14 +0200 Subject: [PATCH 5/5] fmt --- src/voodoo-prep/prep.ml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/voodoo-prep/prep.ml b/src/voodoo-prep/prep.ml index 39ede92b..f37265a1 100644 --- a/src/voodoo-prep/prep.ml +++ b/src/voodoo-prep/prep.ml @@ -15,7 +15,8 @@ let process_package : Fpath.t -> Package.t -> Fpath.t list -> unit = | _ -> assert false); true with _ -> - Format.eprintf "[WARNING] %a ignored: wrong magic number\n%!" Fpath.pp path; + Format.eprintf "[WARNING] %a ignored: wrong magic number\n%!" Fpath.pp + path; false in