From 648e2757744e8536f504696d4a0b86f71c59adab Mon Sep 17 00:00:00 2001 From: Kate Date: Wed, 28 Apr 2021 17:39:46 +0100 Subject: [PATCH 1/3] Make 0install-solver available in the opam 2.1 installation and explicitly pass the git hash for the opam source to keep all the images consistant --- ocaml-dockerfile | 2 +- src/pipeline.ml | 31 +++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/ocaml-dockerfile b/ocaml-dockerfile index 206dd530..695796b2 160000 --- a/ocaml-dockerfile +++ b/ocaml-dockerfile @@ -1 +1 @@ -Subproject commit 206dd530f5ed00b0271976767e280a6a951ad8d5 +Subproject commit 695796b2d33e2f4f354c2bb33b6860f9c98d84d6 diff --git a/src/pipeline.ml b/src/pipeline.ml index 4d758104..535f9859 100644 --- a/src/pipeline.ml +++ b/src/pipeline.ml @@ -8,6 +8,14 @@ let opam_repository () = Current_git.clone ~schedule:weekly "git://github.com/ocaml/opam-repository" |> Current.map Current_git.Commit.id +let get_opam_2_0 () = + Current_git.clone ~schedule:weekly ~gref:"2.0" "git://github.com/ocaml/opam" + |> Current.map Current_git.Commit.id + +let get_opam_master () = + Current_git.clone ~schedule:weekly ~gref:"master" "git://github.com/ocaml/opam" + |> Current.map Current_git.Commit.id + (* [aliases_of d] gives other tags which should point to [d]. e.g. just after the Ubuntu 20.04 release, [aliases_of ubuntu-20.04 = [ ubuntu; ubuntu-lts ]] *) let aliases_of = @@ -77,11 +85,19 @@ let or_die = function (* Pipeline to build the opam base image and the compiler images for a particular architecture. *) module Arch = struct - let install_opam ~arch ~ocluster ~distro ~opam_repository ~push_target = + let install_opam ~arch ~ocluster ~distro ~opam_repository ~push_target ~opam_2_0 ~opam_master = let arch_name = Ocaml_version.string_of_arch arch in + let distro_tag = Dockerfile_distro.tag_of_distro distro in + Current.component "%s@,%s" distro_tag arch_name |> + let> opam_repository = opam_repository + and> opam_2_0 = opam_2_0 + and> opam_master = opam_master + in let dockerfile = + let hash_opam_2_0 = Current_git.Commit_id.hash opam_2_0 in + let hash_opam_master = Current_git.Commit_id.hash opam_master in `Contents ( - let opam = snd @@ Dockerfile_opam.gen_opam2_distro ~arch ~clone_opam_repo:false distro in + let opam = snd @@ Dockerfile_opam.gen_opam2_distro ~arch ~clone_opam_repo:false ~hash_opam_2_0 ~hash_opam_master distro in let open Dockerfile in string_of_t ( opam @@ @@ -93,9 +109,6 @@ module Arch = struct ) ) in - let distro_tag = Dockerfile_distro.tag_of_distro distro in - Current.component "%s@,%s" distro_tag arch_name |> - let> opam_repository = opam_repository in let options = { Cluster_api.Docker.Spec.defaults with squash = true; include_git = true } in let cache_hint = Printf.sprintf "opam-%s" distro_tag in Current_ocluster.Raw.build_and_push ocluster ~src:[opam_repository] dockerfile @@ -131,14 +144,14 @@ module Arch = struct ~pool:(Conf.pool_for_arch `X86_64) (* Build the base image for [distro], plus an image for each compiler version. *) - let pipeline ~ocluster ~opam_repository ~distro arch = + let pipeline ~ocluster ~opam_repository ~opam_2_0 ~opam_master ~distro arch = let opam_image = let push_target = Tag.v distro ~arch |> Cluster_api.Docker.Image_id.of_string |> or_die in - install_opam ~arch ~ocluster ~distro ~opam_repository ~push_target + install_opam ~arch ~ocluster ~distro ~opam_repository ~push_target ~opam_2_0 ~opam_master in let compiler_images = Conf.switches ~arch ~distro |> List.map @@ fun switch -> @@ -194,6 +207,8 @@ let label l t = (* The main pipeline. Builds images for all supported distribution, compiler version and architecture combinations. *) let v ?channel ~ocluster () = let repo = opam_repository () in + let opam_2_0 = get_opam_2_0 () in + let opam_master = get_opam_master () in Current.all ( Conf.distros |> List.map @@ fun distro -> let distro_label = Dockerfile_distro.tag_of_distro distro in @@ -201,7 +216,7 @@ let v ?channel ~ocluster () = Current.collapse ~key:"distro" ~value:distro_label ~input:repo @@ let distro_aliases = aliases_of distro in let arches = Conf.arches_for ~distro in - let arch_results = List.map (Arch.pipeline ~ocluster ~opam_repository:repo ~distro) arches in + let arch_results = List.map (Arch.pipeline ~ocluster ~opam_repository:repo ~opam_2_0 ~opam_master ~distro) arches in let opam_images, ocaml_images, archive_image = List.fold_left (fun (aa,ba,ca) (a,b,c) -> let ca = match ca,c with Some v, _ -> Some v | None, v -> v in From 7664f7d4dae744afd254919644ae7bcf36929e9e Mon Sep 17 00:00:00 2001 From: Kate Date: Mon, 10 May 2021 15:26:01 +0100 Subject: [PATCH 2/3] Ensure the pipeline is ran only once a week instead of 3 times consecutively --- src/dune | 3 +- src/git_repositories.ml | 94 ++++++++++++++++++++++++++++++++++++++++ src/git_repositories.mli | 7 +++ src/pipeline.ml | 36 +++++---------- 4 files changed, 114 insertions(+), 26 deletions(-) create mode 100644 src/git_repositories.ml create mode 100644 src/git_repositories.mli diff --git a/src/dune b/src/dune index e413d6e5..23d364b0 100644 --- a/src/dune +++ b/src/dune @@ -12,4 +12,5 @@ capnp-rpc-unix dockerfile-opam lwt.unix - prometheus-app.unix)) + prometheus-app.unix) + (preprocess (pps ppx_deriving_yojson))) diff --git a/src/git_repositories.ml b/src/git_repositories.ml new file mode 100644 index 00000000..aac5ad9b --- /dev/null +++ b/src/git_repositories.ml @@ -0,0 +1,94 @@ +open Lwt.Infix +open Current.Syntax + +let ( >>!= ) x f = + x >>= function + | Ok y -> f y + | Error _ as e -> Lwt.return e + +module Repositories = struct + type t = No_context + + let id = "git-repositories" + + module Key = struct + type repo = string + + type t = { + opam_repository_master : repo; + opam_2_0 : repo; + opam_master : repo; + } + + let digest {opam_repository_master; opam_2_0; opam_master} = + let json = `Assoc [ + "opam-repository__master", `String opam_repository_master; + "opam__2.0", `String opam_2_0; + "opam__master", `String opam_master; + ] in + Yojson.Safe.to_string json + end + + module Value = struct + type hash = string [@@deriving yojson] + + type t = { + opam_repository_master : hash; + opam_2_0 : hash; + opam_master : hash; + } [@@deriving yojson] + + let marshal t = to_yojson t |> Yojson.Safe.to_string + + let unmarshal s = + match Yojson.Safe.from_string s |> of_yojson with + | Ppx_deriving_yojson_runtime.Result.Ok x -> x + | Ppx_deriving_yojson_runtime.Result.Error _ -> failwith "failed to parse Git_repositories.Repositories.Value.t" + end + + let get_commit_hash ~job ~repo ~branch = + Current.Process.with_tmpdir ~prefix:"git-checkout" @@ fun cwd -> + Current.Process.exec ~cwd ~cancellable:true ~job ("", [|"git"; "clone"; repo; "."|]) >>!= fun () -> + Current.Process.check_output ~cwd ~cancellable:true ~job ("", [|"git"; "rev-parse"; branch|]) >>!= fun hash -> + Lwt.return (Ok (String.trim hash)) + + let build No_context job {Key.opam_repository_master; opam_2_0; opam_master} = + Current.Job.start job ~level:Current.Level.Mostly_harmless >>= fun () -> + get_commit_hash ~job ~repo:opam_repository_master ~branch:"master" >>!= fun opam_repository_master -> + get_commit_hash ~job ~repo:opam_2_0 ~branch:"2.0" >>!= fun opam_2_0 -> + get_commit_hash ~job ~repo:opam_master ~branch:"master" >>!= fun opam_master -> + Lwt.return (Ok {Value.opam_repository_master; opam_2_0; opam_master}) + + let pp f _ = Fmt.string f "Git repositories" + + let auto_cancel = true +end + +module Cache = Current_cache.Make(Repositories) + +type t = { + opam_repository_master : Current_git.Commit_id.t; + opam_2_0 : Current_git.Commit_id.t; + opam_master : Current_git.Commit_id.t; +} + +let get ~schedule = + let key = { + Repositories.Key. + opam_repository_master = "git://github.com/ocaml/opam-repository"; + opam_2_0 = "git://github.com/ocaml/opam"; + opam_master = "git://github.com/ocaml/opam"; + } in + let+ {Repositories.Value.opam_repository_master; opam_2_0; opam_master} = + Current.component "Git-repositories" |> + let> key = Current.return key in + Cache.get ~schedule Repositories.No_context key + in + { + opam_repository_master = + Current_git.Commit_id.v ~repo:key.opam_repository_master ~gref:"master" ~hash:opam_repository_master; + opam_2_0 = + Current_git.Commit_id.v ~repo:key.opam_2_0 ~gref:"2.0" ~hash:opam_2_0; + opam_master = + Current_git.Commit_id.v ~repo:key.opam_master ~gref:"master" ~hash:opam_master; + } diff --git a/src/git_repositories.mli b/src/git_repositories.mli new file mode 100644 index 00000000..7c16efda --- /dev/null +++ b/src/git_repositories.mli @@ -0,0 +1,7 @@ +type t = { + opam_repository_master : Current_git.Commit_id.t; + opam_2_0 : Current_git.Commit_id.t; + opam_master : Current_git.Commit_id.t; +} + +val get : schedule:Current_cache.Schedule.t -> t Current.t diff --git a/src/pipeline.ml b/src/pipeline.ml index 535f9859..871421c9 100644 --- a/src/pipeline.ml +++ b/src/pipeline.ml @@ -4,17 +4,8 @@ module Switch_map = Map.Make(Ocaml_version) let weekly = Current_cache.Schedule.v ~valid_for:(Duration.of_day 7) () -let opam_repository () = - Current_git.clone ~schedule:weekly "git://github.com/ocaml/opam-repository" - |> Current.map Current_git.Commit.id - -let get_opam_2_0 () = - Current_git.clone ~schedule:weekly ~gref:"2.0" "git://github.com/ocaml/opam" - |> Current.map Current_git.Commit.id - -let get_opam_master () = - Current_git.clone ~schedule:weekly ~gref:"master" "git://github.com/ocaml/opam" - |> Current.map Current_git.Commit.id +let git_repositories () = + Git_repositories.get ~schedule:weekly (* [aliases_of d] gives other tags which should point to [d]. e.g. just after the Ubuntu 20.04 release, [aliases_of ubuntu-20.04 = [ ubuntu; ubuntu-lts ]] *) @@ -85,14 +76,11 @@ let or_die = function (* Pipeline to build the opam base image and the compiler images for a particular architecture. *) module Arch = struct - let install_opam ~arch ~ocluster ~distro ~opam_repository ~push_target ~opam_2_0 ~opam_master = + let install_opam ~arch ~ocluster ~distro ~repos ~push_target = let arch_name = Ocaml_version.string_of_arch arch in let distro_tag = Dockerfile_distro.tag_of_distro distro in Current.component "%s@,%s" distro_tag arch_name |> - let> opam_repository = opam_repository - and> opam_2_0 = opam_2_0 - and> opam_master = opam_master - in + let> {Git_repositories.opam_repository_master; opam_2_0; opam_master} = repos in let dockerfile = let hash_opam_2_0 = Current_git.Commit_id.hash opam_2_0 in let hash_opam_master = Current_git.Commit_id.hash opam_master in @@ -111,7 +99,7 @@ module Arch = struct in let options = { Cluster_api.Docker.Spec.defaults with squash = true; include_git = true } in let cache_hint = Printf.sprintf "opam-%s" distro_tag in - Current_ocluster.Raw.build_and_push ocluster ~src:[opam_repository] dockerfile + Current_ocluster.Raw.build_and_push ocluster ~src:[opam_repository_master] dockerfile ~cache_hint ~options ~push_target @@ -144,14 +132,14 @@ module Arch = struct ~pool:(Conf.pool_for_arch `X86_64) (* Build the base image for [distro], plus an image for each compiler version. *) - let pipeline ~ocluster ~opam_repository ~opam_2_0 ~opam_master ~distro arch = + let pipeline ~ocluster ~repos ~distro arch = let opam_image = let push_target = Tag.v distro ~arch |> Cluster_api.Docker.Image_id.of_string |> or_die in - install_opam ~arch ~ocluster ~distro ~opam_repository ~push_target ~opam_2_0 ~opam_master + install_opam ~arch ~ocluster ~distro ~repos ~push_target in let compiler_images = Conf.switches ~arch ~distro |> List.map @@ fun switch -> @@ -206,17 +194,15 @@ let label l t = (* The main pipeline. Builds images for all supported distribution, compiler version and architecture combinations. *) let v ?channel ~ocluster () = - let repo = opam_repository () in - let opam_2_0 = get_opam_2_0 () in - let opam_master = get_opam_master () in + let repos = git_repositories () in Current.all ( Conf.distros |> List.map @@ fun distro -> let distro_label = Dockerfile_distro.tag_of_distro distro in - let repo = label distro_label repo in - Current.collapse ~key:"distro" ~value:distro_label ~input:repo @@ + let repos = label distro_label repos in + Current.collapse ~key:"distro" ~value:distro_label ~input:repos @@ let distro_aliases = aliases_of distro in let arches = Conf.arches_for ~distro in - let arch_results = List.map (Arch.pipeline ~ocluster ~opam_repository:repo ~opam_2_0 ~opam_master ~distro) arches in + let arch_results = List.map (Arch.pipeline ~ocluster ~repos ~distro) arches in let opam_images, ocaml_images, archive_image = List.fold_left (fun (aa,ba,ca) (a,b,c) -> let ca = match ca,c with Some v, _ -> Some v | None, v -> v in From c7d9ded8d51647d23d6c0b0c93b36bb027658184 Mon Sep 17 00:00:00 2001 From: Kate Date: Mon, 10 May 2021 16:01:36 +0100 Subject: [PATCH 3/3] Fix call to git rev-parse --- src/git_repositories.ml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/git_repositories.ml b/src/git_repositories.ml index aac5ad9b..6073d518 100644 --- a/src/git_repositories.ml +++ b/src/git_repositories.ml @@ -48,8 +48,8 @@ module Repositories = struct let get_commit_hash ~job ~repo ~branch = Current.Process.with_tmpdir ~prefix:"git-checkout" @@ fun cwd -> - Current.Process.exec ~cwd ~cancellable:true ~job ("", [|"git"; "clone"; repo; "."|]) >>!= fun () -> - Current.Process.check_output ~cwd ~cancellable:true ~job ("", [|"git"; "rev-parse"; branch|]) >>!= fun hash -> + Current.Process.exec ~cwd ~cancellable:true ~job ("", [|"git"; "clone"; "-b"; branch; repo; "."|]) >>!= fun () -> + Current.Process.check_output ~cwd ~cancellable:true ~job ("", [|"git"; "rev-parse"; "HEAD"|]) >>!= fun hash -> Lwt.return (Ok (String.trim hash)) let build No_context job {Key.opam_repository_master; opam_2_0; opam_master} =