diff --git a/ocaml-dockerfile b/ocaml-dockerfile index 206dd530..695796b2 160000 --- a/ocaml-dockerfile +++ b/ocaml-dockerfile @@ -1 +1 @@ -Subproject commit 206dd530f5ed00b0271976767e280a6a951ad8d5 +Subproject commit 695796b2d33e2f4f354c2bb33b6860f9c98d84d6 diff --git a/src/dune b/src/dune index e413d6e5..23d364b0 100644 --- a/src/dune +++ b/src/dune @@ -12,4 +12,5 @@ capnp-rpc-unix dockerfile-opam lwt.unix - prometheus-app.unix)) + prometheus-app.unix) + (preprocess (pps ppx_deriving_yojson))) diff --git a/src/git_repositories.ml b/src/git_repositories.ml new file mode 100644 index 00000000..6073d518 --- /dev/null +++ b/src/git_repositories.ml @@ -0,0 +1,94 @@ +open Lwt.Infix +open Current.Syntax + +let ( >>!= ) x f = + x >>= function + | Ok y -> f y + | Error _ as e -> Lwt.return e + +module Repositories = struct + type t = No_context + + let id = "git-repositories" + + module Key = struct + type repo = string + + type t = { + opam_repository_master : repo; + opam_2_0 : repo; + opam_master : repo; + } + + let digest {opam_repository_master; opam_2_0; opam_master} = + let json = `Assoc [ + "opam-repository__master", `String opam_repository_master; + "opam__2.0", `String opam_2_0; + "opam__master", `String opam_master; + ] in + Yojson.Safe.to_string json + end + + module Value = struct + type hash = string [@@deriving yojson] + + type t = { + opam_repository_master : hash; + opam_2_0 : hash; + opam_master : hash; + } [@@deriving yojson] + + let marshal t = to_yojson t |> Yojson.Safe.to_string + + let unmarshal s = + match Yojson.Safe.from_string s |> of_yojson with + | Ppx_deriving_yojson_runtime.Result.Ok x -> x + | Ppx_deriving_yojson_runtime.Result.Error _ -> failwith "failed to parse Git_repositories.Repositories.Value.t" + end + + let get_commit_hash ~job ~repo ~branch = + Current.Process.with_tmpdir ~prefix:"git-checkout" @@ fun cwd -> + Current.Process.exec ~cwd ~cancellable:true ~job ("", [|"git"; "clone"; "-b"; branch; repo; "."|]) >>!= fun () -> + Current.Process.check_output ~cwd ~cancellable:true ~job ("", [|"git"; "rev-parse"; "HEAD"|]) >>!= fun hash -> + Lwt.return (Ok (String.trim hash)) + + let build No_context job {Key.opam_repository_master; opam_2_0; opam_master} = + Current.Job.start job ~level:Current.Level.Mostly_harmless >>= fun () -> + get_commit_hash ~job ~repo:opam_repository_master ~branch:"master" >>!= fun opam_repository_master -> + get_commit_hash ~job ~repo:opam_2_0 ~branch:"2.0" >>!= fun opam_2_0 -> + get_commit_hash ~job ~repo:opam_master ~branch:"master" >>!= fun opam_master -> + Lwt.return (Ok {Value.opam_repository_master; opam_2_0; opam_master}) + + let pp f _ = Fmt.string f "Git repositories" + + let auto_cancel = true +end + +module Cache = Current_cache.Make(Repositories) + +type t = { + opam_repository_master : Current_git.Commit_id.t; + opam_2_0 : Current_git.Commit_id.t; + opam_master : Current_git.Commit_id.t; +} + +let get ~schedule = + let key = { + Repositories.Key. + opam_repository_master = "git://github.com/ocaml/opam-repository"; + opam_2_0 = "git://github.com/ocaml/opam"; + opam_master = "git://github.com/ocaml/opam"; + } in + let+ {Repositories.Value.opam_repository_master; opam_2_0; opam_master} = + Current.component "Git-repositories" |> + let> key = Current.return key in + Cache.get ~schedule Repositories.No_context key + in + { + opam_repository_master = + Current_git.Commit_id.v ~repo:key.opam_repository_master ~gref:"master" ~hash:opam_repository_master; + opam_2_0 = + Current_git.Commit_id.v ~repo:key.opam_2_0 ~gref:"2.0" ~hash:opam_2_0; + opam_master = + Current_git.Commit_id.v ~repo:key.opam_master ~gref:"master" ~hash:opam_master; + } diff --git a/src/git_repositories.mli b/src/git_repositories.mli new file mode 100644 index 00000000..7c16efda --- /dev/null +++ b/src/git_repositories.mli @@ -0,0 +1,7 @@ +type t = { + opam_repository_master : Current_git.Commit_id.t; + opam_2_0 : Current_git.Commit_id.t; + opam_master : Current_git.Commit_id.t; +} + +val get : schedule:Current_cache.Schedule.t -> t Current.t diff --git a/src/pipeline.ml b/src/pipeline.ml index 4d758104..871421c9 100644 --- a/src/pipeline.ml +++ b/src/pipeline.ml @@ -4,9 +4,8 @@ module Switch_map = Map.Make(Ocaml_version) let weekly = Current_cache.Schedule.v ~valid_for:(Duration.of_day 7) () -let opam_repository () = - Current_git.clone ~schedule:weekly "git://github.com/ocaml/opam-repository" - |> Current.map Current_git.Commit.id +let git_repositories () = + Git_repositories.get ~schedule:weekly (* [aliases_of d] gives other tags which should point to [d]. e.g. just after the Ubuntu 20.04 release, [aliases_of ubuntu-20.04 = [ ubuntu; ubuntu-lts ]] *) @@ -77,11 +76,16 @@ let or_die = function (* Pipeline to build the opam base image and the compiler images for a particular architecture. *) module Arch = struct - let install_opam ~arch ~ocluster ~distro ~opam_repository ~push_target = + let install_opam ~arch ~ocluster ~distro ~repos ~push_target = let arch_name = Ocaml_version.string_of_arch arch in + let distro_tag = Dockerfile_distro.tag_of_distro distro in + Current.component "%s@,%s" distro_tag arch_name |> + let> {Git_repositories.opam_repository_master; opam_2_0; opam_master} = repos in let dockerfile = + let hash_opam_2_0 = Current_git.Commit_id.hash opam_2_0 in + let hash_opam_master = Current_git.Commit_id.hash opam_master in `Contents ( - let opam = snd @@ Dockerfile_opam.gen_opam2_distro ~arch ~clone_opam_repo:false distro in + let opam = snd @@ Dockerfile_opam.gen_opam2_distro ~arch ~clone_opam_repo:false ~hash_opam_2_0 ~hash_opam_master distro in let open Dockerfile in string_of_t ( opam @@ @@ -93,12 +97,9 @@ module Arch = struct ) ) in - let distro_tag = Dockerfile_distro.tag_of_distro distro in - Current.component "%s@,%s" distro_tag arch_name |> - let> opam_repository = opam_repository in let options = { Cluster_api.Docker.Spec.defaults with squash = true; include_git = true } in let cache_hint = Printf.sprintf "opam-%s" distro_tag in - Current_ocluster.Raw.build_and_push ocluster ~src:[opam_repository] dockerfile + Current_ocluster.Raw.build_and_push ocluster ~src:[opam_repository_master] dockerfile ~cache_hint ~options ~push_target @@ -131,14 +132,14 @@ module Arch = struct ~pool:(Conf.pool_for_arch `X86_64) (* Build the base image for [distro], plus an image for each compiler version. *) - let pipeline ~ocluster ~opam_repository ~distro arch = + let pipeline ~ocluster ~repos ~distro arch = let opam_image = let push_target = Tag.v distro ~arch |> Cluster_api.Docker.Image_id.of_string |> or_die in - install_opam ~arch ~ocluster ~distro ~opam_repository ~push_target + install_opam ~arch ~ocluster ~distro ~repos ~push_target in let compiler_images = Conf.switches ~arch ~distro |> List.map @@ fun switch -> @@ -193,15 +194,15 @@ let label l t = (* The main pipeline. Builds images for all supported distribution, compiler version and architecture combinations. *) let v ?channel ~ocluster () = - let repo = opam_repository () in + let repos = git_repositories () in Current.all ( Conf.distros |> List.map @@ fun distro -> let distro_label = Dockerfile_distro.tag_of_distro distro in - let repo = label distro_label repo in - Current.collapse ~key:"distro" ~value:distro_label ~input:repo @@ + let repos = label distro_label repos in + Current.collapse ~key:"distro" ~value:distro_label ~input:repos @@ let distro_aliases = aliases_of distro in let arches = Conf.arches_for ~distro in - let arch_results = List.map (Arch.pipeline ~ocluster ~opam_repository:repo ~distro) arches in + let arch_results = List.map (Arch.pipeline ~ocluster ~repos ~distro) arches in let opam_images, ocaml_images, archive_image = List.fold_left (fun (aa,ba,ca) (a,b,c) -> let ca = match ca,c with Some v, _ -> Some v | None, v -> v in