diff --git a/bench/irmin-pack/trace_replay.ml b/bench/irmin-pack/trace_replay.ml index b5f5a51db30..e0bed82ebcc 100644 --- a/bench/irmin-pack/trace_replay.ml +++ b/bench/irmin-pack/trace_replay.ml @@ -159,6 +159,7 @@ module Make (Store : Store) = struct mutable commits_since_start_or_gc : int; mutable latest_commit_idx : int; (** the most recent commit idx to be replayed. initial value is -1 *) + mutable gc_count : int; key_per_commit_idx : (int, Store.commit_key) Hashtbl.t; } @@ -322,7 +323,7 @@ module Make (Store : Store) = struct in aux operations 0 - let gc_actions config i commits_since_start_or_gc = + let gc_actions config i commits_since_start_or_gc gc_count = let gc_enabled = (* Is GC enabled at all? *) config.gc_every > 0 @@ -357,10 +358,18 @@ module Make (Store : Store) = struct else false in + let time_to_add_volume = + config.add_volume_every > 0 + && time_to_split + && gc_count > 0 + && gc_count mod config.add_volume_every = 0 + in + let really_split = gc_enabled && time_to_split in let really_start_gc = gc_enabled && time_to_start in let really_wait_gc = gc_wait_enabled && time_to_wait in - (really_wait_gc, really_start_gc, really_split) + let really_add_volume = time_to_add_volume in + (really_wait_gc, really_start_gc, really_split, really_add_volume) let add_commits config repo commit_seq on_commit on_end stats check_hash empty_blobs = @@ -372,6 +381,7 @@ module Make (Store : Store) = struct contexts = Hashtbl.create 3; hash_corresps = Hashtbl.create 3; commits_since_start_or_gc = 0; + gc_count = 0; latest_commit_idx = -1; key_per_commit_idx = Hashtbl.create 3; } @@ -384,11 +394,12 @@ module Make (Store : Store) = struct match commit_seq () with | Seq.Nil -> on_end () >|= fun () -> i | Cons (ops, commit_seq) -> - let really_wait_gc, really_start_gc, really_split = - gc_actions config i t.commits_since_start_or_gc + let really_wait_gc, really_start_gc, really_split, really_add_volume = + gc_actions config i t.commits_since_start_or_gc t.gc_count in (* Split before GC to simulate how it is inteded to be used. *) let () = if really_split then Store.split repo in + let () = if really_add_volume then Store.add_volume repo in let* () = if really_wait_gc then ( [%logs.app @@ -418,6 +429,7 @@ module Make (Store : Store) = struct (Hashtbl.find t.key_per_commit_idx t.latest_commit_idx)]; let finished = function | Ok stats -> + t.gc_count <- t.gc_count + 1; let commit_idx = t.latest_commit_idx in let commit_duration = commit_idx - gc_start_commit_idx in let duration = diff --git a/bench/irmin-pack/trace_replay_intf.ml b/bench/irmin-pack/trace_replay_intf.ml index ef278156d73..93d558d2be9 100644 --- a/bench/irmin-pack/trace_replay_intf.ml +++ b/bench/irmin-pack/trace_replay_intf.ml @@ -33,6 +33,7 @@ module Config = struct gc_every : int; gc_distance_in_the_past : int; gc_wait_after : int; + add_volume_every : int; } (** Replay configuration @@ -102,6 +103,7 @@ module type Store = sig root:string -> store_config -> (Repo.t * on_commit * on_end) Lwt.t val split : repo -> unit + val add_volume : repo -> unit val gc_wait : repo -> unit Lwt.t type stats := Irmin_pack_unix.Stats.Latest_gc.stats diff --git a/bench/irmin-pack/tree.ml b/bench/irmin-pack/tree.ml index 329fbce324e..4430fe45a9d 100644 --- a/bench/irmin-pack/tree.ml +++ b/bench/irmin-pack/tree.ml @@ -39,6 +39,7 @@ type config = { gc_every : int; gc_distance_in_the_past : int; gc_wait_after : int; + add_volume_every : int; } module type Store = sig @@ -61,6 +62,7 @@ module type Store = sig type stats := Irmin_pack_unix.Stats.Latest_gc.stats val split : repo -> unit + val add_volume : repo -> unit val gc_run : ?finished:((stats, string) result -> unit Lwt.t) -> @@ -181,6 +183,7 @@ module Bench_suite (Store : Store) = struct gc_every = config.gc_every; gc_distance_in_the_past = config.gc_distance_in_the_past; gc_wait_after = config.gc_wait_after; + add_volume_every = config.add_volume_every; } in if config.no_summary then @@ -224,6 +227,7 @@ module Make_store_mem (Conf : Irmin_pack.Conf.S) = struct Lwt.return (repo, on_commit, on_end) let split _repo = () + let add_volume _repo = () let gc_wait _repo = Lwt.return_unit let gc_run ?finished:_ _repo _key = Lwt.return_unit end @@ -242,9 +246,14 @@ module Make_store_pack (Conf : Irmin_pack.Conf.S) = struct let indexing_strategy = Irmin_pack.Indexing_strategy.minimal - let create_repo ~root _config = + let create_repo ~root (config : store_config) = + let lower_root = + if config.add_volume_every > 0 then Some (Filename.concat root "lower") + else None + in let conf = - Irmin_pack.config ~readonly:false ~fresh:true ~indexing_strategy root + Irmin_pack.config ~readonly:false ~fresh:true ~indexing_strategy + ~lower_root root in prepare_artefacts_dir root; let* repo = Store.Repo.v conf in @@ -253,6 +262,7 @@ module Make_store_pack (Conf : Irmin_pack.Conf.S) = struct Lwt.return (repo, on_commit, on_end) let split = Store.split + let add_volume = Store.add_volume let gc_wait repo = let* r = Store.Gc.wait repo in @@ -395,7 +405,8 @@ let get_suite suite_filter = let main () ncommits number_of_commits_to_replay suite_filter inode_config store_type freeze_commit path_conversion depth width nchain_trees nlarge_trees replay_trace_path artefacts_path keep_store keep_stat_trace - no_summary empty_blobs gc_every gc_distance_in_the_past gc_wait_after = + no_summary empty_blobs gc_every gc_distance_in_the_past gc_wait_after + add_volume_every = let default = match suite_filter with `Quick -> 10000 | _ -> 13315 in let number_of_commits_to_replay = Option.value ~default number_of_commits_to_replay @@ -422,6 +433,7 @@ let main () ncommits number_of_commits_to_replay suite_filter inode_config gc_every; gc_distance_in_the_past; gc_wait_after; + add_volume_every; } in Printexc.record_backtrace true; @@ -602,6 +614,10 @@ let gc_wait_after = in Arg.(value @@ opt int 0 doc) +let add_volume_every = + let doc = Arg.info ~doc:"Add volume ever N GCs" [ "add-volume-every" ] in + Arg.(value @@ opt int 0 doc) + let main_term = Term.( const main @@ -625,7 +641,8 @@ let main_term = $ empty_blobs $ gc_every $ gc_distance_in_the_past - $ gc_wait_after) + $ gc_wait_after + $ add_volume_every) let deprecated_info = (Term.info [@alert "-deprecated"]) let deprecated_exit = (Term.exit [@alert "-deprecated"]) diff --git a/test/irmin-bench/replay.ml b/test/irmin-bench/replay.ml index 4c09d92cbb7..65495fc5ab7 100644 --- a/test/irmin-bench/replay.ml +++ b/test/irmin-bench/replay.ml @@ -95,6 +95,7 @@ let replay_1_commit () = gc_every = 0; gc_distance_in_the_past = 0; gc_wait_after = 0; + add_volume_every = 0; } in let+ summary = Replay.run () replay_config in @@ -141,6 +142,7 @@ module Store_mem = struct Lwt.return (repo, on_commit, on_end) let split _repo = () + let add_volume _repo = () let gc_wait _repo = Lwt.return_unit let gc_run ?finished:_ _repo _key = Lwt.return_unit end @@ -164,6 +166,7 @@ let replay_1_commit_mem () = gc_every = 0; gc_distance_in_the_past = 0; gc_wait_after = 0; + add_volume_every = 0; } in let+ summary = Replay_mem.run () replay_config in