Skip to content

Commit

Permalink
fixup! irmin-pack: update GC for chunked suffix
Browse files Browse the repository at this point in the history
  • Loading branch information
metanivek committed Nov 2, 2022
1 parent 62dade2 commit 7830b81
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 24 deletions.
6 changes: 3 additions & 3 deletions src/irmin-pack/unix/dispatcher.ml
Original file line number Diff line number Diff line change
Expand Up @@ -85,18 +85,18 @@ module Make (Fm : File_manager.S with module Io = Io.Unix) :

module Suffix_arithmetic = struct
(* Adjust the read in suffix, as the global offset [off] is
[off] = [suffix_start_offset] + [suffix_offset]. *)
[off] = [suffix_start_offset] + [soff] - [suffix_dead_bytes]. *)
let soff_of_off t off =
let open Int63.Syntax in
let suffix_start_offset = suffix_start_offset t in
let suffix_dead_bytes = suffix_dead_bytes t in
off - suffix_start_offset + suffix_dead_bytes

let off_of_soff t suffix_off =
let off_of_soff t soff =
let open Int63.Syntax in
let suffix_start_offset = suffix_start_offset t in
let suffix_dead_bytes = suffix_dead_bytes t in
suffix_off - suffix_dead_bytes + suffix_start_offset
suffix_start_offset + soff - suffix_dead_bytes
end

let offset_of_soff = Suffix_arithmetic.off_of_soff
Expand Down
13 changes: 9 additions & 4 deletions src/irmin-pack/unix/gc.ml
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ module Make (Args : Gc_args.S) = struct
latest_gc_target_offset;
}

let swap_and_purge t removable_chunk_num suffix =
let swap_and_purge t removable_chunk_num suffix_params =
let open Result_syntax in
let { generation; latest_gc_target_offset; _ } = t in
let Worker.
Expand All @@ -117,13 +117,15 @@ module Make (Args : Gc_args.S) = struct
chunk_start_idx;
dead_bytes = suffix_dead_bytes;
} =
suffix
suffix_params
in
(* Calculate chunk num in main process since more chunks could have been
added while GC was running. GC process only tells us how many chunks are
to be removed. *)
let suffix = Fm.suffix t.fm in
let chunk_num = Fm.Suffix.chunk_num suffix - removable_chunk_num in
(* Assert that we have at least one chunk (the appendable chunk), which
is guaranteed by the GC process. *)
assert (chunk_num >= 1);

let* () =
Expand Down Expand Up @@ -233,13 +235,16 @@ module Make (Args : Gc_args.S) = struct
let open Result_syntax in
match (status, gc_output) with
| ( `Success,
Ok { suffix; removable_chunk_idxs; stats = worker_stats } ) ->
Ok { suffix_params; removable_chunk_idxs; stats = worker_stats }
) ->
let partial_stats =
Gc_stats.Main.finish_current_step partial_stats
"swap and purge"
in
let* () =
swap_and_purge t (List.length removable_chunk_idxs) suffix
swap_and_purge t
(List.length removable_chunk_idxs)
suffix_params
in
let partial_stats =
Gc_stats.Main.finish_current_step partial_stats "unlink"
Expand Down
10 changes: 5 additions & 5 deletions src/irmin-pack/unix/gc_worker.ml
Original file line number Diff line number Diff line change
Expand Up @@ -118,15 +118,15 @@ module Make (Args : Gc_args.S) = struct
stats := Gc_stats.Worker.add_file_size !stats "old_prefix" prefix_size;
stats := Gc_stats.Worker.add_file_size !stats "old_mapping" mapping_size

type new_suffix = {
type suffix_params = {
start_offset : int63;
chunk_start_idx : int;
dead_bytes : int63;
}
[@@deriving irmin]

type gc_results = {
suffix : new_suffix;
suffix_params : suffix_params;
removable_chunk_idxs : int list;
stats : Stats.Latest_gc.worker;
}
Expand Down Expand Up @@ -277,8 +277,8 @@ module Make (Args : Gc_args.S) = struct
(Commit_value.parents commit)
in

(* Step 6. Calculate new suffix values. *)
let new_suffix, removable_chunk_idxs =
(* Step 6. Calculate post-GC suffix parameters. *)
let suffix_params, removable_chunk_idxs =
stats :=
Gc_stats.Worker.finish_current_step !stats
"suffix: calculate new values";
Expand Down Expand Up @@ -344,7 +344,7 @@ module Make (Args : Gc_args.S) = struct

(* Step 7. Finalise stats and return. *)
let stats = Gc_stats.Worker.finalise !stats in
{ suffix = new_suffix; removable_chunk_idxs; stats }
{ suffix_params; removable_chunk_idxs; stats }

let write_gc_output ~root ~generation output =
let open Result_syntax in
Expand Down
4 changes: 2 additions & 2 deletions src/irmin-pack/unix/gc_worker.mli
Original file line number Diff line number Diff line change
Expand Up @@ -25,15 +25,15 @@ module Make (Args : Gc_args.S) : sig
val run_and_output_result :
generation:int -> string -> Args.key -> int63 -> unit

type new_suffix = {
type suffix_params = {
start_offset : int63;
chunk_start_idx : int;
dead_bytes : int63;
}
[@@deriving irmin]

type gc_results = {
suffix : new_suffix;
suffix_params : suffix_params;
removable_chunk_idxs : int list;
stats : Stats.Latest_gc.worker;
}
Expand Down
20 changes: 10 additions & 10 deletions test/irmin-pack/test_gc.ml
Original file line number Diff line number Diff line change
Expand Up @@ -923,28 +923,28 @@ module Split = struct
let* () = S.Repo.close t.repo in
S.Repo.close ro_t.repo

let c t s =
let h =
match Irmin.Type.(of_string Schema.Hash.t) s with
let load_commit t h =
let hash =
match Irmin.Type.(of_string Schema.Hash.t) h with
| Error (`Msg s) -> Alcotest.failf "failed hash_of_string %s" s
| Ok h -> h
| Ok hash -> hash
in
let+ c = S.Commit.of_hash t.repo h in
match c with
| None -> Alcotest.failf "Commit %s not found" s
let+ commit = S.Commit.of_hash t.repo hash in
match commit with
| None -> Alcotest.failf "Commit %s not found" h
| Some commit -> commit

let check_preexisting_commit t =
let s = "22e159de13b427226e5901defd17f0c14e744205" in
let* commit = c t s in
let h = "22e159de13b427226e5901defd17f0c14e744205" in
let* commit = load_commit t h in
let tree = S.Commit.tree commit in
let+ got = S.Tree.find tree [ "step-n01"; "step-b01" ] in
Alcotest.(check (option string)) "find blob" (Some "b01") got

let v3_migrated_store_splits_and_gc () =
let root = create_test_env () in
let* t = init ~readonly:false ~fresh:false ~root () in
let* c0 = c t "22e159de13b427226e5901defd17f0c14e744205" in
let* c0 = load_commit t "22e159de13b427226e5901defd17f0c14e744205" in
let* t, c1 = commit_1 t in
let () = S.split t.repo in
let* t = checkout_exn t c1 in
Expand Down

0 comments on commit 7830b81

Please sign in to comment.