From afdb743feaf6a9a081fd023d089a078be9cc44ba Mon Sep 17 00:00:00 2001 From: metanivek Date: Tue, 11 Oct 2022 15:19:11 -0400 Subject: [PATCH 1/3] irmin-pack: add v1 of chunked suffix module --- src/irmin-pack/unix/chunked_suffix.ml | 191 +++++++++++++++++++++ src/irmin-pack/unix/chunked_suffix.mli | 18 ++ src/irmin-pack/unix/chunked_suffix_intf.ml | 94 ++++++++++ src/irmin-pack/unix/control_file_intf.ml | 4 + src/irmin-pack/unix/file_manager.ml | 6 +- src/irmin-pack/unix/import.ml | 14 ++ 6 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 src/irmin-pack/unix/chunked_suffix.ml create mode 100644 src/irmin-pack/unix/chunked_suffix.mli create mode 100644 src/irmin-pack/unix/chunked_suffix_intf.ml diff --git a/src/irmin-pack/unix/chunked_suffix.ml b/src/irmin-pack/unix/chunked_suffix.ml new file mode 100644 index 0000000000..a66a3b7c51 --- /dev/null +++ b/src/irmin-pack/unix/chunked_suffix.ml @@ -0,0 +1,191 @@ +(* + * Copyright (c) 2022-2022 Tarides + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + *) + +open Import +include Chunked_suffix_intf + +module Make (Io : Io.S) (Errs : Io_errors.S with module Io = Io) = struct + module Io = Io + module Errs = Errs + module Ao = Append_only_file.Make (Io) (Errs) + + type chunk = { idx : int; suffix_off : int63; ao : Ao.t } + type create_error = Io.create_error + + type open_error = + [ Io.open_error + | `Closed + | `Invalid_argument + | `Inconsistent_store + | `Read_out_of_bounds ] + + (** A simple container for chunks. *) + module Inventory : sig + type t + + val v : int -> (int -> chunk) -> t + val appendable : t -> chunk + + val find : off:int63 -> t -> chunk * int63 + (** [find ~off t] returns the chunk that contains suffix offset [off], along + with the corresponding [poff] within the chunk. + + Raises `Read_out_of_bounds exception. *) + + val open_ : + start_idx:int -> + chunk_num:int -> + open_chunk: + (chunk_idx:int -> + is_legacy:bool -> + is_appendable:bool -> + (Ao.t, open_error) result) -> + (t, [> open_error ]) result + + val close : t -> (unit, [> Io.close_error | `Pending_flush ]) result + end = struct + type t = chunk Array.t + + exception OpenInventoryError of open_error + + let v = Array.init + let appendable t = Array.get t (Array.length t - 1) + + let find ~off t = + let open Int63.Syntax in + let suffix_off_to_chunk_poff c = off - c.suffix_off in + let find c = + let end_poff = Ao.end_poff c.ao in + let poff = suffix_off_to_chunk_poff c in + Int63.zero <= poff && poff < end_poff + in + match Array.find_opt find t with + | None -> raise (Errors.Pack_error `Read_out_of_bounds) + | Some c -> (c, suffix_off_to_chunk_poff c) + + let open_ ~start_idx ~chunk_num ~open_chunk = + let off_acc = ref Int63.zero in + let create_chunk i = + let suffix_off = !off_acc in + let is_appendable = i = chunk_num - 1 in + let chunk_idx = start_idx + i in + let is_legacy = chunk_idx = 0 in + let open_result = open_chunk ~chunk_idx ~is_legacy ~is_appendable in + match open_result with + | Error err -> raise (OpenInventoryError err) + | Ok ao -> + let chunk_len = Ao.end_poff ao in + (off_acc := Int63.Syntax.(suffix_off + chunk_len)); + { idx = chunk_idx; suffix_off; ao } + in + try Ok (v chunk_num create_chunk) + with OpenInventoryError err -> + Error (err : open_error :> [> open_error ]) + + let close t = + (* Close immutable chunks, ignoring errors. *) + let _ = + Array.sub t 0 (Array.length t - 1) + |> Array.iter @@ fun chunk -> + let _ = Ao.close chunk.ao in + () + in + (* Close appendable chunk and keep error since this + is the one that can have a pending flush. *) + (appendable t).ao |> Ao.close + end + + type t = { inventory : Inventory.t } + + let chunk_path = Layout.V4.suffix_chunk + + let create_rw ~root ~start_idx ~overwrite ~auto_flush_threshold + ~auto_flush_procedure = + let open Result_syntax in + let chunk_idx = start_idx in + let path = chunk_path ~root ~chunk_idx in + let+ ao = + Ao.create_rw ~path ~overwrite ~auto_flush_threshold ~auto_flush_procedure + in + let chunk = { idx = chunk_idx; suffix_off = Int63.zero; ao } in + let inventory = Inventory.v 1 (Fun.const chunk) in + { inventory } + + (** A module to adjust values when mapping from chunks to append-only files *) + module Ao_shim = struct + type t = { dead_header_size : int; end_poff : int63 } + + let v ~path ~end_poff ~dead_header_size ~is_legacy ~is_appendable = + let open Result_syntax in + (* Only use the legacy dead_header_size for legacy chunks. *) + let dead_header_size = if is_legacy then dead_header_size else 0 in + (* The appendable chunk uses the provided [end_poff]; but the others + read their size on disk. TODO: this is needed for the Ao module's current + APIs but could perhaps be removed by future Ao API modifications. *) + let+ end_poff = + if is_appendable then Ok end_poff else Io.size_of_path path + in + { dead_header_size; end_poff } + end + + let open_rw ~root ~end_poff ~start_idx ~chunk_num ~dead_header_size + ~auto_flush_threshold ~auto_flush_procedure = + let open Result_syntax in + let open_chunk ~chunk_idx ~is_legacy ~is_appendable = + let path = chunk_path ~root ~chunk_idx in + let* { dead_header_size; end_poff } = + Ao_shim.v ~path ~end_poff ~dead_header_size ~is_legacy ~is_appendable + in + match is_appendable with + | true -> + Ao.open_rw ~path ~end_poff ~dead_header_size ~auto_flush_threshold + ~auto_flush_procedure + | false -> Ao.open_ro ~path ~end_poff ~dead_header_size + in + let+ inventory = Inventory.open_ ~start_idx ~chunk_num ~open_chunk in + { inventory } + + let open_ro ~root ~end_poff ~dead_header_size ~start_idx ~chunk_num = + let open Result_syntax in + let open_chunk ~chunk_idx ~is_legacy ~is_appendable = + let path = chunk_path ~root ~chunk_idx in + let* { dead_header_size; end_poff } = + Ao_shim.v ~path ~end_poff ~dead_header_size ~is_legacy ~is_appendable + in + Ao.open_ro ~path ~end_poff ~dead_header_size + in + let+ inventory = Inventory.open_ ~start_idx ~chunk_num ~open_chunk in + { inventory } + + let appendable_ao t = (Inventory.appendable t.inventory).ao + let end_poff t = appendable_ao t |> Ao.end_poff + + let read_exn t ~off ~len buf = + let chunk, poff = Inventory.find ~off t.inventory in + Ao.read_exn chunk.ao ~off:poff ~len buf + + let append_exn t s = Ao.append_exn (appendable_ao t) s + let close t = Inventory.close t.inventory + let empty_buffer t = appendable_ao t |> Ao.empty_buffer + let flush t = appendable_ao t |> Ao.flush + let fsync t = appendable_ao t |> Ao.fsync + + let refresh_end_poff t new_end_poff = + Ao.refresh_end_poff (appendable_ao t) new_end_poff + + let readonly t = appendable_ao t |> Ao.readonly + let auto_flush_threshold t = appendable_ao t |> Ao.auto_flush_threshold +end diff --git a/src/irmin-pack/unix/chunked_suffix.mli b/src/irmin-pack/unix/chunked_suffix.mli new file mode 100644 index 0000000000..36e443f191 --- /dev/null +++ b/src/irmin-pack/unix/chunked_suffix.mli @@ -0,0 +1,18 @@ +(* + * Copyright (c) 2022-2022 Tarides + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + *) + +include Chunked_suffix_intf.Sigs +(** @inline *) diff --git a/src/irmin-pack/unix/chunked_suffix_intf.ml b/src/irmin-pack/unix/chunked_suffix_intf.ml new file mode 100644 index 0000000000..f60f86ce04 --- /dev/null +++ b/src/irmin-pack/unix/chunked_suffix_intf.ml @@ -0,0 +1,94 @@ +(* + * Copyright (c) 2022-2022 Tarides + * + * Permission to use, copy, modify, and distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + *) + +open Import + +module type S = sig + (** Abstraction for a chunked suffix. It is functionally equivalent to + {!Append_only_file} but with a chunked implementation that is + parameterized by + + - [start_idx] for {!create_rw} to know the starting file name, and + - [start_idx] and [chunk_num] for the open functions to know the starting + file name and how many files there are. *) + + module Io : Io.S + module Errs : Io_errors.S + module Ao : Append_only_file.S + + type t + type create_error = Io.create_error + + type open_error = + [ Io.open_error + | `Closed + | `Invalid_argument + | `Inconsistent_store + | `Read_out_of_bounds ] + + val create_rw : + root:string -> + start_idx:int -> + overwrite:bool -> + auto_flush_threshold:int -> + auto_flush_procedure:Ao.auto_flush_procedure -> + (t, [> create_error ]) result + + val open_rw : + root:string -> + end_poff:int63 -> + start_idx:int -> + chunk_num:int -> + dead_header_size:int -> + auto_flush_threshold:int -> + auto_flush_procedure:Ao.auto_flush_procedure -> + (t, [> open_error ]) result + + val open_ro : + root:string -> + end_poff:int63 -> + dead_header_size:int -> + start_idx:int -> + chunk_num:int -> + (t, [> open_error ]) result + + val close : t -> (unit, [> Io.close_error | `Pending_flush ]) result + val empty_buffer : t -> bool + val flush : t -> (unit, [> Io.write_error ]) result + val fsync : t -> (unit, [> Io.write_error ]) result + + (* TODO: rename [end_poff] to something that represents what purpose it serves as + a check for what data we know has been written in the appendable chunk. Also + rename the corresponding control file field. + + Possible new names: [consistency_poff], [persisted_poff]. + *) + val end_poff : t -> int63 + val read_exn : t -> off:int63 -> len:int -> bytes -> unit + val append_exn : t -> string -> unit + + (* TODO: rename [refresh_end_poff] to cohere with rename of [end_poff]. *) + val refresh_end_poff : t -> int63 -> (unit, [> `Rw_not_allowed ]) result + val readonly : t -> bool + val auto_flush_threshold : t -> int option +end + +module type Sigs = sig + module type S = S + + module Make (Io : Io.S) (Errs : Io_errors.S with module Io = Io) : + S with module Io = Io and module Errs = Errs +end diff --git a/src/irmin-pack/unix/control_file_intf.ml b/src/irmin-pack/unix/control_file_intf.ml index 6fd1354b7c..ea425d10ac 100644 --- a/src/irmin-pack/unix/control_file_intf.ml +++ b/src/irmin-pack/unix/control_file_intf.ml @@ -151,6 +151,10 @@ module Payload_v4 = struct type t = { dict_end_poff : int63; + (* TODO: rename [suffix_end_poff] to something that clearly communicates its role. + + See corresponding todo in {!Chunked_suffix}. + *) suffix_end_poff : int63; upgraded_from_v3_to_v4 : bool; checksum : int63; diff --git a/src/irmin-pack/unix/file_manager.ml b/src/irmin-pack/unix/file_manager.ml index 890fc5bb9b..3d9ea24d7b 100644 --- a/src/irmin-pack/unix/file_manager.ml +++ b/src/irmin-pack/unix/file_manager.ml @@ -684,7 +684,11 @@ struct let* () = reopen_prefix t ~generation in let* () = reopen_mapping t ~generation in (* Opening the suffix requires passing it its length. We compute it from the - global offsets *) + global offsets + + TODO: this calculation of [suffix_end_poff] only works with one chunk. + this code will be removed once we have chunk GC. + *) let suffix_end_poff = let open Int63.Syntax in new_suffix_end_offset - new_suffix_start_offset diff --git a/src/irmin-pack/unix/import.ml b/src/irmin-pack/unix/import.ml index cfa28189ad..f272f81c4e 100644 --- a/src/irmin-pack/unix/import.ml +++ b/src/irmin-pack/unix/import.ml @@ -20,6 +20,20 @@ let src = Logs.Src.create "irmin-pack.unix" ~doc:"irmin-pack unix backend" module Log = (val Logs.src_log src : Logs.LOG) +module Array = struct + include Array + + let find_opt p a = + let n = length a in + let rec loop i = + if i = n then None + else + let x = get a i in + if p x then Some x else loop (succ i) + in + loop 0 +end + module Int63 = struct include Optint.Int63 From 6ace0bd5872933836bfef8c2d6d619ef39741fe4 Mon Sep 17 00:00:00 2001 From: metanivek Date: Tue, 11 Oct 2022 15:19:38 -0400 Subject: [PATCH 2/3] irmin-pack: replace suffix with chunked suffix --- src/irmin-pack/unix/ext.ml | 6 ++++- src/irmin-pack/unix/file_manager.ml | 33 +++++++++++------------ src/irmin-pack/unix/file_manager_intf.ml | 4 +-- src/irmin-pack/unix/irmin_pack_unix.ml | 1 + src/irmin-tezos-utils/files.ml | 4 +-- test/irmin-pack/common.ml | 3 ++- test/irmin-pack/test_inode.ml | 3 ++- test/irmin-pack/test_pack_version_bump.ml | 3 ++- 8 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/irmin-pack/unix/ext.ml b/src/irmin-pack/unix/ext.ml index d827c0ba08..088f7c95eb 100644 --- a/src/irmin-pack/unix/ext.ml +++ b/src/irmin-pack/unix/ext.ml @@ -35,7 +35,11 @@ module Maker (Config : Conf.S) = struct module Errs = Io_errors.Make (Io) module Control = Control_file.Make (Io) module Aof = Append_only_file.Make (Io) (Errs) - module File_manager = File_manager.Make (Control) (Aof) (Aof) (Index) (Errs) + module Suffix = Chunked_suffix.Make (Io) (Errs) + + module File_manager = + File_manager.Make (Control) (Aof) (Suffix) (Index) (Errs) + module Dict = Dict.Make (File_manager) module Dispatcher = Dispatcher.Make (File_manager) module XKey = Pack_key.Make (H) diff --git a/src/irmin-pack/unix/file_manager.ml b/src/irmin-pack/unix/file_manager.ml index 3d9ea24d7b..02c7f18ad1 100644 --- a/src/irmin-pack/unix/file_manager.ml +++ b/src/irmin-pack/unix/file_manager.ml @@ -24,7 +24,7 @@ let legacy_io_header_size = 16 module Make (Control : Control_file.S with module Io = Io.Unix) (Dict : Append_only_file.S with module Io = Control.Io) - (Suffix : Append_only_file.S with module Io = Control.Io) + (Suffix : Chunked_suffix.S with module Io = Control.Io) (Index : Pack_index.S) (Errs : Io_errors.S with module Io = Control.Io) = struct @@ -240,11 +240,12 @@ struct "reopen_suffix gen:%d end_poff:%d" generation (Int63.to_int end_poff)]; let readonly = Suffix.readonly t.suffix in let* suffix1 = - let path = - Irmin_pack.Layout.V4.suffix_chunk ~root:t.root ~chunk_idx:generation - in - [%log.debug "reload: generation changed, opening %s" path]; - if readonly then Suffix.open_ro ~path ~end_poff ~dead_header_size + let root = t.root in + let start_idx = generation in + let chunk_num = 1 in + [%log.debug "reload: generation changed, opening suffix"]; + if readonly then + Suffix.open_ro ~root ~end_poff ~dead_header_size ~start_idx ~chunk_num else let auto_flush_threshold = match Suffix.auto_flush_threshold t.suffix with @@ -252,8 +253,8 @@ struct | Some x -> x in let cb _ = suffix_requires_a_flush_exn t in - Suffix.open_rw ~path ~end_poff ~dead_header_size ~auto_flush_threshold - ~auto_flush_procedure:(`External cb) + Suffix.open_rw ~root ~end_poff ~dead_header_size ~start_idx ~chunk_num + ~auto_flush_threshold ~auto_flush_procedure:(`External cb) in let suffix0 = t.suffix in t.suffix <- suffix1; @@ -325,14 +326,11 @@ struct in (* 2. Open the other files *) let* suffix = - let path = - Irmin_pack.Layout.V4.suffix_chunk ~root ~chunk_idx:generation - in let auto_flush_threshold = Irmin_pack.Conf.suffix_auto_flush_threshold config in let cb _ = suffix_requires_a_flush_exn (get_instance ()) in - make_suffix ~path ~auto_flush_threshold + make_suffix ~start_idx:generation ~auto_flush_threshold ~auto_flush_procedure:(`External cb) in let* prefix = @@ -458,7 +456,7 @@ struct create_control_file ~overwrite config pl in let make_dict = Dict.create_rw ~overwrite in - let make_suffix = Suffix.create_rw ~overwrite in + let make_suffix = Suffix.create_rw ~root ~overwrite in let make_index ~flush_callback ~readonly ~throttle ~log_size root = (* [overwrite] is ignored for index *) Index.v ~fresh:true ~flush_callback ~readonly ~throttle ~log_size root @@ -489,7 +487,8 @@ struct in let make_dict = Dict.open_rw ~end_poff:pl.dict_end_poff ~dead_header_size in let make_suffix = - Suffix.open_rw ~end_poff:pl.suffix_end_poff ~dead_header_size + Suffix.open_rw ~root ~end_poff:pl.suffix_end_poff ~chunk_num:1 + ~dead_header_size in let make_index ~flush_callback ~readonly ~throttle ~log_size root = Index.v ~fresh:false ~flush_callback ~readonly ~throttle ~log_size root @@ -606,10 +605,8 @@ struct let generation = generation pl.status in (* 2. Open the other files *) let* suffix = - let path = - Irmin_pack.Layout.V4.suffix_chunk ~root ~chunk_idx:generation - in - Suffix.open_ro ~path ~end_poff:pl.suffix_end_poff ~dead_header_size + Suffix.open_ro ~root ~end_poff:pl.suffix_end_poff ~start_idx:generation + ~chunk_num:1 ~dead_header_size in let* prefix = let path = Irmin_pack.Layout.V4.prefix ~root ~generation in diff --git a/src/irmin-pack/unix/file_manager_intf.ml b/src/irmin-pack/unix/file_manager_intf.ml index 0aa9a6776c..f6f54fbff8 100644 --- a/src/irmin-pack/unix/file_manager_intf.ml +++ b/src/irmin-pack/unix/file_manager_intf.ml @@ -61,7 +61,7 @@ module type S = sig module Io : Io.S module Control : Control_file.S with module Io = Io module Dict : Append_only_file.S with module Io = Io - module Suffix : Append_only_file.S with module Io = Io + module Suffix : Chunked_suffix.S with module Io = Io module Index : Pack_index.S module Errs : Io_errors.S with module Io = Io module Mapping_file : Mapping_file.S with module Io = Io @@ -260,7 +260,7 @@ module type Sigs = sig module Make (Control : Control_file.S with module Io = Io.Unix) (Dict : Append_only_file.S with module Io = Control.Io) - (Suffix : Append_only_file.S with module Io = Control.Io) + (Suffix : Chunked_suffix.S with module Io = Control.Io) (Index : Pack_index.S) (Errs : Io_errors.S with module Io = Control.Io) : S diff --git a/src/irmin-pack/unix/irmin_pack_unix.ml b/src/irmin-pack/unix/irmin_pack_unix.ml index a94b46f850..71eef37afe 100644 --- a/src/irmin-pack/unix/irmin_pack_unix.ml +++ b/src/irmin-pack/unix/irmin_pack_unix.ml @@ -31,6 +31,7 @@ module Errors = Errors module Io_errors = Io_errors module Control_file = Control_file module Append_only_file = Append_only_file +module Chunked_suffix = Chunked_suffix module File_manager = File_manager module Maker = Ext.Maker module Mapping_file = Mapping_file diff --git a/src/irmin-tezos-utils/files.ml b/src/irmin-tezos-utils/files.ml index 6a36619ee1..89e0f2df5d 100644 --- a/src/irmin-tezos-utils/files.ml +++ b/src/irmin-tezos-utils/files.ml @@ -18,11 +18,11 @@ module Make (Conf : Irmin_pack.Conf.S) (Schema : Irmin.Schema.Extended) = struct module Errs = Irmin_pack_unix.Io_errors.Make (Io) module Control_file = Irmin_pack_unix.Control_file.Make (Io) module Append_only_file = Irmin_pack_unix.Append_only_file.Make (Io) (Errs) + module Suffix = Irmin_pack_unix.Chunked_suffix.Make (Io) (Errs) module Pack_index = Irmin_pack_unix.Index.Make (Hash) module File_manager = - Irmin_pack_unix.File_manager.Make (Control_file) (Append_only_file) - (Append_only_file) + Irmin_pack_unix.File_manager.Make (Control_file) (Append_only_file) (Suffix) (Pack_index) (Errs) diff --git a/test/irmin-pack/common.ml b/test/irmin-pack/common.ml index 34518368a1..fe060edc9d 100644 --- a/test/irmin-pack/common.ml +++ b/test/irmin-pack/common.ml @@ -81,9 +81,10 @@ module Io = Irmin_pack_unix.Io.Unix module Errs = Irmin_pack_unix.Io_errors.Make (Io) module Control = Irmin_pack_unix.Control_file.Make (Io) module Aof = Irmin_pack_unix.Append_only_file.Make (Io) (Errs) +module Suffix = Irmin_pack_unix.Chunked_suffix.Make (Io) (Errs) module File_manager = - Irmin_pack_unix.File_manager.Make (Control) (Aof) (Aof) (Index) (Errs) + Irmin_pack_unix.File_manager.Make (Control) (Aof) (Suffix) (Index) (Errs) module Dict = Irmin_pack_unix.Dict.Make (File_manager) module Dispatcher = Irmin_pack_unix.Dispatcher.Make (File_manager) diff --git a/test/irmin-pack/test_inode.ml b/test/irmin-pack/test_inode.ml index 271c0a0c67..1037231b61 100644 --- a/test/irmin-pack/test_inode.ml +++ b/test/irmin-pack/test_inode.ml @@ -47,9 +47,10 @@ struct module Errs = Irmin_pack_unix.Io_errors.Make (Io) module Control = Irmin_pack_unix.Control_file.Make (Io) module Aof = Irmin_pack_unix.Append_only_file.Make (Io) (Errs) + module Suffix = Irmin_pack_unix.Chunked_suffix.Make (Io) (Errs) module File_manager = - Irmin_pack_unix.File_manager.Make (Control) (Aof) (Aof) (Index) (Errs) + Irmin_pack_unix.File_manager.Make (Control) (Aof) (Suffix) (Index) (Errs) module Dict = Irmin_pack_unix.Dict.Make (File_manager) module Dispatcher = Irmin_pack_unix.Dispatcher.Make (File_manager) diff --git a/test/irmin-pack/test_pack_version_bump.ml b/test/irmin-pack/test_pack_version_bump.ml index 1750452988..96b4c202e6 100644 --- a/test/irmin-pack/test_pack_version_bump.ml +++ b/test/irmin-pack/test_pack_version_bump.ml @@ -31,9 +31,10 @@ module Private = struct module Errs = Irmin_pack_unix.Io_errors.Make (Io) module Control = Irmin_pack_unix.Control_file.Make (Io) module Aof = Irmin_pack_unix.Append_only_file.Make (Io) (Errs) + module Suffix = Irmin_pack_unix.Chunked_suffix.Make (Io) (Errs) module File_manager = - Irmin_pack_unix.File_manager.Make (Control) (Aof) (Aof) (Index) (Errs) + Irmin_pack_unix.File_manager.Make (Control) (Aof) (Suffix) (Index) (Errs) end module Util = struct From 47f95afa6772d83f0fe1b3ff6ed8e6ce005f6c88 Mon Sep 17 00:00:00 2001 From: metanivek Date: Thu, 20 Oct 2022 09:42:16 -0400 Subject: [PATCH 3/3] Add CHANGES entry for chunked suffix --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 747e2e2ca9..0cb799c789 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,8 @@ - **irmin-pack** - Upgraded on-disk format to version 4. (#2110, @icristescu) - Detecting control file corruption with a checksum (#2119, @art-w) + - Change on-disk layout of the suffix from a single file to a multiple, + chunked file design (#2115, @metanivek) ### Fixed