diff --git a/Makefile-libostree.am b/Makefile-libostree.am index bdc47122f4..eeb0b6c60f 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -184,9 +184,9 @@ libostree_1_la_SOURCES += \ endif # USE_GPGME symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym -#if BUILDOPT_IS_DEVEL_BUILD -#symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym -#endif +if BUILDOPT_IS_DEVEL_BUILD +symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym +endif # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html wl_versionscript_arg = -Wl,--version-script= EXTRA_DIST += \ diff --git a/apidoc/ostree-sections.txt b/apidoc/ostree-sections.txt index c1d6b35ef9..64bc68d234 100644 --- a/apidoc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -412,6 +412,8 @@ OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE ostree_repo_list_objects ostree_repo_list_commit_objects_starting_with ostree_repo_list_static_delta_names +ostree_repo_list_static_delta_indexes +ostree_repo_static_delta_reindex OstreeStaticDeltaGenerateOpt ostree_repo_static_delta_generate ostree_repo_static_delta_execute_offline_with_signature @@ -445,6 +447,7 @@ ostree_repo_pull_default_console_progress_changed ostree_repo_sign_commit ostree_repo_append_gpg_signature ostree_repo_add_gpg_signature_summary +ostree_repo_gpg_sign_data ostree_repo_gpg_verify_data ostree_repo_verify_commit ostree_repo_verify_commit_ext diff --git a/man/ostree.repo-config.xml b/man/ostree.repo-config.xml index 7a01fc0172..e4984430b9 100644 --- a/man/ostree.repo-config.xml +++ b/man/ostree.repo-config.xml @@ -249,6 +249,20 @@ Boston, MA 02111-1307, USA. costly). + + + no-deltas-in-summary + Boolean value controlling whether OSTree should skip + putting an index of available deltas in the summary file. Defaults to false. + + + Since 2020.7 OSTree can use delta indexes outside the summary file, + making the summary file smaller (especially for larger repositories). However + by default we still create the index in the summary file to make older clients + work. If you know all clients will be 2020.7 later you can enable this to + save network bandwidth. + + diff --git a/src/libostree/libostree-devel.sym b/src/libostree/libostree-devel.sym index 8e8d9c8191..435be1908e 100644 --- a/src/libostree/libostree-devel.sym +++ b/src/libostree/libostree-devel.sym @@ -17,9 +17,12 @@ Boston, MA 02111-1307, USA. ***/ -/* Copy the bits below and uncomment the include in Makefile-libostree.am - when adding a symbol. -*/ +LIBOSTREE_2020.8 { +global: + ostree_repo_list_static_delta_indexes; + ostree_repo_static_delta_reindex; + ostree_repo_gpg_sign_data; +} LIBOSTREE_2020.7; /* Stub section for the stable release *after* this development one; don't * edit this other than to update the year. This is just a copy/paste diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 5d9d948ad7..89d95ad9c6 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -135,6 +135,9 @@ _ostree_get_relative_static_delta_part_path (const char *from, const char *to, guint i); +char * +_ostree_get_relative_static_delta_index_path (const char *to); + static inline char * _ostree_get_commitpartial_path (const char *checksum) { return g_strconcat ("state/", checksum, ".commitpartial", NULL); diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 29528fa51f..f822101a36 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -1814,15 +1814,15 @@ _ostree_get_relative_object_path (const char *checksum, return g_string_free (path, FALSE); } -char * -_ostree_get_relative_static_delta_path (const char *from, - const char *to, - const char *target) +static GString * +static_delta_path_base (const char *dir, + const char *from, + const char *to) { guint8 csum_to[OSTREE_SHA256_DIGEST_LEN]; char to_b64[44]; guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN]; - GString *ret = g_string_new ("deltas/"); + GString *ret = g_string_new (dir); ostree_checksum_inplace_to_bytes (to, csum_to); ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64); @@ -1851,6 +1851,16 @@ _ostree_get_relative_static_delta_path (const char *from, g_string_append_c (ret, '/'); g_string_append (ret, to_b64 + 2); + return ret; +} + +char * +_ostree_get_relative_static_delta_path (const char *from, + const char *to, + const char *target) +{ + GString *ret = static_delta_path_base ("deltas/", from, to); + if (target != NULL) { g_string_append_c (ret, '/'); @@ -1883,6 +1893,16 @@ _ostree_get_relative_static_delta_part_path (const char *from, return _ostree_get_relative_static_delta_path (from, to, partstr); } +char * +_ostree_get_relative_static_delta_index_path (const char *to) +{ + GString *ret = static_delta_path_base ("delta-indexes/", NULL, to); + + g_string_append (ret, ".index"); + + return g_string_free (ret, FALSE); +} + gboolean _ostree_parse_delta_name (const char *delta_name, char **out_from, diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index a48feca407..cbbe69717d 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -57,6 +57,7 @@ G_BEGIN_DECLS #define OSTREE_SUMMARY_COLLECTION_MAP "ostree.summary.collection-map" #define OSTREE_SUMMARY_MODE "ostree.summary.mode" #define OSTREE_SUMMARY_TOMBSTONE_COMMITS "ostree.summary.tombstone-commits" +#define OSTREE_SUMMARY_INDEXED_DELTAS "ostree.summary.indexed-deltas" #define _OSTREE_PAYLOAD_LINK_PREFIX "../" #define _OSTREE_PAYLOAD_LINK_PREFIX_LEN (sizeof (_OSTREE_PAYLOAD_LINK_PREFIX) - 1) diff --git a/src/libostree/ostree-repo-pull-private.h b/src/libostree/ostree-repo-pull-private.h index a7ea85cd7e..a827557aed 100644 --- a/src/libostree/ostree-repo-pull-private.h +++ b/src/libostree/ostree-repo-pull-private.h @@ -78,7 +78,9 @@ typedef struct { char *summary_sig_etag; guint64 summary_sig_last_modified; /* seconds since the epoch */ GVariant *summary; - GHashTable *summary_deltas_checksums; + GHashTable *summary_deltas_checksums; /* Filled from summary and delta indexes */ + gboolean summary_has_deltas; /* True if the summary existed and had a delta index */ + gboolean has_indexed_deltas; GHashTable *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */ GHashTable *verified_commits; /* Set of commits that have been verified */ GHashTable *signapi_verified_commits; /* Map of commits that have been signapi verified */ @@ -93,6 +95,7 @@ typedef struct { GHashTable *requested_fallback_content; /* Maps checksum to itself */ GHashTable *pending_fetch_metadata; /* Map */ GHashTable *pending_fetch_content; /* Map */ + GHashTable *pending_fetch_delta_indexes; /* Set */ GHashTable *pending_fetch_delta_superblocks; /* Set */ GHashTable *pending_fetch_deltaparts; /* Set */ guint n_outstanding_metadata_fetches; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index bb14313603..da421db383 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -104,6 +104,14 @@ typedef struct { guint n_retries_remaining; } FetchDeltaSuperData; +typedef struct { + OtPullData *pull_data; + char *from_revision; + char *to_revision; + OstreeCollectionRef *requested_ref; /* (nullable) */ + guint n_retries_remaining; +} FetchDeltaIndexData; + static void variant_or_null_unref (gpointer data) { @@ -116,6 +124,8 @@ static void start_fetch_deltapart (OtPullData *pull_data, FetchStaticDeltaData *fetch); static void start_fetch_delta_superblock (OtPullData *pull_data, FetchDeltaSuperData *fetch_data); +static void start_fetch_delta_index (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data); static gboolean fetcher_queue_is_full (OtPullData *pull_data); static void queue_scan_one_metadata_object (OtPullData *pull_data, const char *csum, @@ -135,6 +145,8 @@ static void queue_scan_one_metadata_object_c (OtPullData *pull_da static void enqueue_one_object_request_s (OtPullData *pull_data, FetchObjectData *fetch_data); +static void enqueue_one_static_delta_index_request_s (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data); static void enqueue_one_static_delta_superblock_request_s (OtPullData *pull_data, FetchDeltaSuperData *fetch_data); static void enqueue_one_static_delta_part_request_s (OtPullData *pull_data, @@ -149,6 +161,11 @@ static gboolean scan_one_metadata_object (OtPullData *pull_data, GCancellable *cancellable, GError **error); static void scan_object_queue_data_free (ScanObjectQueueData *scan_data); +static gboolean initiate_delta_request (OtPullData *pull_data, + const OstreeCollectionRef *ref, + const char *to_revision, + const char *delta_from_revision, + GError **error); static gboolean update_progress (gpointer user_data) @@ -287,6 +304,7 @@ check_outstanding_requests_handle_error (OtPullData *pull_data, g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL); g_queue_clear (&pull_data->scan_object_queue); g_hash_table_remove_all (pull_data->pending_fetch_metadata); + g_hash_table_remove_all (pull_data->pending_fetch_delta_indexes); g_hash_table_remove_all (pull_data->pending_fetch_delta_superblocks); g_hash_table_remove_all (pull_data->pending_fetch_deltaparts); g_hash_table_remove_all (pull_data->pending_fetch_content); @@ -320,6 +338,16 @@ check_outstanding_requests_handle_error (OtPullData *pull_data, g_variant_unref (objname); } + /* Next, process delta index requests */ + g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_indexes); + while (!fetcher_queue_is_full (pull_data) && + g_hash_table_iter_next (&hiter, &key, &value)) + { + FetchDeltaIndexData *fetch = key; + g_hash_table_iter_steal (&hiter); + start_fetch_delta_index (pull_data, g_steal_pointer (&fetch)); + } + /* Next, process delta superblock requests */ g_hash_table_iter_init (&hiter, pull_data->pending_fetch_delta_superblocks); while (!fetcher_queue_is_full (pull_data) && @@ -2117,6 +2145,7 @@ start_fetch_deltapart (OtPullData *pull_data, FetchStaticDeltaData *fetch) { g_autofree char *deltapart_path = _ostree_get_relative_static_delta_part_path (fetch->from_revision, fetch->to_revision, fetch->i); + g_debug ("starting fetch of deltapart %s", deltapart_path); pull_data->n_outstanding_deltapart_fetches++; g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS); _ostree_fetcher_request_to_tmpfile (pull_data->fetcher, @@ -2469,6 +2498,16 @@ fetch_delta_super_data_free (FetchDeltaSuperData *fetch_data) g_free (fetch_data); } +static void +fetch_delta_index_data_free (FetchDeltaIndexData *fetch_data) +{ + g_free (fetch_data->from_revision); + g_free (fetch_data->to_revision); + if (fetch_data->requested_ref) + ostree_collection_ref_free (fetch_data->requested_ref); + g_free (fetch_data); +} + static void set_required_deltas_error (GError **error, const char *from_revision, @@ -2570,6 +2609,7 @@ start_fetch_delta_superblock (OtPullData *pull_data, g_autofree char *delta_name = _ostree_get_relative_static_delta_superblock_path (fetch_data->from_revision, fetch_data->to_revision); + g_debug ("starting fetch of delta superblock %s", delta_name); _ostree_fetcher_request_to_membuf (pull_data->fetcher, pull_data->content_mirrorlist, delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, @@ -2629,6 +2669,147 @@ validate_variant_is_csum (GVariant *csum, return ostree_validate_structureof_csum_v (csum, error); } +static gboolean +collect_available_deltas_for_pull (OtPullData *pull_data, + GVariant *deltas, + GError **error) +{ + gsize n; + + n = deltas ? g_variant_n_children (deltas) : 0; + for (gsize i = 0; i < n; i++) + { + const char *delta; + g_autoptr(GVariant) csum_v = NULL; + g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i); + + g_variant_get_child (ref, 0, "&s", &delta); + g_variant_get_child (ref, 1, "v", &csum_v); + + if (!validate_variant_is_csum (csum_v, error)) + return FALSE; + + guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN); + memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32); + g_hash_table_insert (pull_data->summary_deltas_checksums, + g_strdup (delta), + csum_data); + } + + return TRUE; +} + +static void +on_delta_index_fetched (GObject *src, + GAsyncResult *res, + gpointer data) + +{ + FetchDeltaIndexData *fetch_data = data; + OtPullData *pull_data = fetch_data->pull_data; + g_autoptr(GError) local_error = NULL; + GError **error = &local_error; + g_autoptr(GBytes) delta_index_data = NULL; + const char *from_revision = fetch_data->from_revision; + const char *to_revision = fetch_data->to_revision; + + if (!_ostree_fetcher_request_to_membuf_finish ((OstreeFetcher*)src, + res, + &delta_index_data, + NULL, NULL, NULL, + error)) + { + if (!g_error_matches (local_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND)) + goto out; + g_clear_error (&local_error); + + /* below call to initiate_delta_request() will fail finding the delta and fall back to commit */ + } + else + { + g_autoptr(GVariant) delta_index = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE_VARDICT, delta_index_data, FALSE)); + g_autoptr(GVariant) deltas = g_variant_lookup_value (delta_index, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}")); + + if (!collect_available_deltas_for_pull (pull_data, deltas, error)) + goto out; + } + + if (!initiate_delta_request (pull_data, + fetch_data->requested_ref, + to_revision, + from_revision, + &local_error)) + goto out; + + out: + g_assert (pull_data->n_outstanding_metadata_fetches > 0); + pull_data->n_outstanding_metadata_fetches--; + + if (local_error == NULL) + pull_data->n_fetched_metadata++; + + if (_ostree_fetcher_should_retry_request (local_error, fetch_data->n_retries_remaining--)) + enqueue_one_static_delta_index_request_s (pull_data, g_steal_pointer (&fetch_data)); + else + check_outstanding_requests_handle_error (pull_data, &local_error); + + g_clear_pointer (&fetch_data, fetch_delta_index_data_free); +} + +static void +start_fetch_delta_index (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data) +{ + g_autofree char *delta_name = + _ostree_get_relative_static_delta_index_path (fetch_data->to_revision); + g_debug ("starting fetch of delta index %s", delta_name); + _ostree_fetcher_request_to_membuf (pull_data->fetcher, + pull_data->content_mirrorlist, + delta_name, OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, + OSTREE_MAX_METADATA_SIZE, + 0, pull_data->cancellable, + on_delta_index_fetched, + g_steal_pointer (&fetch_data)); + pull_data->n_outstanding_metadata_fetches++; + pull_data->n_requested_metadata++; +} + +static void +enqueue_one_static_delta_index_request_s (OtPullData *pull_data, + FetchDeltaIndexData *fetch_data) +{ + if (fetcher_queue_is_full (pull_data)) + { + g_debug ("queuing fetch of static delta index to %s", + fetch_data->to_revision); + + g_hash_table_add (pull_data->pending_fetch_delta_indexes, + g_steal_pointer (&fetch_data)); + } + else + { + start_fetch_delta_index (pull_data, g_steal_pointer (&fetch_data)); + } +} + +/* Start a request for a static delta index */ +static void +enqueue_one_static_delta_index_request (OtPullData *pull_data, + const char *to_revision, + const char *from_revision, + const OstreeCollectionRef *ref) +{ + FetchDeltaIndexData *fdata = g_new0(FetchDeltaIndexData, 1); + fdata->pull_data = pull_data; + fdata->from_revision = g_strdup (from_revision); + fdata->to_revision = g_strdup (to_revision); + fdata->requested_ref = (ref != NULL) ? ostree_collection_ref_dup (ref) : NULL; + fdata->n_retries_remaining = pull_data->n_network_retries; + + enqueue_one_static_delta_index_request_s (pull_data, g_steal_pointer (&fdata)); +} + static gboolean _ostree_repo_verify_summary (OstreeRepo *self, const char *name, @@ -3259,6 +3440,65 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, return TRUE; } +static gboolean +initiate_delta_request (OtPullData *pull_data, + const OstreeCollectionRef *ref, + const char *to_revision, + const char *delta_from_revision, + GError **error) +{ + DeltaSearchResult deltares; + + /* Look for a delta to @to_revision in the summary data */ + if (!get_best_static_delta_start_for (pull_data, to_revision, &deltares, + pull_data->cancellable, error)) + return FALSE; + + switch (deltares.result) + { + case DELTA_SEARCH_RESULT_NO_MATCH: + { + if (pull_data->require_static_deltas) /* No deltas found; are they required? */ + { + set_required_deltas_error (error, (ref != NULL) ? ref->ref_name : "", to_revision); + return FALSE; + } + else /* No deltas, fall back to object fetches. */ + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); + } + break; + case DELTA_SEARCH_RESULT_FROM: + enqueue_one_static_delta_superblock_request (pull_data, deltares.from_revision, to_revision, ref); + break; + case DELTA_SEARCH_RESULT_SCRATCH: + { + /* If a from-scratch delta is available, we don’t want to use it if + * the ref already exists locally, since we are likely only a few + * commits out of date; so doing an object pull is likely more + * bandwidth efficient. */ + if (delta_from_revision != NULL) + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); + else + enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, ref); + } + break; + case DELTA_SEARCH_RESULT_UNCHANGED: + { + /* If we already have the commit, here things get a little special; we've historically + * fetched detached metadata, so let's keep doing that. But in the --require-static-deltas + * path, we don't, under the assumption the user wants as little network traffic as + * possible. + */ + if (pull_data->require_static_deltas) + break; + else + queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); + } + } + + return TRUE; +} + /* * initiate_request: * @ref: Optional ref name and collection ID @@ -3298,57 +3538,17 @@ initiate_request (OtPullData *pull_data, return FALSE; } - /* If we have a summary, we can use the newer logic */ - if (pull_data->summary) + /* If we have a summary or delta index, we can use the newer logic. + * We prefer the index as it might have more deltas than the summary + * (i.e. leave some deltas out of summary to make it smaller). */ + if (pull_data->has_indexed_deltas) { - DeltaSearchResult deltares; - - /* Look for a delta to @to_revision in the summary data */ - if (!get_best_static_delta_start_for (pull_data, to_revision, &deltares, - pull_data->cancellable, error)) + enqueue_one_static_delta_index_request (pull_data, to_revision, delta_from_revision, ref); + } + else if (pull_data->summary_has_deltas) + { + if (!initiate_delta_request (pull_data, ref, to_revision, delta_from_revision, error)) return FALSE; - - switch (deltares.result) - { - case DELTA_SEARCH_RESULT_NO_MATCH: - { - if (pull_data->require_static_deltas) /* No deltas found; are they required? */ - { - set_required_deltas_error (error, (ref != NULL) ? ref->ref_name : "", to_revision); - return FALSE; - } - else /* No deltas, fall back to object fetches. */ - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); - } - break; - case DELTA_SEARCH_RESULT_FROM: - enqueue_one_static_delta_superblock_request (pull_data, deltares.from_revision, to_revision, ref); - break; - case DELTA_SEARCH_RESULT_SCRATCH: - { - /* If a from-scratch delta is available, we don’t want to use it if - * the ref already exists locally, since we are likely only a few - * commits out of date; so doing an object pull is likely more - * bandwidth efficient. */ - if (delta_from_revision != NULL) - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); - else - enqueue_one_static_delta_superblock_request (pull_data, NULL, to_revision, ref); - } - break; - case DELTA_SEARCH_RESULT_UNCHANGED: - { - /* If we already have the commit, here things get a little special; we've historically - * fetched detached metadata, so let's keep doing that. But in the --require-static-deltas - * path, we don't, under the assumption the user wants as little network traffic as - * possible. - */ - if (pull_data->require_static_deltas) - break; - else - queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, NULL, 0, ref); - } - } } else if (ref != NULL) { @@ -3392,6 +3592,20 @@ initiate_request (OtPullData *pull_data, return TRUE; } +static gboolean +all_requested_refs_have_commit (GHashTable *requested_refs /* (element-type OstreeCollectionRef utf8) */) +{ + GLNX_HASH_TABLE_FOREACH_KV (requested_refs, const OstreeCollectionRef*, ref, + const char*, override_commitid) + { + /* Note: "" override means whatever is latest */ + if (override_commitid == NULL || *override_commitid == 0) + return FALSE; + } + + return TRUE; +} + /* ------------------------------------------------------------------------------------------ * Below is the libsoup-invariant API; these should match * the stub functions in the #else clause @@ -3497,9 +3711,11 @@ ostree_repo_pull_with_options (OstreeRepo *self, gboolean opt_ref_keyring_map_set = FALSE; gboolean disable_sign_verify = FALSE; gboolean disable_sign_verify_summary = FALSE; + gboolean need_summary = FALSE; const char *main_collection_id = NULL; const char *url_override = NULL; gboolean inherit_transaction = FALSE; + gboolean require_summary_for_mirror = FALSE; g_autoptr(GHashTable) updated_requested_refs_to_fetch = NULL; /* (element-type OstreeCollectionRef utf8) */ gsize i; g_autofree char **opt_localcache_repos = NULL; @@ -3511,6 +3727,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, */ const char *the_ref_to_fetch = NULL; OstreeRepoTransactionStats tstats = { 0, }; + gboolean remote_mode_loaded = FALSE; /* Default */ pull_data->max_metadata_size = OSTREE_MAX_METADATA_SIZE; @@ -3657,6 +3874,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, pull_data->pending_fetch_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, (GDestroyNotify)g_variant_unref, (GDestroyNotify)fetch_object_data_free); + pull_data->pending_fetch_delta_indexes = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_index_data_free, NULL); pull_data->pending_fetch_delta_superblocks = g_hash_table_new_full (NULL, NULL, (GDestroyNotify) fetch_delta_super_data_free, NULL); pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL); @@ -3933,424 +4151,509 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->is_commit_only) pull_data->disable_static_deltas = TRUE; - pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - - { - g_autoptr(GBytes) bytes_sig = NULL; - gboolean summary_sig_not_modified = FALSE; - g_autofree char *summary_sig_etag = NULL; - guint64 summary_sig_last_modified = 0; - gsize n; - g_autoptr(GVariant) refs = NULL; - g_autoptr(GVariant) deltas = NULL; - g_autoptr(GVariant) additional_metadata = NULL; - gboolean summary_from_cache = FALSE; - gboolean remote_mode_loaded = FALSE; - gboolean tombstone_commits = FALSE; - - if (summary_sig_bytes_v) - { - /* Must both be specified */ - g_assert (summary_bytes_v); + /* Compute the set of collection-refs (and optional commit id) to fetch */ - bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v); - bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v); + if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches) + { + require_summary_for_mirror = TRUE; + } + else if (opt_collection_refs_set) + { + const gchar *collection_id, *ref_name, *checksum; - if (!bytes_sig || !bytes_summary) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "summary-bytes or summary-sig-bytes set to invalid value"); + while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum)) + { + if (!ostree_validate_rev (ref_name, error)) goto out; - } + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (collection_id, ref_name), + (*checksum != '\0') ? g_strdup (checksum) : NULL); + } + } + else if (refs_to_fetch != NULL) + { + char **strviter = refs_to_fetch; + char **commitid_strviter = override_commit_ids ?: NULL; - g_debug ("Loaded %s summary from options", remote_name_or_baseurl); - } + while (*strviter) + { + const char *branch = *strviter; - if (!bytes_sig) - { - g_autofree char *summary_sig_if_none_match = NULL; - guint64 summary_sig_if_modified_since = 0; - - /* Load the summary.sig from the network, but send its ETag and - * Last-Modified from the on-disk cache (if it exists) to reduce the - * download size if nothing’s changed. */ - _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, ".sig", - &summary_sig_if_none_match, &summary_sig_if_modified_since); - - g_clear_pointer (&summary_sig_etag, g_free); - summary_sig_last_modified = 0; - if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, - pull_data->meta_mirrorlist, - "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, - summary_sig_if_none_match, summary_sig_if_modified_since, - pull_data->n_network_retries, - &bytes_sig, - &summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified, - OSTREE_MAX_METADATA_SIZE, - cancellable, error)) + if (ostree_validate_checksum_string (branch, NULL)) + { + char *key = g_strdup (branch); + g_hash_table_add (commits_to_fetch, key); + } + else + { + if (!ostree_validate_rev (branch, error)) + goto out; + char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL; + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (NULL, branch), commitid); + } + + strviter++; + if (commitid_strviter) + commitid_strviter++; + } + } + else + { + char **branches_iter; + + branches_iter = configured_branches; + + if (!(branches_iter && *branches_iter)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No configured branches for remote %s", remote_name_or_baseurl); goto out; + } + for (;branches_iter && *branches_iter; branches_iter++) + { + const char *branch = *branches_iter; - /* The server returned HTTP status 304 Not Modified, so we’re clear to - * load summary.sig from the cache. Also load summary, since - * `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */ - if (summary_sig_not_modified) - { - g_clear_pointer (&bytes_sig, g_bytes_unref); - g_clear_pointer (&bytes_summary, g_bytes_unref); - if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, ".sig", - &bytes_sig, - cancellable, error)) - goto out; + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (NULL, branch), NULL); + } + } - if (!bytes_summary && - !pull_data->remote_repo_local && - !_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL, - &bytes_summary, - cancellable, error)) - goto out; - } - } + /* Deltas are necessary when mirroring or resolving a requested ref to a commit. + * We try to avoid loading the potentially large summary if it is not needed. */ + need_summary = require_summary_for_mirror || !all_requested_refs_have_commit (requested_refs_to_fetch) || summary_sig_bytes_v != NULL; - if (bytes_sig && - !bytes_summary && - !pull_data->remote_repo_local && - !_ostree_repo_load_cache_summary_if_same_sig (self, - remote_name_or_baseurl, - bytes_sig, - &bytes_summary, - cancellable, - error)) - goto out; + /* If we don't have indexed deltas, we need the summary for deltas, so check + * the config file for support. + * NOTE: Avoid download if we don't need deltas */ + if (!need_summary && !pull_data->disable_static_deltas) + { + if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error)) + goto out; - if (bytes_summary && !summary_bytes_v) - { - g_debug ("Loaded %s summary from cache", remote_name_or_baseurl); - summary_from_cache = TRUE; - } + /* Check if remote has delta indexes outside summary */ + if (!ot_keyfile_get_boolean_with_default (remote_config, "core", "indexed-deltas", FALSE, + &pull_data->has_indexed_deltas, error)) + goto out; - if (!pull_data->summary && !bytes_summary) - { - g_autofree char *summary_if_none_match = NULL; - guint64 summary_if_modified_since = 0; + if (!pull_data->has_indexed_deltas) + need_summary = TRUE; + } - _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, NULL, - &summary_if_none_match, &summary_if_modified_since); + pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); - g_clear_pointer (&summary_etag, g_free); - summary_last_modified = 0; - if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, - pull_data->meta_mirrorlist, - "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, - summary_if_none_match, summary_if_modified_since, - pull_data->n_network_retries, + if (need_summary) + { + g_autoptr(GBytes) bytes_sig = NULL; + gboolean summary_sig_not_modified = FALSE; + g_autofree char *summary_sig_etag = NULL; + guint64 summary_sig_last_modified = 0; + gsize n; + g_autoptr(GVariant) refs = NULL; + g_autoptr(GVariant) deltas = NULL; + g_autoptr(GVariant) additional_metadata = NULL; + gboolean summary_from_cache = FALSE; + gboolean tombstone_commits = FALSE; + + if (summary_sig_bytes_v) + { + /* Must both be specified */ + g_assert (summary_bytes_v); + + bytes_sig = g_variant_get_data_as_bytes (summary_sig_bytes_v); + bytes_summary = g_variant_get_data_as_bytes (summary_bytes_v); + + if (!bytes_sig || !bytes_summary) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "summary-bytes or summary-sig-bytes set to invalid value"); + goto out; + } + + g_debug ("Loaded %s summary from options", remote_name_or_baseurl); + } + + if (!bytes_sig) + { + g_autofree char *summary_sig_if_none_match = NULL; + guint64 summary_sig_if_modified_since = 0; + + /* Load the summary.sig from the network, but send its ETag and + * Last-Modified from the on-disk cache (if it exists) to reduce the + * download size if nothing’s changed. */ + _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, ".sig", + &summary_sig_if_none_match, &summary_sig_if_modified_since); + + g_clear_pointer (&summary_sig_etag, g_free); + summary_sig_last_modified = 0; + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary.sig", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + summary_sig_if_none_match, summary_sig_if_modified_since, + pull_data->n_network_retries, + &bytes_sig, + &summary_sig_not_modified, &summary_sig_etag, &summary_sig_last_modified, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) + goto out; + + /* The server returned HTTP status 304 Not Modified, so we’re clear to + * load summary.sig from the cache. Also load summary, since + * `_ostree_repo_load_cache_summary_if_same_sig()` would just do that anyway. */ + if (summary_sig_not_modified) + { + g_clear_pointer (&bytes_sig, g_bytes_unref); + g_clear_pointer (&bytes_summary, g_bytes_unref); + if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, ".sig", + &bytes_sig, + cancellable, error)) + goto out; + + if (!bytes_summary && + !pull_data->remote_repo_local && + !_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL, &bytes_summary, - &summary_not_modified, &summary_etag, &summary_last_modified, - OSTREE_MAX_METADATA_SIZE, cancellable, error)) - goto out; + goto out; + } + } - /* The server returned HTTP status 304 Not Modified, so we’re clear to - * load summary from the cache. */ - if (summary_not_modified) - { - g_clear_pointer (&bytes_summary, g_bytes_unref); - if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL, - &bytes_summary, - cancellable, error)) - goto out; - } - } + if (bytes_sig && + !bytes_summary && + !pull_data->remote_repo_local && + !_ostree_repo_load_cache_summary_if_same_sig (self, + remote_name_or_baseurl, + bytes_sig, + &bytes_summary, + cancellable, + error)) + goto out; + + if (bytes_summary && !summary_bytes_v) + { + g_debug ("Loaded %s summary from cache", remote_name_or_baseurl); + summary_from_cache = TRUE; + } + + if (!pull_data->summary && !bytes_summary) + { + g_autofree char *summary_if_none_match = NULL; + guint64 summary_if_modified_since = 0; + + _ostree_repo_load_cache_summary_properties (self, remote_name_or_baseurl, NULL, + &summary_if_none_match, &summary_if_modified_since); + + g_clear_pointer (&summary_etag, g_free); + summary_last_modified = 0; + + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary", OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + summary_if_none_match, summary_if_modified_since, + pull_data->n_network_retries, + &bytes_summary, + &summary_not_modified, &summary_etag, &summary_last_modified, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) + goto out; + + /* The server returned HTTP status 304 Not Modified, so we’re clear to + * load summary from the cache. */ + if (summary_not_modified) + { + g_clear_pointer (&bytes_summary, g_bytes_unref); + if (!_ostree_repo_load_cache_summary_file (self, remote_name_or_baseurl, NULL, + &bytes_summary, + cancellable, error)) + goto out; + } + } #ifndef OSTREE_DISABLE_GPGME - if (!bytes_summary && pull_data->gpg_verify_summary) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "GPG verification enabled, but no summary found (use gpg-verify-summary=false in remote config to disable)"); - goto out; - } + if (!bytes_summary && pull_data->gpg_verify_summary) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "GPG verification enabled, but no summary found (use gpg-verify-summary=false in remote config to disable)"); + goto out; + } #endif /* OSTREE_DISABLE_GPGME */ - if (!bytes_summary && pull_data->require_static_deltas) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, - "Fetch configured to require static deltas, but no summary found"); - goto out; - } + if (!bytes_summary && require_summary_for_mirror) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Fetching all refs was requested in mirror mode, but remote repository does not have a summary"); + goto out; + } #ifndef OSTREE_DISABLE_GPGME - if (!bytes_sig && pull_data->gpg_verify_summary) - { - g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE, - "GPG verification enabled, but no summary.sig found (use gpg-verify-summary=false in remote config to disable)"); - goto out; - } + if (!bytes_sig && pull_data->gpg_verify_summary) + { + g_set_error (error, OSTREE_GPG_ERROR, OSTREE_GPG_ERROR_NO_SIGNATURE, + "GPG verification enabled, but no summary.sig found (use gpg-verify-summary=false in remote config to disable)"); + goto out; + } - if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig) - { - g_autoptr(OstreeGpgVerifyResult) result = NULL; - g_autoptr(GError) temp_error = NULL; + if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig) + { + g_autoptr(OstreeGpgVerifyResult) result = NULL; + g_autoptr(GError) temp_error = NULL; - result = ostree_repo_verify_summary (self, pull_data->remote_name, - bytes_summary, bytes_sig, - cancellable, &temp_error); - if (!ostree_gpg_verify_result_require_valid_signature (result, &temp_error)) - { - if (summary_from_cache) - { - /* The cached summary doesn't match, fetch a new one and verify again. - * Don’t set the cache headers in the HTTP request, to force a - * full download. */ - if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Remote %s cached summary invalid and " - "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified", - pull_data->remote_name); + result = ostree_repo_verify_summary (self, pull_data->remote_name, + bytes_summary, bytes_sig, + cancellable, &temp_error); + if (!ostree_gpg_verify_result_require_valid_signature (result, &temp_error)) + { + if (summary_from_cache) + { + /* The cached summary doesn't match, fetch a new one and verify again. + * Don’t set the cache headers in the HTTP request, to force a + * full download. */ + if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Remote %s cached summary invalid and " + "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified", + pull_data->remote_name); + goto out; + } + else + g_debug ("Remote %s cached summary invalid, pulling new version", + pull_data->remote_name); + + summary_from_cache = FALSE; + g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); + g_clear_pointer (&summary_etag, g_free); + summary_last_modified = 0; + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary", + OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, /* no cache headers */ + pull_data->n_network_retries, + &bytes_summary, + &summary_not_modified, &summary_etag, &summary_last_modified, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) goto out; - } - else - g_debug ("Remote %s cached summary invalid, pulling new version", - pull_data->remote_name); - - summary_from_cache = FALSE; - g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); - g_clear_pointer (&summary_etag, g_free); - summary_last_modified = 0; - if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, - pull_data->meta_mirrorlist, - "summary", - OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, - NULL, 0, /* no cache headers */ - pull_data->n_network_retries, - &bytes_summary, - &summary_not_modified, &summary_etag, &summary_last_modified, - OSTREE_MAX_METADATA_SIZE, - cancellable, error)) - goto out; - g_autoptr(OstreeGpgVerifyResult) retry = - ostree_repo_verify_summary (self, pull_data->remote_name, - bytes_summary, bytes_sig, - cancellable, error); - if (!ostree_gpg_verify_result_require_valid_signature (retry, error)) + g_autoptr(OstreeGpgVerifyResult) retry = + ostree_repo_verify_summary (self, pull_data->remote_name, + bytes_summary, bytes_sig, + cancellable, error); + if (!ostree_gpg_verify_result_require_valid_signature (retry, error)) + goto out; + } + else + { + g_propagate_error (error, g_steal_pointer (&temp_error)); goto out; - } - else - { - g_propagate_error (error, g_steal_pointer (&temp_error)); - goto out; - } - } - } + } + } + } #endif /* OSTREE_DISABLE_GPGME */ - if (pull_data->signapi_summary_verifiers) - { - if (!bytes_sig && pull_data->signapi_summary_verifiers) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)"); - goto out; - } - if (bytes_summary && bytes_sig) - { - g_autoptr(GVariant) signatures = NULL; - g_autoptr(GError) temp_error = NULL; - - signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, - bytes_sig, FALSE); - - - g_assert (pull_data->signapi_summary_verifiers); - if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error)) - { - if (summary_from_cache) - { - /* The cached summary doesn't match, fetch a new one and verify again. - * Don’t set the cache headers in the HTTP request, to force a - * full download. */ - if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Remote %s cached summary invalid and " - "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified", - pull_data->remote_name); + if (pull_data->signapi_summary_verifiers) + { + if (!bytes_sig && pull_data->signapi_summary_verifiers) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Signatures verification enabled, but no summary.sig found (use sign-verify-summary=false in remote config to disable)"); + goto out; + } + if (bytes_summary && bytes_sig) + { + g_autoptr(GVariant) signatures = NULL; + g_autoptr(GError) temp_error = NULL; + + signatures = g_variant_new_from_bytes (OSTREE_SUMMARY_SIG_GVARIANT_FORMAT, + bytes_sig, FALSE); + + g_assert (pull_data->signapi_summary_verifiers); + if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, &temp_error)) + { + if (summary_from_cache) + { + /* The cached summary doesn't match, fetch a new one and verify again. + * Don’t set the cache headers in the HTTP request, to force a + * full download. */ + if ((self->test_error_flags & OSTREE_REPO_TEST_ERROR_INVALID_CACHE) > 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Remote %s cached summary invalid and " + "OSTREE_REPO_TEST_ERROR_INVALID_CACHE specified", + pull_data->remote_name); + goto out; + } + else + g_debug ("Remote %s cached summary invalid, pulling new version", + pull_data->remote_name); + + summary_from_cache = FALSE; + g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); + g_clear_pointer (&summary_etag, g_free); + summary_last_modified = 0; + if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, + pull_data->meta_mirrorlist, + "summary", + OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, + NULL, 0, /* no cache headers */ + pull_data->n_network_retries, + &bytes_summary, + &summary_not_modified, &summary_etag, &summary_last_modified, + OSTREE_MAX_METADATA_SIZE, + cancellable, error)) goto out; - } - else - g_debug ("Remote %s cached summary invalid, pulling new version", - pull_data->remote_name); - - summary_from_cache = FALSE; - g_clear_pointer (&bytes_summary, (GDestroyNotify)g_bytes_unref); - g_clear_pointer (&summary_etag, g_free); - summary_last_modified = 0; - if (!_ostree_fetcher_mirrored_request_to_membuf (pull_data->fetcher, - pull_data->meta_mirrorlist, - "summary", - OSTREE_FETCHER_REQUEST_OPTIONAL_CONTENT, - NULL, 0, /* no cache headers */ - pull_data->n_network_retries, - &bytes_summary, - &summary_not_modified, &summary_etag, &summary_last_modified, - OSTREE_MAX_METADATA_SIZE, - cancellable, error)) - goto out; - if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, error)) + if (!_sign_verify_for_remote (pull_data->signapi_summary_verifiers, bytes_summary, signatures, NULL, error)) goto out; - } - else - { - g_propagate_error (error, g_steal_pointer (&temp_error)); - goto out; - } - } - } - } + } + else + { + g_propagate_error (error, g_steal_pointer (&temp_error)); + goto out; + } + } + } + } - if (bytes_summary) - { - pull_data->summary_data = g_bytes_ref (bytes_summary); - pull_data->summary_etag = g_strdup (summary_etag); - pull_data->summary_last_modified = summary_last_modified; - pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE); + if (bytes_summary) + { + pull_data->summary_data = g_bytes_ref (bytes_summary); + pull_data->summary_etag = g_strdup (summary_etag); + pull_data->summary_last_modified = summary_last_modified; + pull_data->summary = g_variant_new_from_bytes (OSTREE_SUMMARY_GVARIANT_FORMAT, bytes_summary, FALSE); - if (!g_variant_is_normal_form (pull_data->summary)) - { - g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Not normal form"); - goto out; - } - if (!g_variant_is_of_type (pull_data->summary, OSTREE_SUMMARY_GVARIANT_FORMAT)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Doesn't match variant type '%s'", - (char *)OSTREE_SUMMARY_GVARIANT_FORMAT); - goto out; - } + if (!g_variant_is_normal_form (pull_data->summary)) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Not normal form"); + goto out; + } + if (!g_variant_is_of_type (pull_data->summary, OSTREE_SUMMARY_GVARIANT_FORMAT)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Doesn't match variant type '%s'", + (char *)OSTREE_SUMMARY_GVARIANT_FORMAT); + goto out; + } - if (bytes_sig) - { - pull_data->summary_data_sig = g_bytes_ref (bytes_sig); - pull_data->summary_sig_etag = g_strdup (summary_sig_etag); - pull_data->summary_sig_last_modified = summary_sig_last_modified; - } - } + if (bytes_sig) + { + pull_data->summary_data_sig = g_bytes_ref (bytes_sig); + pull_data->summary_sig_etag = g_strdup (summary_sig_etag); + pull_data->summary_sig_last_modified = summary_sig_last_modified; + } + } - if (!summary_from_cache && bytes_summary && bytes_sig) - { - if (!pull_data->remote_repo_local && - !_ostree_repo_cache_summary (self, - remote_name_or_baseurl, - bytes_summary, - summary_etag, summary_last_modified, - bytes_sig, - summary_sig_etag, summary_sig_last_modified, - cancellable, - error)) - goto out; - } + if (!summary_from_cache && bytes_summary && bytes_sig) + { + if (!pull_data->remote_repo_local && + !_ostree_repo_cache_summary (self, + remote_name_or_baseurl, + bytes_summary, + summary_etag, summary_last_modified, + bytes_sig, + summary_sig_etag, summary_sig_last_modified, + cancellable, + error)) + goto out; + } - if (pull_data->summary) - { - additional_metadata = g_variant_get_child_value (pull_data->summary, 1); + if (pull_data->summary) + { + additional_metadata = g_variant_get_child_value (pull_data->summary, 1); - if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id)) - main_collection_id = NULL; - else if (!ostree_validate_collection_id (main_collection_id, error)) - goto out; + if (!g_variant_lookup (additional_metadata, OSTREE_SUMMARY_COLLECTION_ID, "&s", &main_collection_id)) + main_collection_id = NULL; + else if (!ostree_validate_collection_id (main_collection_id, error)) + goto out; - refs = g_variant_get_child_value (pull_data->summary, 0); - for (i = 0, n = g_variant_n_children (refs); i < n; i++) - { - const char *refname; - g_autoptr(GVariant) ref = g_variant_get_child_value (refs, i); + refs = g_variant_get_child_value (pull_data->summary, 0); + for (i = 0, n = g_variant_n_children (refs); i < n; i++) + { + const char *refname; + g_autoptr(GVariant) ref = g_variant_get_child_value (refs, i); - g_variant_get_child (ref, 0, "&s", &refname); + g_variant_get_child (ref, 0, "&s", &refname); - if (!ostree_validate_rev (refname, error)) - goto out; + if (!ostree_validate_rev (refname, error)) + goto out; - if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set) - { - g_hash_table_insert (requested_refs_to_fetch, - ostree_collection_ref_new (main_collection_id, refname), NULL); - } - } + if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set) + { + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (main_collection_id, refname), NULL); + } + } - g_autoptr(GVariant) collection_map = NULL; - collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}")); - if (collection_map != NULL) - { - GVariantIter collection_map_iter; - const char *collection_id; - g_autoptr(GVariant) collection_refs = NULL; + g_autoptr(GVariant) collection_map = NULL; + collection_map = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_COLLECTION_MAP, G_VARIANT_TYPE ("a{sa(s(taya{sv}))}")); + if (collection_map != NULL) + { + GVariantIter collection_map_iter; + const char *collection_id; + g_autoptr(GVariant) collection_refs = NULL; - g_variant_iter_init (&collection_map_iter, collection_map); + g_variant_iter_init (&collection_map_iter, collection_map); - while (g_variant_iter_loop (&collection_map_iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs)) - { - if (!ostree_validate_collection_id (collection_id, error)) - goto out; + while (g_variant_iter_loop (&collection_map_iter, "{&s@a(s(taya{sv}))}", &collection_id, &collection_refs)) + { + if (!ostree_validate_collection_id (collection_id, error)) + goto out; - for (i = 0, n = g_variant_n_children (collection_refs); i < n; i++) - { - const char *refname; - g_autoptr(GVariant) ref = g_variant_get_child_value (collection_refs, i); + for (i = 0, n = g_variant_n_children (collection_refs); i < n; i++) + { + const char *refname; + g_autoptr(GVariant) ref = g_variant_get_child_value (collection_refs, i); - g_variant_get_child (ref, 0, "&s", &refname); + g_variant_get_child (ref, 0, "&s", &refname); - if (!ostree_validate_rev (refname, error)) - goto out; + if (!ostree_validate_rev (refname, error)) + goto out; - if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set) - { - g_hash_table_insert (requested_refs_to_fetch, - ostree_collection_ref_new (collection_id, refname), NULL); - } - } - } - } + if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set) + { + g_hash_table_insert (requested_refs_to_fetch, + ostree_collection_ref_new (collection_id, refname), NULL); + } + } + } + } - deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}")); - n = deltas ? g_variant_n_children (deltas) : 0; - for (i = 0; i < n; i++) - { - const char *delta; - g_autoptr(GVariant) csum_v = NULL; - g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i); + deltas = g_variant_lookup_value (additional_metadata, OSTREE_SUMMARY_STATIC_DELTAS, G_VARIANT_TYPE ("a{sv}")); + pull_data->summary_has_deltas = deltas != NULL && g_variant_n_children (deltas) > 0; + if (!collect_available_deltas_for_pull (pull_data, deltas, error)) + goto out; - g_variant_get_child (ref, 0, "&s", &delta); - g_variant_get_child (ref, 1, "v", &csum_v); + g_variant_lookup (additional_metadata, OSTREE_SUMMARY_INDEXED_DELTAS, "b", &pull_data->has_indexed_deltas); + } - if (!validate_variant_is_csum (csum_v, error)) - goto out; + if (pull_data->summary && + g_variant_lookup (additional_metadata, OSTREE_SUMMARY_MODE, "s", &remote_mode_str) && + g_variant_lookup (additional_metadata, OSTREE_SUMMARY_TOMBSTONE_COMMITS, "b", &tombstone_commits)) + { + if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error)) + goto out; + pull_data->has_tombstone_commits = tombstone_commits; + remote_mode_loaded = TRUE; + } + } - guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN); - memcpy (csum_data, ostree_checksum_bytes_peek (csum_v), 32); - g_hash_table_insert (pull_data->summary_deltas_checksums, - g_strdup (delta), - csum_data); - } - } + if (pull_data->require_static_deltas && !pull_data->has_indexed_deltas && !pull_data->summary_has_deltas) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Fetch configured to require static deltas, but no summary deltas or delta index found"); + goto out; + } - if (pull_data->summary && - g_variant_lookup (additional_metadata, OSTREE_SUMMARY_MODE, "s", &remote_mode_str) && - g_variant_lookup (additional_metadata, OSTREE_SUMMARY_TOMBSTONE_COMMITS, "b", &tombstone_commits)) - { - if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error)) - goto out; - pull_data->has_tombstone_commits = tombstone_commits; - remote_mode_loaded = TRUE; - } - else if (pull_data->remote_repo_local == NULL) + if (remote_mode_loaded && pull_data->remote_repo_local == NULL) { /* Fall-back path which loads the necessary config from the remote’s - * `config` file. Doing so is deprecated since it means an + * `config` file (unless we already read it above). Doing so is deprecated since it means an * additional round trip to the remote for each pull. No need to do * it for local pulls. */ - if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error)) + if (remote_config == NULL && + !load_remote_repo_config (pull_data, &remote_config, cancellable, error)) goto out; if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare", @@ -4367,85 +4670,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, remote_mode_loaded = TRUE; } - if (remote_mode_loaded && pull_data->remote_repo_local == NULL && pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Can't pull from archives with mode \"%s\"", - remote_mode_str); - goto out; - } - } - - if (pull_data->is_mirror && !refs_to_fetch && !opt_collection_refs_set && !configured_branches) - { - if (!bytes_summary) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Fetching all refs was requested in mirror mode, but remote repository does not have a summary"); - goto out; - } - - } - else if (opt_collection_refs_set) - { - const gchar *collection_id, *ref_name, *checksum; - - while (g_variant_iter_loop (collection_refs_iter, "(&s&s&s)", &collection_id, &ref_name, &checksum)) - { - if (!ostree_validate_rev (ref_name, error)) - goto out; - g_hash_table_insert (requested_refs_to_fetch, - ostree_collection_ref_new (collection_id, ref_name), - (*checksum != '\0') ? g_strdup (checksum) : NULL); - } - } - else if (refs_to_fetch != NULL) - { - char **strviter = refs_to_fetch; - char **commitid_strviter = override_commit_ids ?: NULL; - - while (*strviter) - { - const char *branch = *strviter; - - if (ostree_validate_checksum_string (branch, NULL)) - { - char *key = g_strdup (branch); - g_hash_table_add (commits_to_fetch, key); - } - else - { - if (!ostree_validate_rev (branch, error)) - goto out; - char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL; - g_hash_table_insert (requested_refs_to_fetch, - ostree_collection_ref_new (NULL, branch), commitid); - } - - strviter++; - if (commitid_strviter) - commitid_strviter++; - } - } - else + if (remote_mode_loaded && pull_data->remote_repo_local == NULL && pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE) { - char **branches_iter; - - branches_iter = configured_branches; - - if (!(branches_iter && *branches_iter)) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "No configured branches for remote %s", remote_name_or_baseurl); - goto out; - } - for (;branches_iter && *branches_iter; branches_iter++) - { - const char *branch = *branches_iter; - - g_hash_table_insert (requested_refs_to_fetch, - ostree_collection_ref_new (NULL, branch), NULL); - } + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Can't pull from archives with mode \"%s\"", + remote_mode_str); + goto out; } /* Resolve the checksum for each ref. This has to be done into a new hash table, @@ -4900,6 +5130,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_metadata, (GDestroyNotify) g_hash_table_unref); + g_clear_pointer (&pull_data->pending_fetch_delta_indexes, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_delta_superblocks, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->pending_fetch_deltaparts, (GDestroyNotify) g_hash_table_unref); g_queue_foreach (&pull_data->scan_object_queue, (GFunc) scan_object_queue_data_free, NULL); diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index e263e928b0..c253672489 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -168,6 +168,93 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, return TRUE; } +/** + * ostree_repo_list_static_delta_indexes: + * @self: Repo + * @out_indexes: (out) (element-type utf8) (transfer container): String name of delta indexes (checksum) + * @cancellable: Cancellable + * @error: Error + * + * This function synchronously enumerates all static delta indexes in the + * repository, returning its result in @out_indexes. + * + * Since: 2020.7 + */ +gboolean +ostree_repo_list_static_delta_indexes (OstreeRepo *self, + GPtrArray **out_indexes, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GPtrArray) ret_indexes = g_ptr_array_new_with_free_func (g_free); + + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + gboolean exists; + if (!ot_dfd_iter_init_allow_noent (self->repo_dir_fd, "delta-indexes", &dfd_iter, + &exists, error)) + return FALSE; + if (!exists) + { + /* Note early return */ + ot_transfer_out_value (out_indexes, &ret_indexes); + return TRUE; + } + + while (TRUE) + { + g_auto(GLnxDirFdIterator) sub_dfd_iter = { 0, }; + struct dirent *dent; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) + return FALSE; + if (dent == NULL) + break; + if (dent->d_type != DT_DIR) + continue; + if (strlen (dent->d_name) != 2) + continue; + + if (!glnx_dirfd_iterator_init_at (dfd_iter.fd, dent->d_name, FALSE, + &sub_dfd_iter, error)) + return FALSE; + + while (TRUE) + { + struct dirent *sub_dent; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&sub_dfd_iter, &sub_dent, + cancellable, error)) + return FALSE; + if (sub_dent == NULL) + break; + if (sub_dent->d_type != DT_REG) + continue; + + const char *name1 = dent->d_name; + const char *name2 = sub_dent->d_name; + + /* base64 len is 43, but 2 chars are in the parent dir name */ + if (strlen (name2) != 41 + strlen(".index") || + !g_str_has_suffix (name2, ".index")) + continue; + + g_autoptr(GString) out = g_string_new (name1); + g_string_append_len (out, name2, 41); + + char checksum[OSTREE_SHA256_STRING_LEN+1]; + guchar csum[OSTREE_SHA256_DIGEST_LEN]; + + ostree_checksum_b64_inplace_to_bytes (out->str, csum); + ostree_checksum_inplace_from_bytes (csum, checksum); + + g_ptr_array_add (ret_indexes, g_strdup (checksum)); + } + } + + ot_transfer_out_value (out_indexes, &ret_indexes); + return TRUE; +} + gboolean _ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo, GVariant *checksum_array, @@ -1118,3 +1205,207 @@ ostree_repo_static_delta_verify_signature (OstreeRepo *self, return _ostree_repo_static_delta_verify_signature (self, delta_fd, sign, out_success_message, error); } + +static void +null_or_ptr_array_unref (GPtrArray *array) +{ + if (array != NULL) + g_ptr_array_unref (array); +} + +static gboolean +file_has_content (OstreeRepo *repo, + const char *subpath, + GBytes *data, + GCancellable *cancellable) +{ + struct stat stbuf; + glnx_autofd int existing_fd = -1; + + if (!glnx_fstatat (repo->repo_dir_fd, subpath, &stbuf, 0, NULL)) + return FALSE; + + if (stbuf.st_size != g_bytes_get_size (data)) + return FALSE; + + if (!glnx_openat_rdonly (repo->repo_dir_fd, subpath, TRUE, &existing_fd, NULL)) + return FALSE; + + g_autoptr(GBytes) existing_data = glnx_fd_readall_bytes (existing_fd, cancellable, NULL); + if (existing_data == NULL) + return FALSE; + + return g_bytes_equal (existing_data, data); +} + +/** + * ostree_repo_static_delta_reindex: + * @repo: Repo + * @flags: Flags affecting the indexing operation + * @opt_to_commit: ASCII SHA256 checksum of target commit, or %NULL to index all targets + * @cancellable: Cancellable + * @error: Error + * + * The delta index for a particular commit lists all the existing deltas that can be used + * when downloading that commit. This operation regenerates these indexes, either for + * a particular commit (if @opt_to_commit is non-%NULL), or for all commits that + * are reachable by an existing delta (if @opt_to_commit is %NULL). + * + * This is normally called automatically when the summary is updated in ostree_repo_regenerate_summary(). + * + * Locking: shared + */ +gboolean +ostree_repo_static_delta_reindex (OstreeRepo *repo, + OstreeStaticDeltaIndexFlags flags, + const char *opt_to_commit, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GPtrArray) all_deltas = NULL; + g_autoptr(GHashTable) deltas_to_commit_ht = NULL; /* map: to checksum -> ptrarray of from checksums (or NULL) */ + gboolean opt_indexed_deltas; + + /* Protect against parallel prune operation */ + g_autoptr(OstreeRepoAutoLock) lock = + _ostree_repo_auto_lock_push (repo, OSTREE_REPO_LOCK_SHARED, cancellable, error); + if (!lock) + return FALSE; + + /* Enusre that the "indexed-deltas" option is set on the config, so we know this when pulling */ + if (!ot_keyfile_get_boolean_with_default (repo->config, "core", + "indexed-deltas", FALSE, + &opt_indexed_deltas, error)) + return FALSE; + + if (!opt_indexed_deltas) + { + g_autoptr(GKeyFile) config = ostree_repo_copy_config (repo); + g_key_file_set_boolean (config, "core", "indexed-deltas", TRUE); + if (!ostree_repo_write_config (repo, config, error)) + return FALSE; + } + + deltas_to_commit_ht = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, (GDestroyNotify)null_or_ptr_array_unref); + + if (opt_to_commit == NULL) + { + g_autoptr(GPtrArray) old_indexes = NULL; + + /* To ensure all old index files either is regenerated, or + * removed, we initialize all existing indexes to NULL in the + * hashtable. */ + if (!ostree_repo_list_static_delta_indexes (repo, &old_indexes, cancellable, error)) + return FALSE; + + for (int i = 0; i < old_indexes->len; i++) + { + const char *old_index = g_ptr_array_index (old_indexes, i); + g_hash_table_insert (deltas_to_commit_ht, g_strdup (old_index), NULL); + } + } + else + { + if (!ostree_validate_checksum_string (opt_to_commit, error)) + return FALSE; + + /* We ensure the specific old index either is regenerated, or removed */ + g_hash_table_insert (deltas_to_commit_ht, g_strdup (opt_to_commit), NULL); + } + + if (!ostree_repo_list_static_delta_names (repo, &all_deltas, cancellable, error)) + return FALSE; + + for (int i = 0; i < all_deltas->len; i++) + { + const char *delta_name = g_ptr_array_index (all_deltas, i); + g_autofree char *from = NULL; + g_autofree char *to = NULL; + GPtrArray *deltas_to_commit = NULL; + + if (!_ostree_parse_delta_name (delta_name, &from, &to, error)) + return FALSE; + + if (opt_to_commit != NULL && strcmp (to, opt_to_commit) != 0) + continue; + + deltas_to_commit = g_hash_table_lookup (deltas_to_commit_ht, to); + if (deltas_to_commit == NULL) + { + deltas_to_commit = g_ptr_array_new_with_free_func (g_free); + g_hash_table_insert (deltas_to_commit_ht, g_steal_pointer (&to), deltas_to_commit); + } + + g_ptr_array_add (deltas_to_commit, g_steal_pointer (&from)); + } + + GLNX_HASH_TABLE_FOREACH_KV (deltas_to_commit_ht, const char*, to, GPtrArray*, froms) + { + g_autofree char *index_path = _ostree_get_relative_static_delta_index_path (to); + + if (froms == NULL) + { + /* No index to this checksum seen, delete if it exists */ + + g_debug ("Removing delta index for %s", to); + if (!ot_ensure_unlinked_at (repo->repo_dir_fd, index_path, error)) + return FALSE; + } + else + { + g_auto(GVariantDict) index_builder = OT_VARIANT_BUILDER_INITIALIZER; + g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER; + g_autoptr(GVariant) index_variant = NULL; + g_autoptr(GBytes) index = NULL; + + /* We sort on from here so that the index file is reproducible */ + g_ptr_array_sort (froms, (GCompareFunc)g_strcmp0); + + g_variant_dict_init (&deltas_builder, NULL); + + for (int i = 0; i < froms->len; i++) + { + const char *from = g_ptr_array_index (froms, i); + g_autofree char *delta_name = NULL; + GVariant *digest; + + digest = _ostree_repo_static_delta_superblock_digest (repo, from, to, cancellable, error); + if (digest == NULL) + return FALSE; + + if (from != NULL) + delta_name = g_strconcat (from, "-", to, NULL); + else + delta_name = g_strdup (to); + + g_variant_dict_insert_value (&deltas_builder, delta_name, digest); + } + + /* The toplevel of the index is an a{sv} for extensibility, and we use same key name (and format) as when + * storing deltas in the summary. */ + g_variant_dict_init (&index_builder, NULL); + + g_variant_dict_insert_value (&index_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder)); + + index_variant = g_variant_ref_sink (g_variant_dict_end (&index_builder)); + index = g_variant_get_data_as_bytes (index_variant); + + g_autofree char *index_dirname = g_path_get_dirname (index_path); + if (!glnx_shutil_mkdir_p_at (repo->repo_dir_fd, index_dirname, DEFAULT_DIRECTORY_MODE, cancellable, error)) + return FALSE; + + /* delta indexes are generally small and static, so reading it back and comparing is cheap, and it will + lower the write load (and particular sync-load) on the disk during reindexing (i.e. summary updates), */ + if (file_has_content (repo, index_path, index, cancellable)) + continue; + + g_debug ("Updating delta index for %s", to); + if (!glnx_file_replace_contents_at (repo->repo_dir_fd, index_path, + g_bytes_get_data (index, NULL), g_bytes_get_size (index), + 0, cancellable, error)) + return FALSE; + } + } + + return TRUE; +} diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index 5a2e687922..d6c706da3b 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -227,6 +227,11 @@ _ostree_repo_static_delta_delete (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error); +gboolean +_ostree_repo_static_delta_reindex (OstreeRepo *repo, + const char *opt_to_commit, + GCancellable *cancellable, + GError **error); /* Used for static deltas which due to a historical mistake are * inconsistent endian. diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index ba3e877f4f..3bbf5ea0f9 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -5222,6 +5222,67 @@ ostree_repo_add_gpg_signature_summary (OstreeRepo *self, #endif /* OSTREE_DISABLE_GPGME */ } + +/** + * ostree_repo_gpg_sign_data: + * @self: Self + * @data: Data as a #GBytes + * @old_signatures: Existing signatures to append to (or %NULL) + * @key_id: (array zero-terminated=1) (element-type utf8): NULL-terminated array of GPG keys. + * @homedir: (allow-none): GPG home directory, or %NULL + * @out_signature: (out): in case of success will contain signature + * @cancellable: A #GCancellable + * @error: a #GError + * + * Sign the given @data with the specified keys in @key_id. Similar to + * ostree_repo_add_gpg_signature_summary() but can be used on any + * data. + * + * You can use ostree_repo_gpg_verify_data() to verify the signatures. + * + * Returns: @TRUE if @data has been signed successfully, + * @FALSE in case of error (@error will contain the reason). + * + * Since: 2020.8 + */ +gboolean +ostree_repo_gpg_sign_data (OstreeRepo *self, + GBytes *data, + GBytes *old_signatures, + const gchar **key_id, + const gchar *homedir, + GBytes **out_signatures, + GCancellable *cancellable, + GError **error) +{ +#ifndef OSTREE_DISABLE_GPGME + g_autoptr(GVariant) metadata = NULL; + g_autoptr(GVariant) res = NULL; + + if (old_signatures) + metadata = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_SUMMARY_SIG_GVARIANT_STRING), old_signatures, FALSE)); + + for (guint i = 0; key_id[i]; i++) + { + g_autoptr(GBytes) signature_data = NULL; + if (!sign_data (self, data, key_id[i], homedir, + &signature_data, + cancellable, error)) + return FALSE; + + g_autoptr(GVariant) old_metadata = g_steal_pointer (&metadata); + metadata = _ostree_detached_metadata_append_gpg_sig (old_metadata, signature_data); + } + + res = g_variant_get_normal_form (metadata); + *out_signatures = g_variant_get_data_as_bytes (res); + return TRUE; +#else + return glnx_throw (error, "GPG feature is disabled in a build time"); +#endif /* OSTREE_DISABLE_GPGME */ +} + + #ifndef OSTREE_DISABLE_GPGME /* Special remote for _ostree_repo_gpg_verify_with_metadata() */ static const char *OSTREE_ALL_REMOTES = "__OSTREE_ALL_REMOTES__"; @@ -5749,6 +5810,8 @@ ostree_repo_regenerate_summary (OstreeRepo *self, * commits from working. */ g_autoptr(OstreeRepoAutoLock) lock = NULL; + gboolean no_deltas_in_summary = FALSE; + lock = _ostree_repo_auto_lock_push (self, OSTREE_REPO_LOCK_EXCLUSIVE, cancellable, error); if (!lock) @@ -5781,35 +5844,41 @@ ostree_repo_regenerate_summary (OstreeRepo *self, } } - { - g_autoptr(GPtrArray) delta_names = NULL; - g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER; + if (!ot_keyfile_get_boolean_with_default (self->config, "core", + "no-deltas-in-summary", FALSE, + &no_deltas_in_summary, error)) + return FALSE; - if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error)) - return FALSE; + if (!no_deltas_in_summary) + { + g_autoptr(GPtrArray) delta_names = NULL; + g_auto(GVariantDict) deltas_builder = OT_VARIANT_BUILDER_INITIALIZER; - g_variant_dict_init (&deltas_builder, NULL); - for (guint i = 0; i < delta_names->len; i++) - { - g_autofree char *from = NULL; - g_autofree char *to = NULL; - GVariant *digest; + if (!ostree_repo_list_static_delta_names (self, &delta_names, cancellable, error)) + return FALSE; - if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error)) - return FALSE; + g_variant_dict_init (&deltas_builder, NULL); + for (guint i = 0; i < delta_names->len; i++) + { + g_autofree char *from = NULL; + g_autofree char *to = NULL; + GVariant *digest; - digest = _ostree_repo_static_delta_superblock_digest (self, - (from && from[0]) ? from : NULL, - to, cancellable, error); - if (digest == NULL) - return FALSE; + if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error)) + return FALSE; - g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest); - } + digest = _ostree_repo_static_delta_superblock_digest (self, + (from && from[0]) ? from : NULL, + to, cancellable, error); + if (digest == NULL) + return FALSE; - if (delta_names->len > 0) - g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder)); - } + g_variant_dict_insert_value (&deltas_builder, delta_names->pdata[i], digest); + } + + if (delta_names->len > 0) + g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_STATIC_DELTAS, g_variant_dict_end (&deltas_builder)); + } { g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_LAST_MODIFIED, @@ -5834,6 +5903,9 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_new_boolean (tombstone_commits)); } + g_variant_dict_insert_value (&additional_metadata_builder, OSTREE_SUMMARY_INDEXED_DELTAS, + g_variant_new_boolean (TRUE)); + /* Add refs which have a collection specified, which could be in refs/mirrors, * refs/heads, and/or refs/remotes. */ { @@ -5927,6 +5999,9 @@ ostree_repo_regenerate_summary (OstreeRepo *self, g_variant_ref_sink (summary); } + if (!ostree_repo_static_delta_reindex (self, 0, NULL, cancellable, error)) + return FALSE; + if (!_ostree_repo_file_replace_contents (self, self->repo_dir_fd, "summary", diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index d52fc9c3a9..e64c3230ce 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -1046,6 +1046,12 @@ gboolean ostree_repo_list_static_delta_names (OstreeRepo *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_list_static_delta_indexes (OstreeRepo *self, + GPtrArray **out_indexes, + GCancellable *cancellable, + GError **error); + /** * OstreeStaticDeltaGenerateOpt: * @OSTREE_STATIC_DELTA_GENERATE_OPT_LOWLATENCY: Optimize for speed of delta creation over space @@ -1068,6 +1074,23 @@ gboolean ostree_repo_static_delta_generate (OstreeRepo *self, GCancellable *cancellable, GError **error); +/** + * OstreeStaticDeltaIndexFlags: + * @OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE: No special flags + * + * Flags controlling static delta index generation. + */ +typedef enum { + OSTREE_STATIC_DELTA_INDEX_FLAGS_NONE = 0, +} OstreeStaticDeltaIndexFlags; + +_OSTREE_PUBLIC +gboolean ostree_repo_static_delta_reindex (OstreeRepo *repo, + OstreeStaticDeltaIndexFlags flags, + const char *opt_to_commit, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC gboolean ostree_repo_static_delta_execute_offline_with_signature (OstreeRepo *self, GFile *dir_or_file, @@ -1393,6 +1416,16 @@ gboolean ostree_repo_append_gpg_signature (OstreeRepo *self, GCancellable *cancellable, GError **error); +_OSTREE_PUBLIC +gboolean ostree_repo_gpg_sign_data (OstreeRepo *self, + GBytes *data, + GBytes *old_signatures, + const gchar **key_id, + const gchar *homedir, + GBytes **out_signatures, + GCancellable *cancellable, + GError **error); + _OSTREE_PUBLIC OstreeGpgVerifyResult * ostree_repo_verify_commit_ext (OstreeRepo *self, const gchar *commit_checksum, diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index 3e0af5bd96..ff31b574e8 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -53,6 +53,8 @@ BUILTINPROTO(delete); BUILTINPROTO(generate); BUILTINPROTO(apply_offline); BUILTINPROTO(verify); +BUILTINPROTO(indexes); +BUILTINPROTO(reindex); #undef BUILTINPROTO @@ -75,6 +77,12 @@ static OstreeCommand static_delta_subcommands[] = { { "verify", OSTREE_BUILTIN_FLAG_NONE, ot_static_delta_builtin_verify, "Verify static delta signatures" }, + { "indexes", OSTREE_BUILTIN_FLAG_NONE, + ot_static_delta_builtin_indexes, + "List static delta indexes" }, + { "reindex", OSTREE_BUILTIN_FLAG_NONE, + ot_static_delta_builtin_reindex, + "Regenerate static delta indexes" }, { NULL, 0, NULL, NULL } }; @@ -126,6 +134,15 @@ static GOptionEntry verify_options[] = { { NULL } }; +static GOptionEntry indexes_options[] = { + { NULL } +}; + +static GOptionEntry reindex_options[] = { + { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Only update delta index to revision REV", "REV" }, + { NULL } +}; + static void static_delta_usage (char **argv, gboolean is_error) @@ -176,6 +193,46 @@ ot_static_delta_builtin_list (int argc, char **argv, OstreeCommandInvocation *in return TRUE; } +static gboolean +ot_static_delta_builtin_indexes (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) +{ + g_autoptr(OstreeRepo) repo = NULL; + g_autoptr(GOptionContext) context = g_option_context_new (""); + if (!ostree_option_context_parse (context, indexes_options, &argc, &argv, + invocation, &repo, cancellable, error)) + return FALSE; + + g_autoptr(GPtrArray) indexes = NULL; + if (!ostree_repo_list_static_delta_indexes (repo, &indexes, cancellable, error)) + return FALSE; + + if (indexes->len == 0) + g_print ("(No static deltas indexes)\n"); + else + { + for (guint i = 0; i < indexes->len; i++) + g_print ("%s\n", (char*)indexes->pdata[i]); + } + + return TRUE; +} + +static gboolean +ot_static_delta_builtin_reindex (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) +{ + g_autoptr(GOptionContext) context = g_option_context_new (""); + + g_autoptr(OstreeRepo) repo = NULL; + if (!ostree_option_context_parse (context, reindex_options, &argc, &argv, invocation, &repo, cancellable, error)) + return FALSE; + + if (!ostree_repo_static_delta_reindex (repo, 0, opt_to_rev, cancellable, error)) + return FALSE; + + return TRUE; +} + + static gboolean ot_static_delta_builtin_show (int argc, char **argv, OstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error) { diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 082ed31512..0892183859 100644 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -55,10 +55,10 @@ function verify_initial_contents() { } if has_gpgme; then - echo "1..35" + echo "1..36" else # 3 tests needs GPG support - echo "1..32" + echo "1..33" fi # Try both syntaxes @@ -381,6 +381,25 @@ assert_file_has_content err.txt "Upgrade.*is chronologically older" ${CMD_PREFIX} ostree --repo=repo pull --timestamp-check-from-rev=${oldrev} origin main@${middlerev} echo "ok pull timestamp checking" +# test pull without override commit use summary, but with doesn't use summary +# We temporarily replace summary with broken one to detect if it is used +mv ostree-srv/gnomerepo/summary ostree-srv/gnomerepo/summary.backup +echo "broken" > ostree-srv/gnomerepo/summary + +repo_init --no-sign-verify +rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main) +# This will need summary, so will fail +if ${CMD_PREFIX} ostree --repo=repo -v pull origin main; then + assert_not_reached "Should have failed with broken summary" +fi +# This won't need summary so will not fail +${CMD_PREFIX} ostree --repo=repo pull origin main@${rev} + +# Restore summary +mv ostree-srv/gnomerepo/summary.backup ostree-srv/gnomerepo/summary + +echo "ok pull with override id doesn't use summary" + cd ${test_tmpdir} repo_init --no-sign-verify ${CMD_PREFIX} ostree --repo=repo pull origin main diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 557447bc73..bfdec593ff 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -28,7 +28,7 @@ skip_without_user_xattrs bindatafiles="bash true ostree" morebindatafiles="false ls" -echo '1..12' +echo '1..13' mkdir repo ostree_repo_init repo --mode=archive @@ -90,6 +90,11 @@ ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev} || exit 1 ${CMD_PREFIX} ostree --repo=repo prune ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev} || exit 1 +${CMD_PREFIX} ostree --repo=repo static-delta reindex +${CMD_PREFIX} ostree --repo=repo static-delta indexes | wc -l > indexcount +assert_file_has_content indexcount "^1$" +${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${origrev} || exit 1 + permuteDirectory 1 files ${CMD_PREFIX} ostree --repo=repo commit -b test -s test --tree=dir=files @@ -119,6 +124,12 @@ ${CMD_PREFIX} ostree --repo=repo static-delta generate --max-bsdiff-size=10000 - ${CMD_PREFIX} ostree --repo=repo static-delta list | grep ${origrev}-${newrev} || exit 1 +${CMD_PREFIX} ostree --repo=repo static-delta reindex +${CMD_PREFIX} ostree --repo=repo static-delta indexes | wc -l > indexcount +assert_file_has_content indexcount "^2$" +${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${origrev} || exit 1 +${CMD_PREFIX} ostree --repo=repo static-delta indexes | grep ${newrev} || exit 1 + if ${CMD_PREFIX} ostree --repo=repo static-delta generate --from=${origrev} --to=${newrev} --empty 2>>err.txt; then assert_not_reached "static-delta generate --from=${origrev} --empty unexpectedly succeeded" fi @@ -249,6 +260,41 @@ ${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null echo 'ok pull empty delta part' +rm -rf repo/delta-indexes +${CMD_PREFIX} ostree --repo=repo summary -u +${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt +assert_file_has_content summary.txt "ostree\.static\-deltas" + +${CMD_PREFIX} ostree --repo=repo config set core.no-deltas-in-summary true +${CMD_PREFIX} ostree --repo=repo summary -u + +${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt +assert_not_file_has_content summary.txt "ostree\.static\-deltas" + +rm -rf repo2 +mkdir repo2 && ostree_repo_init repo2 --mode=bare-user +${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev} + +rm -rf repo/delta-indexes +if ${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev} &> no-delta.txt; then + assert_not_reached "failing pull with --require-static-deltas unexpectedly succeeded" +fi +assert_file_has_content no-delta.txt "Static deltas required, but none found for" + +${CMD_PREFIX} ostree --repo=repo static-delta reindex +${CMD_PREFIX} ostree --repo=repo2 pull-local --require-static-deltas repo ${samerev} + +${CMD_PREFIX} ostree --repo=repo2 fsck +${CMD_PREFIX} ostree --repo=repo2 ls ${samerev} >/dev/null + +${CMD_PREFIX} ostree --repo=repo config set core.no-deltas-in-summary false +${CMD_PREFIX} ostree --repo=repo summary -u + +${CMD_PREFIX} ostree summary -v --raw --repo=repo > summary.txt +assert_file_has_content summary.txt "ostree\.static\-deltas" + +echo 'ok pull delta part with delta index' + # Make a new branch to test "rebase deltas" echo otherbranch-content > files/otherbranch-content ${CMD_PREFIX} ostree --repo=repo commit -b otherbranch --tree=dir=files