Merge pull request #2188 from alexlarsson/delta-indexes

Add indexes for deltas outside of the summary

Makefile-libostree.am

@@ -184,9 +184,9 @@ libostree_1_la_SOURCES += \
 endif # USE_GPGME
 
 symbol_files = $(top_srcdir)/src/libostree/libostree-released.sym
-#if BUILDOPT_IS_DEVEL_BUILD
-#symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
-#endif
+if BUILDOPT_IS_DEVEL_BUILD
+symbol_files += $(top_srcdir)/src/libostree/libostree-devel.sym
+endif
 # http://blog.jgc.org/2007/06/escaping-comma-and-space-in-gnu-make.html
 wl_versionscript_arg = -Wl,--version-script=
 EXTRA_DIST += \

apidoc/ostree-sections.txt

@@ -412,6 +412,8 @@ OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE
 ostree_repo_list_objects
 ostree_repo_list_commit_objects_starting_with
 ostree_repo_list_static_delta_names
+ostree_repo_list_static_delta_indexes
+ostree_repo_static_delta_reindex
 OstreeStaticDeltaGenerateOpt
 ostree_repo_static_delta_generate
 ostree_repo_static_delta_execute_offline_with_signature
@@ -445,6 +447,7 @@ ostree_repo_pull_default_console_progress_changed
 ostree_repo_sign_commit
 ostree_repo_append_gpg_signature
 ostree_repo_add_gpg_signature_summary
+ostree_repo_gpg_sign_data
 ostree_repo_gpg_verify_data
 ostree_repo_verify_commit
 ostree_repo_verify_commit_ext

man/ostree.repo-config.xml

@@ -249,6 +249,20 @@ Boston, MA 02111-1307, USA.
         costly).
         </para></listitem>
       </varlistentry>
+
+      <varlistentry>
+        <term><varname>no-deltas-in-summary</varname></term>
+        <listitem><para>Boolean value controlling whether OSTree should skip
+        putting an index of available deltas in the summary file. Defaults to false.
+        </para>
+        <para>
+        Since 2020.7 OSTree can use delta indexes outside the summary file,
+        making the summary file smaller (especially for larger repositories). However
+        by default we still create the index in the summary file to make older clients
+        work. If you know all clients will be 2020.7 later you can enable this to
+        save network bandwidth.
+        </para></listitem>
+      </varlistentry>
     </variablelist>
   </refsect1>
 

src/libostree/libostree-devel.sym

@@ -17,9 +17,12 @@
   Boston, MA 02111-1307, USA.
 ***/
 
-/* Copy the bits below and uncomment the include in Makefile-libostree.am
-   when adding a symbol.
-*/
+LIBOSTREE_2020.8 {
+global:
+  ostree_repo_list_static_delta_indexes;
+  ostree_repo_static_delta_reindex;
+  ostree_repo_gpg_sign_data;
+} LIBOSTREE_2020.7;
 
 /* Stub section for the stable release *after* this development one; don't
  * edit this other than to update the year.  This is just a copy/paste

src/libostree/ostree-core-private.h

@@ -135,6 +135,9 @@ _ostree_get_relative_static_delta_part_path (const char        *from,
                                              const char        *to,
                                              guint              i);
 
+char *
+_ostree_get_relative_static_delta_index_path (const char        *to);
+
 static inline char * _ostree_get_commitpartial_path (const char *checksum)
 {
   return g_strconcat ("state/", checksum, ".commitpartial", NULL);

src/libostree/ostree-core.c

@@ -1814,15 +1814,15 @@ _ostree_get_relative_object_path (const char         *checksum,
   return g_string_free (path, FALSE);
 }
 
-char *
-_ostree_get_relative_static_delta_path (const char *from,
-                                        const char *to,
-                                        const char *target)
+static GString *
+static_delta_path_base (const char *dir,
+                        const char *from,
+                        const char *to)
 {
   guint8 csum_to[OSTREE_SHA256_DIGEST_LEN];
   char to_b64[44];
   guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN];
-  GString *ret = g_string_new ("deltas/");
+  GString *ret = g_string_new (dir);
 
   ostree_checksum_inplace_to_bytes (to, csum_to);
   ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64);
@@ -1851,6 +1851,16 @@ _ostree_get_relative_static_delta_path (const char *from,
     g_string_append_c (ret, '/');
   g_string_append (ret, to_b64 + 2);
 
+  return ret;
+}
+
+char *
+_ostree_get_relative_static_delta_path (const char *from,
+                                        const char *to,
+                                        const char *target)
+{
+  GString *ret = static_delta_path_base ("deltas/", from, to);
+
   if (target != NULL)
     {
       g_string_append_c (ret, '/');
@@ -1883,6 +1893,16 @@ _ostree_get_relative_static_delta_part_path (const char        *from,
   return _ostree_get_relative_static_delta_path (from, to, partstr);
 }
 
+char *
+_ostree_get_relative_static_delta_index_path (const char *to)
+{
+  GString *ret = static_delta_path_base ("delta-indexes/", NULL, to);
+
+  g_string_append (ret, ".index");
+
+  return g_string_free (ret, FALSE);
+}
+
 gboolean
 _ostree_parse_delta_name (const char   *delta_name,
                           char        **out_from,

src/libostree/ostree-repo-private.h

@@ -57,6 +57,7 @@ G_BEGIN_DECLS
 #define OSTREE_SUMMARY_COLLECTION_MAP "ostree.summary.collection-map"
 #define OSTREE_SUMMARY_MODE "ostree.summary.mode"
 #define OSTREE_SUMMARY_TOMBSTONE_COMMITS "ostree.summary.tombstone-commits"
+#define OSTREE_SUMMARY_INDEXED_DELTAS "ostree.summary.indexed-deltas"
 
 #define _OSTREE_PAYLOAD_LINK_PREFIX "../"
 #define _OSTREE_PAYLOAD_LINK_PREFIX_LEN (sizeof (_OSTREE_PAYLOAD_LINK_PREFIX) - 1)

src/libostree/ostree-repo-pull-private.h

@@ -78,7 +78,9 @@ typedef struct {
   char             *summary_sig_etag;
   guint64           summary_sig_last_modified;  /* seconds since the epoch */
   GVariant         *summary;
-  GHashTable       *summary_deltas_checksums;
+  GHashTable       *summary_deltas_checksums; /* Filled from summary and delta indexes */
+  gboolean          summary_has_deltas; /* True if the summary existed and had a delta index */
+  gboolean          has_indexed_deltas;
   GHashTable       *ref_original_commits; /* Maps checksum to commit, used by timestamp checks */
   GHashTable       *verified_commits; /* Set<checksum> of commits that have been verified */
   GHashTable       *signapi_verified_commits; /* Map<checksum,verification> of commits that have been signapi verified */
@@ -93,6 +95,7 @@ typedef struct {
   GHashTable       *requested_fallback_content; /* Maps checksum to itself */
   GHashTable       *pending_fetch_metadata; /* Map<ObjectName,FetchObjectData> */
   GHashTable       *pending_fetch_content; /* Map<checksum,FetchObjectData> */
+  GHashTable       *pending_fetch_delta_indexes; /* Set<FetchDeltaIndexData> */
   GHashTable       *pending_fetch_delta_superblocks; /* Set<FetchDeltaSuperData> */
   GHashTable       *pending_fetch_deltaparts; /* Set<FetchStaticDeltaData> */
   guint             n_outstanding_metadata_fetches;