Merge pull request pulp#958 from pulp/patchback/backports/3.1/3a87265…

…9fcb25f813938d94c671c3bc362393168/pr-920

[PR pulp#920/3a872659 backport][3.1] Add pulp to pulp verification tests

CHANGES/919.misc

@@ -0,0 +1 @@
+Added tests that verify the download of content served by ``pulp_deb``.

pulp_deb/tests/functional/api/test_pulp_to_pulp.py

@@ -0,0 +1,207 @@
+"""Tests that verify download of content served by Pulp."""
+import pytest
+
+from pulp_deb.tests.functional.constants import (
+    DEB_P2P_SIMPLE_THEN_STRUCTURED,
+    DEB_PUBLICATION_ARGS_ONLY_SIMPLE,
+    DEB_PUBLICATION_ARGS_ONLY_STRUCTURED,
+    DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED,
+    DEB_P2P_ONLY_SIMPLE,
+    DEB_P2P_ONLY_STRUCTURED,
+    DEB_P2P_SIMPLE_AND_STRUCTURED,
+    DEB_P2P_REMOTE_ARGS_SIMPLE,
+    DEB_P2P_REMOTE_ARGS_STRUCTURED,
+    DEB_P2P_REMOTE_ARGS_BOTH,
+    DEB_P2P_REMOTE_ARGS_VERBATIM,
+)
+from pulp_deb.tests.functional.utils import get_counts_from_content_summary
+
+pulp_to_pulp_test_data = [
+    pytest.param(
+        "immediate",
+        False,
+        DEB_PUBLICATION_ARGS_ONLY_SIMPLE,
+        DEB_P2P_REMOTE_ARGS_SIMPLE,
+        DEB_P2P_ONLY_SIMPLE,
+        id="p2p-immediate-simple",
+    ),
+    pytest.param(
+        "immediate",
+        False,
+        DEB_PUBLICATION_ARGS_ONLY_STRUCTURED,
+        DEB_P2P_REMOTE_ARGS_STRUCTURED,
+        DEB_P2P_ONLY_STRUCTURED,
+        id="p2p-immediate-structured",
+    ),
+    pytest.param(
+        "immediate",
+        False,
+        DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED,
+        DEB_P2P_REMOTE_ARGS_BOTH,
+        DEB_P2P_SIMPLE_AND_STRUCTURED,
+        id="p2p-immediate-both",
+    ),
+    pytest.param(
+        "immediate", True, {}, DEB_P2P_REMOTE_ARGS_VERBATIM, {}, id="p2p-immediate-verbatim"
+    ),
+    pytest.param(
+        "streamed",
+        False,
+        DEB_PUBLICATION_ARGS_ONLY_SIMPLE,
+        DEB_P2P_REMOTE_ARGS_SIMPLE,
+        DEB_P2P_ONLY_SIMPLE,
+        id="p2p-streamed-simple",
+    ),
+    pytest.param(
+        "streamed",
+        False,
+        DEB_PUBLICATION_ARGS_ONLY_STRUCTURED,
+        DEB_P2P_REMOTE_ARGS_STRUCTURED,
+        DEB_P2P_ONLY_STRUCTURED,
+        id="p2p-streamed-structured",
+    ),
+    pytest.param(
+        "streamed",
+        False,
+        DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED,
+        DEB_P2P_REMOTE_ARGS_BOTH,
+        DEB_P2P_SIMPLE_AND_STRUCTURED,
+        id="p2p-streamed-both",
+    ),
+    pytest.param(
+        "streamed", True, {}, DEB_P2P_REMOTE_ARGS_VERBATIM, {}, id="p2p-streamed-verbatim"
+    ),
+    pytest.param(
+        "on_demand",
+        False,
+        DEB_PUBLICATION_ARGS_ONLY_SIMPLE,
+        DEB_P2P_REMOTE_ARGS_SIMPLE,
+        DEB_P2P_ONLY_SIMPLE,
+        id="p2p-on_demand-simple",
+    ),
+    pytest.param(
+        "on_demand",
+        False,
+        DEB_PUBLICATION_ARGS_ONLY_STRUCTURED,
+        DEB_P2P_REMOTE_ARGS_STRUCTURED,
+        DEB_P2P_ONLY_STRUCTURED,
+        id="p2p-on_demand-structured",
+    ),
+    pytest.param(
+        "on_demand",
+        False,
+        DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED,
+        DEB_P2P_REMOTE_ARGS_BOTH,
+        DEB_P2P_SIMPLE_AND_STRUCTURED,
+        id="p2p-on_demand-both",
+    ),
+    pytest.param(
+        "on_demand", True, {}, DEB_P2P_REMOTE_ARGS_VERBATIM, {}, id="p2p-on_demand-verbatim"
+    ),
+]
+
+
+@pytest.mark.parametrize(
+    "policy, is_verbatim, publication_args, remote_args_new, expected_value", pulp_to_pulp_test_data
+)
+def test_pulp_to_pulp_sync(
+    deb_get_content_summary,
+    deb_init_and_sync,
+    deb_distribution_factory,
+    is_verbatim,
+    policy,
+    publication_args,
+    request,
+    remote_args_new,
+    expected_value,
+    delete_orphans_pre,
+):
+    """Verify whether content served by Pulp can be synced.
+
+    The initial sync to Pulp is one of many different download policies, the second sync is
+    immediate in order to exercise downloading all of the files.
+
+    Multiple test cases are covered here which are a combination of different policies and
+    different publish methods:
+
+    * `Test policy=immediate and publish=simple`
+    * `Test policy=immediate and publish=structured`
+    * `Test policy=immediate and publish=simple+structured`
+    * `Test policy=immediate and publish=verbatim`
+    * `Test policy=streamed and publish=simple`
+    * `Test policy=streamed and publish=structured`
+    * `Test policy=streamed and publish=simple+structured`
+    * `Test policy=streamed and publish=verbatim`
+    * `Test policy=on_demand and publish=simple`
+    * `Test policy=on_demand and publish=structured`
+    * `Test policy=on_demand and publish=simple+structured`
+    * `Test policy=on_demand and publish=verbatim`
+    """
+    remote_args = {"policy": policy}
+    publication_factory = (
+        request.getfixturevalue("deb_verbatim_publication_factory")
+        if is_verbatim
+        else request.getfixturevalue("deb_publication_factory")
+    )
+
+    # Create and sync a repository. Then publish and distribute it.
+    repo, _ = deb_init_and_sync(remote_args=remote_args)
+    publication = publication_factory(repo, **publication_args)
+    distribution = deb_distribution_factory(publication)
+
+    # Create and sync a new repository with the published repo as the remote
+    repo_new, _ = deb_init_and_sync(url=distribution.base_url, remote_args=remote_args_new)
+
+    # Assert whether the present content matches the expected data
+    summary_new = deb_get_content_summary(repo_new)
+    summary_orig = deb_get_content_summary(repo)
+    present_new = get_counts_from_content_summary(summary_new.present)
+    if is_verbatim:
+        present_orig = get_counts_from_content_summary(summary_orig.present)
+        assert present_orig == present_new
+    else:
+        assert present_new == expected_value
+
+    # Assert whether the added content matches the expected data
+    added_new = get_counts_from_content_summary(summary_new.added)
+    if is_verbatim:
+        added_orig = get_counts_from_content_summary(summary_orig.added)
+        assert added_orig == added_new
+    else:
+        assert added_new == expected_value
+
+
+def test_pulp_to_pulp_sync_simple_to_structured(
+    deb_init_and_sync,
+    deb_publication_factory,
+    deb_distribution_factory,
+    deb_get_content_summary,
+    delete_orphans_pre,
+):
+    """Verify whether a repository served by Pulp can sync its simple content first
+    and in a concurrent sync simple and structured content without loss of data.
+    """
+    # Create and sync a repository. Then publish and distribute it.
+    repo, _ = deb_init_and_sync(remote_args={"policy": "immediate"})
+    publication = deb_publication_factory(repo, **DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED)
+    distribution = deb_distribution_factory(publication)
+
+    # Create and sync a new repository with published repo as the remote and simple distribution
+    repo_simple, _ = deb_init_and_sync(
+        url=distribution.base_url, remote_args=DEB_P2P_REMOTE_ARGS_SIMPLE
+    )
+    summary_simple = deb_get_content_summary(repo_simple)
+    present_simple = get_counts_from_content_summary(summary_simple.present)
+    added_simple = get_counts_from_content_summary(summary_simple.added)
+    assert present_simple == DEB_P2P_ONLY_SIMPLE
+    assert added_simple == DEB_P2P_ONLY_SIMPLE
+
+    # Use the same respository and sync again with published repo as remote but both distributions
+    repo_both, _ = deb_init_and_sync(
+        repository=repo_simple, url=distribution.base_url, remote_args=DEB_P2P_REMOTE_ARGS_BOTH
+    )
+    summary_both = deb_get_content_summary(repo_both)
+    present_both = get_counts_from_content_summary(summary_both.present)
+    added_both = get_counts_from_content_summary(summary_both.added)
+    assert added_both == DEB_P2P_SIMPLE_THEN_STRUCTURED
+    assert present_both == DEB_P2P_SIMPLE_AND_STRUCTURED

pulp_deb/tests/functional/conftest.py

@@ -603,7 +603,12 @@ def _deb_init_and_sync(
         :param return_task: Whether to include the sync task to the return value. Default: False.
         :returns: A tuple containing the repository and remote and optionally the sync task.
         """
-        url = deb_get_fixture_server_url() if url is None else deb_get_fixture_server_url(url)
+        if url is None:
+            url = deb_get_fixture_server_url()
+        elif url.startswith("http"):
+            url = url
+        else:
+            url = deb_get_fixture_server_url(url)
         if repository is None:
             repository = deb_repository_factory(**repo_args)
         if remote is None:

pulp_deb/tests/functional/constants.py

@@ -46,6 +46,11 @@ def _clean_dict(d):
 DEB_PUBLICATION_ARGS_SIMPLE_AND_STRUCTURED = {"simple": True, "structured": True}
 DEB_PUBLICATION_ARGS_ALL = {"simple": True, "structured": True, "signing_service": ""}
 
+DEB_P2P_REMOTE_ARGS_SIMPLE = {"distributions": "default", "policy": "immediate"}
+DEB_P2P_REMOTE_ARGS_STRUCTURED = {"distributions": "ragnarok nosuite", "policy": "immediate"}
+DEB_P2P_REMOTE_ARGS_BOTH = {"distributions": "ragnarok nosuite default", "policy": "immediate"}
+DEB_P2P_REMOTE_ARGS_VERBATIM = {"distributions": "ragnarok nosuite", "policy": "immediate"}
+
 DEB_FIXTURE_SUMMARY = _clean_dict(
     {
         DEB_RELEASE_NAME: 2,
@@ -130,6 +135,66 @@ def _clean_dict(d):
     }
 )
 
+DEB_P2P_ONLY_SIMPLE = _clean_dict(
+    {
+        DEB_RELEASE_NAME: 1,
+        DEB_RELEASE_ARCHITECTURE_NAME: 3,
+        DEB_RELEASE_COMPONENT_NAME: 1,
+        DEB_RELEASE_FILE_NAME: 1,
+        DEB_PACKAGE_INDEX_NAME: 3,
+        DEB_PACKAGE_RELEASE_COMPONENT_NAME: 4,
+        DEB_INSTALLER_FILE_INDEX_NAME: 0,
+        DEB_PACKAGE_NAME: 4,
+        DEB_INSTALLER_PACKAGE_NAME: 0,
+        DEB_GENERIC_CONTENT_NAME: 0,
+    }
+)
+
+DEB_P2P_ONLY_STRUCTURED = _clean_dict(
+    {
+        DEB_RELEASE_NAME: 2,
+        DEB_RELEASE_ARCHITECTURE_NAME: 5,
+        DEB_RELEASE_COMPONENT_NAME: 3,
+        DEB_RELEASE_FILE_NAME: 2,
+        DEB_PACKAGE_INDEX_NAME: 8,
+        DEB_PACKAGE_RELEASE_COMPONENT_NAME: 7,
+        DEB_INSTALLER_FILE_INDEX_NAME: 0,
+        DEB_PACKAGE_NAME: 4,
+        DEB_INSTALLER_PACKAGE_NAME: 0,
+        DEB_GENERIC_CONTENT_NAME: 0,
+    }
+)
+
+DEB_P2P_SIMPLE_AND_STRUCTURED = _clean_dict(
+    {
+        DEB_RELEASE_NAME: 3,
+        DEB_RELEASE_ARCHITECTURE_NAME: 8,
+        DEB_RELEASE_COMPONENT_NAME: 4,
+        DEB_RELEASE_FILE_NAME: 3,
+        DEB_PACKAGE_INDEX_NAME: 11,
+        DEB_PACKAGE_RELEASE_COMPONENT_NAME: 11,
+        DEB_INSTALLER_FILE_INDEX_NAME: 0,
+        DEB_PACKAGE_NAME: 8,
+        DEB_INSTALLER_PACKAGE_NAME: 0,
+        DEB_GENERIC_CONTENT_NAME: 0,
+    }
+)
+
+DEB_P2P_SIMPLE_THEN_STRUCTURED = _clean_dict(
+    {
+        DEB_RELEASE_NAME: 2,
+        DEB_RELEASE_ARCHITECTURE_NAME: 5,
+        DEB_RELEASE_COMPONENT_NAME: 3,
+        DEB_RELEASE_FILE_NAME: 2,
+        DEB_PACKAGE_INDEX_NAME: 8,
+        DEB_PACKAGE_RELEASE_COMPONENT_NAME: 7,
+        DEB_INSTALLER_FILE_INDEX_NAME: 0,
+        DEB_PACKAGE_NAME: 4,
+        DEB_INSTALLER_PACKAGE_NAME: 0,
+        DEB_GENERIC_CONTENT_NAME: 0,
+    }
+)
+
 DEB_FIXTURE_PACKAGE_COUNT = DEB_FIXTURE_SUMMARY.get(DEB_PACKAGE_NAME, 0)
 
 DEB_REPORT_CODE_SKIP_RELEASE = "sync.release_file.was_skipped"