Skip to content

Commit

Permalink
Skip certain tests if necessary and mark some flaky ones (#657)
Browse files Browse the repository at this point in the history
* Small alignments with upstream

* Skip tests if necessary and mark some flaky ones

* Add extra flaky test
  • Loading branch information
meaksh authored Jun 12, 2024
1 parent e380917 commit c02adf9
Show file tree
Hide file tree
Showing 15 changed files with 67 additions and 14 deletions.
1 change: 1 addition & 0 deletions tests/integration/modules/test_status.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ def test_status_saltmem(self):
self.assertTrue(isinstance(ret, int))

@pytest.mark.slow_test
@pytest.mark.flaky(max_runs=4)
def test_status_diskusage(self):
"""
status.diskusage
Expand Down
7 changes: 1 addition & 6 deletions tests/pytests/functional/cache/test_consul.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,13 @@

import salt.cache
import salt.loader
from salt.utils.versions import Version
from tests.pytests.functional.cache.helpers import run_common_cache_tests

pytest.importorskip(
"consul",
reason="Please install python-consul package to use consul data cache driver",
)
docker = pytest.importorskip("docker")
docker = pytest.importorskip("docker", minversion="4.0.0")

log = logging.getLogger(__name__)

Expand All @@ -25,10 +24,6 @@
pytest.mark.slow_test,
pytest.mark.skip_if_binaries_missing("dockerd"),
pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
pytest.mark.skipif(
Version(docker.__version__) < Version("4.0.0"),
reason="Test does not work in this version of docker-py",
),
]


Expand Down
6 changes: 3 additions & 3 deletions tests/pytests/functional/cache/test_mysql.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@

import salt.cache
import salt.loader
import salt.modules.mysql
from salt.utils.versions import Version
from tests.pytests.functional.cache.helpers import run_common_cache_tests
from tests.support.pytest.mysql import * # pylint: disable=wildcard-import,unused-wildcard-import

docker = pytest.importorskip("docker")
pytest.importorskip("docker", minversion="4.0.0")

log = logging.getLogger(__name__)

Expand All @@ -20,8 +21,7 @@
pytest.mark.skip_if_binaries_missing("dockerd"),
pytest.mark.skipif(INSIDE_CONTAINER, reason="Cannot run in a container"),
pytest.mark.skipif(
Version(docker.__version__) < Version("4.0.0"),
reason="Test does not work in this version of docker-py",
not salt.modules.mysql.MySQLdb, reason="Missing python MySQLdb library"
),
]

Expand Down
3 changes: 3 additions & 0 deletions tests/pytests/functional/modules/state/test_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -714,6 +714,7 @@ def test_retry_option_success(state, state_tree, tmp_path):
@pytest.mark.skip_on_windows(
reason="Skipped until parallel states can be fixed on Windows"
)
@pytest.mark.xfail(reason="This test is flaky")
def test_retry_option_success_parallel(state, state_tree, tmp_path):
"""
test a state with the retry option that should return True immediately (i.e. no retries)
Expand Down Expand Up @@ -753,6 +754,7 @@ def test_retry_option_success_parallel(state, state_tree, tmp_path):


@pytest.mark.slow_test
@pytest.mark.xfail(reason="This test is flaky")
def test_retry_option_eventual_success(state, state_tree, tmp_path):
"""
test a state with the retry option that should return True, eventually
Expand Down Expand Up @@ -801,6 +803,7 @@ def create_testfile(testfile1, testfile2):
reason="Skipped until parallel states can be fixed on Windows"
)
@pytest.mark.slow_test
@pytest.mark.xfail(reason="This test is flaky")
def test_retry_option_eventual_success_parallel(state, state_tree, tmp_path):
"""
test a state with the retry option that should return True, eventually
Expand Down
8 changes: 8 additions & 0 deletions tests/pytests/functional/modules/test_pkg.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,10 @@ def test_which(modules):
@pytest.mark.requires_salt_modules("pkg.version", "pkg.install", "pkg.remove")
@pytest.mark.slow_test
@pytest.mark.requires_network
@pytest.mark.skipif(
bool(salt.utils.path.which("transactional-update")),
reason="Skipping on transactional systems",
)
def test_install_remove(modules, test_pkg, refresh_db):
"""
successfully install and uninstall a package
Expand Down Expand Up @@ -272,6 +276,10 @@ def test_remove():
@pytest.mark.slow_test
@pytest.mark.requires_network
@pytest.mark.requires_salt_states("pkg.installed")
@pytest.mark.skipif(
bool(salt.utils.path.which("transactional-update")),
reason="Skipping on transactional systems",
)
def test_hold_unhold(grains, modules, states, test_pkg, refresh_db):
"""
test holding and unholding a package
Expand Down
5 changes: 5 additions & 0 deletions tests/pytests/functional/modules/test_virtualenv_mod.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import pytest

import salt.utils.path
from salt.modules.virtualenv_mod import KNOWN_BINARY_NAMES

pytestmark = [
Expand Down Expand Up @@ -63,6 +64,10 @@ def test_clear(virtualenv, venv_dir, modules):
assert "pep8" not in packages


@pytest.mark.skipif(
bool(salt.utils.path.which("transactional-update")),
reason="Skipping on transactional systems",
)
def test_virtualenv_ver(virtualenv, venv_dir):
ret = virtualenv.create(str(venv_dir))
assert ret
Expand Down
2 changes: 1 addition & 1 deletion tests/pytests/functional/states/file/test_managed.py
Original file line number Diff line number Diff line change
Expand Up @@ -658,7 +658,6 @@ def test_issue_8947_utf8_sls(modules, tmp_path, state_tree, subtests):
@pytest.mark.skip_if_not_root
@pytest.mark.skip_on_windows(reason="Windows does not support setuid. Skipping.")
def test_owner_after_setuid(file, modules, tmp_path, state_file_account):

"""
Test to check file user/group after setting setuid or setgid.
Because Python os.chown() does reset the setuid/setgid to 0.
Expand Down Expand Up @@ -767,6 +766,7 @@ def test_file_managed_keep_source_false_http(


@pytest.mark.parametrize("verify_ssl", [True, False])
@pytest.mark.flaky(max_runs=4)
def test_verify_ssl_https_source(file, tmp_path, ssl_webserver, verify_ssl):
"""
test verify_ssl when its False and True when managing
Expand Down
10 changes: 10 additions & 0 deletions tests/pytests/functional/states/pkgrepo/test_suse.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import pytest

import salt.utils.path

pytestmark = [
pytest.mark.destructive_test,
pytest.mark.skip_if_not_root,
Expand Down Expand Up @@ -80,6 +82,10 @@ def suse_state_tree(grains, pkgrepo, state_tree):


@pytest.mark.requires_salt_states("pkgrepo.managed", "pkgrepo.absent")
@pytest.mark.skipif(
bool(salt.utils.path.which("transactional-update")),
reason="Skipping on transactional systems",
)
def test_pkgrepo_managed_absent(grains, modules, subtests, suse_state_tree):
"""
Test adding and removing a repository
Expand Down Expand Up @@ -134,6 +140,10 @@ def _run(name, test=False):


@pytest.mark.requires_salt_states("pkgrepo.managed")
@pytest.mark.skipif(
bool(salt.utils.path.which("transactional-update")),
reason="Skipping on transactional systems",
)
def test_pkgrepo_managed_modify(grains, modules, subtests, suse_state_tree):
"""
Test adding and modifying a repository
Expand Down
7 changes: 7 additions & 0 deletions tests/pytests/functional/states/test_docker_container.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
pytest.mark.slow_test,
pytest.mark.skip_on_freebsd(reason="No Docker on FreeBSD available"),
pytest.mark.skip_if_binaries_missing("busybox", reason="Busybox not installed"),
pytest.mark.skip_if_binaries_missing("ldd", reason="ldd is missing"),
pytest.mark.skip_if_binaries_missing(
"docker", "dockerd", reason="Docker not installed"
),
Expand Down Expand Up @@ -172,6 +173,12 @@ def image(tmp_path_factory):
# Somehow the above skip_if_binaries_missing marker for docker
# only get's evaluated after this fixture?!?
pytest.skip("The `docker` binary is not available")
if not salt.modules.cmdmod.retcode(
"ldd {}".format(salt.utils.path.which("busybox"))
):
pytest.skip(
"`busybox` appears to be a dynamic executable, please use busybox-static"
)
container_build_dir = tmp_path_factory.mktemp("busybox")
image_name = random_string("salt-busybox-", uppercase=False)

Expand Down
5 changes: 5 additions & 0 deletions tests/pytests/functional/states/test_npm.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,18 @@

import pytest

import salt.utils.path
from salt.exceptions import CommandExecutionError
from salt.utils.versions import Version

pytestmark = [
pytest.mark.slow_test,
pytest.mark.destructive_test,
pytest.mark.requires_network,
pytest.mark.skipif(
bool(salt.utils.path.which("transactional-update")),
reason="Skipping on transactional systems",
),
]

MAX_NPM_VERSION = "5.0.0"
Expand Down
4 changes: 4 additions & 0 deletions tests/pytests/functional/states/test_pip_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ def run_command(path, **kwargs):


@pytest.mark.slow_test
@pytest.mark.skipif(
bool(salt.utils.path.which("transactional-update")),
reason="Skipping on transactional systems",
)
def test_pip_installed_removed(modules, states):
"""
Tests installed and removed states
Expand Down
11 changes: 10 additions & 1 deletion tests/pytests/functional/states/test_user.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from saltfactories.utils import random_string

import salt.utils.files
import salt.utils.path
import salt.utils.platform

try:
Expand Down Expand Up @@ -137,7 +138,7 @@ def test_user_present_nondefault(grains, modules, states, username, user_home):
if not salt.utils.platform.is_darwin() and not salt.utils.platform.is_windows():
assert user_home.is_dir()

if grains["os_family"] in ("Suse",):
if grains["os_family"] in ("Suse",) and not grains.get("transactional", False):
expected_group_name = "users"
elif grains["os_family"] == "MacOS":
expected_group_name = "staff"
Expand Down Expand Up @@ -380,6 +381,10 @@ def test_user_present_existing(states, username):


@pytest.mark.skip_unless_on_linux(reason="underlying functionality only runs on Linux")
@pytest.mark.skipif(
bool(salt.utils.path.which("transactional-update")),
reason="Skipping on transactional systems",
)
def test_user_present_change_groups(modules, states, username, group_1, group_2):
ret = states.user.present(
name=username,
Expand All @@ -404,6 +409,10 @@ def test_user_present_change_groups(modules, states, username, group_1, group_2)


@pytest.mark.skip_unless_on_linux(reason="underlying functionality only runs on Linux")
@pytest.mark.skipif(
bool(salt.utils.path.which("transactional-update")),
reason="Skipping on transactional systems",
)
def test_user_present_change_optional_groups(
modules, states, username, group_1, group_2
):
Expand Down
2 changes: 0 additions & 2 deletions tests/pytests/integration/cli/test_syndic_eauth.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@

import pytest

from tests.conftest import CODE_DIR

docker = pytest.importorskip("docker", minversion="4.0.0")

INSIDE_CONTAINER = os.getenv("HOSTNAME", "") == "salt-test-container"
Expand Down
2 changes: 1 addition & 1 deletion tests/pytests/integration/daemons/test_memory_leak.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ def file_add_delete_sls(testfile_path, base_env_state_tree_root_dir):

@pytest.mark.skip_on_darwin(reason="MacOS is a spawning platform, won't work")
@pytest.mark.skipif(GITHUB_ACTIONS, reason="Test is failing in GitHub Actions")
@pytest.mark.flaky(max_runs=10)
@pytest.mark.xfail(reason="This test is flaky")
def test_memory_leak(salt_cli, salt_minion, file_add_delete_sls):
max_usg = None

Expand Down
8 changes: 8 additions & 0 deletions tests/support/pytest/mysql.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,19 @@
import pytest
from saltfactories.utils import random_string

import salt.modules.mysql

# This `pytest.importorskip` here actually works because this module
# is imported into test modules, otherwise, the skipping would just fail
pytest.importorskip("docker")
import docker.errors # isort:skip pylint: disable=3rd-party-module-not-gated

pytestmark = [
pytest.mark.skipif(
not salt.modules.mysql.MySQLdb, reason="Missing python MySQLdb library"
)
]

log = logging.getLogger(__name__)


Expand Down

0 comments on commit c02adf9

Please sign in to comment.