From 50020224662a89759386ce03310af152e30c5eaa Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Wed, 20 Nov 2024 00:07:55 +0200 Subject: [PATCH] feature(dependabot): scan docker based loaders for updates since dependabot is expecting very specific format of yaml files it can find and update: * name starts with `values_` * docker image need to be under `image` key since we use same dockerhub repo for most of the loader images and dependabot only handle update of one of them, we had to split it to multiple files that would be scan seperatly. `sct_config` would collect all of those back into the configuration in the same structure it was until this change --- .github/dependabot.yml | 24 +++++++++++++++++++ .../alternator-dns/values_alternator-dns.yaml | 2 ++ .../values_cassandra-stress.yaml | 2 ++ .../cdc-stresser/values_cdc-stresser.yaml | 2 ++ .../values_cql-stress-cassandra-stress.yaml | 2 ++ .../docker_images/gemini/values_gemini.yaml | 2 ++ .../docker_images/harry/values_harry.yaml | 2 ++ defaults/docker_images/kcl/values_kcl.yaml | 2 ++ .../docker_images/latte/values_latte.yaml | 2 ++ .../docker_images/ndbench/values_ndbench.yaml | 2 ++ .../nosqlbench/values_nosqlbench.yaml | 2 ++ .../scylla-bench/values_scylla-bench.yaml | 2 ++ defaults/docker_images/ycsb/values_ycsb.yaml | 2 ++ defaults/test_default.yaml | 16 +++---------- sdcm/sct_config.py | 14 +++++++++++ 15 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 defaults/docker_images/alternator-dns/values_alternator-dns.yaml create mode 100644 defaults/docker_images/cassandra-stress/values_cassandra-stress.yaml create mode 100644 defaults/docker_images/cdc-stresser/values_cdc-stresser.yaml create mode 100644 defaults/docker_images/cql-stress-cassandra-stress/values_cql-stress-cassandra-stress.yaml create mode 100644 defaults/docker_images/gemini/values_gemini.yaml create mode 100644 defaults/docker_images/harry/values_harry.yaml create mode 100644 defaults/docker_images/kcl/values_kcl.yaml create mode 100644 defaults/docker_images/latte/values_latte.yaml create mode 100644 defaults/docker_images/ndbench/values_ndbench.yaml create mode 100644 defaults/docker_images/nosqlbench/values_nosqlbench.yaml create mode 100644 defaults/docker_images/scylla-bench/values_scylla-bench.yaml create mode 100644 defaults/docker_images/ycsb/values_ycsb.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 90366ccc61..4865cbdecf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,3 +14,27 @@ updates: - "scylladb/qa-maintainers" assignees: - "scylladb/qa-maintainers" + + - package-ecosystem: "docker" + directories: + - "/defaults/docker_images/alternator-dns" + - "/defaults/docker_images/cassandra-stress" + - "/defaults/docker_images/cdc-stresser" + - "/defaults/docker_images/cql-stress-cassandra-stress" + - "/defaults/docker_images/gemini" + - "/defaults/docker_images/harry" + - "/defaults/docker_images/kcl" + - "/defaults/docker_images/latte" + - "/defaults/docker_images/nosqlbench" + - "/defaults/docker_images/ndbench" + - "/defaults/docker_images/scylla-bench" + - "/defaults/docker_images/ycsb" + + schedule: + interval: "daily" + time: "03:00" + + reviewers: + - "scylladb/qa-maintainers" + assignees: + - "scylladb/qa-maintainers" diff --git a/defaults/docker_images/alternator-dns/values_alternator-dns.yaml b/defaults/docker_images/alternator-dns/values_alternator-dns.yaml new file mode 100644 index 0000000000..dbcbb429eb --- /dev/null +++ b/defaults/docker_images/alternator-dns/values_alternator-dns.yaml @@ -0,0 +1,2 @@ +alternator-dns: + image: 'scylladb/hydra-loaders:alternator-dns-0.1' diff --git a/defaults/docker_images/cassandra-stress/values_cassandra-stress.yaml b/defaults/docker_images/cassandra-stress/values_cassandra-stress.yaml new file mode 100644 index 0000000000..ee51141d91 --- /dev/null +++ b/defaults/docker_images/cassandra-stress/values_cassandra-stress.yaml @@ -0,0 +1,2 @@ +cassandra-stress: + image: 'scylladb/cassandra-stress:3.13.0' diff --git a/defaults/docker_images/cdc-stresser/values_cdc-stresser.yaml b/defaults/docker_images/cdc-stresser/values_cdc-stresser.yaml new file mode 100644 index 0000000000..44cc5768d9 --- /dev/null +++ b/defaults/docker_images/cdc-stresser/values_cdc-stresser.yaml @@ -0,0 +1,2 @@ +cdc-stresser: + image: 'scylladb/hydra-loaders:cdc-stresser-20210630' diff --git a/defaults/docker_images/cql-stress-cassandra-stress/values_cql-stress-cassandra-stress.yaml b/defaults/docker_images/cql-stress-cassandra-stress/values_cql-stress-cassandra-stress.yaml new file mode 100644 index 0000000000..1eca1c34e8 --- /dev/null +++ b/defaults/docker_images/cql-stress-cassandra-stress/values_cql-stress-cassandra-stress.yaml @@ -0,0 +1,2 @@ +cql-stress-cassandra-stress: + image: 'docker.io/scylladb/hydra-loaders:cql-stress-cassandra-stress-20240718' diff --git a/defaults/docker_images/gemini/values_gemini.yaml b/defaults/docker_images/gemini/values_gemini.yaml new file mode 100644 index 0000000000..24bbe81795 --- /dev/null +++ b/defaults/docker_images/gemini/values_gemini.yaml @@ -0,0 +1,2 @@ +gemini: + image: 'scylladb/hydra-loaders:gemini-v1.8.6' diff --git a/defaults/docker_images/harry/values_harry.yaml b/defaults/docker_images/harry/values_harry.yaml new file mode 100644 index 0000000000..6c416d68fa --- /dev/null +++ b/defaults/docker_images/harry/values_harry.yaml @@ -0,0 +1,2 @@ +harry: + image: 'scylladb/hydra-loaders:cassandra-harry-jdk11-20220816' diff --git a/defaults/docker_images/kcl/values_kcl.yaml b/defaults/docker_images/kcl/values_kcl.yaml new file mode 100644 index 0000000000..8376bfe3a9 --- /dev/null +++ b/defaults/docker_images/kcl/values_kcl.yaml @@ -0,0 +1,2 @@ +kcl: + image: 'scylladb/hydra-loaders:kcl-jdk8-20210526-ShardSyncStrategyType-PERIODIC' diff --git a/defaults/docker_images/latte/values_latte.yaml b/defaults/docker_images/latte/values_latte.yaml new file mode 100644 index 0000000000..d0aa260693 --- /dev/null +++ b/defaults/docker_images/latte/values_latte.yaml @@ -0,0 +1,2 @@ +latte: + image: 'scylladb/hydra-loaders:latte-0.28.1-scylladb' diff --git a/defaults/docker_images/ndbench/values_ndbench.yaml b/defaults/docker_images/ndbench/values_ndbench.yaml new file mode 100644 index 0000000000..a646028486 --- /dev/null +++ b/defaults/docker_images/ndbench/values_ndbench.yaml @@ -0,0 +1,2 @@ +ndbench: + image: 'scylladb/hydra-loaders:ndbench-jdk8-20210720' diff --git a/defaults/docker_images/nosqlbench/values_nosqlbench.yaml b/defaults/docker_images/nosqlbench/values_nosqlbench.yaml new file mode 100644 index 0000000000..75a77c96e4 --- /dev/null +++ b/defaults/docker_images/nosqlbench/values_nosqlbench.yaml @@ -0,0 +1,2 @@ +nosqlbench: + image: 'scylladb/hydra-loaders:nosqlbench-5.21.2' diff --git a/defaults/docker_images/scylla-bench/values_scylla-bench.yaml b/defaults/docker_images/scylla-bench/values_scylla-bench.yaml new file mode 100644 index 0000000000..5496ba7684 --- /dev/null +++ b/defaults/docker_images/scylla-bench/values_scylla-bench.yaml @@ -0,0 +1,2 @@ +scylla-bench: + image: 'scylladb/hydra-loaders:scylla-bench-v0.1.24' diff --git a/defaults/docker_images/ycsb/values_ycsb.yaml b/defaults/docker_images/ycsb/values_ycsb.yaml new file mode 100644 index 0000000000..1918fe36e8 --- /dev/null +++ b/defaults/docker_images/ycsb/values_ycsb.yaml @@ -0,0 +1,2 @@ +ycsb: + image: 'scylladb/hydra-loaders:ycsb-jdk8-20220918' diff --git a/defaults/test_default.yaml b/defaults/test_default.yaml index 1c4b42678b..58d4a64da3 100644 --- a/defaults/test_default.yaml +++ b/defaults/test_default.yaml @@ -205,19 +205,9 @@ aws_fallback_to_next_availability_zone: false enable_argus: true -stress_image: - ndbench: 'scylladb/hydra-loaders:ndbench-jdk8-20210720' - ycsb: 'scylladb/hydra-loaders:ycsb-jdk8-20220918' - nosqlbench: 'scylladb/hydra-loaders:nosqlbench-5.21.2' - cassandra-stress: 'scylladb/cassandra-stress:3.13.0' - scylla-bench: 'scylladb/hydra-loaders:scylla-bench-v0.1.24' - gemini: 'scylladb/hydra-loaders:gemini-v1.8.6' - alternator-dns: 'scylladb/hydra-loaders:alternator-dns-0.1' - cdc-stresser: 'scylladb/hydra-loaders:cdc-stresser-20210630' - kcl: 'scylladb/hydra-loaders:kcl-jdk8-20210526-ShardSyncStrategyType-PERIODIC' - harry: 'scylladb/hydra-loaders:cassandra-harry-jdk11-20220816' - latte: 'scylladb/hydra-loaders:latte-0.28.1-scylladb' - cql-stress-cassandra-stress: 'docker.io/scylladb/hydra-loaders:cql-stress-cassandra-stress-20240718' +# the default stress tools are now defined a separate file for each /default/docker_image/[tool]/values_[tool].yaml +# cause limitation dependabot have related to us putting all of them in same dockerhub repository +stress_image: {} service_level_shares: [1000] diff --git a/sdcm/sct_config.py b/sdcm/sct_config.py index 7a7e97d9d1..5f1bc2671b 100644 --- a/sdcm/sct_config.py +++ b/sdcm/sct_config.py @@ -1797,6 +1797,9 @@ def __init__(self): # noqa: PLR0912, PLR0914, PLR0915 backend_config_files += self.defaults_config_files[str(backend)] self.multi_region_params = self.per_provider_multi_region_params.get(str(backend), []) + # load docker images defaults + self.load_docker_images_defaults() + # 1) load the default backend config files files = anyconfig.load(list(backend_config_files)) anyconfig.merge(self, files) @@ -2103,6 +2106,17 @@ def __init__(self): # noqa: PLR0912, PLR0914, PLR0915 assert len(zero_nodes_num) == len( data_nodes_num), "Config of zero token nodes is not equal config of data nodes for multi dc" + def load_docker_images_defaults(self): + docker_images_dir = pathlib.Path(sct_abs_path('defaults/docker_images')) + if docker_images_dir.is_dir(): + yaml_files = [] + for root, _, files in os.walk(docker_images_dir): + yaml_files.extend([os.path.join(root, f) for f in files if f.endswith('.yaml')]) + if yaml_files: + docker_images_defaults = anyconfig.load(yaml_files) + stress_image = {key: value.get('image') for key, value in docker_images_defaults.items()} + anyconfig.merge(self, dict(stress_image=stress_image)) + def log_config(self): self.log.info(self.dump_config())