From 257268527e5b4e705da49f636a75956bfbb1d8be Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Fri, 15 Nov 2024 10:14:28 +0100 Subject: [PATCH 01/14] fix: adds SSL_ENABLED flag to start scylla cluster in ssl only mode This adds SSL_ENABLED flag to Makefile, so that when you run SSL_ENABLED=true make start-dev-env the scylla cluster will be created with ssl_only config. --- Makefile | 4 +- README.md | 5 + testing/Makefile | 10 + testing/docker-compose.yaml | 24 - testing/scylla/config/scylla-ssl.yaml | 645 +++++++++++++++++++++++++- 5 files changed, 662 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 0029edbd0f..b026f936c9 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,8 @@ SCYLLA_VERSION?=scylla:6.0.1 IP_FAMILY?=IPV4 RAFT_SCHEMA?=none TABLETS?=enabled +# if true starts the scylla cluster with ssl only config +SSL_ENABLED?=false MANAGER_CONFIG := testing/scylla-manager/scylla-manager.yaml PUBLIC_NET := 192.168.200. @@ -171,7 +173,7 @@ start-dev-env: .testing-up deploy-agent build-cli .PHONY: .testing-up .testing-up: - @IPV6=$(IPV6) SCYLLA_VERSION=$(SCYLLA_VERSION) RAFT_SCHEMA=$(RAFT_SCHEMA) TABLETS=$(TABLETS) make -C testing build down up + @IPV6=$(IPV6) SCYLLA_VERSION=$(SCYLLA_VERSION) RAFT_SCHEMA=$(RAFT_SCHEMA) TABLETS=$(TABLETS) SSL_ENABLED=$(SSL_ENABLED) make -C testing build down up .PHONY: dev-env-status dev-env-status: ## Checks status of docker containers and cluster nodes diff --git a/README.md b/README.md index 2910e65e05..1a53ac98f7 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,11 @@ for IPv6 environment: IPV6=true make start-dev-env ``` +for SSL scylla cluster: +```bash +SSL_ENABLED=true make start-dev-env +``` + This command will: 1. Build custom Scylla Docker image (testing/scylla) 2. Compile server, agent and sctool binaries diff --git a/testing/Makefile b/testing/Makefile index 3d9b888f3e..88370d20d8 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -62,8 +62,15 @@ up: @echo "==> Generating encryption files" @cd scylla/certs && ./generate.sh @echo "==> Generating Scylla configuration" + +ifeq ($(SSL_ENABLED),true) + @cp scylla/config/scylla-ssl.yaml scylla/scylla.yaml + @cp scylla/config/cqlshrc-ssl scylla/cqlshrc +else @cp scylla/config/scylla.yaml scylla/scylla.yaml @cp scylla/config/cqlshrc scylla/cqlshrc +endif + ifeq ($(RAFT_SCHEMA),enabled) @$(YQ) write -i scylla/scylla.yaml 'consistent_cluster_management' true endif @@ -108,6 +115,9 @@ endif @until [ 1 -le $$($(SM_NODETOOL) status | grep -c "UN") ]; do echo -n "."; sleep 2; done ; echo "" @./nodes_exec "rm /root/.cqlshrc || true" + @./nodes_exec "mkdir -p /root/.cassandra" + @./nodes_cp "scylla/cqlshrc" "/root/.cassandra/cqlshrc" + @echo "==> Adding Minio user" ./minio/add_user.sh || true @echo "==> Initialising cluster" diff --git a/testing/docker-compose.yaml b/testing/docker-compose.yaml index 03acfd961f..d4c9530c07 100644 --- a/testing/docker-compose.yaml +++ b/testing/docker-compose.yaml @@ -12,9 +12,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -32,9 +29,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -52,9 +46,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs/ - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -72,9 +63,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -92,9 +80,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -112,9 +97,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -132,9 +114,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: @@ -152,9 +131,6 @@ services: - type: bind source: ./scylla/certs/ target: /etc/scylla/certs - - type: bind - source: ./scylla/cqlshrc - target: /root/.cassandra/cqlshrc networks: public: second: diff --git a/testing/scylla/config/scylla-ssl.yaml b/testing/scylla/config/scylla-ssl.yaml index b151d1a330..5fef33dc10 100644 --- a/testing/scylla/config/scylla-ssl.yaml +++ b/testing/scylla/config/scylla-ssl.yaml @@ -1,9 +1,652 @@ -# Scylla SSL enabled configuration. +# Scylla storage config YAML +####################################### +# This file is split to two sections: +# 1. Supported parameters +# 2. Unsupported parameters: reserved for future use or backwards +# compatibility. +# Scylla will only read and use the first segment +####################################### + +### Supported Parameters + +# The name of the cluster. This is mainly used to prevent machines in +# one logical cluster from joining another. +# It is recommended to change the default value when creating a new cluster. +# You can NOT modify this value for an existing cluster +cluster_name: 'Managed Cluster' + +# This defines the number of tokens randomly assigned to this node on the ring +# The more tokens, relative to other nodes, the larger the proportion of data +# that this node will store. You probably want all nodes to have the same number +# of tokens assuming they have equal hardware capability. +num_tokens: 256 + +# Directory where Scylla should store all its files, which are commitlog, +# data, hints, view_hints and saved_caches subdirectories. All of these +# subs can be overridden by the respective options below. +# If unset, the value defaults to /var/lib/scylla +# workdir: /var/lib/scylla + +# Directory where Scylla should store data on disk. +data_file_directories: + - /var/lib/scylla/data + +# commit log. when running on magnetic HDD, this should be a +# separate spindle than the data directories. +commitlog_directory: /var/lib/scylla/commitlog + +# schema commit log. A special commitlog instance +# used for schema and system tables. +# When running on magnetic HDD, this should be a +# separate spindle than the data directories. +# schema_commitlog_directory: /var/lib/scylla/commitlog/schema + +# commitlog_sync may be either "periodic" or "batch." +# +# When in batch mode, Scylla won't ack writes until the commit log +# has been fsynced to disk. It will wait +# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. +# This window should be kept short because the writer threads will +# be unable to do extra work while waiting. (You may need to increase +# concurrent_writes for the same reason.) +# +# commitlog_sync: batch +# commitlog_sync_batch_window_in_ms: 2 +# +# the other option is "periodic" where writes may be acked immediately +# and the CommitLog is simply synced every commitlog_sync_period_in_ms +# milliseconds. +commitlog_sync: periodic +commitlog_sync_period_in_ms: 10000 + +# The size of the individual commitlog file segments. A commitlog +# segment may be archived, deleted, or recycled once all the data +# in it (potentially from each columnfamily in the system) has been +# flushed to sstables. +# +# The default size is 32, which is almost always fine, but if you are +# archiving commitlog segments (see commitlog_archiving.properties), +# then you probably want a finer granularity of archiving; 8 or 16 MB +# is reasonable. +commitlog_segment_size_in_mb: 32 + +# The size of the individual schema commitlog file segments. +# +# The default size is 128, which is 4 times larger than the default +# size of the data commitlog. It's because the segment size puts +# a limit on the mutation size that can be written at once, and some +# schema mutation writes are much larger than average. +schema_commitlog_segment_size_in_mb: 128 + +# seed_provider class_name is saved for future use. +# A seed address is mandatory. +seed_provider: + # The addresses of hosts that will serve as contact points for the joining node. + # It allows the node to discover the cluster ring topology on startup (when + # joining the cluster). + # Once the node has joined the cluster, the seed list has no function. + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + # In a new cluster, provide the address of the first node. + # In an existing cluster, specify the address of at least one existing node. + # If you specify addresses of more than one node, use a comma to separate them. + # For example: ",," + - seeds: "127.0.0.1" + +# Address to bind to and tell other Scylla nodes to connect to. +# You _must_ change this if you want multiple nodes to be able to communicate! +# +# If you leave broadcast_address (below) empty, then setting listen_address +# to 0.0.0.0 is wrong as other nodes will not know how to reach this node. +# If you set broadcast_address, then you can set listen_address to 0.0.0.0. +listen_address: localhost + +# Address to broadcast to other Scylla nodes +# Leaving this blank will set it to the same value as listen_address +# broadcast_address: 1.2.3.4 + + +# When using multiple physical network interfaces, set this to true to listen on broadcast_address +# in addition to the listen_address, allowing nodes to communicate in both interfaces. +# Ignore this property if the network configuration automatically routes between the public and private networks such as EC2. +# +# listen_on_broadcast_address: false + +# port for the CQL native transport to listen for clients on +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# To disable the CQL native transport, remove this option and configure native_transport_port_ssl. +#native_transport_port: 9042 + +# Like native_transport_port, but clients are forwarded to specific shards, based on the +# client-side port numbers. +native_shard_aware_transport_port: 19042 + +# Enabling native transport encryption in client_encryption_options allows you to either use +# encryption for the standard port or to use a dedicated, additional port along with the unencrypted +# standard native_transport_port. +# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption +# for native_transport_port. Setting native_transport_port_ssl to a different value +# from native_transport_port will use encryption for native_transport_port_ssl while +# keeping native_transport_port unencrypted. native_transport_port_ssl: 9142 + +# Like native_transport_port_ssl, but clients are forwarded to specific shards, based on the +# client-side port numbers. +#native_shard_aware_transport_port_ssl: 19142 + +# How long the coordinator should wait for read operations to complete +read_request_timeout_in_ms: 5000 + +# How long the coordinator should wait for writes to complete +write_request_timeout_in_ms: 2000 +# how long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +cas_contention_timeout_in_ms: 1000 + +# phi value that must be reached for a host to be marked down. +# most users should never need to adjust this. +# phi_convict_threshold: 8 + +# IEndpointSnitch. The snitch has two functions: +# - it teaches Scylla enough about your network topology to route +# requests efficiently +# - it allows Scylla to spread replicas around your cluster to avoid +# correlated failures. It does this by grouping machines into +# "datacenters" and "racks." Scylla will do its best not to have +# more than one replica on the same "rack" (which may not actually +# be a physical location) +# +# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER, +# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS +# ARE PLACED. +# +# Out of the box, Scylla provides +# - SimpleSnitch: +# Treats Strategy order as proximity. This can improve cache +# locality when disabling read repair. Only appropriate for +# single-datacenter deployments. +# - GossipingPropertyFileSnitch +# This should be your go-to snitch for production use. The rack +# and datacenter for the local node are defined in +# cassandra-rackdc.properties and propagated to other nodes via +# gossip. If cassandra-topology.properties exists, it is used as a +# fallback, allowing migration from the PropertyFileSnitch. +# - PropertyFileSnitch: +# Proximity is determined by rack and data center, which are +# explicitly configured in cassandra-topology.properties. +# - Ec2Snitch: +# Appropriate for EC2 deployments in a single Region. Loads Region +# and Availability Zone information from the EC2 API. The Region is +# treated as the datacenter, and the Availability Zone as the rack. +# Only private IPs are used, so this will not work across multiple +# Regions. +# - Ec2MultiRegionSnitch: +# Uses public IPs as broadcast_address to allow cross-region +# connectivity. (Thus, you should set seed addresses to the public +# IP as well.) You will need to open the storage_port or +# ssl_storage_port on the public IP firewall. (For intra-Region +# traffic, Scylla will switch to the private IP after +# establishing a connection.) +# - RackInferringSnitch: +# Proximity is determined by rack and data center, which are +# assumed to correspond to the 3rd and 2nd octet of each node's IP +# address, respectively. Unless this happens to match your +# deployment conventions, this is best used as an example of +# writing a custom Snitch class and is provided in that spirit. +# +# You can use a custom Snitch by setting this to the full class name +# of the snitch, which will be assumed to be on your classpath. +endpoint_snitch: GossipingPropertyFileSnitch + +# The address or interface to bind the native transport server to. +# +# Set rpc_address OR rpc_interface, not both. Interfaces must correspond +# to a single address, IP aliasing is not supported. +# +# Leaving rpc_address blank has the same effect as on listen_address +# (i.e. it will be based on the configured hostname of the node). +# +# Note that unlike listen_address, you can specify 0.0.0.0, but you must also +# set broadcast_rpc_address to a value other than 0.0.0.0. +# +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# +# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address +# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 +# address will be used. If true the first ipv6 address will be used. Defaults to false preferring +# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. +rpc_address: localhost +# rpc_interface: eth1 +# rpc_interface_prefer_ipv6: false + +# port for Thrift to listen for clients on +rpc_port: 9160 + +# port for REST API server +api_port: 10000 + +# IP for the REST API server +api_address: 127.0.0.1 + +# Log WARN on any batch size exceeding this value. 128 kiB per batch by default. +# Caution should be taken on increasing the size of this threshold as it can lead to node instability. +batch_size_warn_threshold_in_kb: 128 + +# Fail any multiple-partition batch exceeding this value. 1 MiB (8x warn threshold) by default. +batch_size_fail_threshold_in_kb: 1024 + + # Authentication backend, identifying users + # Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthenticator, + # PasswordAuthenticator}. + # + # - AllowAllAuthenticator performs no checks - set it to disable authentication. + # - PasswordAuthenticator relies on username/password pairs to authenticate + # users. It keeps usernames and hashed passwords in system_auth.credentials table. + # Please increase system_auth keyspace replication factor if you use this authenticator. + # - com.scylladb.auth.TransitionalAuthenticator requires username/password pair + # to authenticate in the same manner as PasswordAuthenticator, but improper credentials + # result in being logged in as an anonymous user. Use for upgrading clusters' auth. +authenticator: PasswordAuthenticator + + # Authorization backend, implementing IAuthorizer; used to limit access/provide permissions + # Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthorizer, + # CassandraAuthorizer}. + # + # - AllowAllAuthorizer allows any action to any user - set it to disable authorization. + # - CassandraAuthorizer stores permissions in system_auth.permissions table. Please + # increase system_auth keyspace replication factor if you use this authorizer. + # - com.scylladb.auth.TransitionalAuthorizer wraps around the CassandraAuthorizer, using it for + # authorizing permission management. Otherwise, it allows all. Use for upgrading + # clusters' auth. +authorizer: CassandraAuthorizer + + # initial_token allows you to specify tokens manually. While you can use # it with + # vnodes (num_tokens > 1, above) -- in which case you should provide a + # comma-separated list -- it's primarily used when adding nodes # to legacy clusters + # that do not have vnodes enabled. + # initial_token: + + # RPC address to broadcast to drivers and other Scylla nodes. This cannot + # be set to 0.0.0.0. If left blank, this will be set to the value of + # rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must + # be set. + # broadcast_rpc_address: 1.2.3.4 + + # Uncomment to enable experimental features + # experimental_features: + # - udf + # - alternator-streams + # - broadcast-tables + # - keyspace-storage-options + + # The directory where hints files are stored if hinted handoff is enabled. + # hints_directory: /var/lib/scylla/hints + +# The directory where hints files are stored for materialized-view updates +# view_hints_directory: /var/lib/scylla/view_hints + +# See https://docs.scylladb.com/architecture/anti-entropy/hinted-handoff +# May either be "true" or "false" to enable globally, or contain a list +# of data centers to enable per-datacenter. +# hinted_handoff_enabled: DC1,DC2 +# hinted_handoff_enabled: true + +# this defines the maximum amount of time a dead host will have hints +# generated. After it has been dead this long, new hints for it will not be +# created until it has been seen alive and gone down again. +# max_hint_window_in_ms: 10800000 # 3 hours + + +# Validity period for permissions cache (fetching permissions can be an +# expensive operation depending on the authorizer, CassandraAuthorizer is +# one example). Defaults to 10000, set to 0 to disable. +# Will be disabled automatically for AllowAllAuthorizer. +# permissions_validity_in_ms: 10000 + +# Refresh interval for permissions cache (if enabled). +# After this interval, cache entries become eligible for refresh. Upon next +# access, an async reload is scheduled and the old value returned until it +# completes. If permissions_validity_in_ms is non-zero, then this also must have +# a non-zero value. Defaults to 2000. It's recommended to set this value to +# be at least 3 times smaller than the permissions_validity_in_ms. +# permissions_update_interval_in_ms: 2000 + +# The partitioner is responsible for distributing groups of rows (by +# partition key) across nodes in the cluster. You should leave this +# alone for new clusters. The partitioner can NOT be changed without +# reloading all data, so when upgrading you should set this to the +# same partitioner you were already using. +# +# Murmur3Partitioner is currently the only supported partitioner, +# +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +# Total space to use for commitlogs. +# +# If space gets above this value (it will round up to the next nearest +# segment multiple), Scylla will flush every dirty CF in the oldest +# segment and remove it. So a small total commitlog space will tend +# to cause more flush activity on less-active columnfamilies. +# +# A value of -1 (default) will automatically equate it to the total amount of memory +# available for Scylla. +commitlog_total_space_in_mb: -1 + +# TCP port, for commands and data +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# storage_port: 7000 + +# SSL port, for encrypted communication. Unused unless enabled in +# encryption_options +# For security reasons, you should not expose this port to the internet. Firewall it if needed. +# ssl_storage_port: 7001 + +# listen_interface: eth0 +# listen_interface_prefer_ipv6: false + +# Whether to start the native transport server. +# Please note that the address on which the native transport is bound is the +# same as the rpc_address. The port however is different and specified below. +# start_native_transport: true + +# The maximum size of allowed frame. Frame (requests) larger than this will +# be rejected as invalid. The default is 256MB. +# native_transport_max_frame_size_in_mb: 256 + +# Whether to start the thrift rpc server. +# start_rpc: true + +# enable or disable keepalive on rpc/native connections +# rpc_keepalive: true + +# Set to true to have Scylla create a hard link to each sstable +# flushed or streamed locally in a backups/ subdirectory of the +# keyspace data. Removing these links is the operator's +# responsibility. +# incremental_backups: false + +# Whether or not to take a snapshot before each compaction. Be +# careful using this option, since Scylla won't clean up the +# snapshots for you. Mostly useful if you're paranoid when there +# is a data format change. +# snapshot_before_compaction: false + +# Whether or not a snapshot is taken of the data before keyspace truncation +# or dropping of column families. The STRONGLY advised default of true +# should be used to provide data safety. If you set this flag to false, you will +# lose data on truncation or drop. +auto_snapshot: false + +# When executing a scan, within or across a partition, we need to keep the +# tombstones seen in memory so we can return them to the coordinator, which +# will use them to make sure other replicas also know about the deleted rows. +# With workloads that generate a lot of tombstones, this can cause performance +# problems and even exhaust the server heap. +# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) +# Adjust the thresholds here if you understand the dangers and want to +# scan more tombstones anyway. These thresholds may also be adjusted at runtime +# using the StorageService mbean. +# tombstone_warn_threshold: 1000 +# tombstone_failure_threshold: 100000 + +# Granularity of the collation index of rows within a partition. +# Increase if your rows are large, or if you have a very large +# number of rows per partition. The competing goals are these: +# 1) a smaller granularity means more index entries are generated +# and looking up rows within the partition by collation column +# is faster +# 2) but, Scylla will keep the collation index in memory for hot +# rows (as part of the key cache), so a larger granularity means +# you can cache more hot rows +# column_index_size_in_kb: 64 + +# Auto-scaling of the promoted index prevents running out of memory +# when the promoted index grows too large (due to partitions with many rows +# vs. too small column_index_size_in_kb). When the serialized representation +# of the promoted index grows by this threshold, the desired block size +# for this partition (initialized to column_index_size_in_kb) +# is doubled, to decrease the sampling resolution by half. +# +# To disable promoted index auto-scaling, set the threshold to 0. +# column_index_auto_scale_threshold_in_kb: 10240 + +# Log a warning when writing partitions larger than this value +# compaction_large_partition_warning_threshold_mb: 1000 + +# Log a warning when writing rows larger than this value +# compaction_large_row_warning_threshold_mb: 10 + +# Log a warning when writing cells larger than this value +# compaction_large_cell_warning_threshold_mb: 1 + +# Log a warning when row number is larger than this value +# compaction_rows_count_warning_threshold: 100000 + +# Log a warning when writing a collection containing more elements than this value +# compaction_collection_elements_count_warning_threshold: 10000 + +# How long the coordinator should wait for seq or index scans to complete +# range_request_timeout_in_ms: 10000 +# How long the coordinator should wait for writes to complete +# counter_write_request_timeout_in_ms: 5000 +# How long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row +# cas_contention_timeout_in_ms: 1000 +# How long the coordinator should wait for truncates to complete +# (This can be much longer, because unless auto_snapshot is disabled +# we need to flush first so we can snapshot before removing the data.) +# truncate_request_timeout_in_ms: 60000 +# The default timeout for other, miscellaneous operations +# request_timeout_in_ms: 10000 + +# Enable or disable inter-node encryption. +# You must also generate keys and provide the appropriate key and trust store locations and passwords. +# +# The available internode options are : all, none, dc, rack +# If set to dc scylla will encrypt the traffic between the DCs +# If set to rack scylla will encrypt the traffic between the racks +# +# SSL/TLS algorithm and ciphers used can be controlled by +# the priority_string parameter. Info on priority string +# syntax and values is available at: +# https://gnutls.org/manual/html_node/Priority-Strings.html +# +# The require_client_auth parameter allows you to +# restrict access to service based on certificate +# validation. Client must provide a certificate +# accepted by the used trust store to connect. +# +# server_encryption_options: +# internode_encryption: none +# certificate: conf/scylla.crt +# keyfile: conf/scylla.key +# truststore: +# certficate_revocation_list: +# require_client_auth: False +# priority_string: + +# enable or disable client/server encryption. client_encryption_options: enabled: true certificate: /etc/scylla/db.crt keyfile: /etc/scylla/db.key truststore: /etc/scylla/ca.crt require_client_auth: true +# truststore: +# require_client_auth: False +# priority_string: + +# internode_compression controls whether traffic between nodes is +# compressed. +# can be: all - all traffic is compressed +# dc - traffic between different datacenters is compressed +# none - nothing is compressed. +# internode_compression: none + +# Enable or disable tcp_nodelay for inter-dc communication. +# Disabling it will result in larger (but fewer) network packets being sent, +# reducing overhead from the TCP protocol itself, at the cost of increasing +# latency if you block for cross-datacenter responses. +# inter_dc_tcp_nodelay: false + +# Relaxation of environment checks. +# +# Scylla places certain requirements on its environment. If these requirements are +# not met, performance and reliability can be degraded. +# +# These requirements include: +# - A filesystem with good support for asynchronous I/O (AIO). Currently, +# this means XFS. +# +# false: strict environment checks are in place; do not start if they are not met. +# true: relaxed environment checks; performance and reliability may degraade. +# +# developer_mode: false + + +# Idle-time background processing +# +# Scylla can perform certain jobs in the background while the system is otherwise idle, +# freeing processor resources when there is other work to be done. +# +# defragment_memory_on_idle: true +# +# prometheus port +# By default, Scylla opens prometheus API port on port 9180 +# setting the port to 0 will disable the prometheus API. +# prometheus_port: 9180 +# +# prometheus address +# Leaving this blank will set it to the same value as listen_address. +# This means that by default, Scylla listens to the prometheus API on the same +# listening address (and therefore network interface) used to listen for +# internal communication. If the monitoring node is not in this internal +# network, you can override prometheus_address explicitly - e.g., setting +# it to 0.0.0.0 to listen on all interfaces. +# prometheus_address: 1.2.3.4 + +# Distribution of data among cores (shards) within a node +# +# Scylla distributes data within a node among shards, using a round-robin +# strategy: +# [shard0] [shard1] ... [shardN-1] [shard0] [shard1] ... [shardN-1] ... +# +# Scylla versions 1.6 and below used just one repetition of the pattern; +# this interfered with data placement among nodes (vnodes). +# +# Scylla versions 1.7 and above use 4096 repetitions of the pattern; this +# provides for better data distribution. +# +# the value below is log (base 2) of the number of repetitions. +# +# Set to 0 to avoid rewriting all data when upgrading from Scylla 1.6 and +# below. +# +# Keep at 12 for new clusters. +murmur3_partitioner_ignore_msb_bits: 12 + +# Bypass in-memory data cache (the row cache) when performing reversed queries. +# reversed_reads_auto_bypass_cache: false + +# Use a new optimized algorithm for performing reversed reads. +# Set to `false` to fall-back to the old algorithm. +# enable_optimized_reversed_reads: true + +# Use on a new, parallel algorithm for performing aggregate queries. +# Set to `false` to fall-back to the old algorithm. +# enable_parallelized_aggregation: true + +# When enabled, the node will start using separate commit log for schema changes +# right from the boot. Without this, it only happens following a restart after +# all nodes in the cluster were upgraded. +# +# Having this option ensures that new installations don't need a rolling restart +# to use the feature, but upgrades do. +# +# WARNING: It's unsafe to set this to false if the node previously booted +# with the schema commit log enabled. In such case, some schema changes +# may be lost if the node was not cleanly stopped. +force_schema_commit_log: true + +# Time for which task manager task is kept in memory after it completes. +task_ttl_in_seconds: 10 + +# Use Raft to consistently manage schema information in the cluster. +# Refer to https://docs.scylladb.com/master/architecture/raft.html for more details. +# The 'Handling Failures' section is especially important. +# +# Once enabled in a cluster, this cannot be turned off. +# If you want to bootstrap a new cluster without Raft, make sure to set this to `false` +# before starting your nodes for the first time. +# +# A cluster not using Raft can be 'upgraded' to use Raft. Refer to the aforementioned +# documentation, section 'Enabling Raft in ScyllaDB 5.2 and further', for the procedure. +consistent_cluster_management: true + +# In materialized views, restrictions are allowed only on the view's primary key columns. +# In old versions Scylla mistakenly allowed IS NOT NULL restrictions on columns which were not part +# of the view's primary key. These invalid restrictions were ignored. +# This option controls the behavior when someone tries to create a view with such invalid IS NOT NULL restrictions. +# +# Can be true, false, or warn. +# * `true`: IS NOT NULL is allowed only on the view's primary key columns, +# trying to use it on other columns will cause an error, as it should. +# * `false`: Scylla accepts IS NOT NULL restrictions on regular columns, but they're silently ignored. +# It's useful for backwards compatibility. +# * `warn`: The same as false, but there's a warning about invalid view restrictions. +# +# To preserve backwards compatibility on old clusters, Scylla's default setting is `warn`. +# New clusters have this option set to `true` by scylla.yaml (which overrides the default `warn`) +# to make sure that trying to create an invalid view causes an error. +strict_is_not_null_in_views: true + +# The Unix Domain Socket the node uses for maintenance socket. +# The possible options are: +# * ignore: the node will not open the maintenance socket, +# * workdir: the node will open the maintenance socket on the path /cql.m, +# where is a path defined by the workdir configuration option, +# * : the node will open the maintenance socket on the path . +maintenance_socket: ignore + +# If set to true, configuration parameters defined with LiveUpdate option can be updated in runtime with CQL +# by updating system.config virtual table. If we don't want any configuration parameter to be changed in runtime +# via CQL, this option should be set to false. This parameter doesn't impose any limits on other mechanisms updating +# configuration parameters in runtime, e.g. sending SIGHUP or using API. This option should be set to false +# e.g. for cloud users, for whom scylla's configuration should be changed only by support engineers. +# live_updatable_config_params_changeable_via_cql: true + +# **************** +# * GUARDRAILS * +# **************** + +# Guardrails to warn or fail when Replication Factor is smaller/greater than the threshold. +# Please note that the value of 0 is always allowed, +# which means that having no replication at all, i.e. RF = 0, is always valid. +# A guardrail value smaller than 0, e.g. -1, means that the guardrail is disabled. +# Commenting out a guardrail also means it is disabled. +# minimum_replication_factor_fail_threshold: -1 +# minimum_replication_factor_warn_threshold: 3 +# maximum_replication_factor_warn_threshold: -1 +# maximum_replication_factor_fail_threshold: -1 + +# Guardrails to warn about or disallow creating a keyspace with specific replication strategy. +# Each of these 2 settings is a list storing replication strategies considered harmful. +# The replication strategies to choose from are: +# 1) SimpleStrategy, +# 2) NetworkTopologyStrategy, +# 3) LocalStrategy, +# 4) EverywhereStrategy +# +# replication_strategy_warn_list: +# - SimpleStrategy +# replication_strategy_fail_list: + +api_ui_dir: /usr/lib/scylla/swagger-ui/dist/ +api_doc_dir: /usr/lib/scylla/api/api-doc/ +alternator_port: 8000 +alternator_write_isolation: only_rmw_uses_lwt +alternator_enforce_authorization: true +enable_ipv6_dns_lookup: true + +uuid_sstable_identifiers_enabled: false From 7b7e591cb0724256d0ee6c6cc0fb20a197360053 Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Fri, 15 Nov 2024 10:20:53 +0100 Subject: [PATCH 02/14] fix: handle ssl only scylla clusters This fixes how SM decides which port to use when connecting to Scylla nodes. --- pkg/ping/cqlping/cqlping_integration_test.go | 5 ++- pkg/service/cluster/service.go | 42 +++++++++---------- .../service_restore_integration_test.go | 2 +- pkg/testutils/db/db.go | 2 +- 4 files changed, 25 insertions(+), 26 deletions(-) diff --git a/pkg/ping/cqlping/cqlping_integration_test.go b/pkg/ping/cqlping/cqlping_integration_test.go index 7cc007bf7f..52b5a15c14 100644 --- a/pkg/ping/cqlping/cqlping_integration_test.go +++ b/pkg/ping/cqlping/cqlping_integration_test.go @@ -8,10 +8,11 @@ package cqlping import ( "context" "crypto/tls" - "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" "testing" "time" + "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" + "github.com/scylladb/go-log" "github.com/scylladb/scylla-manager/v3/pkg/ping" "github.com/scylladb/scylla-manager/v3/pkg/scyllaclient" @@ -24,7 +25,7 @@ func TestPingIntegration(t *testing.T) { client := newTestClient(t, log.NewDevelopmentWithLevel(zapcore.InfoLevel).Named("client"), nil) defer client.Close() - sessionHosts, err := cluster.GetRPCAddresses(context.Background(), client, []string{testconfig.ManagedClusterHost()}) + sessionHosts, err := cluster.GetRPCAddresses(context.Background(), client, []string{testconfig.ManagedClusterHost()}, false) if err != nil { t.Fatal(err) } diff --git a/pkg/service/cluster/service.go b/pkg/service/cluster/service.go index 0a9bb70778..bb7d92cc87 100644 --- a/pkg/service/cluster/service.go +++ b/pkg/service/cluster/service.go @@ -617,9 +617,12 @@ func SingleHostSessionConfigOption(host string) SessionConfigOption { return errors.Wrapf(err, "fetch node (%s) info", host) } cqlAddr := ni.CQLAddr(host) + if ni.ClientEncryptionEnabled { + cqlAddr = ni.CQLSSLAddr(host) + } cfg.Hosts = []string{cqlAddr} - cfg.HostFilter = gocql.WhiteListHostFilter(cqlAddr) cfg.DisableInitialHostLookup = true + cfg.HostFilter = gocql.WhiteListHostFilter(cqlAddr) return nil } } @@ -643,9 +646,14 @@ func (s *Service) GetSession(ctx context.Context, clusterID uuid.UUID, opts ...S return session, err } } - // Fill hosts if they weren't specified by the options + + clusterInfo, err := s.GetClusterByID(ctx, clusterID) + if err != nil { + return session, errors.Wrap(err, "cluster by id") + } + // Fill hosts if they weren't specified by the options or make sure that they use correct rpc address. if len(cfg.Hosts) == 0 { - sessionHosts, err := GetRPCAddresses(ctx, client, client.Config().Hosts) + sessionHosts, err := GetRPCAddresses(ctx, client, client.Config().Hosts, clusterInfo.ForceTLSDisabled || clusterInfo.ForceNonSSLSessionPort) if err != nil { s.logger.Info(ctx, "Gets session", "err", err) if errors.Is(err, ErrNoRPCAddressesFound) { @@ -662,7 +670,7 @@ func (s *Service) GetSession(ctx context.Context, clusterID uuid.UUID, opts ...S if err := s.extendClusterConfigWithAuthentication(clusterID, ni, cfg); err != nil { return session, err } - if err := s.extendClusterConfigWithTLS(ctx, clusterID, ni, cfg); err != nil { + if err := s.extendClusterConfigWithTLS(clusterInfo, ni, cfg); err != nil { return session, err } @@ -695,24 +703,15 @@ func (s *Service) extendClusterConfigWithAuthentication(clusterID uuid.UUID, ni return nil } -func (s *Service) extendClusterConfigWithTLS(ctx context.Context, clusterID uuid.UUID, ni *scyllaclient.NodeInfo, cfg *gocql.ClusterConfig) error { - cluster, err := s.GetClusterByID(ctx, clusterID) - if err != nil { - return errors.Wrap(err, "get cluster by id") - } - - cqlPort := ni.CQLPort() +func (s *Service) extendClusterConfigWithTLS(cluster *Cluster, ni *scyllaclient.NodeInfo, cfg *gocql.ClusterConfig) error { if ni.ClientEncryptionEnabled && !cluster.ForceTLSDisabled { - if !cluster.ForceNonSSLSessionPort { - cqlPort = ni.CQLSSLPort() - } cfg.SslOpts = &gocql.SslOptions{ Config: &tls.Config{ InsecureSkipVerify: true, }, } if ni.ClientEncryptionRequireAuth { - keyPair, err := s.loadTLSIdentity(clusterID) + keyPair, err := s.loadTLSIdentity(cluster.ID) if err != nil { return err } @@ -720,11 +719,6 @@ func (s *Service) extendClusterConfigWithTLS(ctx context.Context, clusterID uuid } } - p, err := strconv.Atoi(cqlPort) - if err != nil { - return errors.Wrap(err, "parse cql port") - } - cfg.Port = p return nil } @@ -770,7 +764,7 @@ var ErrNoRPCAddressesFound = errors.New("no RPC addresses found") // GetRPCAddresses accepts client and hosts parameters that are used later on to query client.NodeInfo endpoint // returning RPC addresses for given hosts. // RPC addresses are the ones that scylla uses to accept CQL connections. -func GetRPCAddresses(ctx context.Context, client *scyllaclient.Client, hosts []string) ([]string, error) { +func GetRPCAddresses(ctx context.Context, client *scyllaclient.Client, hosts []string, clusterSSLDisabled bool) ([]string, error) { var sessionHosts []string var combinedError error for _, h := range hosts { @@ -779,7 +773,11 @@ func GetRPCAddresses(ctx context.Context, client *scyllaclient.Client, hosts []s combinedError = multierr.Append(combinedError, err) continue } - sessionHosts = append(sessionHosts, ni.CQLAddr(h)) + addr := ni.CQLAddr(h) + if ni.ClientEncryptionEnabled && !clusterSSLDisabled { + addr = ni.CQLSSLAddr(h) + } + sessionHosts = append(sessionHosts, addr) } if len(sessionHosts) == 0 { diff --git a/pkg/service/restore/service_restore_integration_test.go b/pkg/service/restore/service_restore_integration_test.go index d3a79ff39f..1892128ee0 100644 --- a/pkg/service/restore/service_restore_integration_test.go +++ b/pkg/service/restore/service_restore_integration_test.go @@ -1737,7 +1737,7 @@ func (h *restoreTestHelper) restartScylla() { b := backoff.WithContext(backoff.WithMaxRetries( backoff.NewConstantBackOff(500*time.Millisecond), 10), ctx) if err := backoff.Retry(func() error { - sessionHosts, err = cluster.GetRPCAddresses(ctx, h.Client, []string{host}) + sessionHosts, err = cluster.GetRPCAddresses(ctx, h.Client, []string{host}, false) return err }, b); err != nil { h.T.Fatal(err) diff --git a/pkg/testutils/db/db.go b/pkg/testutils/db/db.go index 5d59b6ac5d..05fff3bced 100644 --- a/pkg/testutils/db/db.go +++ b/pkg/testutils/db/db.go @@ -83,7 +83,7 @@ func CreateManagedClusterSession(tb testing.TB, empty bool, client *scyllaclient tb.Helper() ctx := context.Background() - sessionHosts, err := cluster.GetRPCAddresses(ctx, client, client.Config().Hosts) + sessionHosts, err := cluster.GetRPCAddresses(ctx, client, client.Config().Hosts, false) if err != nil { tb.Log(err) if errors.Is(err, cluster.ErrNoRPCAddressesFound) { From 9d47e7752869fa761aebaf63e7f3c42583320e0b Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Tue, 19 Nov 2024 17:43:33 +0100 Subject: [PATCH 03/14] fix: CQLAddr provides ssl or non-ssl addr depending on cluster conf. This replaces CQLAddr and CQLSSLAddr with one function which returns correct cql addr depending on cluster configuration. Also backup worker is modified a little bit to get cluster configuration with tls related info. --- pkg/cmd/scylla-manager/server.go | 2 +- pkg/scyllaclient/client_agent.go | 36 ++--- pkg/scyllaclient/client_agent_test.go | 123 ++++++++++++------ pkg/service/backup/service.go | 47 +++---- .../backup/service_backup_integration_test.go | 9 +- pkg/service/backup/worker.go | 17 +-- pkg/service/backup/worker_schema.go | 2 +- pkg/service/cluster/service.go | 14 +- pkg/service/configcache/tlsconfig.go | 5 +- pkg/service/healthcheck/service.go | 6 +- .../repair/service_repair_integration_test.go | 2 +- .../restore/helper_integration_test.go | 9 +- .../service_restore_integration_test.go | 9 +- 13 files changed, 171 insertions(+), 110 deletions(-) diff --git a/pkg/cmd/scylla-manager/server.go b/pkg/cmd/scylla-manager/server.go index 8c9a40c228..cb4c54ad0f 100644 --- a/pkg/cmd/scylla-manager/server.go +++ b/pkg/cmd/scylla-manager/server.go @@ -112,7 +112,7 @@ func (s *server) makeServices(ctx context.Context) error { s.session, s.config.Backup, metrics.NewBackupMetrics().MustRegister(), - s.clusterSvc.GetClusterName, + s.clusterSvc.GetClusterByID, s.clusterSvc.Client, s.clusterSvc.GetSession, s.logger.Named("backup"), diff --git a/pkg/scyllaclient/client_agent.go b/pkg/scyllaclient/client_agent.go index d79daf8094..67ccef3301 100644 --- a/pkg/scyllaclient/client_agent.go +++ b/pkg/scyllaclient/client_agent.go @@ -70,27 +70,31 @@ func (c *Client) PinCPU(ctx context.Context, host string) error { return err } -// CQLAddr returns CQL address from NodeInfo. -// Scylla can have separate rpc_address (CQL), listen_address and respectfully -// broadcast_rpc_address and broadcast_address if some 3rd party routing -// is added. -// `fallback` argument is used in case any of above addresses is zero address. -func (ni *NodeInfo) CQLAddr(fallback string) string { - addr, port := ni.cqlAddr(fallback), ni.CQLPort() +// cqlAddr returns CQL address from NodeInfo. +func (ni *NodeInfo) cqlAddr(fallback string) string { + addr, port := ni.cqlListenAddr(fallback), ni.cqlPort() return net.JoinHostPort(addr, port) } -// CQLSSLAddr returns CQL SSL address from NodeInfo. +// cqlSSLAddr returns CQL SSL address from NodeInfo. +func (ni *NodeInfo) cqlSSLAddr(fallback string) string { + addr, port := ni.cqlListenAddr(fallback), ni.cqlSSLPort() + return net.JoinHostPort(addr, port) +} + +// CQLAddr returns either CQL or CQL SSL address from Node Info depending on the cluster configuration. // Scylla can have separate rpc_address (CQL), listen_address and respectfully // broadcast_rpc_address and broadcast_address if some 3rd party routing // is added. // `fallback` argument is used in case any of above addresses is zero address. -func (ni *NodeInfo) CQLSSLAddr(fallback string) string { - addr, port := ni.cqlAddr(fallback), ni.CQLSSLPort() - return net.JoinHostPort(addr, port) +func (ni *NodeInfo) CQLAddr(fallback string, clusterTLSAddrDisabled bool) string { + if ni.ClientEncryptionEnabled && !clusterTLSAddrDisabled { + return ni.cqlSSLAddr(fallback) + } + return ni.cqlAddr(fallback) } -func (ni *NodeInfo) cqlAddr(fallback string) string { +func (ni *NodeInfo) cqlListenAddr(fallback string) string { const ipv4Zero, ipv6Zero = "0.0.0.0", "::0" if ni.BroadcastRPCAddress != "" { @@ -109,13 +113,13 @@ func (ni *NodeInfo) cqlAddr(fallback string) string { return ni.ListenAddress } -// CQLPort returns CQL port from NodeInfo. -func (ni *NodeInfo) CQLPort() string { +// cqlPort returns CQL port from NodeInfo. +func (ni *NodeInfo) cqlPort() string { return ni.NativeTransportPort } -// CQLSSLPort returns CQL SSL port from NodeInfo. -func (ni *NodeInfo) CQLSSLPort() string { +// cqlSSLPort returns CQL SSL port from NodeInfo. +func (ni *NodeInfo) cqlSSLPort() string { return ni.NativeTransportPortSsl } diff --git a/pkg/scyllaclient/client_agent_test.go b/pkg/scyllaclient/client_agent_test.go index 9e0a48196b..bac5fa8712 100644 --- a/pkg/scyllaclient/client_agent_test.go +++ b/pkg/scyllaclient/client_agent_test.go @@ -17,9 +17,10 @@ func TestNodeInfoCQLAddr(t *testing.T) { t.Parallel() table := []struct { - Name string - NodeInfo *scyllaclient.NodeInfo - GoldenAddress string + Name string + NodeInfo *scyllaclient.NodeInfo + ClusterDisableSSL bool + GoldenAddress string }{ { Name: "Broadcast RPC address is set", @@ -29,7 +30,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { ListenAddress: "1.2.3.6", NativeTransportPort: "1234", }, - GoldenAddress: "1.2.3.4:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.4:1234", }, { Name: "RPC address is set", @@ -38,7 +40,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { RPCAddress: "1.2.3.5", ListenAddress: "1.2.3.6", }, - GoldenAddress: "1.2.3.5:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.5:1234", }, { Name: "Listen Address is set", @@ -46,7 +49,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { NativeTransportPort: "1234", ListenAddress: "1.2.3.6", }, - GoldenAddress: "1.2.3.6:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.6:1234", }, { Name: "Fallback is returned when RPC Address is IPv4 zero", @@ -54,7 +58,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { NativeTransportPort: "1234", RPCAddress: "0.0.0.0", }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when RPC Address is IPv6 zero", @@ -62,7 +67,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { NativeTransportPort: "1234", RPCAddress: "::0", }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when Listen Address is IPv4 zero", @@ -70,7 +76,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { NativeTransportPort: "1234", ListenAddress: "0.0.0.0", }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when Listen Address is IPv6 zero", @@ -78,7 +85,8 @@ func TestNodeInfoCQLAddr(t *testing.T) { NativeTransportPort: "1234", ListenAddress: "::0", }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, } @@ -87,7 +95,7 @@ func TestNodeInfoCQLAddr(t *testing.T) { t.Run(test.Name, func(t *testing.T) { t.Parallel() - addr := test.NodeInfo.CQLAddr(fallback) + addr := test.NodeInfo.CQLAddr(fallback, test.ClusterDisableSSL) if addr != test.GoldenAddress { t.Errorf("expected %s address, got %s", test.GoldenAddress, addr) } @@ -99,68 +107,105 @@ func TestNodeInfoCQLSSLAddr(t *testing.T) { t.Parallel() table := []struct { - Name string - NodeInfo *scyllaclient.NodeInfo - GoldenAddress string + Name string + NodeInfo *scyllaclient.NodeInfo + ClusterDisableSSL bool + GoldenAddress string }{ { Name: "Broadcast RPC address is set", NodeInfo: &scyllaclient.NodeInfo{ - BroadcastRPCAddress: "1.2.3.4", - RPCAddress: "1.2.3.5", - ListenAddress: "1.2.3.6", - NativeTransportPortSsl: "1234", + BroadcastRPCAddress: "1.2.3.4", + RPCAddress: "1.2.3.5", + ListenAddress: "1.2.3.6", + NativeTransportPortSsl: "1234", + ClientEncryptionEnabled: true, }, - GoldenAddress: "1.2.3.4:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.4:1234", }, { Name: "RPC address is set", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - RPCAddress: "1.2.3.5", - ListenAddress: "1.2.3.6", + NativeTransportPortSsl: "1234", + RPCAddress: "1.2.3.5", + ListenAddress: "1.2.3.6", + ClientEncryptionEnabled: true, }, - GoldenAddress: "1.2.3.5:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.5:1234", }, { Name: "Listen Address is set", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - ListenAddress: "1.2.3.6", + NativeTransportPortSsl: "1234", + ListenAddress: "1.2.3.6", + ClientEncryptionEnabled: true, }, - GoldenAddress: "1.2.3.6:1234", + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.6:1234", }, { Name: "Fallback is returned when RPC Address is IPv4 zero", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - RPCAddress: "0.0.0.0", + NativeTransportPortSsl: "1234", + RPCAddress: "0.0.0.0", + ClientEncryptionEnabled: true, }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when RPC Address is IPv6 zero", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - RPCAddress: "::0", + NativeTransportPortSsl: "1234", + RPCAddress: "::0", + ClientEncryptionEnabled: true, }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when Listen Address is IPv4 zero", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - ListenAddress: "0.0.0.0", + NativeTransportPortSsl: "1234", + ListenAddress: "0.0.0.0", + ClientEncryptionEnabled: true, }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), }, { Name: "Fallback is returned when Listen Address is IPv6 zero", NodeInfo: &scyllaclient.NodeInfo{ - NativeTransportPortSsl: "1234", - ListenAddress: "::0", + NativeTransportPortSsl: "1234", + ListenAddress: "::0", + ClientEncryptionEnabled: true, }, - GoldenAddress: net.JoinHostPort(fallback, "1234"), + ClusterDisableSSL: false, + GoldenAddress: net.JoinHostPort(fallback, "1234"), + }, + { + Name: "NativeTransportPort is returned when ssl is disabled on cluster level", + NodeInfo: &scyllaclient.NodeInfo{ + NativeTransportPort: "4321", + NativeTransportPortSsl: "1234", + ListenAddress: "1.2.3.4", + ClientEncryptionEnabled: true, + }, + ClusterDisableSSL: true, + GoldenAddress: "1.2.3.4:4321", + }, + { + Name: "NativeTransportPort is returned when Node Client Encryption is not enabled", + NodeInfo: &scyllaclient.NodeInfo{ + NativeTransportPort: "4321", + NativeTransportPortSsl: "1234", + ListenAddress: "1.2.3.4", + ClientEncryptionEnabled: false, + }, + ClusterDisableSSL: false, + GoldenAddress: "1.2.3.4:4321", }, } @@ -169,7 +214,7 @@ func TestNodeInfoCQLSSLAddr(t *testing.T) { t.Run(test.Name, func(t *testing.T) { t.Parallel() - addr := test.NodeInfo.CQLSSLAddr(fallback) + addr := test.NodeInfo.CQLAddr(fallback, test.ClusterDisableSSL) if addr != test.GoldenAddress { t.Errorf("expected %s address, got %s", test.GoldenAddress, addr) } diff --git a/pkg/service/backup/service.go b/pkg/service/backup/service.go index 4456517106..509ebee22c 100644 --- a/pkg/service/backup/service.go +++ b/pkg/service/backup/service.go @@ -42,16 +42,16 @@ type Service struct { config Config metrics metrics.BackupMetrics - clusterName cluster.NameFunc - scyllaClient scyllaclient.ProviderFunc - clusterSession cluster.SessionFunc - logger log.Logger + clusterProvider cluster.ProviderFunc + scyllaClient scyllaclient.ProviderFunc + clusterSession cluster.SessionFunc + logger log.Logger dth deduplicateTestHooks } func NewService(session gocqlx.Session, config Config, metrics metrics.BackupMetrics, - clusterName cluster.NameFunc, scyllaClient scyllaclient.ProviderFunc, clusterSession cluster.SessionFunc, logger log.Logger, + clusterProvider cluster.ProviderFunc, scyllaClient scyllaclient.ProviderFunc, clusterSession cluster.SessionFunc, logger log.Logger, ) (*Service, error) { if session.Session == nil || session.Closed() { return nil, errors.New("invalid session") @@ -61,7 +61,7 @@ func NewService(session gocqlx.Session, config Config, metrics metrics.BackupMet return nil, errors.Wrap(err, "invalid config") } - if clusterName == nil { + if clusterProvider == nil { return nil, errors.New("invalid cluster name provider") } @@ -70,13 +70,13 @@ func NewService(session gocqlx.Session, config Config, metrics metrics.BackupMet } return &Service{ - session: session, - config: config, - metrics: metrics, - clusterName: clusterName, - scyllaClient: scyllaClient, - clusterSession: clusterSession, - logger: logger, + session: session, + config: config, + metrics: metrics, + clusterProvider: clusterProvider, + scyllaClient: scyllaClient, + clusterSession: clusterSession, + logger: logger, }, nil } @@ -667,22 +667,23 @@ func (s *Service) Backup(ctx context.Context, clusterID, taskID, runID uuid.UUID return errors.Wrap(err, "initialize: register the run") } - // Get cluster name - clusterName, err := s.clusterName(ctx, run.ClusterID) + // Get cluster info by id + cluster, err := s.clusterProvider(ctx, run.ClusterID) if err != nil { - return errors.Wrap(err, "invalid cluster") + return errors.Wrap(err, "cluster provider") } // Create a worker w := &worker{ workerTools: workerTools{ - ClusterID: clusterID, - ClusterName: clusterName, - TaskID: taskID, - RunID: runID, - SnapshotTag: run.SnapshotTag, - Config: s.config, - Client: client, + ClusterID: clusterID, + ClusterName: cluster.Name, + ClusterTLSAddrDisabled: cluster.ForceTLSDisabled || cluster.ForceNonSSLSessionPort, + TaskID: taskID, + RunID: runID, + SnapshotTag: run.SnapshotTag, + Config: s.config, + Client: client, }, PrevStage: run.Stage, Metrics: s.metrics, diff --git a/pkg/service/backup/service_backup_integration_test.go b/pkg/service/backup/service_backup_integration_test.go index 610ab0f637..b4b7d8365a 100644 --- a/pkg/service/backup/service_backup_integration_test.go +++ b/pkg/service/backup/service_backup_integration_test.go @@ -115,8 +115,13 @@ func newTestServiceWithUser(t *testing.T, session gocqlx.Session, client *scylla session, c, metrics.NewBackupMetrics(), - func(_ context.Context, id uuid.UUID) (string, error) { - return "test_cluster", nil + func(_ context.Context, id uuid.UUID) (*cluster.Cluster, error) { + return &cluster.Cluster{ + ID: id, + Name: "test_cluster", + ForceTLSDisabled: false, + ForceNonSSLSessionPort: false, + }, nil }, func(context.Context, uuid.UUID) (*scyllaclient.Client, error) { return client, nil diff --git a/pkg/service/backup/worker.go b/pkg/service/backup/worker.go index 3555f80412..465f0db647 100644 --- a/pkg/service/backup/worker.go +++ b/pkg/service/backup/worker.go @@ -51,14 +51,15 @@ func (sd snapshotDir) String() string { // workerTools is an intersection of fields and methods // useful for both worker and restoreWorker. type workerTools struct { - ClusterID uuid.UUID - ClusterName string - TaskID uuid.UUID - RunID uuid.UUID - SnapshotTag string - Config Config - Client *scyllaclient.Client - Logger log.Logger + ClusterID uuid.UUID + ClusterName string + ClusterTLSAddrDisabled bool + TaskID uuid.UUID + RunID uuid.UUID + SnapshotTag string + Config Config + Client *scyllaclient.Client + Logger log.Logger } // worker is responsible for coordinating backup procedure. diff --git a/pkg/service/backup/worker_schema.go b/pkg/service/backup/worker_schema.go index e22d421d76..0338a95d66 100644 --- a/pkg/service/backup/worker_schema.go +++ b/pkg/service/backup/worker_schema.go @@ -141,7 +141,7 @@ func (w *worker) UploadSchema(ctx context.Context, hosts []hostInfo) (stepError func (w *worker) createSingleHostSessionToAnyHost(ctx context.Context, hosts []string, sessionFunc cluster.SessionFunc) (session gocqlx.Session, host string, err error) { var retErr error for _, h := range hosts { - session, err := sessionFunc(ctx, w.ClusterID, cluster.SingleHostSessionConfigOption(h)) + session, err := sessionFunc(ctx, w.ClusterID, cluster.SingleHostSessionConfigOption(h, w.ClusterTLSAddrDisabled)) if err != nil { w.Logger.Error(ctx, "Couldn't connect to host via CQL", "host", h, "error", err) retErr = err diff --git a/pkg/service/cluster/service.go b/pkg/service/cluster/service.go index bb7d92cc87..b10433292b 100644 --- a/pkg/service/cluster/service.go +++ b/pkg/service/cluster/service.go @@ -610,16 +610,13 @@ func (s *Service) ListNodes(ctx context.Context, clusterID uuid.UUID) ([]Node, e type SessionConfigOption func(ctx context.Context, clusterID uuid.UUID, client *scyllaclient.Client, cfg *gocql.ClusterConfig) error // SingleHostSessionConfigOption ensures that session will be connected only to the single, provided host. -func SingleHostSessionConfigOption(host string) SessionConfigOption { +func SingleHostSessionConfigOption(host string, clusterTLSAddrDisabled bool) SessionConfigOption { return func(ctx context.Context, _ uuid.UUID, client *scyllaclient.Client, cfg *gocql.ClusterConfig) error { ni, err := client.NodeInfo(ctx, host) if err != nil { return errors.Wrapf(err, "fetch node (%s) info", host) } - cqlAddr := ni.CQLAddr(host) - if ni.ClientEncryptionEnabled { - cqlAddr = ni.CQLSSLAddr(host) - } + cqlAddr := ni.CQLAddr(host, clusterTLSAddrDisabled) cfg.Hosts = []string{cqlAddr} cfg.DisableInitialHostLookup = true cfg.HostFilter = gocql.WhiteListHostFilter(cqlAddr) @@ -764,7 +761,7 @@ var ErrNoRPCAddressesFound = errors.New("no RPC addresses found") // GetRPCAddresses accepts client and hosts parameters that are used later on to query client.NodeInfo endpoint // returning RPC addresses for given hosts. // RPC addresses are the ones that scylla uses to accept CQL connections. -func GetRPCAddresses(ctx context.Context, client *scyllaclient.Client, hosts []string, clusterSSLDisabled bool) ([]string, error) { +func GetRPCAddresses(ctx context.Context, client *scyllaclient.Client, hosts []string, clusterTLSAddrDisabled bool) ([]string, error) { var sessionHosts []string var combinedError error for _, h := range hosts { @@ -773,10 +770,7 @@ func GetRPCAddresses(ctx context.Context, client *scyllaclient.Client, hosts []s combinedError = multierr.Append(combinedError, err) continue } - addr := ni.CQLAddr(h) - if ni.ClientEncryptionEnabled && !clusterSSLDisabled { - addr = ni.CQLSSLAddr(h) - } + addr := ni.CQLAddr(h, clusterTLSAddrDisabled) sessionHosts = append(sessionHosts, addr) } diff --git a/pkg/service/configcache/tlsconfig.go b/pkg/service/configcache/tlsconfig.go index a0d3613389..8042405cb1 100644 --- a/pkg/service/configcache/tlsconfig.go +++ b/pkg/service/configcache/tlsconfig.go @@ -26,10 +26,7 @@ func newCQLTLSConfigIfEnabled(c *cluster.Cluster, nodeInfo *scyllaclient.NodeInf if !cqlTLSEnabled || c.ForceTLSDisabled { return nil, nil // nolint: nilnil } - cqlAddress := nodeInfo.CQLAddr(host) - if !c.ForceNonSSLSessionPort { - cqlAddress = nodeInfo.CQLSSLAddr(host) - } + cqlAddress := nodeInfo.CQLAddr(host, c.ForceTLSDisabled || c.ForceNonSSLSessionPort) tlsConfig := &tls.Config{ InsecureSkipVerify: true, } diff --git a/pkg/service/healthcheck/service.go b/pkg/service/healthcheck/service.go index 1213682963..8407e0e281 100644 --- a/pkg/service/healthcheck/service.go +++ b/pkg/service/healthcheck/service.go @@ -331,9 +331,13 @@ func (s *Service) decorateNodeStatus(status *NodeStatus, ni configcache.NodeConf } func (s *Service) pingCQL(ctx context.Context, clusterID uuid.UUID, host string, timeout time.Duration, ni configcache.NodeConfig) (rtt time.Duration, err error) { + cluster, err := s.clusterProvider(ctx, clusterID) + if err != nil { + return rtt, errors.Wrap(err, "cluster provider") + } // Try to connect directly to host address. config := cqlping.Config{ - Addr: ni.CQLAddr(host), + Addr: ni.CQLAddr(host, cluster.ForceTLSDisabled || cluster.ForceNonSSLSessionPort), Timeout: timeout, } diff --git a/pkg/service/repair/service_repair_integration_test.go b/pkg/service/repair/service_repair_integration_test.go index f2bc2aedfe..00c1735a8c 100644 --- a/pkg/service/repair/service_repair_integration_test.go +++ b/pkg/service/repair/service_repair_integration_test.go @@ -1231,7 +1231,7 @@ func TestServiceRepairIntegration(t *testing.T) { } cfg := cqlping.Config{ - Addr: ni.CQLAddr(ignored), + Addr: ni.CQLAddr(ignored, false), Timeout: time.Minute, } diff --git a/pkg/service/restore/helper_integration_test.go b/pkg/service/restore/helper_integration_test.go index 517884efcb..0281e20d72 100644 --- a/pkg/service/restore/helper_integration_test.go +++ b/pkg/service/restore/helper_integration_test.go @@ -126,8 +126,13 @@ func newBackupSvc(t *testing.T, mgrSession gocqlx.Session, client *scyllaclient. mgrSession, defaultBackupTestConfig(), metrics.NewBackupMetrics(), - func(_ context.Context, id uuid.UUID) (string, error) { - return "test_cluster", nil + func(_ context.Context, id uuid.UUID) (*cluster.Cluster, error) { + return &cluster.Cluster{ + ID: id, + Name: "test_cluster", + ForceTLSDisabled: false, + ForceNonSSLSessionPort: false, + }, nil }, func(context.Context, uuid.UUID) (*scyllaclient.Client, error) { return client, nil diff --git a/pkg/service/restore/service_restore_integration_test.go b/pkg/service/restore/service_restore_integration_test.go index 1892128ee0..7a2072c030 100644 --- a/pkg/service/restore/service_restore_integration_test.go +++ b/pkg/service/restore/service_restore_integration_test.go @@ -128,8 +128,13 @@ func newTestService(t *testing.T, session gocqlx.Session, client *scyllaclient.C session, defaultBackupTestConfig(), metrics.NewBackupMetrics(), - func(_ context.Context, id uuid.UUID) (string, error) { - return "test_cluster", nil + func(_ context.Context, id uuid.UUID) (*cluster.Cluster, error) { + return &cluster.Cluster{ + ID: id, + Name: "test_cluster", + ForceTLSDisabled: false, + ForceNonSSLSessionPort: false, + }, nil }, func(context.Context, uuid.UUID) (*scyllaclient.Client, error) { return client, nil From 735d68910ff98c1aaa9e22daafa36f8b76744066 Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Tue, 19 Nov 2024 18:27:09 +0100 Subject: [PATCH 04/14] fix(Makefile): use yq to produce scylla config with ssl enabled This uses yq to delete non ssl port from scylla.yaml config and also merges it with scylla-ssl.yaml which contains requried parameters to enable ssl in scylla cluster. --- testing/Makefile | 7 +- testing/scylla/config/scylla-ssl.yaml | 646 +------------------------- 2 files changed, 7 insertions(+), 646 deletions(-) diff --git a/testing/Makefile b/testing/Makefile index 88370d20d8..fadbf85582 100644 --- a/testing/Makefile +++ b/testing/Makefile @@ -62,12 +62,15 @@ up: @echo "==> Generating encryption files" @cd scylla/certs && ./generate.sh @echo "==> Generating Scylla configuration" + @cp scylla/config/scylla.yaml scylla/scylla.yaml ifeq ($(SSL_ENABLED),true) - @cp scylla/config/scylla-ssl.yaml scylla/scylla.yaml + # disable non-ssl port + @$(YQ) delete -i scylla/scylla.yaml 'native_transport_port' + # merge into scylla.yaml values from config/scylla-ssl.yaml with overwrite option (-x) + @$(YQ) merge -i -x scylla/scylla.yaml scylla/config/scylla-ssl.yaml @cp scylla/config/cqlshrc-ssl scylla/cqlshrc else - @cp scylla/config/scylla.yaml scylla/scylla.yaml @cp scylla/config/cqlshrc scylla/cqlshrc endif diff --git a/testing/scylla/config/scylla-ssl.yaml b/testing/scylla/config/scylla-ssl.yaml index 5fef33dc10..65f7245c55 100644 --- a/testing/scylla/config/scylla-ssl.yaml +++ b/testing/scylla/config/scylla-ssl.yaml @@ -1,652 +1,10 @@ -# Scylla storage config YAML +# Scylla SSL enabled configuration. +# This file merged with testing/config/scylla.yaml during start-dev-env process when SSL_ENABLED=true. -####################################### -# This file is split to two sections: -# 1. Supported parameters -# 2. Unsupported parameters: reserved for future use or backwards -# compatibility. -# Scylla will only read and use the first segment -####################################### - -### Supported Parameters - -# The name of the cluster. This is mainly used to prevent machines in -# one logical cluster from joining another. -# It is recommended to change the default value when creating a new cluster. -# You can NOT modify this value for an existing cluster -cluster_name: 'Managed Cluster' - -# This defines the number of tokens randomly assigned to this node on the ring -# The more tokens, relative to other nodes, the larger the proportion of data -# that this node will store. You probably want all nodes to have the same number -# of tokens assuming they have equal hardware capability. -num_tokens: 256 - -# Directory where Scylla should store all its files, which are commitlog, -# data, hints, view_hints and saved_caches subdirectories. All of these -# subs can be overridden by the respective options below. -# If unset, the value defaults to /var/lib/scylla -# workdir: /var/lib/scylla - -# Directory where Scylla should store data on disk. -data_file_directories: - - /var/lib/scylla/data - -# commit log. when running on magnetic HDD, this should be a -# separate spindle than the data directories. -commitlog_directory: /var/lib/scylla/commitlog - -# schema commit log. A special commitlog instance -# used for schema and system tables. -# When running on magnetic HDD, this should be a -# separate spindle than the data directories. -# schema_commitlog_directory: /var/lib/scylla/commitlog/schema - -# commitlog_sync may be either "periodic" or "batch." -# -# When in batch mode, Scylla won't ack writes until the commit log -# has been fsynced to disk. It will wait -# commitlog_sync_batch_window_in_ms milliseconds between fsyncs. -# This window should be kept short because the writer threads will -# be unable to do extra work while waiting. (You may need to increase -# concurrent_writes for the same reason.) -# -# commitlog_sync: batch -# commitlog_sync_batch_window_in_ms: 2 -# -# the other option is "periodic" where writes may be acked immediately -# and the CommitLog is simply synced every commitlog_sync_period_in_ms -# milliseconds. -commitlog_sync: periodic -commitlog_sync_period_in_ms: 10000 - -# The size of the individual commitlog file segments. A commitlog -# segment may be archived, deleted, or recycled once all the data -# in it (potentially from each columnfamily in the system) has been -# flushed to sstables. -# -# The default size is 32, which is almost always fine, but if you are -# archiving commitlog segments (see commitlog_archiving.properties), -# then you probably want a finer granularity of archiving; 8 or 16 MB -# is reasonable. -commitlog_segment_size_in_mb: 32 - -# The size of the individual schema commitlog file segments. -# -# The default size is 128, which is 4 times larger than the default -# size of the data commitlog. It's because the segment size puts -# a limit on the mutation size that can be written at once, and some -# schema mutation writes are much larger than average. -schema_commitlog_segment_size_in_mb: 128 - -# seed_provider class_name is saved for future use. -# A seed address is mandatory. -seed_provider: - # The addresses of hosts that will serve as contact points for the joining node. - # It allows the node to discover the cluster ring topology on startup (when - # joining the cluster). - # Once the node has joined the cluster, the seed list has no function. - - class_name: org.apache.cassandra.locator.SimpleSeedProvider - parameters: - # In a new cluster, provide the address of the first node. - # In an existing cluster, specify the address of at least one existing node. - # If you specify addresses of more than one node, use a comma to separate them. - # For example: ",," - - seeds: "127.0.0.1" - -# Address to bind to and tell other Scylla nodes to connect to. -# You _must_ change this if you want multiple nodes to be able to communicate! -# -# If you leave broadcast_address (below) empty, then setting listen_address -# to 0.0.0.0 is wrong as other nodes will not know how to reach this node. -# If you set broadcast_address, then you can set listen_address to 0.0.0.0. -listen_address: localhost - -# Address to broadcast to other Scylla nodes -# Leaving this blank will set it to the same value as listen_address -# broadcast_address: 1.2.3.4 - - -# When using multiple physical network interfaces, set this to true to listen on broadcast_address -# in addition to the listen_address, allowing nodes to communicate in both interfaces. -# Ignore this property if the network configuration automatically routes between the public and private networks such as EC2. -# -# listen_on_broadcast_address: false - -# port for the CQL native transport to listen for clients on -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# To disable the CQL native transport, remove this option and configure native_transport_port_ssl. -#native_transport_port: 9042 - -# Like native_transport_port, but clients are forwarded to specific shards, based on the -# client-side port numbers. -native_shard_aware_transport_port: 19042 - -# Enabling native transport encryption in client_encryption_options allows you to either use -# encryption for the standard port or to use a dedicated, additional port along with the unencrypted -# standard native_transport_port. -# Enabling client encryption and keeping native_transport_port_ssl disabled will use encryption -# for native_transport_port. Setting native_transport_port_ssl to a different value -# from native_transport_port will use encryption for native_transport_port_ssl while -# keeping native_transport_port unencrypted. native_transport_port_ssl: 9142 - -# Like native_transport_port_ssl, but clients are forwarded to specific shards, based on the -# client-side port numbers. -#native_shard_aware_transport_port_ssl: 19142 - -# How long the coordinator should wait for read operations to complete -read_request_timeout_in_ms: 5000 - -# How long the coordinator should wait for writes to complete -write_request_timeout_in_ms: 2000 -# how long a coordinator should continue to retry a CAS operation -# that contends with other proposals for the same row -cas_contention_timeout_in_ms: 1000 - -# phi value that must be reached for a host to be marked down. -# most users should never need to adjust this. -# phi_convict_threshold: 8 - -# IEndpointSnitch. The snitch has two functions: -# - it teaches Scylla enough about your network topology to route -# requests efficiently -# - it allows Scylla to spread replicas around your cluster to avoid -# correlated failures. It does this by grouping machines into -# "datacenters" and "racks." Scylla will do its best not to have -# more than one replica on the same "rack" (which may not actually -# be a physical location) -# -# IF YOU CHANGE THE SNITCH AFTER DATA IS INSERTED INTO THE CLUSTER, -# YOU MUST RUN A FULL REPAIR, SINCE THE SNITCH AFFECTS WHERE REPLICAS -# ARE PLACED. -# -# Out of the box, Scylla provides -# - SimpleSnitch: -# Treats Strategy order as proximity. This can improve cache -# locality when disabling read repair. Only appropriate for -# single-datacenter deployments. -# - GossipingPropertyFileSnitch -# This should be your go-to snitch for production use. The rack -# and datacenter for the local node are defined in -# cassandra-rackdc.properties and propagated to other nodes via -# gossip. If cassandra-topology.properties exists, it is used as a -# fallback, allowing migration from the PropertyFileSnitch. -# - PropertyFileSnitch: -# Proximity is determined by rack and data center, which are -# explicitly configured in cassandra-topology.properties. -# - Ec2Snitch: -# Appropriate for EC2 deployments in a single Region. Loads Region -# and Availability Zone information from the EC2 API. The Region is -# treated as the datacenter, and the Availability Zone as the rack. -# Only private IPs are used, so this will not work across multiple -# Regions. -# - Ec2MultiRegionSnitch: -# Uses public IPs as broadcast_address to allow cross-region -# connectivity. (Thus, you should set seed addresses to the public -# IP as well.) You will need to open the storage_port or -# ssl_storage_port on the public IP firewall. (For intra-Region -# traffic, Scylla will switch to the private IP after -# establishing a connection.) -# - RackInferringSnitch: -# Proximity is determined by rack and data center, which are -# assumed to correspond to the 3rd and 2nd octet of each node's IP -# address, respectively. Unless this happens to match your -# deployment conventions, this is best used as an example of -# writing a custom Snitch class and is provided in that spirit. -# -# You can use a custom Snitch by setting this to the full class name -# of the snitch, which will be assumed to be on your classpath. -endpoint_snitch: GossipingPropertyFileSnitch - -# The address or interface to bind the native transport server to. -# -# Set rpc_address OR rpc_interface, not both. Interfaces must correspond -# to a single address, IP aliasing is not supported. -# -# Leaving rpc_address blank has the same effect as on listen_address -# (i.e. it will be based on the configured hostname of the node). -# -# Note that unlike listen_address, you can specify 0.0.0.0, but you must also -# set broadcast_rpc_address to a value other than 0.0.0.0. -# -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# -# If you choose to specify the interface by name and the interface has an ipv4 and an ipv6 address -# you can specify which should be chosen using rpc_interface_prefer_ipv6. If false the first ipv4 -# address will be used. If true the first ipv6 address will be used. Defaults to false preferring -# ipv4. If there is only one address it will be selected regardless of ipv4/ipv6. -rpc_address: localhost -# rpc_interface: eth1 -# rpc_interface_prefer_ipv6: false - -# port for Thrift to listen for clients on -rpc_port: 9160 - -# port for REST API server -api_port: 10000 - -# IP for the REST API server -api_address: 127.0.0.1 - -# Log WARN on any batch size exceeding this value. 128 kiB per batch by default. -# Caution should be taken on increasing the size of this threshold as it can lead to node instability. -batch_size_warn_threshold_in_kb: 128 - -# Fail any multiple-partition batch exceeding this value. 1 MiB (8x warn threshold) by default. -batch_size_fail_threshold_in_kb: 1024 - - # Authentication backend, identifying users - # Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthenticator, - # PasswordAuthenticator}. - # - # - AllowAllAuthenticator performs no checks - set it to disable authentication. - # - PasswordAuthenticator relies on username/password pairs to authenticate - # users. It keeps usernames and hashed passwords in system_auth.credentials table. - # Please increase system_auth keyspace replication factor if you use this authenticator. - # - com.scylladb.auth.TransitionalAuthenticator requires username/password pair - # to authenticate in the same manner as PasswordAuthenticator, but improper credentials - # result in being logged in as an anonymous user. Use for upgrading clusters' auth. -authenticator: PasswordAuthenticator - - # Authorization backend, implementing IAuthorizer; used to limit access/provide permissions - # Out of the box, Scylla provides org.apache.cassandra.auth.{AllowAllAuthorizer, - # CassandraAuthorizer}. - # - # - AllowAllAuthorizer allows any action to any user - set it to disable authorization. - # - CassandraAuthorizer stores permissions in system_auth.permissions table. Please - # increase system_auth keyspace replication factor if you use this authorizer. - # - com.scylladb.auth.TransitionalAuthorizer wraps around the CassandraAuthorizer, using it for - # authorizing permission management. Otherwise, it allows all. Use for upgrading - # clusters' auth. -authorizer: CassandraAuthorizer - - # initial_token allows you to specify tokens manually. While you can use # it with - # vnodes (num_tokens > 1, above) -- in which case you should provide a - # comma-separated list -- it's primarily used when adding nodes # to legacy clusters - # that do not have vnodes enabled. - # initial_token: - - # RPC address to broadcast to drivers and other Scylla nodes. This cannot - # be set to 0.0.0.0. If left blank, this will be set to the value of - # rpc_address. If rpc_address is set to 0.0.0.0, broadcast_rpc_address must - # be set. - # broadcast_rpc_address: 1.2.3.4 - - # Uncomment to enable experimental features - # experimental_features: - # - udf - # - alternator-streams - # - broadcast-tables - # - keyspace-storage-options - - # The directory where hints files are stored if hinted handoff is enabled. - # hints_directory: /var/lib/scylla/hints - -# The directory where hints files are stored for materialized-view updates -# view_hints_directory: /var/lib/scylla/view_hints - -# See https://docs.scylladb.com/architecture/anti-entropy/hinted-handoff -# May either be "true" or "false" to enable globally, or contain a list -# of data centers to enable per-datacenter. -# hinted_handoff_enabled: DC1,DC2 -# hinted_handoff_enabled: true - -# this defines the maximum amount of time a dead host will have hints -# generated. After it has been dead this long, new hints for it will not be -# created until it has been seen alive and gone down again. -# max_hint_window_in_ms: 10800000 # 3 hours - - -# Validity period for permissions cache (fetching permissions can be an -# expensive operation depending on the authorizer, CassandraAuthorizer is -# one example). Defaults to 10000, set to 0 to disable. -# Will be disabled automatically for AllowAllAuthorizer. -# permissions_validity_in_ms: 10000 - -# Refresh interval for permissions cache (if enabled). -# After this interval, cache entries become eligible for refresh. Upon next -# access, an async reload is scheduled and the old value returned until it -# completes. If permissions_validity_in_ms is non-zero, then this also must have -# a non-zero value. Defaults to 2000. It's recommended to set this value to -# be at least 3 times smaller than the permissions_validity_in_ms. -# permissions_update_interval_in_ms: 2000 - -# The partitioner is responsible for distributing groups of rows (by -# partition key) across nodes in the cluster. You should leave this -# alone for new clusters. The partitioner can NOT be changed without -# reloading all data, so when upgrading you should set this to the -# same partitioner you were already using. -# -# Murmur3Partitioner is currently the only supported partitioner, -# -partitioner: org.apache.cassandra.dht.Murmur3Partitioner - -# Total space to use for commitlogs. -# -# If space gets above this value (it will round up to the next nearest -# segment multiple), Scylla will flush every dirty CF in the oldest -# segment and remove it. So a small total commitlog space will tend -# to cause more flush activity on less-active columnfamilies. -# -# A value of -1 (default) will automatically equate it to the total amount of memory -# available for Scylla. -commitlog_total_space_in_mb: -1 - -# TCP port, for commands and data -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# storage_port: 7000 - -# SSL port, for encrypted communication. Unused unless enabled in -# encryption_options -# For security reasons, you should not expose this port to the internet. Firewall it if needed. -# ssl_storage_port: 7001 - -# listen_interface: eth0 -# listen_interface_prefer_ipv6: false - -# Whether to start the native transport server. -# Please note that the address on which the native transport is bound is the -# same as the rpc_address. The port however is different and specified below. -# start_native_transport: true - -# The maximum size of allowed frame. Frame (requests) larger than this will -# be rejected as invalid. The default is 256MB. -# native_transport_max_frame_size_in_mb: 256 - -# Whether to start the thrift rpc server. -# start_rpc: true - -# enable or disable keepalive on rpc/native connections -# rpc_keepalive: true - -# Set to true to have Scylla create a hard link to each sstable -# flushed or streamed locally in a backups/ subdirectory of the -# keyspace data. Removing these links is the operator's -# responsibility. -# incremental_backups: false - -# Whether or not to take a snapshot before each compaction. Be -# careful using this option, since Scylla won't clean up the -# snapshots for you. Mostly useful if you're paranoid when there -# is a data format change. -# snapshot_before_compaction: false - -# Whether or not a snapshot is taken of the data before keyspace truncation -# or dropping of column families. The STRONGLY advised default of true -# should be used to provide data safety. If you set this flag to false, you will -# lose data on truncation or drop. -auto_snapshot: false - -# When executing a scan, within or across a partition, we need to keep the -# tombstones seen in memory so we can return them to the coordinator, which -# will use them to make sure other replicas also know about the deleted rows. -# With workloads that generate a lot of tombstones, this can cause performance -# problems and even exhaust the server heap. -# (http://www.datastax.com/dev/blog/cassandra-anti-patterns-queues-and-queue-like-datasets) -# Adjust the thresholds here if you understand the dangers and want to -# scan more tombstones anyway. These thresholds may also be adjusted at runtime -# using the StorageService mbean. -# tombstone_warn_threshold: 1000 -# tombstone_failure_threshold: 100000 - -# Granularity of the collation index of rows within a partition. -# Increase if your rows are large, or if you have a very large -# number of rows per partition. The competing goals are these: -# 1) a smaller granularity means more index entries are generated -# and looking up rows within the partition by collation column -# is faster -# 2) but, Scylla will keep the collation index in memory for hot -# rows (as part of the key cache), so a larger granularity means -# you can cache more hot rows -# column_index_size_in_kb: 64 - -# Auto-scaling of the promoted index prevents running out of memory -# when the promoted index grows too large (due to partitions with many rows -# vs. too small column_index_size_in_kb). When the serialized representation -# of the promoted index grows by this threshold, the desired block size -# for this partition (initialized to column_index_size_in_kb) -# is doubled, to decrease the sampling resolution by half. -# -# To disable promoted index auto-scaling, set the threshold to 0. -# column_index_auto_scale_threshold_in_kb: 10240 - -# Log a warning when writing partitions larger than this value -# compaction_large_partition_warning_threshold_mb: 1000 - -# Log a warning when writing rows larger than this value -# compaction_large_row_warning_threshold_mb: 10 - -# Log a warning when writing cells larger than this value -# compaction_large_cell_warning_threshold_mb: 1 - -# Log a warning when row number is larger than this value -# compaction_rows_count_warning_threshold: 100000 - -# Log a warning when writing a collection containing more elements than this value -# compaction_collection_elements_count_warning_threshold: 10000 - -# How long the coordinator should wait for seq or index scans to complete -# range_request_timeout_in_ms: 10000 -# How long the coordinator should wait for writes to complete -# counter_write_request_timeout_in_ms: 5000 -# How long a coordinator should continue to retry a CAS operation -# that contends with other proposals for the same row -# cas_contention_timeout_in_ms: 1000 -# How long the coordinator should wait for truncates to complete -# (This can be much longer, because unless auto_snapshot is disabled -# we need to flush first so we can snapshot before removing the data.) -# truncate_request_timeout_in_ms: 60000 -# The default timeout for other, miscellaneous operations -# request_timeout_in_ms: 10000 - -# Enable or disable inter-node encryption. -# You must also generate keys and provide the appropriate key and trust store locations and passwords. -# -# The available internode options are : all, none, dc, rack -# If set to dc scylla will encrypt the traffic between the DCs -# If set to rack scylla will encrypt the traffic between the racks -# -# SSL/TLS algorithm and ciphers used can be controlled by -# the priority_string parameter. Info on priority string -# syntax and values is available at: -# https://gnutls.org/manual/html_node/Priority-Strings.html -# -# The require_client_auth parameter allows you to -# restrict access to service based on certificate -# validation. Client must provide a certificate -# accepted by the used trust store to connect. -# -# server_encryption_options: -# internode_encryption: none -# certificate: conf/scylla.crt -# keyfile: conf/scylla.key -# truststore: -# certficate_revocation_list: -# require_client_auth: False -# priority_string: - -# enable or disable client/server encryption. client_encryption_options: enabled: true certificate: /etc/scylla/db.crt keyfile: /etc/scylla/db.key truststore: /etc/scylla/ca.crt require_client_auth: true -# truststore: -# require_client_auth: False -# priority_string: - -# internode_compression controls whether traffic between nodes is -# compressed. -# can be: all - all traffic is compressed -# dc - traffic between different datacenters is compressed -# none - nothing is compressed. -# internode_compression: none - -# Enable or disable tcp_nodelay for inter-dc communication. -# Disabling it will result in larger (but fewer) network packets being sent, -# reducing overhead from the TCP protocol itself, at the cost of increasing -# latency if you block for cross-datacenter responses. -# inter_dc_tcp_nodelay: false - -# Relaxation of environment checks. -# -# Scylla places certain requirements on its environment. If these requirements are -# not met, performance and reliability can be degraded. -# -# These requirements include: -# - A filesystem with good support for asynchronous I/O (AIO). Currently, -# this means XFS. -# -# false: strict environment checks are in place; do not start if they are not met. -# true: relaxed environment checks; performance and reliability may degraade. -# -# developer_mode: false - - -# Idle-time background processing -# -# Scylla can perform certain jobs in the background while the system is otherwise idle, -# freeing processor resources when there is other work to be done. -# -# defragment_memory_on_idle: true -# -# prometheus port -# By default, Scylla opens prometheus API port on port 9180 -# setting the port to 0 will disable the prometheus API. -# prometheus_port: 9180 -# -# prometheus address -# Leaving this blank will set it to the same value as listen_address. -# This means that by default, Scylla listens to the prometheus API on the same -# listening address (and therefore network interface) used to listen for -# internal communication. If the monitoring node is not in this internal -# network, you can override prometheus_address explicitly - e.g., setting -# it to 0.0.0.0 to listen on all interfaces. -# prometheus_address: 1.2.3.4 - -# Distribution of data among cores (shards) within a node -# -# Scylla distributes data within a node among shards, using a round-robin -# strategy: -# [shard0] [shard1] ... [shardN-1] [shard0] [shard1] ... [shardN-1] ... -# -# Scylla versions 1.6 and below used just one repetition of the pattern; -# this interfered with data placement among nodes (vnodes). -# -# Scylla versions 1.7 and above use 4096 repetitions of the pattern; this -# provides for better data distribution. -# -# the value below is log (base 2) of the number of repetitions. -# -# Set to 0 to avoid rewriting all data when upgrading from Scylla 1.6 and -# below. -# -# Keep at 12 for new clusters. -murmur3_partitioner_ignore_msb_bits: 12 - -# Bypass in-memory data cache (the row cache) when performing reversed queries. -# reversed_reads_auto_bypass_cache: false - -# Use a new optimized algorithm for performing reversed reads. -# Set to `false` to fall-back to the old algorithm. -# enable_optimized_reversed_reads: true - -# Use on a new, parallel algorithm for performing aggregate queries. -# Set to `false` to fall-back to the old algorithm. -# enable_parallelized_aggregation: true - -# When enabled, the node will start using separate commit log for schema changes -# right from the boot. Without this, it only happens following a restart after -# all nodes in the cluster were upgraded. -# -# Having this option ensures that new installations don't need a rolling restart -# to use the feature, but upgrades do. -# -# WARNING: It's unsafe to set this to false if the node previously booted -# with the schema commit log enabled. In such case, some schema changes -# may be lost if the node was not cleanly stopped. -force_schema_commit_log: true - -# Time for which task manager task is kept in memory after it completes. -task_ttl_in_seconds: 10 - -# Use Raft to consistently manage schema information in the cluster. -# Refer to https://docs.scylladb.com/master/architecture/raft.html for more details. -# The 'Handling Failures' section is especially important. -# -# Once enabled in a cluster, this cannot be turned off. -# If you want to bootstrap a new cluster without Raft, make sure to set this to `false` -# before starting your nodes for the first time. -# -# A cluster not using Raft can be 'upgraded' to use Raft. Refer to the aforementioned -# documentation, section 'Enabling Raft in ScyllaDB 5.2 and further', for the procedure. -consistent_cluster_management: true - -# In materialized views, restrictions are allowed only on the view's primary key columns. -# In old versions Scylla mistakenly allowed IS NOT NULL restrictions on columns which were not part -# of the view's primary key. These invalid restrictions were ignored. -# This option controls the behavior when someone tries to create a view with such invalid IS NOT NULL restrictions. -# -# Can be true, false, or warn. -# * `true`: IS NOT NULL is allowed only on the view's primary key columns, -# trying to use it on other columns will cause an error, as it should. -# * `false`: Scylla accepts IS NOT NULL restrictions on regular columns, but they're silently ignored. -# It's useful for backwards compatibility. -# * `warn`: The same as false, but there's a warning about invalid view restrictions. -# -# To preserve backwards compatibility on old clusters, Scylla's default setting is `warn`. -# New clusters have this option set to `true` by scylla.yaml (which overrides the default `warn`) -# to make sure that trying to create an invalid view causes an error. -strict_is_not_null_in_views: true - -# The Unix Domain Socket the node uses for maintenance socket. -# The possible options are: -# * ignore: the node will not open the maintenance socket, -# * workdir: the node will open the maintenance socket on the path /cql.m, -# where is a path defined by the workdir configuration option, -# * : the node will open the maintenance socket on the path . -maintenance_socket: ignore - -# If set to true, configuration parameters defined with LiveUpdate option can be updated in runtime with CQL -# by updating system.config virtual table. If we don't want any configuration parameter to be changed in runtime -# via CQL, this option should be set to false. This parameter doesn't impose any limits on other mechanisms updating -# configuration parameters in runtime, e.g. sending SIGHUP or using API. This option should be set to false -# e.g. for cloud users, for whom scylla's configuration should be changed only by support engineers. -# live_updatable_config_params_changeable_via_cql: true - -# **************** -# * GUARDRAILS * -# **************** - -# Guardrails to warn or fail when Replication Factor is smaller/greater than the threshold. -# Please note that the value of 0 is always allowed, -# which means that having no replication at all, i.e. RF = 0, is always valid. -# A guardrail value smaller than 0, e.g. -1, means that the guardrail is disabled. -# Commenting out a guardrail also means it is disabled. -# minimum_replication_factor_fail_threshold: -1 -# minimum_replication_factor_warn_threshold: 3 -# maximum_replication_factor_warn_threshold: -1 -# maximum_replication_factor_fail_threshold: -1 - -# Guardrails to warn about or disallow creating a keyspace with specific replication strategy. -# Each of these 2 settings is a list storing replication strategies considered harmful. -# The replication strategies to choose from are: -# 1) SimpleStrategy, -# 2) NetworkTopologyStrategy, -# 3) LocalStrategy, -# 4) EverywhereStrategy -# -# replication_strategy_warn_list: -# - SimpleStrategy -# replication_strategy_fail_list: - -api_ui_dir: /usr/lib/scylla/swagger-ui/dist/ -api_doc_dir: /usr/lib/scylla/api/api-doc/ -alternator_port: 8000 -alternator_write_isolation: only_rmw_uses_lwt -alternator_enforce_authorization: true -enable_ipv6_dns_lookup: true - -uuid_sstable_identifiers_enabled: false From f821fcc6fdbb7e4c8d9841998b7cc3161a39c3ec Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Wed, 20 Nov 2024 09:18:34 +0100 Subject: [PATCH 05/14] fix: typo in testing/scylla/config/scylla-ssl.yaml Co-authored-by: karol-kokoszka <50329145+karol-kokoszka@users.noreply.github.com> --- testing/scylla/config/scylla-ssl.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scylla/config/scylla-ssl.yaml b/testing/scylla/config/scylla-ssl.yaml index 65f7245c55..8ee09b0af2 100644 --- a/testing/scylla/config/scylla-ssl.yaml +++ b/testing/scylla/config/scylla-ssl.yaml @@ -1,5 +1,5 @@ # Scylla SSL enabled configuration. -# This file merged with testing/config/scylla.yaml during start-dev-env process when SSL_ENABLED=true. +# This file is merged with testing/config/scylla.yaml during start-dev-env process when SSL_ENABLED=true. native_transport_port_ssl: 9142 client_encryption_options: From 461ef995895ddf4c18608bf7c7260f344784b815 Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Wed, 20 Nov 2024 19:46:15 +0100 Subject: [PATCH 06/14] fix(test): use scylla cluster with SSL for integration tests This enables ssl only scylla cluster for the most of our integration tests in ci. This also fixes cqlping test so it supports a scylla cluster with ssl. --- .github/actions/test-setup/action.yml | 8 +- .github/cfg/integration-test-cfg.yaml | 11 ++- .github/cfg/integration-test-core.yaml | 19 +++-- .github/cfg/main.go | 5 ++ ...ation-tests-2023.1.11-IPV4-raftschema.yaml | 17 +++-- .../integration-tests-2023.1.11-IPV4.yaml | 17 +++-- ...ation-tests-2023.1.11-IPV6-raftschema.yaml | 17 +++-- .../integration-tests-2024.1.12-IPV4.yaml | 17 +++-- .../integration-tests-2024.1.12-IPV6.yaml | 17 +++-- .../integration-tests-6.2.0-IPV4-tablets.yaml | 17 +++-- .../integration-tests-6.2.0-IPV4.yaml | 17 +++-- ...ation-tests-6.2.0-IPV6-tablets-nossl.yaml} | 19 +++-- ...latest-enterprise-IPV4-tablets-nossl.yaml} | 19 +++-- ...egration-tests-latest-enterprise-IPV4.yaml | 16 ++-- Makefile | 1 + README.md | 8 +- pkg/ping/cqlping/cqlping_integration_test.go | 73 ++++++++++++++++++- 17 files changed, 230 insertions(+), 68 deletions(-) rename .github/workflows/{integration-tests-6.2.0-IPV6-tablets.yaml => integration-tests-6.2.0-IPV6-tablets-nossl.yaml} (82%) rename .github/workflows/{integration-tests-latest-enterprise-IPV4-tablets.yaml => integration-tests-latest-enterprise-IPV4-tablets-nossl.yaml} (82%) diff --git a/.github/actions/test-setup/action.yml b/.github/actions/test-setup/action.yml index bba816cc24..9ffb9bd674 100644 --- a/.github/actions/test-setup/action.yml +++ b/.github/actions/test-setup/action.yml @@ -18,6 +18,10 @@ inputs: description: "Should this action run 'make start-dev-env'" required: false default: 'true' + ssl-enabled: + description: "Specifies if Scylla cluster should use ssl only configuration or not" + required: false + default: 'true' runs: using: "composite" @@ -42,5 +46,5 @@ runs: - name: Start dev env if: inputs.start-dev-env == 'true' - run: make start-dev-env SCYLLA_VERSION=${{ inputs.scylla-version }} IP_FAMILY=${{ inputs.ip-family }} RAFT_SCHEMA=${{ inputs.raft-schema }} TABLETS=${{ inputs.tablets }} - shell: bash \ No newline at end of file + run: make start-dev-env SCYLLA_VERSION=${{ inputs.scylla-version }} IP_FAMILY=${{ inputs.ip-family }} RAFT_SCHEMA=${{ inputs.raft-schema }} TABLETS=${{ inputs.tablets }} SSL_ENABLED=${{ inputs.ssl-enabled }} + shell: bash diff --git a/.github/cfg/integration-test-cfg.yaml b/.github/cfg/integration-test-cfg.yaml index e1a34719e8..9a8b28f920 100644 --- a/.github/cfg/integration-test-cfg.yaml +++ b/.github/cfg/integration-test-cfg.yaml @@ -2,41 +2,49 @@ ip-family: IPV4 raft-schema: disabled tablets: none + ssl-enabled: true - scylla-version: scylla-enterprise:2023.1.11 ip-family: IPV4 raft-schema: enabled tablets: none + ssl-enabled: true - scylla-version: scylla-enterprise:2023.1.11 ip-family: IPV6 raft-schema: enabled tablets: none + ssl-enabled: true - scylla-version: scylla-enterprise:2024.1.12 ip-family: IPV4 raft-schema: none tablets: none + ssl-enabled: true - scylla-version: scylla-enterprise:2024.1.12 ip-family: IPV6 raft-schema: none tablets: none + ssl-enabled: true - scylla-version: scylla:6.2.0 ip-family: IPV4 raft-schema: none tablets: disabled + ssl-enabled: true - scylla-version: scylla:6.2.0 ip-family: IPV4 raft-schema: none tablets: enabled + ssl-enabled: true - scylla-version: scylla:6.2.0 ip-family: IPV6 raft-schema: none tablets: enabled + ssl-enabled: false - scylla-version: scylla-enterprise-nightly:latest-enterprise ip-family: IPV4 @@ -46,4 +54,5 @@ - scylla-version: scylla-enterprise-nightly:latest-enterprise ip-family: IPV4 raft-schema: none - tablets: enabled \ No newline at end of file + tablets: enabled + ssl-enabled: false diff --git a/.github/cfg/integration-test-core.yaml b/.github/cfg/integration-test-core.yaml index f4433455b6..579a2a2139 100644 --- a/.github/cfg/integration-test-core.yaml +++ b/.github/cfg/integration-test-core.yaml @@ -26,9 +26,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} tablets: ${{ env.tablets }} + ssl-enabled: ${{ env.ssl-enabled }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' restore-schema: name: Test restore schema @@ -44,11 +45,11 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} tablets: ${{ env.tablets }} - + ssl-enabled: ${{ env.ssl-enabled }} # Go does not support negative lookahead in regex expressions, so it has to be done manually. # This regex ensures that all restore tests that didn't match restore-tables job will be run here. - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' backup: name: Test backup @@ -64,9 +65,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} tablets: ${{ env.tablets }} + ssl-enabled: ${{ env.ssl-enabled }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair @@ -82,9 +84,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} tablets: ${{ env.tablets }} + ssl-enabled: ${{ env.ssl-enabled }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair small-pkg: name: Test other, smaller packages @@ -93,9 +96,10 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup @@ -104,6 +108,7 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} tablets: ${{ env.tablets }} + ssl-enabled: ${{ env.ssl-enabled }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping @@ -127,4 +132,4 @@ jobs: run: make pkg-integration-test PKG=./pkg/store - name: Run migrate tests - run: make pkg-integration-test PKG=./pkg/schema/migrate \ No newline at end of file + run: make pkg-integration-test PKG=./pkg/schema/migrate diff --git a/.github/cfg/main.go b/.github/cfg/main.go index 96c002841d..15182efc47 100644 --- a/.github/cfg/main.go +++ b/.github/cfg/main.go @@ -14,6 +14,7 @@ type integrationTestCfg struct { IPFamily string `yaml:"ip-family"` RaftSchema string `yaml:"raft-schema"` Tablets string `yaml:"tablets"` + SSLEnabled string `yaml:"ssl-enabled,omitempty"` } func (cfg integrationTestCfg) name() string { @@ -29,6 +30,10 @@ func (cfg integrationTestCfg) name() string { if cfg.Tablets == "enabled" { parts = append(parts, "tablets") } + if cfg.SSLEnabled == "false" { + parts = append(parts, "nossl") + + } return strings.Join(parts, "-") } diff --git a/.github/workflows/integration-tests-2023.1.11-IPV4-raftschema.yaml b/.github/workflows/integration-tests-2023.1.11-IPV4-raftschema.yaml index 5fec04a396..04e53937a4 100644 --- a/.github/workflows/integration-tests-2023.1.11-IPV4-raftschema.yaml +++ b/.github/workflows/integration-tests-2023.1.11-IPV4-raftschema.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: enabled tablets: none + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-2023.1.11-IPV4.yaml b/.github/workflows/integration-tests-2023.1.11-IPV4.yaml index 83f8750b73..b8bf790a6c 100644 --- a/.github/workflows/integration-tests-2023.1.11-IPV4.yaml +++ b/.github/workflows/integration-tests-2023.1.11-IPV4.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: disabled tablets: none + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-2023.1.11-IPV6-raftschema.yaml b/.github/workflows/integration-tests-2023.1.11-IPV6-raftschema.yaml index a20ba2634a..02dee7238e 100644 --- a/.github/workflows/integration-tests-2023.1.11-IPV6-raftschema.yaml +++ b/.github/workflows/integration-tests-2023.1.11-IPV6-raftschema.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV6 raft-schema: enabled tablets: none + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-2024.1.12-IPV4.yaml b/.github/workflows/integration-tests-2024.1.12-IPV4.yaml index 7d0a94c329..d0b5a60128 100644 --- a/.github/workflows/integration-tests-2024.1.12-IPV4.yaml +++ b/.github/workflows/integration-tests-2024.1.12-IPV4.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: none tablets: none + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-2024.1.12-IPV6.yaml b/.github/workflows/integration-tests-2024.1.12-IPV6.yaml index 03e4ae174a..ac781fa308 100644 --- a/.github/workflows/integration-tests-2024.1.12-IPV6.yaml +++ b/.github/workflows/integration-tests-2024.1.12-IPV6.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV6 raft-schema: none tablets: none + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-6.2.0-IPV4-tablets.yaml b/.github/workflows/integration-tests-6.2.0-IPV4-tablets.yaml index e35915f741..f1c4836a09 100644 --- a/.github/workflows/integration-tests-6.2.0-IPV4-tablets.yaml +++ b/.github/workflows/integration-tests-6.2.0-IPV4-tablets.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: none tablets: enabled + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-6.2.0-IPV4.yaml b/.github/workflows/integration-tests-6.2.0-IPV4.yaml index 25e34459a2..0758c835f8 100644 --- a/.github/workflows/integration-tests-6.2.0-IPV4.yaml +++ b/.github/workflows/integration-tests-6.2.0-IPV4.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: none tablets: disabled + ssl-enabled: "true" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/.github/workflows/integration-tests-6.2.0-IPV6-tablets.yaml b/.github/workflows/integration-tests-6.2.0-IPV6-tablets-nossl.yaml similarity index 82% rename from .github/workflows/integration-tests-6.2.0-IPV6-tablets.yaml rename to .github/workflows/integration-tests-6.2.0-IPV6-tablets-nossl.yaml index 829585560e..745a89c680 100644 --- a/.github/workflows/integration-tests-6.2.0-IPV6-tablets.yaml +++ b/.github/workflows/integration-tests-6.2.0-IPV6-tablets-nossl.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV6 raft-schema: none tablets: enabled + ssl-enabled: "false" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping @@ -99,7 +106,7 @@ jobs: run: make pkg-integration-test PKG=./pkg/store - name: Run migrate tests run: make pkg-integration-test PKG=./pkg/schema/migrate -name: integration-tests-6.2.0-IPV6-tablets +name: integration-tests-6.2.0-IPV6-tablets-nossl "on": pull_request: types: diff --git a/.github/workflows/integration-tests-latest-enterprise-IPV4-tablets.yaml b/.github/workflows/integration-tests-latest-enterprise-IPV4-tablets-nossl.yaml similarity index 82% rename from .github/workflows/integration-tests-latest-enterprise-IPV4-tablets.yaml rename to .github/workflows/integration-tests-latest-enterprise-IPV4-tablets-nossl.yaml index cc9663318d..a7146f9ae6 100644 --- a/.github/workflows/integration-tests-latest-enterprise-IPV4-tablets.yaml +++ b/.github/workflows/integration-tests-latest-enterprise-IPV4-tablets-nossl.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: none tablets: enabled + ssl-enabled: "false" jobs: backup: name: Test backup @@ -19,9 +20,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +36,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +52,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +68,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping @@ -99,7 +106,7 @@ jobs: run: make pkg-integration-test PKG=./pkg/store - name: Run migrate tests run: make pkg-integration-test PKG=./pkg/schema/migrate -name: integration-tests-latest-enterprise-IPV4-tablets +name: integration-tests-latest-enterprise-IPV4-tablets-nossl "on": pull_request: types: diff --git a/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml b/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml index 9e7e64989f..2344ba5ba6 100644 --- a/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml +++ b/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml @@ -19,9 +19,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/backup + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/backup repair: name: Test repair runs-on: ubuntu-latest @@ -34,9 +35,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/repair + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/repair restore-schema: name: Test restore schema runs-on: ubuntu-latest @@ -49,9 +51,10 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestore([^T]|.{1}[^a]|.{2}[^b]|.{3}[^l]|.{4}[^e]|.{5}[^s]).*Integration"' restore-tables: name: Test restore tables runs-on: ubuntu-latest @@ -64,24 +67,27 @@ jobs: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run tests - run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' + run: make pkg-integration-test IP_FAMILY=${{ env.ip-family }} SSL_ENABLED=${{ env.ssl-enabled}} PKG=./pkg/service/restore RUN='"TestRestoreTables.*Integration"' small-pkg: name: Test other, smaller packages runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Set IP_FAMILY var for all tests + - name: Set IP_FAMILY and SSL_ENABLED var for all tests run: | echo "IP_FAMILY=${{ env.ip-family }}" >> $GITHUB_ENV + echo "SSL_ENABLED=${{ env.ssl-enabled }}" >> $GITHUB_ENV - name: Setup testing dependencies uses: ./.github/actions/test-setup with: ip-family: ${{ env.ip-family }} raft-schema: ${{ env.raft-schema }} scylla-version: ${{ env.scylla-version }} + ssl-enabled: ${{ env.ssl-enabled }} tablets: ${{ env.tablets }} - name: Run cqlping tests run: make pkg-integration-test PKG=./pkg/ping/cqlping diff --git a/Makefile b/Makefile index b026f936c9..569ca52b3e 100644 --- a/Makefile +++ b/Makefile @@ -139,6 +139,7 @@ pkg-integration-test: -v "$(PWD)/testing:/integration-test/testing" \ -v "$(PWD)/$(PKG)/testdata:/integration-test/testdata" \ -w "/integration-test" \ + -e "SSL_ENABLED=$(SSL_ENABLED)" \ -u $(CURRENT_UID):$(CURRENT_GID) \ -i --read-only --rm ubuntu integration-test -test.v -test.run $(RUN) $(INTEGRATION_TEST_ARGS) $(SSL_FLAGS) $(ARGS) diff --git a/README.md b/README.md index 1a53ac98f7..24c794eee2 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ Scylla Manager consists of tree components: |-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------| | **2024.1.12** | ![integration-tests-2024.1.12-IPV4]
![integration-tests-2024.1.12-IPV6] | Restoration of schema into cluster with `consistant_cluster_management: true` is not supported | | **2023.1.11** | ![integration-tests-2023.1.11-IPV4]
![integration-tests-2023.1.11-IPV4-raftschema]
![integration-tests-2023.1.11-IPV6-raftschema] | Restoration of schema into cluster with `consistant_cluster_management: true` is not supported | -| **6.2.0** | ![integration-tests-6.2.0-IPV4]
![integration-tests-6.2.0-IPV4-tablets]
![integration-tests-6.2.0-IPV6-tablets] | Restoration of **Authentication** and **Service Levels** is not supported
Restoration of schema containing **Alternator** tables is not supported | -| **latest-enterprise** | ![integration-tests-latest-enterprise-IPV4]
![integration-tests-latest-enterprise-IPV4-tablets] | Restoration of **Authentication** and **Service Levels** is not supported
Restoration of schema containing **Alternator** tables is not supported | +| **6.2.0** | ![integration-tests-6.2.0-IPV4]
![integration-tests-6.2.0-IPV4-tablets]
![integration-tests-6.2.0-IPV6-tablets-nossl] | Restoration of **Authentication** and **Service Levels** is not supported
Restoration of schema containing **Alternator** tables is not supported | +| **latest-enterprise** | ![integration-tests-latest-enterprise-IPV4]
![integration-tests-latest-enterprise-IPV4-tablets-nossl] | Restoration of **Authentication** and **Service Levels** is not supported
Restoration of schema containing **Alternator** tables is not supported | [integration-tests-2024.1.12-IPV4]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-2024.1.12-IPV4.yaml/badge.svg?branch=master [integration-tests-2024.1.12-IPV6]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-2024.1.12-IPV6.yaml/badge.svg?branch=master @@ -29,9 +29,9 @@ Scylla Manager consists of tree components: [integration-tests-2023.1.11-IPV6-raftschema]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-2023.1.11-IPV6-raftschema.yaml/badge.svg?branch=master [integration-tests-6.2.0-IPV4]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-6.2.0-IPV4.yaml/badge.svg?branch=master [integration-tests-6.2.0-IPV4-tablets]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-6.2.0-IPV4-tablets.yaml/badge.svg?branch=master -[integration-tests-6.2.0-IPV6-tablets]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-6.2.0-IPV6-tablets.yaml/badge.svg?branch=master +[integration-tests-6.2.0-IPV6-tablets-nossl]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-6.2.0-IPV6-tablets-nossl.yaml/badge.svg?branch=master [integration-tests-latest-enterprise-IPV4]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-latest-enterprise-IPV4.yaml/badge.svg?branch=master -[integration-tests-latest-enterprise-IPV4-tablets]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-latest-enterprise-IPV4-tablets.yaml/badge.svg?branch=master +[integration-tests-latest-enterprise-IPV4-tablets-nossl]: https://github.com/scylladb/scylla-manager/actions/workflows/integration-tests-latest-enterprise-IPV4-tablets-nossl.yaml/badge.svg?branch=master ## Installing and updating Go diff --git a/pkg/ping/cqlping/cqlping_integration_test.go b/pkg/ping/cqlping/cqlping_integration_test.go index 52b5a15c14..5b991b0f0d 100644 --- a/pkg/ping/cqlping/cqlping_integration_test.go +++ b/pkg/ping/cqlping/cqlping_integration_test.go @@ -8,9 +8,15 @@ package cqlping import ( "context" "crypto/tls" + "crypto/x509" + "fmt" + "os" + "strconv" "testing" "time" + "github.com/gocql/gocql" + "github.com/pkg/errors" "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" "github.com/scylladb/go-log" @@ -25,17 +31,30 @@ func TestPingIntegration(t *testing.T) { client := newTestClient(t, log.NewDevelopmentWithLevel(zapcore.InfoLevel).Named("client"), nil) defer client.Close() - sessionHosts, err := cluster.GetRPCAddresses(context.Background(), client, []string{testconfig.ManagedClusterHost()}, false) + sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) if err != nil { - t.Fatal(err) + t.Fatalf("parse SSL_ENABLED env var: %v\n", err) } + sessionHosts, err := cluster.GetRPCAddresses(context.Background(), client, []string{testconfig.ManagedClusterHost()}, !sslEnabled) + if err != nil { + t.Fatal(err) + } user, password := testconfig.ManagedClusterCredentials() config := Config{ Addr: sessionHosts[0], Timeout: 250 * time.Millisecond, } + if sslEnabled { + testSSLConfig := testconfig.CQLSSLOptions() + tlsConfig, err := setupTLSConfig(testSSLConfig) + if err != nil { + t.Fatalf("setup tls config: %v", err) + } + config.TLSConfig = tlsConfig + } + t.Run("simple", func(t *testing.T) { d, err := NativeCQLPing(context.Background(), config, log.NopLogger) if err != nil { @@ -106,3 +125,53 @@ func newTestClient(t *testing.T, logger log.Logger, config *scyllaclient.Config) } return c } + +// setupTLSConfig copied from github.com/gocql/gocql/connectionpool.go to convert gocql.SslOptions into *tls.Config +func setupTLSConfig(sslOpts *gocql.SslOptions) (*tls.Config, error) { + // Config.InsecureSkipVerify | EnableHostVerification | Result + // Config is nil | true | verify host + // Config is nil | false | do not verify host + // false | false | verify host + // true | false | do not verify host + // false | true | verify host + // true | true | verify host + var tlsConfig *tls.Config + if sslOpts.Config == nil { + tlsConfig = &tls.Config{ + InsecureSkipVerify: !sslOpts.EnableHostVerification, + } + } else { + // use clone to avoid race. + tlsConfig = sslOpts.Config.Clone() + } + + if tlsConfig.InsecureSkipVerify && sslOpts.EnableHostVerification { + tlsConfig.InsecureSkipVerify = false + } + + // ca cert is optional + if sslOpts.CaPath != "" { + if tlsConfig.RootCAs == nil { + tlsConfig.RootCAs = x509.NewCertPool() + } + + pem, err := os.ReadFile(sslOpts.CaPath) + if err != nil { + return nil, fmt.Errorf("connectionpool: unable to open CA certs: %v", err) + } + + if !tlsConfig.RootCAs.AppendCertsFromPEM(pem) { + return nil, errors.New("connectionpool: failed parsing or CA certs") + } + } + + if sslOpts.CertPath != "" || sslOpts.KeyPath != "" { + mycert, err := tls.LoadX509KeyPair(sslOpts.CertPath, sslOpts.KeyPath) + if err != nil { + return nil, fmt.Errorf("connectionpool: unable to load X509 key pair: %v", err) + } + tlsConfig.Certificates = append(tlsConfig.Certificates, mycert) + } + + return tlsConfig, nil +} From 62eb540bffe9cc1829f2622aa0dead31e484d20b Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Wed, 20 Nov 2024 19:56:59 +0100 Subject: [PATCH 07/14] fix(cluster): simplifies SingleHostSessionOption when dealing with SSL This changes the signature of SessionConfigOption so that SingleHostSession func can be simplified when Scylla cluster uses SSL. --- pkg/cmd/scylla-manager/server.go | 2 +- pkg/service/backup/service.go | 47 +++++++++---------- .../backup/service_backup_integration_test.go | 9 +--- pkg/service/backup/worker.go | 17 ++++--- pkg/service/backup/worker_schema.go | 2 +- pkg/service/cluster/service.go | 19 ++++---- .../restore/helper_integration_test.go | 9 +--- .../service_restore_integration_test.go | 9 +--- 8 files changed, 49 insertions(+), 65 deletions(-) diff --git a/pkg/cmd/scylla-manager/server.go b/pkg/cmd/scylla-manager/server.go index cb4c54ad0f..8c9a40c228 100644 --- a/pkg/cmd/scylla-manager/server.go +++ b/pkg/cmd/scylla-manager/server.go @@ -112,7 +112,7 @@ func (s *server) makeServices(ctx context.Context) error { s.session, s.config.Backup, metrics.NewBackupMetrics().MustRegister(), - s.clusterSvc.GetClusterByID, + s.clusterSvc.GetClusterName, s.clusterSvc.Client, s.clusterSvc.GetSession, s.logger.Named("backup"), diff --git a/pkg/service/backup/service.go b/pkg/service/backup/service.go index 509ebee22c..4456517106 100644 --- a/pkg/service/backup/service.go +++ b/pkg/service/backup/service.go @@ -42,16 +42,16 @@ type Service struct { config Config metrics metrics.BackupMetrics - clusterProvider cluster.ProviderFunc - scyllaClient scyllaclient.ProviderFunc - clusterSession cluster.SessionFunc - logger log.Logger + clusterName cluster.NameFunc + scyllaClient scyllaclient.ProviderFunc + clusterSession cluster.SessionFunc + logger log.Logger dth deduplicateTestHooks } func NewService(session gocqlx.Session, config Config, metrics metrics.BackupMetrics, - clusterProvider cluster.ProviderFunc, scyllaClient scyllaclient.ProviderFunc, clusterSession cluster.SessionFunc, logger log.Logger, + clusterName cluster.NameFunc, scyllaClient scyllaclient.ProviderFunc, clusterSession cluster.SessionFunc, logger log.Logger, ) (*Service, error) { if session.Session == nil || session.Closed() { return nil, errors.New("invalid session") @@ -61,7 +61,7 @@ func NewService(session gocqlx.Session, config Config, metrics metrics.BackupMet return nil, errors.Wrap(err, "invalid config") } - if clusterProvider == nil { + if clusterName == nil { return nil, errors.New("invalid cluster name provider") } @@ -70,13 +70,13 @@ func NewService(session gocqlx.Session, config Config, metrics metrics.BackupMet } return &Service{ - session: session, - config: config, - metrics: metrics, - clusterProvider: clusterProvider, - scyllaClient: scyllaClient, - clusterSession: clusterSession, - logger: logger, + session: session, + config: config, + metrics: metrics, + clusterName: clusterName, + scyllaClient: scyllaClient, + clusterSession: clusterSession, + logger: logger, }, nil } @@ -667,23 +667,22 @@ func (s *Service) Backup(ctx context.Context, clusterID, taskID, runID uuid.UUID return errors.Wrap(err, "initialize: register the run") } - // Get cluster info by id - cluster, err := s.clusterProvider(ctx, run.ClusterID) + // Get cluster name + clusterName, err := s.clusterName(ctx, run.ClusterID) if err != nil { - return errors.Wrap(err, "cluster provider") + return errors.Wrap(err, "invalid cluster") } // Create a worker w := &worker{ workerTools: workerTools{ - ClusterID: clusterID, - ClusterName: cluster.Name, - ClusterTLSAddrDisabled: cluster.ForceTLSDisabled || cluster.ForceNonSSLSessionPort, - TaskID: taskID, - RunID: runID, - SnapshotTag: run.SnapshotTag, - Config: s.config, - Client: client, + ClusterID: clusterID, + ClusterName: clusterName, + TaskID: taskID, + RunID: runID, + SnapshotTag: run.SnapshotTag, + Config: s.config, + Client: client, }, PrevStage: run.Stage, Metrics: s.metrics, diff --git a/pkg/service/backup/service_backup_integration_test.go b/pkg/service/backup/service_backup_integration_test.go index b4b7d8365a..610ab0f637 100644 --- a/pkg/service/backup/service_backup_integration_test.go +++ b/pkg/service/backup/service_backup_integration_test.go @@ -115,13 +115,8 @@ func newTestServiceWithUser(t *testing.T, session gocqlx.Session, client *scylla session, c, metrics.NewBackupMetrics(), - func(_ context.Context, id uuid.UUID) (*cluster.Cluster, error) { - return &cluster.Cluster{ - ID: id, - Name: "test_cluster", - ForceTLSDisabled: false, - ForceNonSSLSessionPort: false, - }, nil + func(_ context.Context, id uuid.UUID) (string, error) { + return "test_cluster", nil }, func(context.Context, uuid.UUID) (*scyllaclient.Client, error) { return client, nil diff --git a/pkg/service/backup/worker.go b/pkg/service/backup/worker.go index 465f0db647..3555f80412 100644 --- a/pkg/service/backup/worker.go +++ b/pkg/service/backup/worker.go @@ -51,15 +51,14 @@ func (sd snapshotDir) String() string { // workerTools is an intersection of fields and methods // useful for both worker and restoreWorker. type workerTools struct { - ClusterID uuid.UUID - ClusterName string - ClusterTLSAddrDisabled bool - TaskID uuid.UUID - RunID uuid.UUID - SnapshotTag string - Config Config - Client *scyllaclient.Client - Logger log.Logger + ClusterID uuid.UUID + ClusterName string + TaskID uuid.UUID + RunID uuid.UUID + SnapshotTag string + Config Config + Client *scyllaclient.Client + Logger log.Logger } // worker is responsible for coordinating backup procedure. diff --git a/pkg/service/backup/worker_schema.go b/pkg/service/backup/worker_schema.go index 0338a95d66..e22d421d76 100644 --- a/pkg/service/backup/worker_schema.go +++ b/pkg/service/backup/worker_schema.go @@ -141,7 +141,7 @@ func (w *worker) UploadSchema(ctx context.Context, hosts []hostInfo) (stepError func (w *worker) createSingleHostSessionToAnyHost(ctx context.Context, hosts []string, sessionFunc cluster.SessionFunc) (session gocqlx.Session, host string, err error) { var retErr error for _, h := range hosts { - session, err := sessionFunc(ctx, w.ClusterID, cluster.SingleHostSessionConfigOption(h, w.ClusterTLSAddrDisabled)) + session, err := sessionFunc(ctx, w.ClusterID, cluster.SingleHostSessionConfigOption(h)) if err != nil { w.Logger.Error(ctx, "Couldn't connect to host via CQL", "host", h, "error", err) retErr = err diff --git a/pkg/service/cluster/service.go b/pkg/service/cluster/service.go index b10433292b..353d67fea0 100644 --- a/pkg/service/cluster/service.go +++ b/pkg/service/cluster/service.go @@ -607,16 +607,16 @@ func (s *Service) ListNodes(ctx context.Context, clusterID uuid.UUID) ([]Node, e } // SessionConfigOption defines function modifying cluster config that can be used when creating session. -type SessionConfigOption func(ctx context.Context, clusterID uuid.UUID, client *scyllaclient.Client, cfg *gocql.ClusterConfig) error +type SessionConfigOption func(ctx context.Context, cluster *Cluster, client *scyllaclient.Client, cfg *gocql.ClusterConfig) error // SingleHostSessionConfigOption ensures that session will be connected only to the single, provided host. -func SingleHostSessionConfigOption(host string, clusterTLSAddrDisabled bool) SessionConfigOption { - return func(ctx context.Context, _ uuid.UUID, client *scyllaclient.Client, cfg *gocql.ClusterConfig) error { +func SingleHostSessionConfigOption(host string) SessionConfigOption { + return func(ctx context.Context, cluster *Cluster, client *scyllaclient.Client, cfg *gocql.ClusterConfig) error { ni, err := client.NodeInfo(ctx, host) if err != nil { return errors.Wrapf(err, "fetch node (%s) info", host) } - cqlAddr := ni.CQLAddr(host, clusterTLSAddrDisabled) + cqlAddr := ni.CQLAddr(host, cluster.ForceTLSDisabled || cluster.ForceNonSSLSessionPort) cfg.Hosts = []string{cqlAddr} cfg.DisableInitialHostLookup = true cfg.HostFilter = gocql.WhiteListHostFilter(cqlAddr) @@ -637,17 +637,18 @@ func (s *Service) GetSession(ctx context.Context, clusterID uuid.UUID, opts ...S } defer logutil.LogOnError(ctx, s.logger, client.Close, "Couldn't close scylla client") + clusterInfo, err := s.GetClusterByID(ctx, clusterID) + if err != nil { + return session, errors.Wrap(err, "cluster by id") + } + cfg := gocql.NewCluster() for _, opt := range opts { - if err := opt(ctx, clusterID, client, cfg); err != nil { + if err := opt(ctx, clusterInfo, client, cfg); err != nil { return session, err } } - clusterInfo, err := s.GetClusterByID(ctx, clusterID) - if err != nil { - return session, errors.Wrap(err, "cluster by id") - } // Fill hosts if they weren't specified by the options or make sure that they use correct rpc address. if len(cfg.Hosts) == 0 { sessionHosts, err := GetRPCAddresses(ctx, client, client.Config().Hosts, clusterInfo.ForceTLSDisabled || clusterInfo.ForceNonSSLSessionPort) diff --git a/pkg/service/restore/helper_integration_test.go b/pkg/service/restore/helper_integration_test.go index 0281e20d72..517884efcb 100644 --- a/pkg/service/restore/helper_integration_test.go +++ b/pkg/service/restore/helper_integration_test.go @@ -126,13 +126,8 @@ func newBackupSvc(t *testing.T, mgrSession gocqlx.Session, client *scyllaclient. mgrSession, defaultBackupTestConfig(), metrics.NewBackupMetrics(), - func(_ context.Context, id uuid.UUID) (*cluster.Cluster, error) { - return &cluster.Cluster{ - ID: id, - Name: "test_cluster", - ForceTLSDisabled: false, - ForceNonSSLSessionPort: false, - }, nil + func(_ context.Context, id uuid.UUID) (string, error) { + return "test_cluster", nil }, func(context.Context, uuid.UUID) (*scyllaclient.Client, error) { return client, nil diff --git a/pkg/service/restore/service_restore_integration_test.go b/pkg/service/restore/service_restore_integration_test.go index 7a2072c030..1892128ee0 100644 --- a/pkg/service/restore/service_restore_integration_test.go +++ b/pkg/service/restore/service_restore_integration_test.go @@ -128,13 +128,8 @@ func newTestService(t *testing.T, session gocqlx.Session, client *scyllaclient.C session, defaultBackupTestConfig(), metrics.NewBackupMetrics(), - func(_ context.Context, id uuid.UUID) (*cluster.Cluster, error) { - return &cluster.Cluster{ - ID: id, - Name: "test_cluster", - ForceTLSDisabled: false, - ForceNonSSLSessionPort: false, - }, nil + func(_ context.Context, id uuid.UUID) (string, error) { + return "test_cluster", nil }, func(context.Context, uuid.UUID) (*scyllaclient.Client, error) { return client, nil From 87cdcc0dfc15ab4cc99cd7a19b7a9b04ad1b7f64 Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Thu, 21 Nov 2024 11:00:50 +0100 Subject: [PATCH 08/14] fix(test): adds ssl support to cqlping integration tests This adds ssl related configuration options to cqlping integration tests config when ssl is enabled. --- pkg/ping/cqlping/cqlping_integration_test.go | 58 +------------------- pkg/testutils/testconfig/testconfig.go | 55 +++++++++++++++++++ 2 files changed, 57 insertions(+), 56 deletions(-) diff --git a/pkg/ping/cqlping/cqlping_integration_test.go b/pkg/ping/cqlping/cqlping_integration_test.go index 5b991b0f0d..476e19be85 100644 --- a/pkg/ping/cqlping/cqlping_integration_test.go +++ b/pkg/ping/cqlping/cqlping_integration_test.go @@ -8,15 +8,11 @@ package cqlping import ( "context" "crypto/tls" - "crypto/x509" - "fmt" "os" "strconv" "testing" "time" - "github.com/gocql/gocql" - "github.com/pkg/errors" "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" "github.com/scylladb/go-log" @@ -47,8 +43,8 @@ func TestPingIntegration(t *testing.T) { } if sslEnabled { - testSSLConfig := testconfig.CQLSSLOptions() - tlsConfig, err := setupTLSConfig(testSSLConfig) + sslOpts := testconfig.CQLSSLOptions() + tlsConfig, err := testconfig.TLSConfig(sslOpts) if err != nil { t.Fatalf("setup tls config: %v", err) } @@ -125,53 +121,3 @@ func newTestClient(t *testing.T, logger log.Logger, config *scyllaclient.Config) } return c } - -// setupTLSConfig copied from github.com/gocql/gocql/connectionpool.go to convert gocql.SslOptions into *tls.Config -func setupTLSConfig(sslOpts *gocql.SslOptions) (*tls.Config, error) { - // Config.InsecureSkipVerify | EnableHostVerification | Result - // Config is nil | true | verify host - // Config is nil | false | do not verify host - // false | false | verify host - // true | false | do not verify host - // false | true | verify host - // true | true | verify host - var tlsConfig *tls.Config - if sslOpts.Config == nil { - tlsConfig = &tls.Config{ - InsecureSkipVerify: !sslOpts.EnableHostVerification, - } - } else { - // use clone to avoid race. - tlsConfig = sslOpts.Config.Clone() - } - - if tlsConfig.InsecureSkipVerify && sslOpts.EnableHostVerification { - tlsConfig.InsecureSkipVerify = false - } - - // ca cert is optional - if sslOpts.CaPath != "" { - if tlsConfig.RootCAs == nil { - tlsConfig.RootCAs = x509.NewCertPool() - } - - pem, err := os.ReadFile(sslOpts.CaPath) - if err != nil { - return nil, fmt.Errorf("connectionpool: unable to open CA certs: %v", err) - } - - if !tlsConfig.RootCAs.AppendCertsFromPEM(pem) { - return nil, errors.New("connectionpool: failed parsing or CA certs") - } - } - - if sslOpts.CertPath != "" || sslOpts.KeyPath != "" { - mycert, err := tls.LoadX509KeyPair(sslOpts.CertPath, sslOpts.KeyPath) - if err != nil { - return nil, fmt.Errorf("connectionpool: unable to load X509 key pair: %v", err) - } - tlsConfig.Certificates = append(tlsConfig.Certificates, mycert) - } - - return tlsConfig, nil -} diff --git a/pkg/testutils/testconfig/testconfig.go b/pkg/testutils/testconfig/testconfig.go index 2ac2231c51..ad905245f1 100644 --- a/pkg/testutils/testconfig/testconfig.go +++ b/pkg/testutils/testconfig/testconfig.go @@ -3,7 +3,12 @@ package testconfig import ( + "crypto/tls" + "crypto/x509" + "errors" "flag" + "fmt" + "os" "strings" "time" @@ -123,3 +128,53 @@ func ScyllaManagerDBCluster() string { } return *flagCluster } + +// TLSConfig returns tls.Config to work ssl enabled scylla cluster. +// this function is almost an exact copy of setupTLSConfig from github.com/gocql/gocql/connectionpool.go. +func TLSConfig(sslOpts *gocql.SslOptions) (*tls.Config, error) { + // Config.InsecureSkipVerify | EnableHostVerification | Result + // Config is nil | true | verify host + // Config is nil | false | do not verify host + // false | false | verify host + // true | false | do not verify host + // false | true | verify host + // true | true | verify host + var tlsConfig *tls.Config + if sslOpts.Config == nil { + tlsConfig = &tls.Config{ + InsecureSkipVerify: !sslOpts.EnableHostVerification, + } + } else { + // use clone to avoid race. + tlsConfig = sslOpts.Config.Clone() + } + + if tlsConfig.InsecureSkipVerify && sslOpts.EnableHostVerification { + tlsConfig.InsecureSkipVerify = false + } + // ca cert is optional. + if sslOpts.CaPath != "" { + if tlsConfig.RootCAs == nil { + tlsConfig.RootCAs = x509.NewCertPool() + } + + pem, err := os.ReadFile(sslOpts.CaPath) + if err != nil { + return nil, fmt.Errorf("connectionpool: unable to open CA certs: %w", err) + } + + if !tlsConfig.RootCAs.AppendCertsFromPEM(pem) { + return nil, errors.New("connectionpool: failed parsing or CA certs") + } + } + + if sslOpts.CertPath != "" || sslOpts.KeyPath != "" { + mycert, err := tls.LoadX509KeyPair(sslOpts.CertPath, sslOpts.KeyPath) + if err != nil { + return nil, fmt.Errorf("connectionpool: unable to load X509 key pair: %w", err) + } + tlsConfig.Certificates = append(tlsConfig.Certificates, mycert) + } + + return tlsConfig, nil +} From a3124d8717b2a520d49b8d4d87a67f78fa53dbfa Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Thu, 21 Nov 2024 11:02:41 +0100 Subject: [PATCH 09/14] fix(test): adds ssl support to repair integration test This adds ssl support to repair integartion test case that uses cqlping --- .../repair/service_repair_integration_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/service/repair/service_repair_integration_test.go b/pkg/service/repair/service_repair_integration_test.go index 00c1735a8c..cc995aad6f 100644 --- a/pkg/service/repair/service_repair_integration_test.go +++ b/pkg/service/repair/service_repair_integration_test.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "net/http" + "os" "regexp" "sort" "strconv" @@ -33,6 +34,7 @@ import ( "github.com/scylladb/scylla-manager/v3/pkg/schema/table" "github.com/scylladb/scylla-manager/v3/pkg/service/cluster" "github.com/scylladb/scylla-manager/v3/pkg/service/scheduler" + "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" . "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" . "github.com/scylladb/scylla-manager/v3/pkg/testutils/testhelper" "github.com/scylladb/scylla-manager/v3/pkg/util" @@ -1210,6 +1212,10 @@ func TestServiceRepairIntegration(t *testing.T) { }) t.Run("repair ignore hosts", func(t *testing.T) { + sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) + if err != nil { + t.Fatalf("parse SSL_ENABLED env var: %v\n", err) + } h := newRepairTestHelper(t, session, defaultConfig()) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1234,6 +1240,14 @@ func TestServiceRepairIntegration(t *testing.T) { Addr: ni.CQLAddr(ignored, false), Timeout: time.Minute, } + if sslEnabled { + sslOpts := testconfig.CQLSSLOptions() + tlsConfig, err := testconfig.TLSConfig(sslOpts) + if err != nil { + t.Fatalf("setup tls config: %v", err) + } + cfg.TLSConfig = tlsConfig + } cond := func() bool { if _, err = cqlping.QueryPing(ctx, cfg, TestDBUsername(), TestDBPassword()); err != nil { From 22412dfe0a52f13accbc1981dc6aa9a58a73d005 Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Thu, 21 Nov 2024 11:03:41 +0100 Subject: [PATCH 10/14] fix(test): adds ssl support to healthcheck integration tests --- .../healthcheck/service_integration_test.go | 147 ++++++++++++------ pkg/testutils/exec.go | 6 + 2 files changed, 102 insertions(+), 51 deletions(-) diff --git a/pkg/service/healthcheck/service_integration_test.go b/pkg/service/healthcheck/service_integration_test.go index 7dbe0b4376..069acd428b 100644 --- a/pkg/service/healthcheck/service_integration_test.go +++ b/pkg/service/healthcheck/service_integration_test.go @@ -11,6 +11,8 @@ import ( "io" "net" "net/http" + "os" + "strconv" "strings" "testing" "time" @@ -77,6 +79,17 @@ func TestStatus_Ping_Independent_From_REST_Integration(t *testing.T) { Host: hostWithUnresponsiveREST, AuthToken: "token", } + sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) + if err != nil { + t.Fatalf("parse SSL_ENABLED env var: %v\n", err) + } + if sslEnabled { + sslOpts := CQLSSLOptions() + userKey, _ := os.ReadFile(sslOpts.KeyPath) + userCrt, _ := os.ReadFile(sslOpts.CertPath) + testCluster.SSLUserKeyFile = userKey + testCluster.SSLUserCertFile = userCrt + } err = clusterSvc.PutCluster(context.Background(), testCluster) if err != nil { t.Fatal(err) @@ -161,12 +174,25 @@ func TestStatusIntegration(t *testing.T) { Host: "192.168.200.11", AuthToken: "token", } + + sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) + if err != nil { + t.Fatalf("parse SSL_ENABLED env var: %v\n", err) + } + if sslEnabled { + sslOpts := CQLSSLOptions() + userKey, _ := os.ReadFile(sslOpts.KeyPath) + userCrt, _ := os.ReadFile(sslOpts.CertPath) + c.SSLUserKeyFile = userKey + c.SSLUserCertFile = userCrt + } + err = clusterSvc.PutCluster(context.Background(), c) if err != nil { t.Fatal(err) } - testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s) + testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s, sslEnabled) } func TestStatusWithCQLCredentialsIntegration(t *testing.T) { @@ -190,15 +216,26 @@ func TestStatusWithCQLCredentialsIntegration(t *testing.T) { Username: username, Password: password, } + sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) + if err != nil { + t.Fatalf("parse SSL_ENABLED env var: %v\n", err) + } + if sslEnabled { + sslOpts := CQLSSLOptions() + userKey, _ := os.ReadFile(sslOpts.KeyPath) + userCrt, _ := os.ReadFile(sslOpts.CertPath) + c.SSLUserKeyFile = userKey + c.SSLUserCertFile = userCrt + } err = clusterSvc.PutCluster(context.Background(), c) if err != nil { t.Fatal(err) } - testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s) + testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s, sslEnabled) } -func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster.Servicer, clusterProvider cluster.ProviderFunc, secretsStore store.Store) { +func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster.Servicer, clusterProvider cluster.ProviderFunc, secretsStore store.Store, sslEnabled bool) { logger := log.NewDevelopmentWithLevel(zapcore.InfoLevel).Named("healthcheck") // Tests here do not test the dynamic t/o functionality @@ -264,12 +301,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -290,20 +327,20 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "TIMEOUT", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "TIMEOUT", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) t.Run("node CQL TIMEOUT", func(t *testing.T) { host := IPFromTestNet("12") - blockCQL(t, host) - defer unblockCQL(t, host) + blockCQL(t, host, sslEnabled) + defer unblockCQL(t, host, sslEnabled) status, err := s.Status(context.Background(), clusterID) if err != nil { @@ -316,12 +353,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "TIMEOUT", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "TIMEOUT", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -342,12 +379,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "TIMEOUT"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "TIMEOUT", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -368,12 +405,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "DOWN", RESTCause: "dial tcp " + URLEncodeIP(ToCanonicalIP(IPFromTestNet("12"))) + ":10001: connect: connection refused", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "DOWN", RESTCause: "dial tcp " + URLEncodeIP(ToCanonicalIP(IPFromTestNet("12"))) + ":10001: connect: connection refused", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -393,12 +430,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UNAUTHORIZED", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "UNAUTHORIZED", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -418,12 +455,12 @@ func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster } golden := []NodeStatus{ - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "HTTP 502", AlternatorStatus: "UP"}, - {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, - {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP"}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("11")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("12")), CQLStatus: "UP", RESTStatus: "HTTP 502", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc1", Host: ToCanonicalIP(IPFromTestNet("13")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("21")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("22")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, + {Datacenter: "dc2", Host: ToCanonicalIP(IPFromTestNet("23")), CQLStatus: "UP", RESTStatus: "UP", AlternatorStatus: "UP", SSL: sslEnabled}, } assertEqual(t, golden, status) }) @@ -482,16 +519,24 @@ func tryUnblockREST(t *testing.T, hosts []string) { } } -func blockCQL(t *testing.T, h string) { +func blockCQL(t *testing.T, h string, sslEnabled bool) { t.Helper() - if err := RunIptablesCommand(h, CmdBlockScyllaCQL); err != nil { + cmd := CmdBlockScyllaCQL + if sslEnabled { + cmd = CmdBlockScyllaCQLSSL + } + if err := RunIptablesCommand(h, cmd); err != nil { t.Error(err) } } -func unblockCQL(t *testing.T, h string) { +func unblockCQL(t *testing.T, h string, sslEnabled bool) { t.Helper() - if err := RunIptablesCommand(h, CmdUnblockScyllaCQL); err != nil { + cmd := CmdUnblockScyllaCQL + if sslEnabled { + cmd = CmdUnblockScyllaCQLSSL + } + if err := RunIptablesCommand(h, cmd); err != nil { t.Error(err) } } diff --git a/pkg/testutils/exec.go b/pkg/testutils/exec.go index 0b768a2f4e..624dea384f 100644 --- a/pkg/testutils/exec.go +++ b/pkg/testutils/exec.go @@ -24,6 +24,12 @@ const ( // CmdUnblockScyllaCQL defines the command used for unblocking the Scylla CQL access. CmdUnblockScyllaCQL = "iptables -D INPUT -p tcp --destination-port 9042 -j DROP" + // CmdBlockScyllaCQLSSL defines the command used for blocking the Scylla CQL access. + CmdBlockScyllaCQLSSL = "iptables -A INPUT -p tcp --destination-port 9142 -j DROP" + + // CmdUnblockScyllaCQLSSL defines the command used for unblocking the Scylla CQL access. + CmdUnblockScyllaCQLSSL = "iptables -D INPUT -p tcp --destination-port 9142 -j DROP" + // CmdBlockScyllaAlternator defines the command used for blocking the Scylla Alternator access. CmdBlockScyllaAlternator = "iptables -A INPUT -p tcp --destination-port 8000 -j DROP" From 689580345e54399b658c292b67bfccca0f56b85c Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Thu, 21 Nov 2024 11:26:02 +0100 Subject: [PATCH 11/14] fix(test): unifies how SSL_ENABLED is used in testconfig --- pkg/testutils/db/db.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/testutils/db/db.go b/pkg/testutils/db/db.go index 05fff3bced..a7e39bb67f 100644 --- a/pkg/testutils/db/db.go +++ b/pkg/testutils/db/db.go @@ -101,7 +101,7 @@ func CreateManagedClusterSession(tb testing.TB, empty bool, client *scyllaclient Username: user, Password: pass, } - if os.Getenv("SSL_ENABLED") != "" { + if os.Getenv("SSL_ENABLED") != "false" { cluster.SslOpts = testconfig.CQLSSLOptions() cluster.Port = testconfig.CQLPort() } From 53889e0fda8c4da2665361deab4043003a0f4b90 Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Thu, 21 Nov 2024 11:40:53 +0100 Subject: [PATCH 12/14] fix(ci): adds missing ssl-enabled option for a one entry in ci config --- .github/cfg/integration-test-cfg.yaml | 1 + .github/workflows/integration-tests-latest-enterprise-IPV4.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/cfg/integration-test-cfg.yaml b/.github/cfg/integration-test-cfg.yaml index 9a8b28f920..5f7e51e0fe 100644 --- a/.github/cfg/integration-test-cfg.yaml +++ b/.github/cfg/integration-test-cfg.yaml @@ -50,6 +50,7 @@ ip-family: IPV4 raft-schema: none tablets: disabled + ssl-enabled: true - scylla-version: scylla-enterprise-nightly:latest-enterprise ip-family: IPV4 diff --git a/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml b/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml index 2344ba5ba6..b910a1f7b2 100644 --- a/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml +++ b/.github/workflows/integration-tests-latest-enterprise-IPV4.yaml @@ -6,6 +6,7 @@ env: ip-family: IPV4 raft-schema: none tablets: disabled + ssl-enabled: "true" jobs: backup: name: Test backup From 83690f977582e890f467213b1268eadaf3f5b900 Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Thu, 21 Nov 2024 13:08:28 +0100 Subject: [PATCH 13/14] refactor: moves parsing of SSL_ENABLED env var to the testconfig pkg This refactor some parts of the tests that are using SSL_ENABLED env var. --- pkg/ping/cqlping/cqlping_integration_test.go | 11 +--- .../healthcheck/service_integration_test.go | 62 ++++++++----------- .../repair/service_repair_integration_test.go | 7 +-- pkg/testutils/db/db.go | 2 +- pkg/testutils/testconfig/testconfig.go | 11 ++++ 5 files changed, 40 insertions(+), 53 deletions(-) diff --git a/pkg/ping/cqlping/cqlping_integration_test.go b/pkg/ping/cqlping/cqlping_integration_test.go index 476e19be85..8b6bec6d4b 100644 --- a/pkg/ping/cqlping/cqlping_integration_test.go +++ b/pkg/ping/cqlping/cqlping_integration_test.go @@ -8,8 +8,6 @@ package cqlping import ( "context" "crypto/tls" - "os" - "strconv" "testing" "time" @@ -27,12 +25,7 @@ func TestPingIntegration(t *testing.T) { client := newTestClient(t, log.NewDevelopmentWithLevel(zapcore.InfoLevel).Named("client"), nil) defer client.Close() - sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) - if err != nil { - t.Fatalf("parse SSL_ENABLED env var: %v\n", err) - } - - sessionHosts, err := cluster.GetRPCAddresses(context.Background(), client, []string{testconfig.ManagedClusterHost()}, !sslEnabled) + sessionHosts, err := cluster.GetRPCAddresses(context.Background(), client, []string{testconfig.ManagedClusterHost()}, !testconfig.IsSSLEnabled()) if err != nil { t.Fatal(err) } @@ -42,7 +35,7 @@ func TestPingIntegration(t *testing.T) { Timeout: 250 * time.Millisecond, } - if sslEnabled { + if testconfig.IsSSLEnabled() { sslOpts := testconfig.CQLSSLOptions() tlsConfig, err := testconfig.TLSConfig(sslOpts) if err != nil { diff --git a/pkg/service/healthcheck/service_integration_test.go b/pkg/service/healthcheck/service_integration_test.go index 069acd428b..4744e24148 100644 --- a/pkg/service/healthcheck/service_integration_test.go +++ b/pkg/service/healthcheck/service_integration_test.go @@ -12,7 +12,6 @@ import ( "net" "net/http" "os" - "strconv" "strings" "testing" "time" @@ -79,17 +78,8 @@ func TestStatus_Ping_Independent_From_REST_Integration(t *testing.T) { Host: hostWithUnresponsiveREST, AuthToken: "token", } - sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) - if err != nil { - t.Fatalf("parse SSL_ENABLED env var: %v\n", err) - } - if sslEnabled { - sslOpts := CQLSSLOptions() - userKey, _ := os.ReadFile(sslOpts.KeyPath) - userCrt, _ := os.ReadFile(sslOpts.CertPath) - testCluster.SSLUserKeyFile = userKey - testCluster.SSLUserCertFile = userCrt - } + clusterWithSSL(t, testCluster, IsSSLEnabled()) + err = clusterSvc.PutCluster(context.Background(), testCluster) if err != nil { t.Fatal(err) @@ -174,25 +164,14 @@ func TestStatusIntegration(t *testing.T) { Host: "192.168.200.11", AuthToken: "token", } - - sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) - if err != nil { - t.Fatalf("parse SSL_ENABLED env var: %v\n", err) - } - if sslEnabled { - sslOpts := CQLSSLOptions() - userKey, _ := os.ReadFile(sslOpts.KeyPath) - userCrt, _ := os.ReadFile(sslOpts.CertPath) - c.SSLUserKeyFile = userKey - c.SSLUserCertFile = userCrt - } + clusterWithSSL(t, c, IsSSLEnabled()) err = clusterSvc.PutCluster(context.Background(), c) if err != nil { t.Fatal(err) } - testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s, sslEnabled) + testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s, IsSSLEnabled()) } func TestStatusWithCQLCredentialsIntegration(t *testing.T) { @@ -216,23 +195,14 @@ func TestStatusWithCQLCredentialsIntegration(t *testing.T) { Username: username, Password: password, } - sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) - if err != nil { - t.Fatalf("parse SSL_ENABLED env var: %v\n", err) - } - if sslEnabled { - sslOpts := CQLSSLOptions() - userKey, _ := os.ReadFile(sslOpts.KeyPath) - userCrt, _ := os.ReadFile(sslOpts.CertPath) - c.SSLUserKeyFile = userKey - c.SSLUserCertFile = userCrt - } + clusterWithSSL(t, c, IsSSLEnabled()) + err = clusterSvc.PutCluster(context.Background(), c) if err != nil { t.Fatal(err) } - testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s, sslEnabled) + testStatusIntegration(t, c.ID, clusterSvc, clusterSvc.GetClusterByID, s, IsSSLEnabled()) } func testStatusIntegration(t *testing.T, clusterID uuid.UUID, clusterSvc cluster.Servicer, clusterProvider cluster.ProviderFunc, secretsStore store.Store, sslEnabled bool) { @@ -622,3 +592,21 @@ func fakeHealthCheckStatus(host string, code int) http.RoundTripper { return nil, nil }) } + +func clusterWithSSL(t *testing.T, cluster *cluster.Cluster, sslEnabled bool) { + t.Helper() + if !sslEnabled { + return + } + sslOpts := CQLSSLOptions() + userKey, err := os.ReadFile(sslOpts.KeyPath) + if err != nil { + t.Fatalf("read file (%s) err: %v", sslOpts.KeyPath, err) + } + userCrt, err := os.ReadFile(sslOpts.CertPath) + if err != nil { + t.Fatalf("read file (%s) err: %v", sslOpts.CertPath, err) + } + cluster.SSLUserKeyFile = userKey + cluster.SSLUserCertFile = userCrt +} diff --git a/pkg/service/repair/service_repair_integration_test.go b/pkg/service/repair/service_repair_integration_test.go index cc995aad6f..6c8d5c5493 100644 --- a/pkg/service/repair/service_repair_integration_test.go +++ b/pkg/service/repair/service_repair_integration_test.go @@ -12,7 +12,6 @@ import ( "fmt" "io" "net/http" - "os" "regexp" "sort" "strconv" @@ -1212,10 +1211,6 @@ func TestServiceRepairIntegration(t *testing.T) { }) t.Run("repair ignore hosts", func(t *testing.T) { - sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) - if err != nil { - t.Fatalf("parse SSL_ENABLED env var: %v\n", err) - } h := newRepairTestHelper(t, session, defaultConfig()) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1240,7 +1235,7 @@ func TestServiceRepairIntegration(t *testing.T) { Addr: ni.CQLAddr(ignored, false), Timeout: time.Minute, } - if sslEnabled { + if testconfig.IsSSLEnabled() { sslOpts := testconfig.CQLSSLOptions() tlsConfig, err := testconfig.TLSConfig(sslOpts) if err != nil { diff --git a/pkg/testutils/db/db.go b/pkg/testutils/db/db.go index a7e39bb67f..598470e25d 100644 --- a/pkg/testutils/db/db.go +++ b/pkg/testutils/db/db.go @@ -101,7 +101,7 @@ func CreateManagedClusterSession(tb testing.TB, empty bool, client *scyllaclient Username: user, Password: pass, } - if os.Getenv("SSL_ENABLED") != "false" { + if os.Getenv("SSL_ENABLED") == "true" { cluster.SslOpts = testconfig.CQLSSLOptions() cluster.Port = testconfig.CQLPort() } diff --git a/pkg/testutils/testconfig/testconfig.go b/pkg/testutils/testconfig/testconfig.go index ad905245f1..040218fdd5 100644 --- a/pkg/testutils/testconfig/testconfig.go +++ b/pkg/testutils/testconfig/testconfig.go @@ -9,6 +9,7 @@ import ( "flag" "fmt" "os" + "strconv" "strings" "time" @@ -129,6 +130,16 @@ func ScyllaManagerDBCluster() string { return *flagCluster } +// IsSSLEnabled is a helper function to parse SSL_ENABLED env var. +// SSL_ENABLED env var indicates if scylla cluster is configured to use ssl or not. +func IsSSLEnabled() bool { + sslEnabled, err := strconv.ParseBool(os.Getenv("SSL_ENABLED")) + if err != nil { + panic("parse SSL_ENABLED env var:" + err.Error()) + } + return sslEnabled +} + // TLSConfig returns tls.Config to work ssl enabled scylla cluster. // this function is almost an exact copy of setupTLSConfig from github.com/gocql/gocql/connectionpool.go. func TLSConfig(sslOpts *gocql.SslOptions) (*tls.Config, error) { From 63edf64203a0b1a9daa64e717844de6fa2b931af Mon Sep 17 00:00:00 2001 From: Vasil Averyanau Date: Thu, 21 Nov 2024 13:09:35 +0100 Subject: [PATCH 14/14] fix(test): use cqlping with ssl for the restore test of old scylla ver This fixes how restore integration tests handle old Scylla versions: old versions require a restart after schema restoration. To ensure Scylla is up and running, the tests perform a CQL ping, which should be initialized correctly when SSL is enabled. --- pkg/service/restore/service_restore_integration_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/service/restore/service_restore_integration_test.go b/pkg/service/restore/service_restore_integration_test.go index 1892128ee0..810edbe934 100644 --- a/pkg/service/restore/service_restore_integration_test.go +++ b/pkg/service/restore/service_restore_integration_test.go @@ -30,6 +30,7 @@ import ( "github.com/scylladb/scylla-manager/v3/pkg/service/backup" "github.com/scylladb/scylla-manager/v3/pkg/service/repair" . "github.com/scylladb/scylla-manager/v3/pkg/service/restore" + "github.com/scylladb/scylla-manager/v3/pkg/testutils/testconfig" . "github.com/scylladb/scylla-manager/v3/pkg/testutils/testhelper" "github.com/scylladb/scylla-manager/v3/pkg/util/jsonutil" "go.uber.org/atomic" @@ -1744,6 +1745,13 @@ func (h *restoreTestHelper) restartScylla() { } cfg.Addr = sessionHosts[0] + if testconfig.IsSSLEnabled() { + sslOpts := testconfig.CQLSSLOptions() + cfg.TLSConfig, err = testconfig.TLSConfig(sslOpts) + if err != nil { + h.T.Fatalf("tls config: %v", err) + } + } cond := func() bool { if _, err = cqlping.QueryPing(ctx, cfg, TestDBUsername(), TestDBPassword()); err != nil { return false