switch to use ruamel.yaml

since ruamel.yaml can preserve yaml comment
so we can have the original yaml files
of scylla

(cherry picked from commit 2747515)

README.md

@@ -55,12 +55,11 @@ ccm create my_cluster -n 3 --scylla --vnodes \
 Requirements
 ------------
 
-- A working python installation (tested to work with python 3.11).
-- pyYAML (http://pyyaml.org/ -- `sudo easy_install pyYaml`)
-- ant (http://ant.apache.org/, on Mac OS X, `brew install ant`)
-- psutil (https://pypi.python.org/pypi/psutil)
-- Java (which version depends on the version of Cassandra you plan to use. If
-  unsure, use Java 7 as it is known to work with current versions of Cassandra).
+- A working python installation (tested to work with python 3.12).
+- `pip install -e .` to install the required dependencies.
+- Java if cassandra is used or older scylla < 6.0 (which version depends on the version 
+  of Cassandra you plan to use. If unsure, use Java 8 as it is known to 
+  work with current versions of Cassandra).
 - ccm only works on localhost for now. If you want to create multiple
   node clusters, the simplest way is to use multiple loopback aliases. On
   modern linux distributions you probably don't need to do anything, but
@@ -75,17 +74,8 @@ Requirements
 
 Known issues
 ------------
-Windows only:
-  - `node start` pops up a window, stealing focus.
-  - cli and cqlsh started from ccm show incorrect prompts on command-prompt
-  - non nodetool-based command-line options fail (sstablesplit, scrub, etc)
-  - cli_session does not accept commands.
-  - To install psutil, you must use the .msi from pypi. pip install psutil will not work
-  - You will need ant.bat in your PATH in order to build C* from source
-  - You must run with an Unrestricted Powershell Execution-Policy if using Cassandra 2.1.0+
-  - Ant installed via [chocolatey](https://chocolatey.org/) will not be found by ccm, so you must create a symbolic
-    link in order to fix the issue (as administrator):
-    - cmd /c mklink C:\ProgramData\chocolatey\bin\ant.bat C:\ProgramData\chocolatey\bin\ant.exe
+
+- this fork of ccm doesn't support Windows
 
 Installation
 ------------

ccmlib/cluster.py

@@ -8,13 +8,15 @@
 from collections import OrderedDict, defaultdict
 from concurrent.futures import ThreadPoolExecutor
 
-import yaml
+from ruamel.yaml import YAML
 
 from ccmlib import common, repository
 from ccmlib.node import Node, NodeError
 from ccmlib.common import logger
 from ccmlib.utils.version import parse_version
 
+yaml = YAML()
+yaml.default_flow_style = False
 
 class Cluster(object):
 
@@ -675,7 +677,7 @@ def _update_config(self, install_dir=None):
             cluster_config['sni_proxy_listen_port'] = self.sni_proxy_listen_port
 
         with open(filename, 'w') as f:
-            yaml.safe_dump(cluster_config, f)
+            yaml.dump(cluster_config, f)
 
     def __update_pids(self, started):
         for node, p, _ in started:

ccmlib/cluster_factory.py

@@ -1,6 +1,6 @@
 import os
 
-import yaml
+from ruamel.yaml import YAML
 
 from ccmlib import common, repository
 from ccmlib.cluster import Cluster
@@ -10,6 +10,8 @@
 from ccmlib import repository
 from ccmlib.node import Node
 
+yaml = YAML()
+yaml.default_flow_style = False
 
 class ClusterFactory():
 
@@ -18,7 +20,7 @@ def load(path, name):
         cluster_path = os.path.join(path, name)
         filename = os.path.join(cluster_path, 'cluster.conf')
         with open(filename, 'r') as f:
-            data = yaml.safe_load(f)
+            data = yaml.load(f)
         try:
             install_dir = None
             scylla_manager_install_path = data.get('scylla_manager_install_path')

ccmlib/common.py

@@ -20,11 +20,18 @@
 from typing import Callable, Optional, TextIO, Union, List
 from pathlib import Path
 
+<<<<<<< HEAD
 import yaml
+=======
+import psutil
+from ruamel.yaml import YAML
+>>>>>>> 2747515 (switch to use ruamel.yaml)
 from boto3.session import Session
 from botocore import UNSIGNED
 from botocore.client import Config
 
+yaml = YAML()
+yaml.default_flow_style = False
 
 BIN_DIR = "bin"
 CASSANDRA_CONF_DIR = "conf"
@@ -246,7 +253,7 @@ def get_config():
         return {}
 
     with open(config_path, 'r') as f:
-        return yaml.safe_load(f)
+        return yaml.load(f)
 
 
 def now_ms():
@@ -787,7 +794,7 @@ def parse_settings(args):
         elif val.lower() == "false":
             val = False
         else:
-            val = yaml.safe_load(val)
+            val = yaml.load(val)
         splitted = key.split('.')
         if len(splitted) == 2:
             try:
@@ -924,7 +931,7 @@ def is_dse_cluster(path):
             cluster_path = os.path.join(path, name)
             filename = os.path.join(cluster_path, 'cluster.conf')
             with open(filename, 'r') as f:
-                data = yaml.safe_load(f)
+                data = yaml.load(f)
             if 'dse_dir' in data:
                 return True
     except IOError:

ccmlib/dse_node.py

@@ -9,11 +9,14 @@
 import subprocess
 import time
 
-import yaml
+from ruamel.yaml import YAML
 
 from ccmlib import common
 from ccmlib.node import Node, NodeError
 
+yaml = YAML()
+yaml.default_flow_style = False
+
 
 class DseNode(Node):
 
@@ -293,7 +296,7 @@ def import_bin_files(self):
     def __update_yaml(self):
         conf_file = os.path.join(self.get_path(), 'resources', 'dse', 'conf', 'dse.yaml')
         with open(conf_file, 'r') as f:
-            data = yaml.safe_load(f)
+            data = yaml.load(f)
 
         data['system_key_directory'] = os.path.join(self.get_path(), 'keys')
 
@@ -310,7 +313,7 @@ def __update_yaml(self):
                 data[name] = full_options[name]
 
         with open(conf_file, 'w') as f:
-            yaml.safe_dump(data, f, default_flow_style=False)
+            yaml.dump(data, f)
 
     def _update_log4j(self):
         super(DseNode, self)._update_log4j()

ccmlib/node.py

@@ -17,13 +17,16 @@
 import locale
 from collections import namedtuple
 
-import yaml
+from ruamel.yaml import YAML
 
 from ccmlib import common
 from ccmlib.cli_session import CliSession
 from ccmlib.repository import setup
 from ccmlib.utils.version import parse_version
 
+yaml = YAML()
+yaml.default_flow_style = False
+
 class Status():
     UNINITIALIZED = "UNINITIALIZED"
     UP = "UP"
@@ -135,7 +138,7 @@ def load(path, name, cluster):
         node_path = os.path.join(path, name)
         filename = os.path.join(node_path, 'node.conf')
         with open(filename, 'r') as f:
-            data = yaml.safe_load(f)
+            data = yaml.load(f)
         try:
             itf = data['interfaces']
             initial_token = None
@@ -1581,12 +1584,12 @@ def _update_config(self):
         if self.workload is not None:
             values['workload'] = self.workload
         with open(filename, 'w') as f:
-            yaml.safe_dump(values, f)
+            yaml.dump(values, f)
 
     def __update_yaml(self):
         conf_file = os.path.join(self.get_conf_dir(), common.CASSANDRA_CONF)
         with open(conf_file, 'r') as f:
-            data = yaml.safe_load(f)
+            data = yaml.load(f)
 
         with open(conf_file, 'r') as f:
             yaml_text = f.read()
@@ -1636,7 +1639,7 @@ def __update_yaml(self):
                     data[name] = full_options[name]
 
         with open(conf_file, 'w') as f:
-            yaml.safe_dump(data, f, default_flow_style=False)
+            yaml.dump(data, f)
 
     def _update_log4j(self):
         append_pattern = 'log4j.appender.R.File='
@@ -1958,7 +1961,7 @@ def _clean_win_jmx(self):
     def get_conf_option(self, option):
         conf_file = os.path.join(self.get_conf_dir(), common.CASSANDRA_CONF)
         with open(conf_file, 'r') as f:
-            data = yaml.safe_load(f)
+            data = yaml.load(f)
 
         if option in data:
             return data[option]

ccmlib/scylla_cluster.py

@@ -4,10 +4,10 @@
 import time
 import subprocess
 import signal
-import yaml
 import uuid
 import datetime
 
+from ruamel.yaml import YAML
 
 from ccmlib import common
 from ccmlib.cluster import Cluster
@@ -19,6 +19,8 @@
 
 SNITCH = 'org.apache.cassandra.locator.GossipingPropertyFileSnitch'
 
+yaml = YAML()
+yaml.default_flow_style = False
 
 class ScyllaCluster(Cluster):
 
@@ -244,7 +246,7 @@ def _update_config(self, install_dir=None):
         filename = os.path.join(self.get_path(), 'cluster.conf')
 
         with open(filename, 'r') as f:
-            data = yaml.safe_load(f)
+            data = yaml.load(f)
 
         if self.is_scylla_reloc():
             data['scylla_version'] = self.scylla_version
@@ -253,7 +255,7 @@ def _update_config(self, install_dir=None):
             data['scylla_manager_install_path'] = self._scylla_manager.install_dir
 
         with open(filename, 'w') as f:
-            yaml.safe_dump(data, f)
+            yaml.dump(data, f)
 
     def sctool(self, cmd):
         if self._scylla_manager == None:
@@ -313,7 +315,7 @@ def _get_api_address(self):
     def _update_config(self, install_dir=None):
         conf_file = os.path.join(self._get_path(), common.SCYLLAMANAGER_CONF)
         with open(conf_file, 'r') as f:
-            data = yaml.safe_load(f)
+            data = yaml.load(f)
         data['http'] = self._get_api_address()
         if not 'database' in data:
             data['database'] = {}
@@ -349,7 +351,7 @@ def _update_config(self, install_dir=None):
         for key in keys_to_delete:
             del data[key]
         with open(conf_file, 'w') as f:
-            yaml.safe_dump(data, f, default_flow_style=False)
+            yaml.dump(data, f)
 
     def _copy_config_files(self, install_dir):
         conf_dir = os.path.join(install_dir, 'etc')

ccmlib/scylla_docker_cluster.py

@@ -3,7 +3,7 @@
 from subprocess import check_call, run, PIPE
 import logging
 
-import yaml
+from ruamel.yaml import YAML
 
 from ccmlib.scylla_cluster import ScyllaCluster
 from ccmlib.scylla_node import ScyllaNode
@@ -12,6 +12,8 @@
 
 LOGGER = logging.getLogger("ccm")
 
+yaml = YAML()
+yaml.default_flow_style = False
 
 class ScyllaDockerCluster(ScyllaCluster):
     def __init__(self, *args, **kwargs):

ccmlib/scylla_node.py

@@ -19,7 +19,7 @@
 import logging
 
 import psutil
-import yaml
+from ruamel.yaml import YAML
 import glob
 import re
 import requests
@@ -35,6 +35,9 @@
 from ccmlib.utils.version import parse_version
 
 
+yaml = YAML()
+yaml.default_flow_style = False
+
 class ScyllaNode(Node):
 
     """
@@ -372,18 +375,18 @@ def _create_agent_config(self):
         data['s3'] = {"endpoint": os.getenv("AWS_S3_ENDPOINT"), "provider": "Minio"}
 
         with open(conf_file, 'w') as f:
-            yaml.safe_dump(data, f, default_flow_style=False)
+            yaml.dump(data, f)
         return conf_file
 
     def update_agent_config(self, new_settings, restart_agent_after_change=True):
         conf_file = os.path.join(self.get_conf_dir(), 'scylla-manager-agent.yaml')
         with open(conf_file, 'r') as f:
-            current_config = yaml.safe_load(f)
+            current_config = yaml.load(f)
 
         current_config.update(new_settings)
 
         with open(conf_file, 'w') as f:
-            yaml.safe_dump(current_config, f, default_flow_style=False)
+            yaml.dump(current_config, f)
 
         if restart_agent_after_change:
             self.restart_scylla_manager_agent(gently=True, recreate_config=False)
@@ -416,7 +419,7 @@ def start_scylla_manager_agent(self, create_config=True):
             pid_file.write(str(self._process_agent.pid))
 
         with open(config_file, 'r') as f:
-            current_config = yaml.safe_load(f)
+            current_config = yaml.load(f)
             # Extracting currently configured port
             current_listening_port = int(current_config['https'].split(":")[1])
 
@@ -569,7 +572,7 @@ def start(self, join_ring=True, no_wait=False, verbose=False,
         # from config file scylla#59
         conf_file = os.path.join(self.get_conf_dir(), common.SCYLLA_CONF)
         with open(conf_file, 'r') as f:
-            data = yaml.safe_load(f)
+            data = yaml.load(f)
         jvm_args = jvm_args + ['--api-address', data['api_address']]
 
         args = [launch_bin, '--options-file', options_file, '--log-to-stdout', '1']
@@ -1142,7 +1145,7 @@ def update_yaml(self):
         # TODO: copied from node.py
         conf_file = os.path.join(self.get_conf_dir(), common.SCYLLA_CONF)
         with open(conf_file, 'r') as f:
-            data = yaml.safe_load(f)
+            data = yaml.load(f)
 
         data['cluster_name'] = self.cluster.name
         data['auto_bootstrap'] = self.auto_bootstrap
@@ -1203,7 +1206,7 @@ def update_yaml(self):
         if 'alternator_port' in data or 'alternator_https_port' in data:
             data['alternator_address'] = data['listen_address']
         with open(conf_file, 'w') as f:
-            yaml.safe_dump(data, f, default_flow_style=False)
+            yaml.dump(data, f)
 
         # TODO: - for now create a cassandra conf file leaving only
         # cassandra config items - this should be removed once tools are
@@ -1313,7 +1316,7 @@ def update_yaml(self):
                 cassandra_data[key] = data[key]
 
         with open(cassandra_conf_file, 'w') as f:
-            yaml.safe_dump(cassandra_data, f, default_flow_style=False)
+            yaml.dump(cassandra_data, f)
 
     def __update_yaml_dse(self):
         raise NotImplementedError('ScyllaNode.__update_yaml_dse')