diff --git a/bootstrap_test.py b/bootstrap_test.py index 9739a63e3c..c3942d8437 100644 --- a/bootstrap_test.py +++ b/bootstrap_test.py @@ -1016,3 +1016,54 @@ def test_bootstrap_binary_disabled(self): self.assert_log_had_msg(node3, "Leaving write survey mode and joining ring at operator request") assert_bootstrap_state(self, node3, 'COMPLETED', user='cassandra', password='cassandra') node3.wait_for_binary_interface() + + @since('4.0') + @pytest.mark.no_vnodes + def test_simple_bootstrap_with_everywhere_strategy(self): + cluster = self.cluster + tokens = cluster.balanced_tokens(2) + cluster.set_configuration_options(values={'num_tokens': 1}) + + logger.debug("[node1, node2] tokens: %r" % (tokens,)) + + keys = 10000 + + # Create a single node cluster + cluster.populate(1) + node1 = cluster.nodelist()[0] + node1.set_configuration_options(values={'initial_token': tokens[0]}) + cluster.start() + + session = self.patient_cql_connection(node1) + create_ks(session, 'ks', 'EverywhereStrategy') + create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) + + insert_statement = session.prepare("INSERT INTO ks.cf (key, c1, c2) VALUES (?, 'value1', 'value2')") + execute_concurrent_with_args(session, insert_statement, [['k%d' % k] for k in range(keys)]) + + node1.flush() + node1.compact() + + # Reads inserted data all during the bootstrap process. We shouldn't + # get any error + query_c1c2(session, random.randint(0, keys - 1), ConsistencyLevel.ONE) + session.shutdown() + + # Bootstrapping a new node in the current version + node2 = new_node(cluster) + node2.set_configuration_options(values={'initial_token': tokens[1]}) + node2.start(wait_for_binary_proto=True) + node2.compact() + + node1.cleanup() + logger.debug("node1 size for ks.cf after cleanup: %s" % float(data_size(node1,'ks','cf'))) + node1.compact() + logger.debug("node1 size for ks.cf after compacting: %s" % float(data_size(node1,'ks','cf'))) + + logger.debug("node2 size for ks.cf after compacting: %s" % float(data_size(node2,'ks','cf'))) + + size1 = float(data_size(node1,'ks','cf')) + size2 = float(data_size(node2,'ks','cf')) + assert_almost_equal(size1, size2, error=0.3) + + assert_bootstrap_state(self, node2, 'COMPLETED') diff --git a/conftest.py b/conftest.py index 33e188fd52..79b9714191 100644 --- a/conftest.py +++ b/conftest.py @@ -43,6 +43,9 @@ def check_required_loopback_interfaces_available(): def pytest_addoption(parser): + parser.addoption("--sstable-format", action="store", default="bti", + help="SSTable format to be used by default for all newly created SSTables: " + "big or bti (default: bti)") parser.addoption("--use-vnodes", action="store_true", default=False, help="Determines wither or not to setup clusters using vnodes for tests") parser.addoption("--use-off-heap-memtables", action="store_true", default=False, diff --git a/cqlsh_tests/test_cqlsh.py b/cqlsh_tests/test_cqlsh.py index 85333d5111..c700954254 100644 --- a/cqlsh_tests/test_cqlsh.py +++ b/cqlsh_tests/test_cqlsh.py @@ -1080,6 +1080,7 @@ def get_test_table_output(self, has_val=True, has_val_idx=True): AND comment = '' AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} + AND memtable = {} AND crc_check_chance = 1.0 AND default_time_to_live = 0 AND extensions = {} @@ -1169,6 +1170,7 @@ def get_users_table_output(self): AND comment = '' AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} + AND memtable = {} AND crc_check_chance = 1.0 AND default_time_to_live = 0 AND extensions = {} @@ -1275,6 +1277,7 @@ def get_users_by_state_mv_output(self): AND comment = '' AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} + AND memtable = {} AND crc_check_chance = 1.0 AND default_time_to_live = 0 AND extensions = {} diff --git a/cqlsh_tests/test_cqlsh_copy.py b/cqlsh_tests/test_cqlsh_copy.py index 32db55fd69..6ba54360db 100644 --- a/cqlsh_tests/test_cqlsh_copy.py +++ b/cqlsh_tests/test_cqlsh_copy.py @@ -3330,3 +3330,79 @@ def _test_invalid_data_for_maps(): _test_invalid_data_for_sets() _test_invalid_data_for_lists() _test_invalid_data_for_maps() + + @since('4.0') + def test_geotypes_copy(self): + """ + Tests whether cqlsh COPY properly handles geo types with empty and null values. + + @since 4.0.0 + + Steps: + * insert several geoTypes with null and empty values among them into a table + * cqlsh copy contents to .csv file and save them in a list + * wipe the table comletely of all data + * cqlsh copy contents from .csv back into the table + * get the contents of the table into a list + * assert the pre wiped data is the same as the newly copied data + :return + """ + self.prepare() + + self.session.execute("create table geo (k int primary key, point 'PointType', line 'LineStringType', poly 'PolygonType');") + self.session.execute("insert into geo (k, point, line, poly) VALUES (0, 'point(1.2 3.4)', 'linestring(1.0 1.1, 2.0 2.1, 3.0 3.1)', 'POLYGON ((10.1 10.0, 110.0 10.0, 110.0 110.0, 10.0 110.0, 10.0 10.0), (20.0 20.0, 20.0 30.0, 30.0 30.0, 30.0 20.0, 20.0 20.0))');") + self.session.execute("insert into geo (k, point, line, poly) VALUES (2, 'point(1.2 3.4)', 'linestring EMPTY', 'POLYGON EMPTY');") + self.session.execute("insert into geo (k) VALUES (1);") + + # make sure data is inserted + data_actual = rows_to_list(self.session.execute("select * from geo;")) + assert len(data_actual) == 3 + + # dump data to CSV and truncate + tempfile = self.get_temp_file() + self.run_cqlsh(cmds="COPY ks.geo TO '{name}'".format(name=tempfile.name)) + self.run_cqlsh(cmds="truncate ks.geo;") + + # import data back + self.run_cqlsh(cmds="COPY ks.geo FROM '{name}'".format(name=tempfile.name)) + data_copy = rows_to_list(self.session.execute("select * from geo;")) + + assert data_actual == data_copy + + @since("4.0") + def test_date_range_copy(self): + """ + DateRangeTests.test_copy_command + + Tests whether cqlsh COPY properly handles date_range types, including null values. + @note we cannot insert empty value ('') as it is not presented as null in cqlsh but it is in COPY + """ + self.prepare() + + self.session.execute("create table incomes (org text, period 'DateRangeType', incomes int, ver 'DateRangeType', primary key (org, period));") + # insert some data + self.session.execute("insert into incomes(org, period, incomes) values ('A','2014', 20140);") + self.session.execute("insert into incomes(org, period, incomes) values ('A','2015', 201500);") + self.session.execute("insert into incomes(org, period, incomes) values ('A','[2016-01-01 TO 2016-06-30]', 1007);") + self.session.execute("insert into incomes(org, period, incomes) values ('B','[2017-02-12 12:30:07 TO 2017-02-17 13:39:43.789]', 777);") + self.session.execute("insert into incomes(org, period, incomes, ver) values ('X','2011', 0, null);") + self.session.execute("insert into incomes(org, period, incomes) values ('C','*', 996);") + self.session.execute("insert into incomes(org, period, incomes) values ('C','[* TO *]', 997);") + self.session.execute("insert into incomes(org, period, incomes) values ('C','[* TO 2015-01]', 998);") + self.session.execute("insert into incomes(org, period, incomes) values ('C','[2015-01 TO *]', 999);") + + # make sure data is inserted + data_actual = rows_to_list(self.session.execute("select * from incomes;")) + assert len(data_actual) == 9 + + # dump data to CSV and truncate + tempfile = self.get_temp_file() + self.run_cqlsh(cmds="COPY ks.incomes TO '{name}'".format(name=tempfile.name)) + self.run_cqlsh(cmds="truncate ks.incomes;") + + # import data back + self.run_cqlsh(cmds="COPY ks.incomes FROM '{name}'".format(name=tempfile.name)) + data_copy = rows_to_list(self.session.execute("select * from incomes;")) + + assert data_actual == data_copy + diff --git a/cqlsh_tests/test_cqlsh_types.py b/cqlsh_tests/test_cqlsh_types.py new file mode 100644 index 0000000000..11e4604c7e --- /dev/null +++ b/cqlsh_tests/test_cqlsh_types.py @@ -0,0 +1,67 @@ +import logging +import pytest + +from dtest import Tester, create_ks + +logger = logging.getLogger(__name__) +since = pytest.mark.since + + +@since("4.0") +class TestCqlshTypes(Tester): + + def prepare(self, workload=None): + if not self.cluster.nodelist(): + self.allow_log_errors = True + self.cluster.populate(1) + if workload is not None: + for node in self.cluster.nodelist(): + node.set_workload(workload) + logger.debug('About to start cluster') + self.cluster.start() + logger.debug('Cluster started') + for node in self.cluster.nodelist(): + node.watch_log_for('Starting listening for CQL clients', timeout=60) + self.cluster.nodelist()[0].watch_log_for('Created default superuser') + self.node = self.cluster.nodelist()[0] + + conn = self.patient_cql_connection(self.node) + create_ks(conn, 'ks', 1) + + logger.debug('prepare completed') + + def test_point(self): + self.prepare() + + expected = 'POINT (1.2 2.3)' + self.node.run_cqlsh("CREATE TABLE ks.point_tbl (k INT PRIMARY KEY, point 'PointType');") + self.node.run_cqlsh("INSERT INTO ks.point_tbl (k, point) VALUES (1, '{}')".format(expected)) + result = self.node.run_cqlsh("SELECT * FROM ks.point_tbl;") + assert expected in result[0], result + + def test_linestring(self): + self.prepare() + + expected = 'LINESTRING (30.0 10.0, 10.0 30.0, 40.0 40.0)' + self.node.run_cqlsh("CREATE TABLE ks.line_tbl (k INT PRIMARY KEY, linestring 'LineStringType');") + self.node.run_cqlsh("INSERT INTO ks.line_tbl (k, linestring) VALUES (1, '{}')".format(expected)) + result = self.node.run_cqlsh("SELECT * FROM ks.line_tbl;") + assert expected in result[0], result + + def test_polygon(self): + self.prepare() + + expected = 'POLYGON ((30.0 10.0, 40.0 40.0, 20.0 40.0, 10.0 20.0, 30.0 10.0))' + self.node.run_cqlsh("CREATE TABLE ks.polygon_tbl (k INT PRIMARY KEY, polygon 'PolygonType');") + self.node.run_cqlsh("INSERT INTO ks.polygon_tbl (k, polygon) VALUES (1, '{}')".format(expected)) + result = self.node.run_cqlsh("SELECT * FROM ks.polygon_tbl;") + assert expected in result[0], result + + def test_date_range(self): + self.prepare() + + expected = '[2015-01 TO *]' + self.node.run_cqlsh("CREATE TABLE ks.date_range_tbl (k INT PRIMARY KEY, date_range_tbl 'DateRangeType');") + self.node.run_cqlsh("INSERT INTO ks.date_range_tbl (k, date_range_tbl) VALUES (1, '{}')".format(expected)) + result = self.node.run_cqlsh("SELECT * FROM ks.date_range_tbl;") + assert expected in result[0], result diff --git a/dtest.py b/dtest.py index f2c89b21dc..cb2470cbf3 100644 --- a/dtest.py +++ b/dtest.py @@ -359,6 +359,8 @@ def create_ks(session, name, rf): if isinstance(rf, int): # we assume simpleStrategy query = query % (name, "'class':'SimpleStrategy', 'replication_factor':%d" % rf) + elif 'EverywhereStrategy' in rf: + query = query % (name, "'class':'org.apache.cassandra.locator.EverywhereStrategy'") else: assert len(rf) >= 0, "At least one datacenter/rf pair is needed" # we assume networkTopologyStrategy diff --git a/dtest_config.py b/dtest_config.py index 86e8c96b25..116566b25c 100644 --- a/dtest_config.py +++ b/dtest_config.py @@ -11,6 +11,7 @@ class DTestConfig: def __init__(self): + self.sstable_format = "bti" self.use_vnodes = True self.use_off_heap_memtables = False self.num_tokens = -1 @@ -41,6 +42,12 @@ def setup(self, config): self.cassandra_version_from_build = self.get_version_from_build() return + self.sstable_format = config.getoption("--sstable-format") + if self.sstable_format: + assert self.sstable_format in ['bti', 'big'], "SSTable format {} is invalid - must be either bti or big".format(self.sstable_format) + default_sstable_format_prop = " -Dcassandra.sstable.format.default=" + self.sstable_format + os.environ.update({"JVM_EXTRA_OPTS": (os.environ.get("JVM_EXTRA_OPTS") or "") + default_sstable_format_prop}) + self.use_vnodes = config.getoption("--use-vnodes") self.use_off_heap_memtables = config.getoption("--use-off-heap-memtables") self.num_tokens = config.getoption("--num-tokens") diff --git a/offline_tools_test.py b/offline_tools_test.py index 8f6f98dcc1..502bebf054 100644 --- a/offline_tools_test.py +++ b/offline_tools_test.py @@ -266,9 +266,9 @@ def test_sstableverify(self): hashcomputed = False for line in outlines: if sstable in line: - if "Verifying BigTableReader" in line: + if "Verifying " in line: verified = True - elif "Checking computed hash of BigTableReader" in line: + elif "Checking computed hash of " in line: hashcomputed = True else: logger.debug(line) diff --git a/run_dtests.py b/run_dtests.py index 34dd5af766..3f3e1fda91 100755 --- a/run_dtests.py +++ b/run_dtests.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """ -usage: run_dtests.py [-h] [--use-vnodes] [--use-off-heap-memtables] [--num-tokens=NUM_TOKENS] [--data-dir-count-per-instance=DATA_DIR_COUNT_PER_INSTANCE] +usage: run_dtests.py [-h] [--sstable-format=FORMAT] [--use-vnodes] [--use-off-heap-memtables] [--num-tokens=NUM_TOKENS] [--data-dir-count-per-instance=DATA_DIR_COUNT_PER_INSTANCE] [--force-resource-intensive-tests] [--skip-resource-intensive-tests] [--cassandra-dir=CASSANDRA_DIR] [--cassandra-version=CASSANDRA_VERSION] [--delete-logs] [--execute-upgrade-tests] [--execute-upgrade-tests-only] [--disable-active-log-watching] [--keep-test-dir] [--enable-jacoco-code-coverage] [--dtest-enable-debug-logging] [--dtest-print-tests-only] [--dtest-print-tests-output=DTEST_PRINT_TESTS_OUTPUT] @@ -8,6 +8,7 @@ optional arguments: -h, --help show this help message and exit + --sstable-format SSTable format to be used by default for all newly created SSTables: big or bti (default: bti) --use-vnodes Determines wither or not to setup clusters using vnodes for tests (default: False) --use-off-heap-memtables Enable Off Heap Memtables when creating test clusters for tests (default: False) --num-tokens=NUM_TOKENS Number of tokens to set num_tokens yaml setting to when creating instances with vnodes enabled (default: 256) diff --git a/update-history/STAR-617-2/5-8c968011 STAR-247: Allow to easily run tests against big or bti sstable format b/update-history/STAR-617-2/5-8c968011 STAR-247: Allow to easily run tests against big or bti sstable format new file mode 100644 index 0000000000..c8bdd3897e --- /dev/null +++ b/update-history/STAR-617-2/5-8c968011 STAR-247: Allow to easily run tests against big or bti sstable format @@ -0,0 +1,27 @@ +commit 8c9680112dc2ba92b04460e695b74fa0273b2791 +Author: jacek-lewandowski +AuthorDate: Fri Apr 2 13:51:45 2021 +0200 +Commit: jacek-lewandowski +CommitDate: Mon Apr 12 17:01:32 2021 +0200 + + STAR-247: Allow to easily run tests against big or bti sstable format + +Resolutions: +diff --git a/dtest_config.py b/dtest_config.py +index fbe15a9f..116566b2 100644 +--- a/dtest_config.py ++++ b/dtest_config.py +@@ -43,9 +43,10 @@ class DTestConfig: + return + + self.sstable_format = config.getoption("--sstable-format") +- assert self.sstable_format in ['bti', 'big'], "SSTable format {} is invalid - must be either bti or big".format(self.sstable_format) +- default_sstable_format_prop = " -Dcassandra.sstable.format.default=" + self.sstable_format +- os.environ.update({"JVM_EXTRA_OPTS": (os.environ.get("JVM_EXTRA_OPTS") or "") + default_sstable_format_prop}) ++ if self.sstable_format: ++ assert self.sstable_format in ['bti', 'big'], "SSTable format {} is invalid - must be either bti or big".format(self.sstable_format) ++ default_sstable_format_prop = " -Dcassandra.sstable.format.default=" + self.sstable_format ++ os.environ.update({"JVM_EXTRA_OPTS": (os.environ.get("JVM_EXTRA_OPTS") or "") + default_sstable_format_prop}) + + self.use_vnodes = config.getoption("--use-vnodes") + self.use_off_heap_memtables = config.getoption("--use-off-heap-memtables")