Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

STAR-617-2: Changes from DSEDB - DO NOT MERGE #15

Draft
wants to merge 5 commits into
base: trunk
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions bootstrap_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1016,3 +1016,54 @@ def test_bootstrap_binary_disabled(self):
self.assert_log_had_msg(node3, "Leaving write survey mode and joining ring at operator request")
assert_bootstrap_state(self, node3, 'COMPLETED', user='cassandra', password='cassandra')
node3.wait_for_binary_interface()

@since('4.0')
@pytest.mark.no_vnodes
def test_simple_bootstrap_with_everywhere_strategy(self):
cluster = self.cluster
tokens = cluster.balanced_tokens(2)
cluster.set_configuration_options(values={'num_tokens': 1})

logger.debug("[node1, node2] tokens: %r" % (tokens,))

keys = 10000

# Create a single node cluster
cluster.populate(1)
node1 = cluster.nodelist()[0]
node1.set_configuration_options(values={'initial_token': tokens[0]})
cluster.start()

session = self.patient_cql_connection(node1)
create_ks(session, 'ks', 'EverywhereStrategy')
create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})

insert_statement = session.prepare("INSERT INTO ks.cf (key, c1, c2) VALUES (?, 'value1', 'value2')")
execute_concurrent_with_args(session, insert_statement, [['k%d' % k] for k in range(keys)])

node1.flush()
node1.compact()

# Reads inserted data all during the bootstrap process. We shouldn't
# get any error
query_c1c2(session, random.randint(0, keys - 1), ConsistencyLevel.ONE)
session.shutdown()

# Bootstrapping a new node in the current version
node2 = new_node(cluster)
node2.set_configuration_options(values={'initial_token': tokens[1]})
node2.start(wait_for_binary_proto=True)
node2.compact()

node1.cleanup()
logger.debug("node1 size for ks.cf after cleanup: %s" % float(data_size(node1,'ks','cf')))
node1.compact()
logger.debug("node1 size for ks.cf after compacting: %s" % float(data_size(node1,'ks','cf')))

logger.debug("node2 size for ks.cf after compacting: %s" % float(data_size(node2,'ks','cf')))

size1 = float(data_size(node1,'ks','cf'))
size2 = float(data_size(node2,'ks','cf'))
assert_almost_equal(size1, size2, error=0.3)

assert_bootstrap_state(self, node2, 'COMPLETED')
3 changes: 3 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ def check_required_loopback_interfaces_available():


def pytest_addoption(parser):
parser.addoption("--sstable-format", action="store", default="bti",
help="SSTable format to be used by default for all newly created SSTables: "
"big or bti (default: bti)")
parser.addoption("--use-vnodes", action="store_true", default=False,
help="Determines wither or not to setup clusters using vnodes for tests")
parser.addoption("--use-off-heap-memtables", action="store_true", default=False,
Expand Down
3 changes: 3 additions & 0 deletions cqlsh_tests/test_cqlsh.py
Original file line number Diff line number Diff line change
Expand Up @@ -1080,6 +1080,7 @@ def get_test_table_output(self, has_val=True, has_val_idx=True):
AND comment = ''
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND memtable = {}
AND crc_check_chance = 1.0
AND default_time_to_live = 0
AND extensions = {}
Expand Down Expand Up @@ -1169,6 +1170,7 @@ def get_users_table_output(self):
AND comment = ''
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND memtable = {}
AND crc_check_chance = 1.0
AND default_time_to_live = 0
AND extensions = {}
Expand Down Expand Up @@ -1275,6 +1277,7 @@ def get_users_by_state_mv_output(self):
AND comment = ''
AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
AND compression = {'chunk_length_in_kb': '16', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
AND memtable = {}
AND crc_check_chance = 1.0
AND default_time_to_live = 0
AND extensions = {}
Expand Down
76 changes: 76 additions & 0 deletions cqlsh_tests/test_cqlsh_copy.py
Original file line number Diff line number Diff line change
Expand Up @@ -3330,3 +3330,79 @@ def _test_invalid_data_for_maps():
_test_invalid_data_for_sets()
_test_invalid_data_for_lists()
_test_invalid_data_for_maps()

@since('4.0')
def test_geotypes_copy(self):
"""
Tests whether cqlsh COPY properly handles geo types with empty and null values.

@since 4.0.0

Steps:
* insert several geoTypes with null and empty values among them into a table
* cqlsh copy contents to .csv file and save them in a list
* wipe the table comletely of all data
* cqlsh copy contents from .csv back into the table
* get the contents of the table into a list
* assert the pre wiped data is the same as the newly copied data
:return
"""
self.prepare()

self.session.execute("create table geo (k int primary key, point 'PointType', line 'LineStringType', poly 'PolygonType');")
self.session.execute("insert into geo (k, point, line, poly) VALUES (0, 'point(1.2 3.4)', 'linestring(1.0 1.1, 2.0 2.1, 3.0 3.1)', 'POLYGON ((10.1 10.0, 110.0 10.0, 110.0 110.0, 10.0 110.0, 10.0 10.0), (20.0 20.0, 20.0 30.0, 30.0 30.0, 30.0 20.0, 20.0 20.0))');")
self.session.execute("insert into geo (k, point, line, poly) VALUES (2, 'point(1.2 3.4)', 'linestring EMPTY', 'POLYGON EMPTY');")
self.session.execute("insert into geo (k) VALUES (1);")

# make sure data is inserted
data_actual = rows_to_list(self.session.execute("select * from geo;"))
assert len(data_actual) == 3

# dump data to CSV and truncate
tempfile = self.get_temp_file()
self.run_cqlsh(cmds="COPY ks.geo TO '{name}'".format(name=tempfile.name))
self.run_cqlsh(cmds="truncate ks.geo;")

# import data back
self.run_cqlsh(cmds="COPY ks.geo FROM '{name}'".format(name=tempfile.name))
data_copy = rows_to_list(self.session.execute("select * from geo;"))

assert data_actual == data_copy

@since("4.0")
def test_date_range_copy(self):
"""
DateRangeTests.test_copy_command

Tests whether cqlsh COPY properly handles date_range types, including null values.
@note we cannot insert empty value ('') as it is not presented as null in cqlsh but it is in COPY
"""
self.prepare()

self.session.execute("create table incomes (org text, period 'DateRangeType', incomes int, ver 'DateRangeType', primary key (org, period));")
# insert some data
self.session.execute("insert into incomes(org, period, incomes) values ('A','2014', 20140);")
self.session.execute("insert into incomes(org, period, incomes) values ('A','2015', 201500);")
self.session.execute("insert into incomes(org, period, incomes) values ('A','[2016-01-01 TO 2016-06-30]', 1007);")
self.session.execute("insert into incomes(org, period, incomes) values ('B','[2017-02-12 12:30:07 TO 2017-02-17 13:39:43.789]', 777);")
self.session.execute("insert into incomes(org, period, incomes, ver) values ('X','2011', 0, null);")
self.session.execute("insert into incomes(org, period, incomes) values ('C','*', 996);")
self.session.execute("insert into incomes(org, period, incomes) values ('C','[* TO *]', 997);")
self.session.execute("insert into incomes(org, period, incomes) values ('C','[* TO 2015-01]', 998);")
self.session.execute("insert into incomes(org, period, incomes) values ('C','[2015-01 TO *]', 999);")

# make sure data is inserted
data_actual = rows_to_list(self.session.execute("select * from incomes;"))
assert len(data_actual) == 9

# dump data to CSV and truncate
tempfile = self.get_temp_file()
self.run_cqlsh(cmds="COPY ks.incomes TO '{name}'".format(name=tempfile.name))
self.run_cqlsh(cmds="truncate ks.incomes;")

# import data back
self.run_cqlsh(cmds="COPY ks.incomes FROM '{name}'".format(name=tempfile.name))
data_copy = rows_to_list(self.session.execute("select * from incomes;"))

assert data_actual == data_copy

67 changes: 67 additions & 0 deletions cqlsh_tests/test_cqlsh_types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import logging
import pytest

from dtest import Tester, create_ks

logger = logging.getLogger(__name__)
since = pytest.mark.since


@since("4.0")
class TestCqlshTypes(Tester):

def prepare(self, workload=None):
if not self.cluster.nodelist():
self.allow_log_errors = True
self.cluster.populate(1)
if workload is not None:
for node in self.cluster.nodelist():
node.set_workload(workload)
logger.debug('About to start cluster')
self.cluster.start()
logger.debug('Cluster started')
for node in self.cluster.nodelist():
node.watch_log_for('Starting listening for CQL clients', timeout=60)
self.cluster.nodelist()[0].watch_log_for('Created default superuser')
self.node = self.cluster.nodelist()[0]

conn = self.patient_cql_connection(self.node)
create_ks(conn, 'ks', 1)

logger.debug('prepare completed')

def test_point(self):
self.prepare()

expected = 'POINT (1.2 2.3)'
self.node.run_cqlsh("CREATE TABLE ks.point_tbl (k INT PRIMARY KEY, point 'PointType');")
self.node.run_cqlsh("INSERT INTO ks.point_tbl (k, point) VALUES (1, '{}')".format(expected))
result = self.node.run_cqlsh("SELECT * FROM ks.point_tbl;")
assert expected in result[0], result

def test_linestring(self):
self.prepare()

expected = 'LINESTRING (30.0 10.0, 10.0 30.0, 40.0 40.0)'
self.node.run_cqlsh("CREATE TABLE ks.line_tbl (k INT PRIMARY KEY, linestring 'LineStringType');")
self.node.run_cqlsh("INSERT INTO ks.line_tbl (k, linestring) VALUES (1, '{}')".format(expected))
result = self.node.run_cqlsh("SELECT * FROM ks.line_tbl;")
assert expected in result[0], result

def test_polygon(self):
self.prepare()

expected = 'POLYGON ((30.0 10.0, 40.0 40.0, 20.0 40.0, 10.0 20.0, 30.0 10.0))'
self.node.run_cqlsh("CREATE TABLE ks.polygon_tbl (k INT PRIMARY KEY, polygon 'PolygonType');")
self.node.run_cqlsh("INSERT INTO ks.polygon_tbl (k, polygon) VALUES (1, '{}')".format(expected))
result = self.node.run_cqlsh("SELECT * FROM ks.polygon_tbl;")
assert expected in result[0], result

def test_date_range(self):
self.prepare()

expected = '[2015-01 TO *]'
self.node.run_cqlsh("CREATE TABLE ks.date_range_tbl (k INT PRIMARY KEY, date_range_tbl 'DateRangeType');")
self.node.run_cqlsh("INSERT INTO ks.date_range_tbl (k, date_range_tbl) VALUES (1, '{}')".format(expected))
result = self.node.run_cqlsh("SELECT * FROM ks.date_range_tbl;")
assert expected in result[0], result
2 changes: 2 additions & 0 deletions dtest.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,8 @@ def create_ks(session, name, rf):
if isinstance(rf, int):
# we assume simpleStrategy
query = query % (name, "'class':'SimpleStrategy', 'replication_factor':%d" % rf)
elif 'EverywhereStrategy' in rf:
query = query % (name, "'class':'org.apache.cassandra.locator.EverywhereStrategy'")
else:
assert len(rf) >= 0, "At least one datacenter/rf pair is needed"
# we assume networkTopologyStrategy
Expand Down
7 changes: 7 additions & 0 deletions dtest_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

class DTestConfig:
def __init__(self):
self.sstable_format = "bti"
self.use_vnodes = True
self.use_off_heap_memtables = False
self.num_tokens = -1
Expand Down Expand Up @@ -41,6 +42,12 @@ def setup(self, config):
self.cassandra_version_from_build = self.get_version_from_build()
return

self.sstable_format = config.getoption("--sstable-format")
if self.sstable_format:
assert self.sstable_format in ['bti', 'big'], "SSTable format {} is invalid - must be either bti or big".format(self.sstable_format)
default_sstable_format_prop = " -Dcassandra.sstable.format.default=" + self.sstable_format
os.environ.update({"JVM_EXTRA_OPTS": (os.environ.get("JVM_EXTRA_OPTS") or "") + default_sstable_format_prop})

self.use_vnodes = config.getoption("--use-vnodes")
self.use_off_heap_memtables = config.getoption("--use-off-heap-memtables")
self.num_tokens = config.getoption("--num-tokens")
Expand Down
4 changes: 2 additions & 2 deletions offline_tools_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,9 +266,9 @@ def test_sstableverify(self):
hashcomputed = False
for line in outlines:
if sstable in line:
if "Verifying BigTableReader" in line:
if "Verifying " in line:
verified = True
elif "Checking computed hash of BigTableReader" in line:
elif "Checking computed hash of " in line:
hashcomputed = True
else:
logger.debug(line)
Expand Down
3 changes: 2 additions & 1 deletion run_dtests.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
#!/usr/bin/env python
"""
usage: run_dtests.py [-h] [--use-vnodes] [--use-off-heap-memtables] [--num-tokens=NUM_TOKENS] [--data-dir-count-per-instance=DATA_DIR_COUNT_PER_INSTANCE]
usage: run_dtests.py [-h] [--sstable-format=FORMAT] [--use-vnodes] [--use-off-heap-memtables] [--num-tokens=NUM_TOKENS] [--data-dir-count-per-instance=DATA_DIR_COUNT_PER_INSTANCE]
[--force-resource-intensive-tests] [--skip-resource-intensive-tests] [--cassandra-dir=CASSANDRA_DIR] [--cassandra-version=CASSANDRA_VERSION]
[--delete-logs] [--execute-upgrade-tests] [--execute-upgrade-tests-only] [--disable-active-log-watching] [--keep-test-dir]
[--enable-jacoco-code-coverage] [--dtest-enable-debug-logging] [--dtest-print-tests-only] [--dtest-print-tests-output=DTEST_PRINT_TESTS_OUTPUT]
[--pytest-options=PYTEST_OPTIONS] [--dtest-tests=DTEST_TESTS]

optional arguments:
-h, --help show this help message and exit
--sstable-format SSTable format to be used by default for all newly created SSTables: big or bti (default: bti)
--use-vnodes Determines wither or not to setup clusters using vnodes for tests (default: False)
--use-off-heap-memtables Enable Off Heap Memtables when creating test clusters for tests (default: False)
--num-tokens=NUM_TOKENS Number of tokens to set num_tokens yaml setting to when creating instances with vnodes enabled (default: 256)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
commit 8c9680112dc2ba92b04460e695b74fa0273b2791
Author: jacek-lewandowski <[email protected]>
AuthorDate: Fri Apr 2 13:51:45 2021 +0200
Commit: jacek-lewandowski <[email protected]>
CommitDate: Mon Apr 12 17:01:32 2021 +0200

STAR-247: Allow to easily run tests against big or bti sstable format

Resolutions:
diff --git a/dtest_config.py b/dtest_config.py
index fbe15a9f..116566b2 100644
--- a/dtest_config.py
+++ b/dtest_config.py
@@ -43,9 +43,10 @@ class DTestConfig:
return

self.sstable_format = config.getoption("--sstable-format")
- assert self.sstable_format in ['bti', 'big'], "SSTable format {} is invalid - must be either bti or big".format(self.sstable_format)
- default_sstable_format_prop = " -Dcassandra.sstable.format.default=" + self.sstable_format
- os.environ.update({"JVM_EXTRA_OPTS": (os.environ.get("JVM_EXTRA_OPTS") or "") + default_sstable_format_prop})
+ if self.sstable_format:
+ assert self.sstable_format in ['bti', 'big'], "SSTable format {} is invalid - must be either bti or big".format(self.sstable_format)
+ default_sstable_format_prop = " -Dcassandra.sstable.format.default=" + self.sstable_format
+ os.environ.update({"JVM_EXTRA_OPTS": (os.environ.get("JVM_EXTRA_OPTS") or "") + default_sstable_format_prop})

self.use_vnodes = config.getoption("--use-vnodes")
self.use_off_heap_memtables = config.getoption("--use-off-heap-memtables")