diff --git a/python-regression/installZmq.sh b/python-regression/installZmq.sh new file mode 100644 index 0000000000..e69de29bb2 diff --git a/python-regression/setup.py b/python-regression/setup.py index 834f184bdf..fa2ebfeb36 100644 --- a/python-regression/setup.py +++ b/python-regression/setup.py @@ -9,6 +9,7 @@ 'pyota', 'aloe', 'pyyaml', + 'pyzmq' ] ) diff --git a/python-regression/tests/features/machine1/1_api_tests.feature b/python-regression/tests/features/machine1/1_api_tests.feature index 1033fe58a9..48c41f07c6 100644 --- a/python-regression/tests/features/machine1/1_api_tests.feature +++ b/python-regression/tests/features/machine1/1_api_tests.feature @@ -113,7 +113,12 @@ Feature: Test API calls on Machine 1 Scenario: GetTransactionsToApprove is called - Given "getTransactionsToApprove" is called on "nodeA-m1" with: + #Subscribe to zmq stream for walker topic + Given "nodeA-m1" is subscribed to the following zmq topics: + |keys | + |mctn | + + And "getTransactionsToApprove" is called on "nodeA-m1" with: |keys |values |type | |depth |3 |int | @@ -123,6 +128,10 @@ Feature: Test API calls on Machine 1 |duration | |trunkTransaction | + And the zmq stream for "nodeA-m1" contains a response for following topics: + |keys | + |mctn | + Scenario: CheckConsistency is called Given "checkConsistency" is called on "nodeA-m1" with: diff --git a/python-regression/tests/features/machine2/2_transaction_tests.feature b/python-regression/tests/features/machine2/2_transaction_tests.feature index 5883eb50f3..d2e06a6a08 100644 --- a/python-regression/tests/features/machine2/2_transaction_tests.feature +++ b/python-regression/tests/features/machine2/2_transaction_tests.feature @@ -3,7 +3,7 @@ Feature: Test transaction confirmation Scenario: Zero Value Transactions are confirmed In this test, a number of zero value transactions will be made to a specified node. A milestone will be issued that references these transactions, and this should - confirm the transations. + confirm the transactions. Given "10" transactions are issued on "nodeA-m2" with: |keys |values |type | @@ -30,18 +30,18 @@ Feature: Test transaction confirmation When a transaction is generated and attached on "nodeA-m2" with: - | keys | values | type | - | address | TEST_ADDRESS | staticValue | - | value | 0 | int | + | keys | values | type | + | address | TEST_ADDRESS | staticValue | + | value | 0 | int | And "getInclusionStates" is called on "nodeA-m2" with: - | keys | values | type | - | transactions | TEST_STORE_ADDRESS | staticList | - | tips | latestMilestone | configValue | + | keys | values | type | + | transactions | TEST_STORE_ADDRESS | staticList | + | tips | latestMilestone | configValue | Then the response for "getInclusionStates" should return with: - | keys | values | type | - | states | False | boolListMixed | + | keys | values | type | + | states | False | boolListMixed | Scenario: Value Transactions are confirmed @@ -87,3 +87,35 @@ Feature: Test transaction confirmation | keys | values | type | | states | False | boolListMixed | + + Scenario: ZMQ receives transaction streams + Sends a predefined transaction object and a milestone that references it. This should trigger a series + of zmq stream publications. The responses for these streams are then checked against the expected contents of + the stream. + + #Subscribe to zmq stream transaction topics + Given "nodeA-m2" is subscribed to the following zmq topics: + | keys | + | sn | + | sn_trytes | + | tx | + | tx_trytes | + + Then "storeTransactions" is called on "nodeA-m2" with: + |keys |values |type | + |trytes |TRANSACTION_TEST_TRANSACTION_TRYTES |staticList | + + #In the default test, the latest sent index will be 52. The next milestone issued should be 53. + When a milestone is issued with index 53 and references: + |keys |values |type | + |transactions |TRANSACTION_TEST_TRANSACTION_HASH |staticValue | + + #Give the node time to solidify the milestone + And we wait "10" second/seconds + + Then the zmq stream for "nodeA-m2" contains a response for following responses: + | keys | values | type | + |sn | TRANSACTION_TEST_TRANSACTION_HASH | staticValue | + |sn_trytes | TRANSACTION_TEST_TRANSACTION_TRYTES | staticValue | + |tx | TRANSACTION_TEST_TRANSACTION_HASH | staticValue | + |tx_trytes | TRANSACTION_TEST_TRANSACTION_TRYTES | staticValue | diff --git a/python-regression/tests/features/machine6/6_local_snapshots_tests.feature b/python-regression/tests/features/machine6/6_local_snapshots_tests.feature index 11ea4a12d1..5151da7486 100644 --- a/python-regression/tests/features/machine6/6_local_snapshots_tests.feature +++ b/python-regression/tests/features/machine6/6_local_snapshots_tests.feature @@ -9,6 +9,14 @@ Feature: Test Bootstrapping With LS Scenario: PermaNode is synced Check that the permanode has been started correctly and is synced. + #Subscribe to zmq streams for milestones + Given "nodeA-m6" is subscribed to the following zmq topics: + |keys | + |lmi | + |lmsi | + |lmhs | + + #First make sure nodes are neighbored Given "nodeA-m6" and "nodeB-m6" are neighbors And "nodeA-m6" and "nodeC-m6" are neighbors @@ -18,6 +26,12 @@ Feature: Test Bootstrapping With LS And we wait "30" second/seconds Then "nodeA-m6" is synced up to milestone 10322 + And the zmq stream for "nodeA-m6" contains a response for following topics: + |keys | + |lmi | + |lmsi | + |lmhs | + Scenario: DB node is synced, and files contain expected values Check that the node started with just a DB is synced correctly, and that the proper addresses and hashes have been @@ -27,6 +41,12 @@ Feature: Test Bootstrapping With LS Given "nodeB-m6" and "nodeA-m6" are neighbors And "nodeB-m6" and "nodeC-m6" are neighbors + #Subscribe to zmq streams for milestones + Then "nodeA-m6" is subscribed to the following zmq topics: + |keys | + |lmi | + |lmsi | + # Default for test is to issue 10323 When milestone 10323 is issued on "nodeA-m6" #Give the node time to finish syncing properly, then make sure that the node is synced to the latest milestone. @@ -42,6 +62,11 @@ Feature: Test Bootstrapping With LS |keys |values |type | |hashes |LS_TEST_MILESTONE_HASHES |staticValue | + Then the zmq stream for "nodeA-m6" contains a response for following topics: + |keys |values |type | + |lmi |10323 |int | + |lmsi |10323 |int | + Scenario: LS DB node is synced Check that the node started with just a LS DB is synced correctly. diff --git a/python-regression/tests/features/steps/zmq_steps.py b/python-regression/tests/features/steps/zmq_steps.py new file mode 100644 index 0000000000..88e0071101 --- /dev/null +++ b/python-regression/tests/features/steps/zmq_steps.py @@ -0,0 +1,124 @@ +from aloe import world, step +from util.test_logic import api_test_logic as api_utils +import zmq + +import logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +context = zmq.Context() +socket = context.socket(zmq.SUB) +poller = zmq.Poller() + + +@step(r'"([^"]+)" is subscribed to the following zmq topics:') +def subscribe_zmq(step, node): + """ + Subscribe to the given topics on the indicated node. + + :param step.hashes: List of topics to subscribe to + :param node: The node to subscribe to + """ + arg_list = step.hashes + + host = world.machine['nodes'][node]['podip'] + port = world.machine['nodes'][node]['clusterip_ports']['zmq-feed'] + socket.connect("tcp://{}:{}".format(host, port)) + + for arg in arg_list: + socket.setsockopt_string(zmq.SUBSCRIBE, arg['keys']) + + poller.register(socket, zmq.POLLIN) + + +@step(r'the zmq stream for "([^"]+)" contains a response for following topics:') +def check_zmq_stream(step, node): + """" + Read the zmq stream on the indicated node, and ensure that all the provided topics are present in the + stream response. + + :param step.hashes: List of topics to check response for + :param node: The node the stream is being read from (Not used in test, provided for clarity) + """ + arg_list = step.hashes + + keys = [] + for arg in arg_list: + keys.append(arg['keys']) + + contains_response = False + checked_args = [] + while len(poller.poll(timeout=1000)) != 0: + received = socket.recv().split() + contains_response = False + for arg in keys: + if arg in checked_args: + contains_response = True + break + + if received[0].decode() == arg: + contains_response = True + checked_args.append(arg) + break + + assert contains_response is True, \ + "The expected response in the zmq subscription '{}' is missing".format(arg) + + assert contains_response is True, "Expected ZMQ data not found" + logger.info("Expected ZMQ data was found") + + +@step(r'the zmq stream for "([^"]+)" contains a response for following responses:') +def check_zmq_responses(step, node): + """" + Read the zmq stream on the indicated node, and ensure that all the provided responses are present in the + stream response. + + :param step.hashes: List of topics to check response for + :param node: The node the stream is being read from (Not used in test, provided for clarity) + """ + expected_values = {} + args = step.hashes + api_utils.prepare_options(args, expected_values) + + keys = [] + for arg in args: + keys.append(arg['keys']) + + contains_response = False + checked_args = [] + + while len(poller.poll(timeout=1000)) != 0: + received = socket.recv().split() + contains_response = False + for arg in keys: + if arg in checked_args: + contains_response = True + break + + if received[0].decode() == arg: + value = fetch_value_from_response(received, arg) + if value == expected_values[arg]: + contains_response = True + checked_args.append(arg) + break + + assert contains_response is True, \ + "The expected response in the zmq subscription '{}' is missing".format(arg) + + assert contains_response is True, "Expected ZMQ data not found" + logger.info("Expected ZMQ data was found") + + +def fetch_value_from_response(response, arg): + first_arg = ['tx', 'tx_trytes', 'sn_trytes', 'lmhs', 'mctn'] + second_arg = ['lmi', 'lmsi', 'sn'] + + if arg in first_arg: + value = response[1].decode() + elif arg in second_arg: + value = response[2].decode() + else: + value = response[1].decode() + + return value diff --git a/python-regression/util/static_vals.py b/python-regression/util/static_vals.py index a4d332bd50..5f7031361a 100644 --- a/python-regression/util/static_vals.py +++ b/python-regression/util/static_vals.py @@ -101,6 +101,31 @@ 99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ 99999999999999999999999999999999999999999999999999999999999999999999999999999999999" +TRANSACTION_TEST_TRANSACTION_HASH = "GOCZFAPJYTSUFSTBNNAAFPFN9EPLQSANBWBCSBZYEZZMQGGMPTSDEAGZCVFRSSTG9PRQZNIGNA9H9I999" + +TRANSACTION_TEST_TRANSACTION_TRYTES = "99999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +99999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999\ +999999999999999999999999999999999999999999999999999999999999999999999999999999999999TEST9TRANSACTION9TEST9TRANSACTION9T\ +EST9TRANSACTION9TEST9TRANSACTION9TEST999999999999999999999999999999999999LA9999999999999999999999999PENPEZD999999999999\ +99999999ZKJDCAXDVILDLFAPDQZCKROIQRDKHZZIX9QQ9RQICWYLH9EUCFZUBKWAAREIXSIPLNQBGXAACBZAKCWLCKPXJCNAGRRONLISRTGMFXFPPCLXOBR\ +JKOKBSVKSPTKPZGDSPCWIGBXMGXDWOFANHGGXTTUYVETETEA999KPXJCNAGRRONLISRTGMFXFPPCLXOBRJKOKBSVKSPTKPZGDSPCWIGBXMGXDWOFANHGGXT\ +TUYVETETEA999LA9999999999999999999999999ZKCMPXBPF999999999MMMMMMMMMK9RXJPQRYYVLYLYOFHFMWDTUIOB" # Local Snapshot Test files LS_TEST_STATE_ADDRESSES = [