Add data accuracy tests (#17)

* Add native_script test * Clean up * Switch to monitoring schema * add pool/account tests * Add pool/account tests * Add asset/address/epoch endpoint tests * Add test ordering * Missing import * Fix request query * Fix spec: active_stake field * Fix random asset selection * setup chnages * Add headers for POST request * Network section tests * Fix api schema for pre-shelley block pools * Fix totals select * Fix endpoint in totals test * Fix totals compare request * Fix totals assertion format * Switch to CMD params * Fix URLs and test filename * Update setup-tests Usage section, change haproxy default port to 8053 for http, and update schema (pool ticker is nullable) * Remove tip endpoint test * Add install continue flag Co-authored-by: rdlrt <[email protected]>
cardano-community · Apr 5, 2022 · a0f19c9 · a0f19c9
1 parent 4dd53f0
commit a0f19c9
Show file tree

Hide file tree

Showing 15 changed files with 284 additions and 17 deletions.
diff --git a/specs/results/koiosapi-guild.yaml b/specs/results/koiosapi-guild.yaml
@@ -1465,6 +1465,7 @@ components:
             example: pool1z5uqdk7dzdxaae5633fqfcu2eqzy3a3rgtuvy087fdld7yws0xt
           ticker:
             type: string
+            nullable: true
             description: Pool ticker
             example: JAZZ
     pool_history_info:
@@ -1805,8 +1806,9 @@ components:
             example: "2021-10-02T21:50:11"
           active_stake:
             type: string
-            description: Rewards accumulated as of given epoch (in lovelaces)
+            description: Total active stake in epoch stake snapshot (null for pre-Shelley epochs)
             example: 23395112387185880
+            nullable: true
     epoch_params:
       type: array
       items:
@@ -1989,8 +1991,9 @@ components:
             example: "vrf_vk1pmxyz8efuyj6eq6zkk373f28u47v06nwp5t59jr5fcmcusaazlmqhxu8k2"
           pool:
             type: string
-            description: Pool ID in bech32 format
+            description: Pool ID in bech32 format (null for pre-Shelley blocks)
             example: pool155efqn9xpcf73pphkk88cmlkdwx4ulkg606tne970qswczg3asc
+            nullable: true
           op_cert_counter:
             type: integer
             description: Counter value of the operational certificate used to create this block

diff --git a/specs/results/koiosapi-mainnet.yaml b/specs/results/koiosapi-mainnet.yaml
@@ -1465,6 +1465,7 @@ components:
             example: pool1z5uqdk7dzdxaae5633fqfcu2eqzy3a3rgtuvy087fdld7yws0xt
           ticker:
             type: string
+            nullable: true
             description: Pool ticker
             example: JAZZ
     pool_history_info:
@@ -1805,8 +1806,9 @@ components:
             example: "2021-10-02T21:50:11"
           active_stake:
             type: string
-            description: Rewards accumulated as of given epoch (in lovelaces)
+            description: Total active stake in epoch stake snapshot (null for pre-Shelley epochs)
             example: 23395112387185880
+            nullable: true
     epoch_params:
       type: array
       items:
@@ -1989,8 +1991,9 @@ components:
             example: "vrf_vk1pmxyz8efuyj6eq6zkk373f28u47v06nwp5t59jr5fcmcusaazlmqhxu8k2"
           pool:
             type: string
-            description: Pool ID in bech32 format
+            description: Pool ID in bech32 format (null for pre-Shelley blocks)
             example: pool155efqn9xpcf73pphkk88cmlkdwx4ulkg606tne970qswczg3asc
+            nullable: true
           op_cert_counter:
             type: integer
             description: Counter value of the operational certificate used to create this block

diff --git a/specs/results/koiosapi-testnet.yaml b/specs/results/koiosapi-testnet.yaml
@@ -1465,6 +1465,7 @@ components:
             example: pool1z5uqdk7dzdxaae5633fqfcu2eqzy3a3rgtuvy087fdld7yws0xt
           ticker:
             type: string
+            nullable: true
             description: Pool ticker
             example: JAZZ
     pool_history_info:
@@ -1805,8 +1806,9 @@ components:
             example: "2021-10-02T21:50:11"
           active_stake:
             type: string
-            description: Rewards accumulated as of given epoch (in lovelaces)
+            description: Total active stake in epoch stake snapshot (null for pre-Shelley epochs)
             example: 23395112387185880
+            nullable: true
     epoch_params:
       type: array
       items:
@@ -1989,8 +1991,9 @@ components:
             example: "vrf_vk1pmxyz8efuyj6eq6zkk373f28u47v06nwp5t59jr5fcmcusaazlmqhxu8k2"
           pool:
             type: string
-            description: Pool ID in bech32 format
+            description: Pool ID in bech32 format (null for pre-Shelley blocks)
             example: pool155efqn9xpcf73pphkk88cmlkdwx4ulkg606tne970qswczg3asc
+            nullable: true
           op_cert_counter:
             type: integer
             description: Counter value of the operational certificate used to create this block

diff --git a/specs/templates/4-api-schemas.yaml b/specs/templates/4-api-schemas.yaml
@@ -118,6 +118,7 @@ schemas:
             example: pool1z5uqdk7dzdxaae5633fqfcu2eqzy3a3rgtuvy087fdld7yws0xt
           ticker:
             type: string
+            nullable: true
             description: Pool ticker
             example: JAZZ
     pool_history_info:
@@ -458,8 +459,9 @@ schemas:
             example: "2021-10-02T21:50:11"
           active_stake:
             type: string
-            description: Rewards accumulated as of given epoch (in lovelaces)
+            description: Total active stake in epoch stake snapshot (null for pre-Shelley epochs)
             example: 23395112387185880
+            nullable: true
     epoch_params:
       type: array
       items:
@@ -642,8 +644,9 @@ schemas:
             example: "vrf_vk1pmxyz8efuyj6eq6zkk373f28u47v06nwp5t59jr5fcmcusaazlmqhxu8k2"
           pool:
             type: string
-            description: Pool ID in bech32 format
+            description: Pool ID in bech32 format (null for pre-Shelley blocks)
             example: pool155efqn9xpcf73pphkk88cmlkdwx4ulkg606tne970qswczg3asc
+            nullable: true
           op_cert_counter:
             type: integer
             description: Counter value of the operational certificate used to create this block

diff --git a/tests/__init__.py b/tests/__init__.py
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+import pytest
+import schemathesis
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--local-url", action="store", default="http://127.0.0.1:8053/api/v0"
+    )
+    parser.addoption(
+        "--compare-url", action="store", default="https://guild.koios.rest/api/v0"
+    )
+    parser.addoption(
+        "--api-schema-file",
+        action="store",
+        default="../specs/results/koiosapi-guild.yaml",
+    )
+
+
+@pytest.fixture
+def local_url(request):
+    return request.config.getoption("--local-url")
+
+
+@pytest.fixture
+def compare_url(request):
+    return request.config.getoption("--compare-url")
+
+
+@pytest.fixture
+def api_schema(request):
+    schema = schemathesis.from_path(request.config.getoption("--api-schema-file"))
+
+    return schema
diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py
diff --git a/tests/helpers/requests.py b/tests/helpers/requests.py
@@ -0,0 +1,13 @@
+import requests
+
+
+def get_request(url=""):
+    response = requests.get(url)
+    return response
+
+
+def post_request(url="", data={}):
+    headers = {"Content-Type": "application/json", "Accept": "application/json"}
+    response = requests.post(url, json=data, headers=headers)
+
+    return response
diff --git a/tests/setup-tests.sh b/tests/setup-tests.sh
@@ -1,14 +1,32 @@
 #!/usr/bin/env bash
 
-python3 -m pip install virtualenv
+sudo apt-get install python3 python3-pip -y >/dev/null
+python3 -m pip install virtualenv >/dev/null
 export PATH=$PATH:~/.local/bin/
-virtualenv koios-tests
+virtualenv koios-tests >/dev/null
 source koios-tests/bin/activate
-python3 -m pip install schemathesis
-curl -sfL https://raw.githubusercontent.com/cardano-community/koios-artifacts/master/tests/not_empty_response.py -o not_empty_response.py
-echo " To run the current test, use the below:"
-echo "    schemathesis --pre-run not_empty_response run --request-timeout 5000 --hypothesis-seed 1 https://guild.koios.rest/koiosapi.yaml --hypothesis-phases=explicit -v --hypothesis-verbosity quiet -b http://127.0.0.1:8053/api/v0 -c not_empty_response"
-echo "        where http://127.0.0.1:8053/api/v0 is the URL of instance you want to test, and guild.koios.rest is the enviornment you want to test against" 
-echo "    *[Not Ready] - Schema output format matching is not yet ready, but if you'd like to still run all the tests, you can use '-c all' in the above command"
+python3 -m pip install schemathesis pytest-order >/dev/null
+curl -sfL https://raw.githubusercontent.com/cardano-community/koios-artifacts/master/tests/not_empty_response.py -o not_empty_response.py >/dev/null
 
-echo " To quit from Python virtualenv, you can run 'deactivate' "
+cat <<-EOF
+	
+	To run the endpoint validation tests, use the below:
+	
+	  schemathesis --pre-run not_empty_response run --request-timeout 5000 --hypothesis-seed 1 https://guild.koios.rest/koiosapi.yaml \\
+	               --hypothesis-phases=explicit -v --hypothesis-verbosity quiet -b http://127.0.0.1:8053/api/v0 -c all
+	
+	      where http://127.0.0.1:8053/api/v0 is the URL of instance you want to test, and guild.koios.rest is the target enviornment for testing
+	
+	To run the data validations tests, use the below:
+	
+	  pytest --local-url http://127.0.0.1:8053/api/v0 --compare-url https://guild.koios.rest/api/v0 --api-schema-file ../specs/results/koiosapi-guild.yaml -x -v
+	
+	  Arguments:
+	      local-run		:	URL of instance you want to test"
+	      compare-url	:	Source-of-truth instance to compare returned data against"
+	      api-schema-file	:	The API specs/schema file you want to use as input for validation"
+	
+	To enter Python virtualenv, type 'source koios-tests/bin/activate'
+	To exit from Python virtualenv, you can run 'deactivate' 
+	
+	EOF
diff --git a/tests/test_asset_address_endpoints.py b/tests/test_asset_address_endpoints.py
@@ -0,0 +1,52 @@
+import pytest
+import random
+from tests.helpers.requests import get_request
+
+random_asset_address = ""
+
+
+@pytest.mark.order(1)
+def test_asset_list_endpoint(api_schema, local_url, compare_url):
+    global random_asset_address
+
+    asset_list_response = get_request(f"{local_url}/asset_list")
+    api_schema["/asset_list"]["GET"].validate_response(asset_list_response)
+
+    random_asset = random.choice(asset_list_response.json())
+    random_asset_policy, random_asset_name = (
+        random_asset["policy_id"],
+        random_asset["asset_names"]["hex"][0],
+    )
+
+    asset_info_response = get_request(
+        f"{local_url}/asset_info?_asset_policy={random_asset_policy}&_asset_name={random_asset_name}"
+    )
+    api_schema["/asset_info"]["GET"].validate_response(asset_info_response)
+
+    compare_asset_info_response = get_request(
+        f"{compare_url}/asset_info?_asset_policy={random_asset_policy}&_asset_name={random_asset_name}"
+    )
+
+    assert asset_info_response.json() == compare_asset_info_response.json()
+
+    asset_address_list_response = get_request(
+        f"{local_url}/asset_address_list?_asset_policy={random_asset_policy}&_asset_name={random_asset_name}"
+    )
+
+    random_asset_address = random.choice(asset_address_list_response.json())
+
+
+@pytest.mark.order(2)
+def test_address_info_endpoint(api_schema, compare_url, local_url):
+    global random_asset_address
+
+    address_info_response = get_request(
+        f"{local_url}/address_info?_address={random_asset_address}"
+    )
+    api_schema["/address_info"]["GET"].validate_response(address_info_response)
+
+    compare_address_info_response = get_request(
+        f"{compare_url}/address_info?_address={random_asset_address}"
+    )
+
+    assert address_info_response.json() == compare_address_info_response.json()
diff --git a/tests/test_epoch_endpoints.py b/tests/test_epoch_endpoints.py
@@ -0,0 +1,39 @@
+import pytest
+import random
+
+from tests.helpers.requests import get_request
+
+random_epoch_no = ""
+
+
+@pytest.mark.order(1)
+def test_epoch_info_endpoint(api_schema, local_url, compare_url):
+    global random_epoch_no
+
+    epoch_info_response = get_request(f"{local_url}/epoch_info")
+    api_schema["/epoch_info"]["GET"].validate_response(epoch_info_response)
+
+    random_epoch = random.choice(epoch_info_response.json())
+    random_epoch_no = random_epoch["epoch_no"]
+
+    compare_epoch_info_response = get_request(
+        f"{compare_url}/epoch_info?_epoch_no={random_epoch_no}"
+    )
+
+    assert random_epoch == compare_epoch_info_response.json()[0]
+
+
+@pytest.mark.order(2)
+def test_epoch_params_endpoint(api_schema, compare_url, local_url):
+    global random_epoch_no
+
+    epoch_params_response = get_request(
+        f"{local_url}/epoch_params?_epoch_no={random_epoch_no}"
+    )
+    api_schema["/epoch_params"]["GET"].validate_response(epoch_params_response)
+
+    compare_epoch_params_response = get_request(
+        f"{compare_url}/epoch_params?_epoch_no={random_epoch_no}"
+    )
+
+    assert epoch_params_response.json() == compare_epoch_params_response.json()
diff --git a/tests/test_genesis_endpoint.py b/tests/test_genesis_endpoint.py
@@ -0,0 +1,10 @@
+from tests.helpers.requests import get_request
+
+
+def test_genesis_endpoint(api_schema, local_url, compare_url):
+    genesis_response = get_request(f"{local_url}/genesis")
+    api_schema["/genesis"]["GET"].validate_response(genesis_response)
+
+    compare_genesis_response = get_request(f"{compare_url}/genesis")
+
+    assert genesis_response.json() == compare_genesis_response.json()
diff --git a/tests/test_pool_account_endpoints.py b/tests/test_pool_account_endpoints.py
@@ -0,0 +1,45 @@
+import pytest
+import random
+from tests.helpers.requests import get_request, post_request
+
+pool_reward_address = ""
+
+
+@pytest.mark.order(1)
+def test_pool_list_endpoint(api_schema, local_url, compare_url):
+    global pool_reward_address
+
+    pool_list_response = get_request(f"{local_url}/pool_list")
+    api_schema["/pool_list"]["GET"].validate_response(pool_list_response)
+
+    random_pool = random.choice(pool_list_response.json())
+    random_pool_id = random_pool["pool_id_bech32"]
+
+    pool_info_response = post_request(
+        f"{local_url}/pool_info", {"_pool_bech32_ids": [random_pool_id]}
+    )
+    api_schema["/pool_info"]["POST"].validate_response(pool_info_response)
+
+    compare_pool_info_response = post_request(
+        f"{compare_url}/pool_info", {"_pool_bech32_ids": [random_pool_id]}
+    )
+
+    assert pool_info_response.json() == compare_pool_info_response.json()
+
+    pool_reward_address = pool_info_response.json()[0]["reward_addr"]
+
+
+@pytest.mark.order(2)
+def test_account_info_endpoint(api_schema, local_url, compare_url):
+    global pool_reward_address
+
+    account_info_response = get_request(
+        f"{local_url}/account_info?_address={pool_reward_address}"
+    )
+    api_schema["/account_info"]["GET"].validate_response(account_info_response)
+
+    compare_account_info_response = get_request(
+        f"{compare_url}/account_info?_address={pool_reward_address}"
+    )
+
+    assert account_info_response.json() == compare_account_info_response.json()
diff --git a/tests/test_script_tx_block_endpoints.py b/tests/test_script_tx_block_endpoints.py
@@ -0,0 +1,28 @@
+import random
+from tests.helpers.requests import get_request, post_request
+
+
+def test_native_script_list_endpoint(api_schema, local_url, compare_url):
+    script_response = get_request(f"{local_url}/native_script_list")
+    api_schema["/native_script_list"]["GET"].validate_response(script_response)
+
+    random_script = random.choice(script_response.json())
+    random_script_creation_tx_hash = random_script["creation_tx_hash"]
+
+    tx_response = post_request(
+        f"{local_url}/tx_info", {"_tx_hashes": [random_script_creation_tx_hash]}
+    )
+    api_schema["/tx_info"]["POST"].validate_response(tx_response)
+
+    tx_block_hash = tx_response.json()[0]["block_hash"]
+
+    block_response = post_request(
+        f"{local_url}/block_info", {"_block_hashes": [tx_block_hash]}
+    )
+    api_schema["/block_info"]["POST"].validate_response(block_response)
+
+    compare_block_response = post_request(
+        f"{compare_url}/block_info", {"_block_hashes": [tx_block_hash]}
+    )
+
+    assert block_response.json() == compare_block_response.json()