diff --git a/ops/ccs-ops-misc/load_test/README.md b/ops/ccs-ops-misc/load_test/README.md index 38e202d958..8b86e555cc 100644 --- a/ops/ccs-ops-misc/load_test/README.md +++ b/ops/ccs-ops-misc/load_test/README.md @@ -70,7 +70,19 @@ The tests are run with parameters that specify all the test params and test file A single-line test will look like this (placeholders to replace in brackets): - python3 runtests.py --homePath="" --clientCertPath="/bluebutton-backend-test-data-server-client-test-keypair.pem" --databaseHost="" --databaseUsername="" --databasePassword="" --testHost="https://test.bfd.cms.gov" --testFile="./v2/eob_test_id_count.py" --testRunTime="1m" --maxClients="100" --clientsPerSecond="5" + python3 runtests.py --homePath="" --clientCertPath="/bluebutton-backend-test-data-server-client-test-keypair.pem" --databaseUri="postgres://:@:/" --testHost="https://test.bfd.cms.gov" --testFile="./v2/eob_test_id_count.py" --testRunTime="1m" --maxClients="100" --clientsPerSecond="5" + +If you have existing configuration in `./config.yml`, you can also run the tests via: + +``` +python3 runtests.py --testFile="" +``` + +Or, if you have some YAML configuration in a different file (note that these values will be saved to the root `./config.yml`, so subsequent runs can omit the `configPath` if you are not changing anything): + +``` +python3 runtests.py --configPath="config/.yml" --testFile="" ... +``` Essentially, all the items you would set up in the config file are set in a single line. There are some optional, and some required parameters here: @@ -78,16 +90,14 @@ Essentially, all the items you would set up in the config file are set in a sing **--clientCertPath** : (Required) : The path to the PEM file we copied/modified earlier. Should be located in your home directory like this: ~/bluebutton-backend-test-data-server-client-test-keypair.pem -**--databaseHost** : (Required) : The database host, used to pull data. Get this from the database endpoint we noted in the earlier step "Getting The Database Instance" - -**--databaseUsername** : (Required) : The username for connecting to the database, can be found in Keybase. Make sure to use the correct one for the environment you're connecting to. - -**--databasePassword** : (Required) : The password for connecting to the database, can be found in Keybase. Make sure to use the correct one for the environment you're connecting to. +**--databaseUri** : (Required) : The URI used for connecting to the database server. Needs to include username, password (both can be found in Keybase, make sure to use the correct one for the environment you're connecting to), hostname and port. **--testHost** : (Required) : The load balancer or single node to run the tests against. The environment used here should match the same environment (test, prod-sbx, prod) as the database, so we pull the appropriate data for the environment tested against. Note that when using a single node, you should specity the Ip AND the port for the application. **--testFile** : (Required) : The path to the test file we want to run. +**--configPath** : (Optional) : The path to a YAML configuration file that will be read from for the values specified here. The values in this configuration file will be merged with values from the CLI, with the CLI values taking priority. The resulting merged values will be written to the repository's root `config.yml`, so if `--configPath` is specified as a YAML file other than `config.yml` the YAML file at that path will not be modified (only read from). If not provided, defaults to `config.yml` (the root YAML configuration file). + **--serverPublicKey** : (Optional) : To allow the tests to trust the server responses, you can add the path to the public certificate here. This is not required to run the tests successfully, and may not be needed as a parameter at all. If not provided, defaults to an empty string (does not cause test issues.) **--testRunTime** : (Optional) : How long the test will run. This uses values such as 1m, 30s, 1h, or combinations of these such as 1m 30s. If not provided, defaults to 1m. **Note**: We automatically adjust the run time so that the test runs for the specified amount of time *after* spawning all clients (see `--maxClients` and `--clientsPerSecond` below). For example, with the defaults of a one-minute test, 100 clients, and a spawn rate of 5 clients per second, then the script will spend twenty seconds ramping up its clients and *then* run for the one minute specified. It is optional whether or not to use the `--resetStats` flag to drop the statistics covering this ramp-up period. @@ -104,6 +114,12 @@ Essentially, all the items you would set up in the config file are set in a sing There is one special type of test that requires a data setup script to be run beforehand; this is the coverage contract test. This test runs through every list of pages for a set of contracts, but the urls for hitting all the pages of whatever contracts need to be set up before the test runs. This can be done by calling the data setup script common/write_contract_cursors.py similar to this: +> **Note:** You _must_ generate the contract URLs via the below script if you are running any of the following test suites: +> * `v1/patient_test_coverageContract.py` +> * `v1/regression_suite.py` +> * `v2/patient_test_coverageContract.py` +> * `v2/regression_suite.py` + python3 ./common/write_contract_cursors.py --contracts="Z9997,Z9998,Z9999" --year="2020" --month="01" --count="5" --version="v2" The script will write out the results to homePath specified in the config (and expect them there during runtime.) diff --git a/ops/ccs-ops-misc/load_test/common/config.py b/ops/ccs-ops-misc/load_test/common/config.py index b3c04562a7..034fffec43 100644 --- a/ops/ccs-ops-misc/load_test/common/config.py +++ b/ops/ccs-ops-misc/load_test/common/config.py @@ -1,19 +1,20 @@ +import os import yaml ''' Saves a config file using the input file data. ''' def save(fileData): - configFile = open("config.yml", 'w') - configFile.write("homePath: \"%s\"\n" % fileData.homePath) - configFile.write("clientCertPath: \"%s\"\n" % fileData.clientCertPath) - configFile.write("serverPublicKey: \"%s\"\n" % fileData.serverPublicKey) - configFile.write("dbUri: \"%s\"\n" % fileData.dbUri) - configFile.write("testHost: \"%s\"\n" % fileData.testHost) - configFile.write("testRunTime: \"%s\"\n" % fileData.testRunTime) - configFile.write("testNumTotalClients: \"%s\"\n" % fileData.testNumTotalClients) - configFile.write("testCreatedClientsPerSecond: \"%s\"\n" % fileData.testCreatedClientsPerSecond) - configFile.write("resetStatsAfterClientSpawn: \"%s\"" % fileData.resetStatsAfterClientSpawn) + configFile = open('config.yml', 'w') + configFile.write("homePath: \"%s\"\n" % fileData["homePath"]) + configFile.write("clientCertPath: \"%s\"\n" % fileData["clientCertPath"]) + configFile.write("serverPublicKey: \"%s\"\n" % fileData["serverPublicKey"]) + configFile.write("dbUri: \"%s\"\n" % fileData["dbUri"]) + configFile.write("testHost: \"%s\"\n" % fileData["testHost"]) + configFile.write("testRunTime: \"%s\"\n" % fileData["testRunTime"]) + configFile.write("testNumTotalClients: \"%s\"\n" % fileData["testNumTotalClients"]) + configFile.write("testCreatedClientsPerSecond: \"%s\"\n" % fileData["testCreatedClientsPerSecond"]) + configFile.write("resetStatsAfterClientSpawn: \"%s\"" % fileData["resetStatsAfterClientSpawn"]) configFile.close() ''' @@ -23,23 +24,23 @@ def save(fileData): ''' def create(): - ## Create a small data object for holding the input data - class fileData: pass + ## Create a dictionary for holding the input data + fileData = {} ## Prompt user for 4 config values and write to file - fileData.homePath = input("Input full path to the home directory: ") - fileData.clientCertPath = input("Input full path to the client cert file (pem): ") - fileData.serverPublicKey = input("Input server public key (optional, hit enter to skip): ") - fileData.dbUri = input("Input database uri for environment under test: ") - fileData.testHost = input("Input desired test host (BFD server ip+port to test against, ex: https://10.235.16.152:7443 or load balancer address ex. https://test.bfd.cms.gov): ") - fileData.testRunTime = input("Input desired test run time (eg. 30s, 1m): ") - fileData.testNumTotalClients = input("Input total number of clients to create: ") - fileData.testCreatedClientsPerSecond = input("Input number of clients to create per second (ramp-up speed): ") - fileData.resetStatsAfterClientSpawn = (input("Reset statistics after spawning clients? [y/N]: ").lower == 'y') + fileData["homePath"] = input("Input full path to the home directory: ") + fileData["clientCertPath"] = input("Input full path to the client cert file (pem): ") + fileData["serverPublicKey"] = input("Input server public key (optional, hit enter to skip): ") + fileData["dbUri"] = input("Input database uri for environment under test: ") + fileData["testHost"] = input("Input desired test host (BFD server ip+port to test against, ex: https://10.235.16.152:7443 or load balancer address ex. https://test.bfd.cms.gov): ") + fileData["testRunTime"] = input("Input desired test run time (eg. 30s, 1m): ") + fileData["testNumTotalClients"] = input("Input total number of clients to create: ") + fileData["testCreatedClientsPerSecond"] = input("Input number of clients to create per second (ramp-up speed): ") + fileData["resetStatsAfterClientSpawn"] = (input("Reset statistics after spawning clients? [y/N]: ").lower == 'y') save(fileData) ## Attempt to read the new file try: - config = yaml.safe_load(open("config.yml")) + config = yaml.safe_load(open('config.yml')) return config except yaml.YAMLError as err: print("Unable to parse YAML configuration file; please check/create the file manually from the sample file.") @@ -47,13 +48,23 @@ class fileData: pass print("Could not read the new file; please try again.") ''' -Loads a config from the config file; if no file exists, will attempt to create one via user prompts. +Loads a config from the default config file (./config.yml); if no file exists, will attempt +to create one via user prompts. Returns the loaded config, or None if nothing could be loaded or an error occurred. ''' def load(): + return load_from_path('config.yml') + +''' +Loads a config from the specified config file path; if no file exists, will attempt to +create one via user prompts. + +Returns the loaded config, or None if nothing could be loaded or an error occurred. +''' +def load_from_path(path: str): try: - return yaml.safe_load(open("config.yml")) + return yaml.safe_load(open(path)) except yaml.YAMLError as err: print("Unable to parse YAML configuration file; please ensure the format matches the example file.") return diff --git a/ops/ccs-ops-misc/load_test/common/url_path.py b/ops/ccs-ops-misc/load_test/common/url_path.py new file mode 100644 index 0000000000..e56e851a96 --- /dev/null +++ b/ops/ccs-ops-misc/load_test/common/url_path.py @@ -0,0 +1,12 @@ +from typing import Dict +from urllib.parse import urlencode + +''' +Creates a query path from a base path (i.e. /v2/fhir/Coverage) and a dictionary of query parameters +''' +def create_url_path(path: str, query_params: Dict[str, str] = {}) -> str: + if not query_params: + return path + + params = urlencode(query_params) + return "{}?{}".format(path, params) \ No newline at end of file diff --git a/ops/ccs-ops-misc/load_test/common/validation.py b/ops/ccs-ops-misc/load_test/common/validation.py index 8a99533b80..f06941f6d0 100644 --- a/ops/ccs-ops-misc/load_test/common/validation.py +++ b/ops/ccs-ops-misc/load_test/common/validation.py @@ -9,13 +9,22 @@ SLA_PATIENT = "SLA_PATIENT" SLA_EOB_WITH_SINCE = "SLA_EOB_WITH_SINCE" SLA_EOB_WITHOUT_SINCE = "SLA_EOB_WITHOUT_SINCE" +SLA_V1_BASELINE = "SLA_V1_BASELINE" +SLA_V2_BASELINE = "SLA_V2_BASELINE" ## Values are for 50%, 95%, 99%, and the failsafe limit in order, in milliseconds +## TODO: Pull these values from production metrics (i.e. New Relic) slas = { SLA_COVERAGE : [10, 100, 250, 500], SLA_PATIENT : [1000, 3000, 5000, 8000], SLA_EOB_WITH_SINCE : [100, 250, 1000, 3000], - SLA_EOB_WITHOUT_SINCE : [500, 1000, 3000, 6000] + SLA_EOB_WITHOUT_SINCE : [500, 1000, 3000, 6000], + # The following values were selected for 5 concurrent users against + # a target c5xlarge AWS instance running bfd-server + SLA_V1_BASELINE : [10, 325, 550, 10000], + # The following values were selected for 3 concurrent users against + # a target c5xlarge AWS instance running bfd-server + SLA_V2_BASELINE : [10, 325, 550, 10000], } ''' diff --git a/ops/ccs-ops-misc/load_test/config/config.regression_v1_c5xlarge.yml b/ops/ccs-ops-misc/load_test/config/config.regression_v1_c5xlarge.yml new file mode 100644 index 0000000000..8b045bdb0f --- /dev/null +++ b/ops/ccs-ops-misc/load_test/config/config.regression_v1_c5xlarge.yml @@ -0,0 +1,7 @@ +# This configuration was created for targeting a c5xlarge instance with the V1 performance regression suite +# in v1/regression_suite.py. It was determined that 5 concurrent users puts approximately +# 50% - 60% load on a machine of that size. + +testRunTime: "3m" +testNumTotalClients: "5" +testCreatedClientsPerSecond: "5" \ No newline at end of file diff --git a/ops/ccs-ops-misc/load_test/config/config.regression_v2_c5xlarge.yml b/ops/ccs-ops-misc/load_test/config/config.regression_v2_c5xlarge.yml new file mode 100644 index 0000000000..8a81e3ae5e --- /dev/null +++ b/ops/ccs-ops-misc/load_test/config/config.regression_v2_c5xlarge.yml @@ -0,0 +1,7 @@ +# This configuration was created for targeting a c5xlarge instance with the V2 performance regression suite +# in v2/regression_suite.py. It was determined that 3 concurrent users puts approximately +# 50% - 60% load on a machine of that size. + +testRunTime: "3m" +testNumTotalClients: "3" +testCreatedClientsPerSecond: "3" \ No newline at end of file diff --git a/ops/ccs-ops-misc/load_test/runtests.py b/ops/ccs-ops-misc/load_test/runtests.py index c5e3024e9b..1217db2dae 100644 --- a/ops/ccs-ops-misc/load_test/runtests.py +++ b/ops/ccs-ops-misc/load_test/runtests.py @@ -52,18 +52,25 @@ def adjusted_run_time(runTime, maxClients, clientsPerSecond): Runs a specified test via the input args. ''' def run_with_params(argv): + ## Dictionary that holds the default values of each config value + defaultConfigData = { + 'homePath': '', + 'clientCertPath': '', + 'databaseUri': '', + 'testHost': '', + 'configPath': 'config.yml', + 'serverPublicKey': '', + 'testRunTime': "1m", + 'testNumTotalClients': "100", + 'testCreatedClientsPerSecond': "5", + 'resetStatsAfterClientSpawn': False + } + + ## Dictionary to hold data passed in via the CLI that will be stored + ## in the root config.yml file + configData = {} testFile = '' - - ## container to hold config data - class configData: pass - - ## Optional Params with defaults - configData.serverPublicKey = '' - configData.testRunTime = "1m" - configData.testNumTotalClients = "100" - configData.testCreatedClientsPerSecond = "5" - configData.resetStatsAfterClientSpawn = False workerThreads = "1" helpString = ('runtests.py \n--homePath="" (Required) ' @@ -71,6 +78,7 @@ class configData: pass '\n--databaseUri="postgres://@.rds.amazonaws.com:port/" (Required)' '\n--testHost="https://:7443 or https://.bfd.cms.gov" (Required)' '\n--testFile="//test_to_run.py" (Required)' + '\n--configPath="" (Optional, Default: "./config.yml")' '\n--serverPublicKey="" (Optional, Default: "")' '\n--testRunTime="" (Optional, Default 1m)' '\n--maxClients="" (Optional, Default 100)' @@ -80,8 +88,8 @@ class configData: pass try: opts, args = getopt.getopt(argv,"h",["homePath=", "clientCertPath=", "databaseUri=", - "testHost=", "serverPublicKey=", "testRunTime=", "maxClients=", "clientsPerSecond=", - "testFile=","workerThreads=","resetStats"]) + "testHost=", "serverPublicKey=", "configPath=", "testRunTime=", "maxClients=", + "clientsPerSecond=", "testFile=", "workerThreads=", "resetStats"]) except getopt.GetoptError: print(helpString) sys.exit(2) @@ -91,48 +99,60 @@ class configData: pass print(helpString) sys.exit() elif opt == "--homePath": - configData.homePath = arg + configData["homePath"] = arg elif opt == "--clientCertPath": - configData.clientCertPath = arg + configData["clientCertPath"] = arg elif opt == "--databaseUri": - configData.dbUri = arg + configData["dbUri"] = arg elif opt == "--testHost": - configData.testHost = arg + configData["testHost"] = arg + elif opt == "--configPath": + configData["configPath"] = arg elif opt == "--serverPublicKey": - configData.serverPublicKey = arg + configData["serverPublicKey"] = arg elif opt == "--testRunTime": - configData.testRunTime = arg + configData["testRunTime"] = arg elif opt == "--maxClients": - configData.testNumTotalClients = arg + configData["testNumTotalClients"] = arg elif opt == "--clientsPerSecond": - configData.testCreatedClientsPerSecond = arg + configData["testCreatedClientsPerSecond"] = arg elif opt == "--testFile": testFile = arg elif opt == "--workerThreads": workerThreads = arg elif opt == "--resetStats": - configData.resetStatsAfterClientSpawn = True + configData["resetStatsAfterClientSpawn"] = True else: print(helpString) sys.exit() + ## Read the specified configuration file + yamlConfig = config.load_from_path(configData.get("configPath", defaultConfigData["configPath"])) + ## Merge the stored data with data passed in via the CLI, with the + ## CLI data taking priority + configData = {**yamlConfig, **configData} + ## Finally, merge the merged configuration values with the defaults, + ## in case any optional arguments were not set via the CLI or the specified + ## YAML configuration file + configData = {**defaultConfigData, **configData} + ## Add on extra time to the run-time to account for ramp-up of clients. - adjusted_time = adjusted_run_time(configData.testRunTime, - configData.testNumTotalClients, configData.testCreatedClientsPerSecond) + adjusted_time = adjusted_run_time(configData["testRunTime"], + configData["testNumTotalClients"], configData["testCreatedClientsPerSecond"]) if adjusted_time is None: print("Could not determine adjusted run time. Please use a format " + "like \"1m 30s\" for the --testRunTime option") sys.exit(1) - configData.testRunTime = f"{adjusted_time}s" + configData["testRunTime"] = f"{adjusted_time}s" print('Run time adjusted to account for ramp-up time. New run time: %s' % timedelta(seconds=adjusted_time)) ## Check if all required params are set - if not all([configData.homePath, configData.clientCertPath, configData.dbUri, configData.testHost, testFile]): + if not all([configData["homePath"], configData["clientCertPath"], configData["dbUri"], configData["testHost"], testFile]): print("Missing required arg (See -h for help on params)") sys.exit(2) - ## write out config file + ## write out to repository root config file (_NOT_ the file specified by "configPath") config.save(configData) setup.set_locust_test_name(testFile) diff --git a/ops/ccs-ops-misc/load_test/v1/regression_suite.py b/ops/ccs-ops-misc/load_test/v1/regression_suite.py new file mode 100644 index 0000000000..aadd444779 --- /dev/null +++ b/ops/ccs-ops-misc/load_test/v1/regression_suite.py @@ -0,0 +1,178 @@ +"""Regression test suite for V1 BFD Server endpoints. + +The tests within this Locust test suite hit various endpoints that were +determined to be representative of typical V1 endpoint loads. When running +this test suite, all tests in this suite will be run in parallel, with +equal weighting being applied to each. +""" + +import random +from typing import Dict +import urllib3 +import common.config as config +import common.data as data +import common.errors as errors +import common.test_setup as setup +from common.url_path import create_url_path +import common.validation as validation +from locust import HttpUser, task, events, tag + +client_cert = setup.getClientCert() +server_public_key = setup.loadServerPublicKey() +setup.disable_no_cert_warnings(server_public_key, urllib3) +setup.set_locust_env(config.load()) + +mbis = data.load_mbis() +bene_ids = data.load_bene_ids() +last_updated = data.get_last_updated() +cursor_list = data.load_cursors("v1") + +class BFDUser(HttpUser): + def on_start(self): + """Run once when a BFDUser is initialized by Locust. + + This method copies the necessary test data (lists of + MBIs, beneficiary IDs, and contract cursor URLs) as + members of this particular BFDUser instance. We then + shuffle these copied lists such that concurrent BFDUsers + are not querying the same data at the same time. + """ + + copied_bene_ids = bene_ids.copy() + random.shuffle(copied_bene_ids) + + copied_mbis = mbis.copy() + random.shuffle(copied_mbis) + + copied_cursor_list = cursor_list.copy() + random.shuffle(copied_cursor_list) + + self.bene_ids = copied_bene_ids + self.mbis = copied_mbis + self.cursor_list = copied_cursor_list + + def get_bene_id(self) -> int: + """Returns the next beneficiary ID in this BFDUser's list of IDs. + + This method pops (that is, takes the topmost item) the next beneficiary + ID in this instance's list of beneficiary IDs and returns it. If no + more IDs are available, this method will stop the test run. + """ + + if len(self.bene_ids) == 0: + errors.no_data_stop_test(self) + + return self.bene_ids.pop() + + def get_mbi(self) -> int: + """Returns the next MBI in this BFDUser's list of MBIs. + + This method pops (that is, takes the topmost item) the next MBI + in this instance's list of MBIs and returns it. If no + more MBIs are available, this method will stop the test run. + """ + + if len(self.mbis) == 0: + errors.no_data_stop_test(self) + + return self.mbis.pop() + + def get_cursor_path(self) -> str: + """Returns the next cursor path in this BFDUser's list of URL paths. + + This method pops (that is, takes the topmost item) the next contract cursor + URL path in this instance's list of contract paths and returns it. If no + more paths are available, this method will stop the test run. + """ + + if len(self.cursor_list) == 0: + errors.no_data_stop_test(self) + + return self.cursor_list.pop() + + def get(self, base_path: str, params: Dict[str, str] = {}, headers: Dict[str, str] = {}, name: str = ''): + """Sends a GET request to the endpoint at base_path with the various query string parameters and headers specified. + + This method extends Locust's HttpUser::client.get() method to make creating the requests + nicer. Specifically, the query string parameters are specified as a separate dictionary + opposed to part of the path, the cert and verify arguments (which will never change) + are already set, and Cache-Control headers are automatically set to ensure caching is + disabled. + """ + + self.client.get(create_url_path(base_path, params), + cert=client_cert, + verify=server_public_key, + headers={**headers, 'Cache-Control': 'no-store, no-cache'}, + name=name) + + @task + def coverage_test_id_count(self): + self.get(f'/v1/fhir/Coverage', params={'beneficiary': f'{self.get_bene_id()}', '_count':'10'}, + name='/v1/fhir/Coverage search by id / count=10') + + @task + def coverage_test_id_lastUpdated(self): + self.get(f'/v1/fhir/Coverage', params={'beneficiary': f'{self.get_bene_id()}', '_lastUpdated': f'gt{last_updated}'}, + name='/v1/fhir/Coverage search by id / lastUpdated (2 weeks)') + + @task + def eob_test_id(self): + self.get(f'/v1/fhir/ExplanationOfBenefit', params={'patient': f'{self.get_bene_id()}', '_format': 'json'}, + name='/v1/fhir/ExplanationOfBenefit search by id') + + @task + def eob_test_id_count_typePDE(self): + self.get(f'/v1/fhir/ExplanationOfBenefit', params={'patient': f'{self.get_bene_id()}', '_format': 'json', '_count': '50', '_types': 'PDE'}, + name='/v1/fhir/ExplanationOfBenefit search by id / type = PDE / count = 50') + + @task + def eob_test_id_lastUpdated_count(self): + self.get(f'/v1/fhir/ExplanationOfBenefit', params={'patient': f'{self.get_bene_id()}', '_format': 'json', '_count': '100', '_lastUpdated': f'gt{last_updated}',}, + name='/v1/fhir/ExplanationOfBenefit search by id / lastUpdated / count = 100') + + @task + def eob_test_id_lastUpdated_includeTaxNumbers(self): + self.get(f'/v1/fhir/ExplanationOfBenefit', params={'patient': f'{self.get_bene_id()}', '_format': 'json', '_lastUpdated': f'gt{last_updated}', '_IncludeTaxNumbers': 'true'}, + name='/v1/fhir/ExplanationOfBenefit search by id / lastUpdated / includeTaxNumbers = true') + + @task + def eob_test_id_lastUpdated(self): + self.get(f'/v1/fhir/ExplanationOfBenefit', params={'patient': f'{self.get_bene_id()}', '_format': 'json', '_lastUpdated': f'gt{last_updated}'}, + name='/v1/fhir/ExplanationOfBenefit search by id / lastUpdated') + + @task + def patient_test_id(self): + self.get(f'/v1/fhir/Patient/{self.get_bene_id()}', + name='/v1/fhir/Patient/id') + + @task + def patient_test_id_lastUpdated_includeMbi_includeAddress(self): + self.get(f'/v1/fhir/Patient', params={'_id': f'{self.get_bene_id()}', '_lastUpdated': f'gt{last_updated}', '_IncludeIdentifiers': 'mbi', '_IncludeTaxNumbers': 'true'}, + name='/v1/fhir/Patient/id search by id / lastUpdated (2 weeks) / includeTaxNumbers = true / includeIdentifiers = mbi') + + @task + def patient_test_coverageContract(self): + self.get(self.get_cursor_path(), headers={"IncludeIdentifiers": "mbi"}, + name='/v1/fhir/Patient search by coverage contract (all pages)') + + @task + def patient_test_hashedMbi(self): + self.get(f'/v1/fhir/Patient', params={'identifier': f'https://bluebutton.cms.gov/resources/identifier/mbi-hash|{self.get_mbi()}', '_IncludeIdentifiers': 'mbi'}, + name='/v1/fhir/Patient search by hashed mbi / includeIdentifiers = mbi') + +''' +Adds a global failsafe check to ensure that if this test overwhelms the +database, we bail out and stop hitting the server. +''' +@events.init.add_listener +def on_locust_init(environment, **_kwargs): + validation.setup_failsafe_event(environment, validation.SLA_V1_BASELINE) + +''' +Adds a listener that will run when the test ends which checks the various +response time percentiles against the SLA for this endpoint. +''' +@events.test_stop.add_listener +def on_locust_quit(environment, **_kwargs): + validation.check_sla_validation(environment, validation.SLA_V1_BASELINE) \ No newline at end of file diff --git a/ops/ccs-ops-misc/load_test/v2/regression_suite.py b/ops/ccs-ops-misc/load_test/v2/regression_suite.py new file mode 100644 index 0000000000..b55f224351 --- /dev/null +++ b/ops/ccs-ops-misc/load_test/v2/regression_suite.py @@ -0,0 +1,173 @@ +"""Regression test suite for V2 BFD Server endpoints. + +The tests within this Locust test suite hit various endpoints that were +determined to be representative of typical V2 endpoint loads. When running +this test suite, all tests in this suite will be run in parallel, with +equal weighting being applied to each. +""" + +import random +from typing import Dict +import urllib3 +import common.config as config +import common.data as data +import common.errors as errors +import common.test_setup as setup +from common.url_path import create_url_path +import common.validation as validation +from locust import HttpUser, task, events, tag + +client_cert = setup.getClientCert() +server_public_key = setup.loadServerPublicKey() +setup.disable_no_cert_warnings(server_public_key, urllib3) +setup.set_locust_env(config.load()) + +mbis = data.load_mbis() +bene_ids = data.load_bene_ids() +last_updated = data.get_last_updated() +cursor_list = data.load_cursors("v2") + +class BFDUser(HttpUser): + def on_start(self): + """Run once when a BFDUser is initialized by Locust. + + This method copies the necessary test data (lists of + MBIs, beneficiary IDs, and contract cursor URLs) as + members of this particular BFDUser instance. We then + shuffle these copied lists such that concurrent BFDUsers + are not querying the same data at the same time. + """ + + copied_bene_ids = bene_ids.copy() + random.shuffle(copied_bene_ids) + + copied_mbis = mbis.copy() + random.shuffle(copied_mbis) + + copied_cursor_list = cursor_list.copy() + random.shuffle(copied_cursor_list) + + self.bene_ids = copied_bene_ids + self.mbis = copied_mbis + self.cursor_list = copied_cursor_list + + def get_bene_id(self) -> int: + """Returns the next beneficiary ID in this BFDUser's list of IDs. + + This method pops (that is, takes the topmost item) the next beneficiary + ID in this instance's list of beneficiary IDs and returns it. If no + more IDs are available, this method will stop the test run. + """ + + if len(self.bene_ids) == 0: + errors.no_data_stop_test(self) + + return self.bene_ids.pop() + + def get_mbi(self) -> int: + """Returns the next MBI in this BFDUser's list of MBIs. + + This method pops (that is, takes the topmost item) the next MBI + in this instance's list of MBIs and returns it. If no + more MBIs are available, this method will stop the test run. + """ + + if len(self.mbis) == 0: + errors.no_data_stop_test(self) + + return self.mbis.pop() + + def get_cursor_path(self) -> str: + """Returns the next cursor path in this BFDUser's list of URL paths. + + This method pops (that is, takes the topmost item) the next contract cursor + URL path in this instance's list of contract paths and returns it. If no + more paths are available, this method will stop the test run. + """ + + if len(self.cursor_list) == 0: + errors.no_data_stop_test(self) + + return self.cursor_list.pop() + + def get(self, base_path: str, params: Dict[str, str] = {}, headers: Dict[str, str] = {}, name: str = ''): + """Sends a GET request to the endpoint at base_path with the various query string parameters and headers specified. + + This method extends Locust's HttpUser::client.get() method to make creating the requests + nicer. Specifically, the query string parameters are specified as a separate dictionary + opposed to part of the path, the cert and verify arguments (which will never change) + are already set, and Cache-Control headers are automatically set to ensure caching is + disabled. + """ + + self.client.get(create_url_path(base_path, params), + cert=client_cert, + verify=server_public_key, + headers={**headers, 'Cache-Control': 'no-store, no-cache'}, + name=name) + + @task + def coverage_test_id_count(self): + self.get('/v2/fhir/Coverage', params={'beneficiary': self.get_bene_id(), '_count': '10'}, + name='/v2/fhir/Coverage search by id / count=10') + + @task + def coverage_test_id_lastUpdated(self): + self.get('/v2/fhir/Coverage', params={'_lastUpdated': f'gt{last_updated}', 'beneficiary': self.get_bene_id()}, + name='/v2/fhir/Coverage search by id / lastUpdated (2 weeks)') + + @task + def coverage_test_id(self): + self.get('/v2/fhir/Coverage', params={'beneficiary': self.get_bene_id()}, + name='/v2/fhir/Coverage search by id') + + @task + def eob_test_id_count(self): + self.get('/v2/fhir/ExplanationOfBenefit', params={'patient': self.get_bene_id(), '_count': '10', '_format': 'application/fhir+json'}, + name='/v2/fhir/ExplanationOfBenefit search by id / count=10') + + @task + def eob_test_id_includeTaxNumber(self): + self.get('/v2/fhir/ExplanationOfBenefit', params={'_lastUpdated': f'gt{last_updated}', 'patient': self.get_bene_id(), '_IncludeTaxNumbers': 'true', '_format': 'application/fhir+json'}, + name='/v2/fhir/ExplanationOfBenefit search by id / lastUpdated / includeTaxNumbers = true') + + @task + def eob_test_id(self): + self.get('/v2/fhir/ExplanationOfBenefit', params={'patient': self.get_bene_id(), '_format': 'application/fhir+json'}, + name='/v2/fhir/ExplanationOfBenefit search by id') + + @task + def patient_test_coverageContract(self): + self.get(self.get_cursor_path(), headers={"IncludeIdentifiers": "mbi"}, + name='/v2/fhir/Patient search by coverage contract (all pages)') + + @task + def patient_test_hashedMbi(self): + self.get('/v2/fhir/Patient', params={'identifier': f'https://bluebutton.cms.gov/resources/identifier/mbi-hash|{self.get_mbi()}', '_IncludeIdentifiers': 'mbi'}, + name='/v2/fhir/Patient search by hashed mbi / _IncludeIdentifiers=mbi') + + @task + def patient_test_id_lastUpdated(self): + self.get('/v2/fhir/Patient', params={'_id': self.get_bene_id(), '_format': 'application/fhir+json', '_IncludeIdentifiers': 'mbi', '_lastUpdated': f'gt{last_updated}'}, + name='/v2/fhir/Patient search by id / _IncludeIdentifiers=mbi / last updated (2 weeks)') + + @task + def patient_test_id(self): + self.get('/v2/fhir/Patient', params={'_id': self.get_bene_id(), '_format': 'application/fhir+json'}, + name='/v2/fhir/Patient search by id') + +''' +Adds a global failsafe check to ensure that if this test overwhelms the +database, we bail out and stop hitting the server. +''' +@events.init.add_listener +def on_locust_init(environment, **_kwargs): + validation.setup_failsafe_event(environment, validation.SLA_V2_BASELINE) + +''' +Adds a listener that will run when the test ends which checks the various +response time percentiles against the SLA for this endpoint. +''' +@events.test_stop.add_listener +def on_locust_quit(environment, **_kwargs): + validation.check_sla_validation(environment, validation.SLA_V2_BASELINE) \ No newline at end of file