diff --git a/docs/new_endpoint_tutorial.md b/docs/new_endpoint_tutorial.md index 6e6094161..abf8a5fa1 100644 --- a/docs/new_endpoint_tutorial.md +++ b/docs/new_endpoint_tutorial.md @@ -123,11 +123,9 @@ Here's what we add to each client: def fluview_meta(): """Fetch FluView metadata.""" # Set up request - params = { - 'endpoint': 'fluview_meta', - } + params = {} # Make the API call - return Epidata._request(params) + return Epidata._request("fluview_meta", params) ``` - [`delphi_epidata.R`](https://github.com/cmu-delphi/delphi-epidata/blob/dev/src/client/delphi_epidata.R) diff --git a/integrations/acquisition/covid_hosp/facility/test_scenarios.py b/integrations/acquisition/covid_hosp/facility/test_scenarios.py index c6c51e2f5..44ee3572d 100644 --- a/integrations/acquisition/covid_hosp/facility/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/facility/test_scenarios.py @@ -28,7 +28,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py b/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py index 2054d19c8..8636295bc 100644 --- a/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/state_daily/test_scenarios.py @@ -32,7 +32,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py b/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py index 8565b8e7f..46bdeebcd 100644 --- a/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py +++ b/integrations/acquisition/covid_hosp/state_timeseries/test_scenarios.py @@ -28,7 +28,7 @@ def setUp(self): self.test_utils = UnitTestUtils(__file__) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py index 16e2506fc..30b6bbcf2 100644 --- a/integrations/acquisition/covidcast/test_covidcast_meta_caching.py +++ b/integrations/acquisition/covidcast/test_covidcast_meta_caching.py @@ -22,7 +22,7 @@ ) # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' class CovidcastMetaCacheTests(unittest.TestCase): @@ -82,8 +82,8 @@ def tearDown(self): @staticmethod def _make_request(): - params = {'endpoint': 'covidcast_meta', 'cached': 'true'} - response = requests.get(Epidata.BASE_URL, params=params, auth=Epidata.auth) + params = {'cached': 'true'} + response = requests.get(f"{Epidata.BASE_URL}/covidcast_meta", params=params, auth=Epidata.auth) response.raise_for_status() return response.json() diff --git a/integrations/acquisition/covidcast/test_csv_uploading.py b/integrations/acquisition/covidcast/test_csv_uploading.py index e9c066f6a..dab35f414 100644 --- a/integrations/acquisition/covidcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast/test_csv_uploading.py @@ -69,7 +69,7 @@ def setUp(self): epidata_cnx.close() # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def tearDown(self): @@ -132,7 +132,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_values = pd.concat([values, pd.DataFrame({ "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") + expected_values = pd.concat([values, pd.DataFrame({ "geo_type": "state", "source": "src-name", "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3})], axis=1).rename(columns=uploader_column_rename).to_dict(orient="records") expected_response = {'result': 1, 'epidata': self.apply_lag(expected_values), 'message': 'success'} self.assertEqual(response, expected_response) @@ -161,6 +161,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419] * 3, "signal": [signal_name] * 3, "direction": [None] * 3, @@ -194,7 +197,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_response = {'result': -2, 'message': 'no results'} + expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} self.assertEqual(response, expected_response) self.verify_timestamps_and_defaults() @@ -220,6 +223,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values_df = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419], "signal": [signal_name], "direction": [None]})], axis=1).rename(columns=uploader_column_rename) @@ -253,6 +259,9 @@ def test_uploading(self): response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') expected_values = pd.concat([values, pd.DataFrame({ + "geo_type": "state", + "source": "src-name", + "time_type": "day", "time_value": [20200419], "signal": [signal_name], "direction": [None] @@ -283,7 +292,7 @@ def test_uploading(self): main(args) response = Epidata.covidcast('src-name', signal_name, 'day', 'state', 20200419, '*') - expected_response = {'result': -2, 'message': 'no results'} + expected_response = {'epidata': [], 'result': -2, 'message': 'no results'} self.assertEqual(response, expected_response) self.verify_timestamps_and_defaults() diff --git a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py index 1299c6144..bf1a0f9a0 100644 --- a/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py +++ b/integrations/acquisition/covidcast_nowcast/test_csv_uploading.py @@ -55,7 +55,7 @@ def setUp(self): secrets.db.epi = ('user', 'pass') # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def tearDown(self): diff --git a/integrations/client/test_delphi_epidata.py b/integrations/client/test_delphi_epidata.py index 798ca5f1a..2af923b2b 100644 --- a/integrations/client/test_delphi_epidata.py +++ b/integrations/client/test_delphi_epidata.py @@ -24,9 +24,9 @@ def fake_epidata_endpoint(func): """This can be used as a decorator to enable a bogus Epidata endpoint to return 404 responses.""" def wrapper(*args): - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/fake_api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/fake_epidata' func(*args) - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' return wrapper class DelphiEpidataPythonClientTests(CovidcastBase): @@ -39,7 +39,7 @@ def localSetUp(self): self._db._cursor.execute('update covidcast_meta_cache set timestamp = 0, epidata = "[]"') # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database @@ -65,8 +65,8 @@ def test_covidcast(self): ) expected = [ - row_latest_issue.as_api_compatibility_row_dict(), - rows[-1].as_api_compatibility_row_dict() + row_latest_issue.as_api_row_dict(), + rows[-1].as_api_row_dict() ] self.assertEqual(response['epidata'], expected) @@ -85,10 +85,10 @@ def test_covidcast(self): expected = [{ rows[0].signal: [ - row_latest_issue.as_api_compatibility_row_dict(ignore_fields=['signal']), + row_latest_issue.as_api_row_dict(ignore_fields=['signal']), ], rows[-1].signal: [ - rows[-1].as_api_compatibility_row_dict(ignore_fields=['signal']), + rows[-1].as_api_row_dict(ignore_fields=['signal']), ], }] @@ -105,7 +105,7 @@ def test_covidcast(self): **self.params_from_row(rows[0]) ) - expected = [row_latest_issue.as_api_compatibility_row_dict()] + expected = [row_latest_issue.as_api_row_dict()] # check result self.assertEqual(response_1, { @@ -120,7 +120,7 @@ def test_covidcast(self): **self.params_from_row(rows[0], as_of=rows[1].issue) ) - expected = [rows[1].as_api_compatibility_row_dict()] + expected = [rows[1].as_api_row_dict()] # check result self.maxDiff=None @@ -145,8 +145,8 @@ def test_covidcast(self): ) expected = [ - rows[0].as_api_compatibility_row_dict(), - rows[1].as_api_compatibility_row_dict() + rows[0].as_api_row_dict(), + rows[1].as_api_row_dict() ] # check result @@ -162,7 +162,7 @@ def test_covidcast(self): **self.params_from_row(rows[0], lag=2) ) - expected = [row_latest_issue.as_api_compatibility_row_dict()] + expected = [row_latest_issue.as_api_row_dict()] # check result self.assertDictEqual(response_3, { @@ -178,7 +178,7 @@ def test_covidcast(self): ) # check result - self.assertEqual(response_1, {'message': 'no results', 'result': -2}) + self.assertEqual(response_1, {'epidata': [], 'message': 'no results', 'result': -2}) @patch('requests.post') @patch('requests.get') @@ -204,7 +204,7 @@ def test_retry_request(self, get): mock_response = MagicMock() mock_response.status_code = 200 get.side_effect = [JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None) + response = Epidata._request("") self.assertEqual(get.call_count, 2) self.assertEqual(response, mock_response.json()) @@ -215,7 +215,7 @@ def test_retry_request(self, get): get.side_effect = [JSONDecodeError('Expecting value', "", 0), JSONDecodeError('Expecting value', "", 0), mock_response] - response = Epidata._request(None) + response = Epidata._request("") self.assertEqual(get.call_count, 2) # 2 from previous test + 2 from this one self.assertEqual(response, {'result': 0, 'message': 'error: Expecting value: line 1 column 1 (char 0)'} @@ -236,7 +236,7 @@ def test_geo_value(self): self._insert_rows(rows) counties = [ - rows[i].as_api_compatibility_row_dict() for i in range(N) + rows[i].as_api_row_dict() for i in range(N) ] def fetch(geo): @@ -347,13 +347,15 @@ def test_async_epidata(self): self.params_from_row(rows[0], source='covidcast'), self.params_from_row(rows[1], source='covidcast') ]*12, batch_size=10) - responses = [i[0] for i in test_output] - # check response is same as standard covidcast call, using 24 calls to test batch sizing + responses = [i[0]["epidata"] for i in test_output] + # check response is same as standard covidcast call (minus fields omitted by the api.php endpoint), + # using 24 calls to test batch sizing + ignore_fields = CovidcastTestRow._api_row_compatibility_ignore_fields self.assertEqual( responses, [ - Epidata.covidcast(**self.params_from_row(rows[0])), - Epidata.covidcast(**self.params_from_row(rows[1])), + [{k: row[k] for k in row.keys() - ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[0]))["epidata"]], + [{k: row[k] for k in row.keys() - ignore_fields} for row in Epidata.covidcast(**self.params_from_row(rows[1]))["epidata"]], ]*12 ) diff --git a/integrations/client/test_nowcast.py b/integrations/client/test_nowcast.py index f5124e021..84fc0e080 100644 --- a/integrations/client/test_nowcast.py +++ b/integrations/client/test_nowcast.py @@ -39,7 +39,7 @@ def setUp(self): self.cur = cnx.cursor() # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database @@ -133,4 +133,4 @@ def test_covidcast_nowcast(self): response = Epidata.covidcast_nowcast( 'src', 'sig1', 'sensor', 'day', 'county', 22222222, '01001') - self.assertEqual(response, {'result': -2, 'message': 'no results'}) + self.assertEqual(response, {'epidata': [], 'result': -2, 'message': 'no results'}) diff --git a/integrations/server/test_api_keys.py b/integrations/server/test_api_keys.py index 418e265ac..8cb50016a 100644 --- a/integrations/server/test_api_keys.py +++ b/integrations/server/test_api_keys.py @@ -11,24 +11,21 @@ class APIKeysTets(DelphiTestBase): def localSetUp(self): self.role_name = "cdc" - def _make_request(self, url: str = None, params: dict = {}, auth: tuple = None): - if not url: - url = self.epidata_client.BASE_URL + def _make_request(self, endpoint: str, params: dict = {}, auth: tuple = None): + url = f"{self.epidata_client.BASE_URL}/{endpoint}" response = requests.get(url, params=params, auth=auth) return response def test_public_route(self): """Test public route""" - public_route = "http://delphi_web_epidata/epidata/version" status_codes = set() for _ in range(10): - status_codes.add(self._make_request(public_route).status_code) + status_codes.add(self._make_request("version").status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_data_source(self): """Test requests with no multiples and with provided `data_source` and `signal` as a separate query params.""" params = { - "source": "covidcast", "data_source": "fb-survey", "signal": "smoothed_wcli", "time_type": "day", @@ -38,13 +35,12 @@ def test_no_multiples_data_source(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200}) def test_no_multiples_source_signal(self): """Test requests with colon-delimited source-signal param presentation.""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -53,13 +49,12 @@ def test_no_multiples_source_signal(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_two_multiples(self): """Test requests with 2 multiples and allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -68,13 +63,12 @@ def test_multiples_allowed_signal_two_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200}) def test_multiples_non_allowed_signal(self): """Test requests with 2 multiples and non-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "hospital-admissions:smoothed_adj_covid19_from_claims", "time_type": "day", "geo_type": "state", @@ -83,13 +77,12 @@ def test_multiples_non_allowed_signal(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200, 429}) def test_multiples_mixed_allowed_signal_two_multiples(self): """Test requests with 2 multiples and mixed-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,hospital-admissions:smoothed_adj_covid19_from_claims", "time_type": "day", "geo_type": "state", @@ -98,13 +91,12 @@ def test_multiples_mixed_allowed_signal_two_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {200, 429}) def test_multiples_allowed_signal_three_multiples(self): """Test requests with 3 multiples and allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -113,13 +105,12 @@ def test_multiples_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_three_multiples(self): """Test requests with 3 multiples and mixed-allowed dashboard signal""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", "time_type": "day", "geo_type": "state", @@ -128,13 +119,12 @@ def test_multiples_mixed_allowed_signal_three_multiples(self): } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("covidcast", params=params).status_code) self.assertEqual(status_codes, {401}) def test_multiples_mixed_allowed_signal_api_key(self): """Test requests with 3 multiples and mixed-allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli1", "time_type": "day", "geo_type": "state", @@ -144,14 +134,13 @@ def test_multiples_mixed_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) def test_multiples_allowed_signal_api_key(self): """Test requests with 3 multiples and allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli,fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -161,14 +150,13 @@ def test_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) def test_no_multiples_allowed_signal_api_key(self): """Test requests with no multiples and allowed dashboard signal + valid API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -178,14 +166,13 @@ def test_no_multiples_allowed_signal_api_key(self): status_codes = set() for _ in range(10): status_codes.add( - self._make_request(params=params, auth=self.epidata_client.auth).status_code + self._make_request("covidcast", params=params, auth=self.epidata_client.auth).status_code ) self.assertEqual(status_codes, {200}) def test_no_multiples_allowed_signal_bad_api_key(self): """Test requests with no multiples and allowed dashboard signal + bad API Key""" params = { - "source": "covidcast", "signal": "fb-survey:smoothed_wcli", "time_type": "day", "geo_type": "state", @@ -196,54 +183,51 @@ def test_no_multiples_allowed_signal_bad_api_key(self): for _ in range(10): status_codes.add( self._make_request( - params=params, auth=("bad_key", "bad_email") + "covidcast", params=params, auth=("bad_key", "bad_email") ).status_code ) self.assertEqual(status_codes, {200}) def test_restricted_endpoint_no_key(self): """Test restricted endpoint with no auth key""" - params = {"source": "cdc", "regions": "1as", "epiweeks": "202020"} + params = {"regions": "1as", "epiweeks": "202020"} status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_invalid_key(self): """Test restricted endpoint with invalid auth key""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "invalid_key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_no_roles_key(self): """Test restricted endpoint with no roles key""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {401}) def test_restricted_endpoint_valid_roles_key(self): """Test restricted endpoint with valid auth key with required role""" params = { - "source": "cdc", "regions": "1as", "epiweeks": "202020", "auth": "cdc_key", } status_codes = set() for _ in range(10): - status_codes.add(self._make_request(params=params).status_code) + status_codes.add(self._make_request("cdc", params=params).status_code) self.assertEqual(status_codes, {200}) diff --git a/integrations/server/test_covid_hosp.py b/integrations/server/test_covid_hosp.py index 37aa77363..100d961c4 100644 --- a/integrations/server/test_covid_hosp.py +++ b/integrations/server/test_covid_hosp.py @@ -16,7 +16,7 @@ def setUp(self): """Perform per-test setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') # use the local instance of the epidata database diff --git a/integrations/server/test_covidcast.py b/integrations/server/test_covidcast.py index 73787d664..4963ca9a8 100644 --- a/integrations/server/test_covidcast.py +++ b/integrations/server/test_covidcast.py @@ -23,7 +23,7 @@ def localSetUp(self): def request_based_on_row(self, row: CovidcastTestRow, **kwargs): params = self.params_from_row(row, endpoint='covidcast', **kwargs) # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') response = Epidata.covidcast(**params) @@ -90,7 +90,7 @@ def test_round_trip(self): # make the request response = self.request_based_on_row(row) - expected = [row.as_api_compatibility_row_dict()] + expected = [row.as_api_row_dict()] self.assertEqual(response, { 'result': 1, @@ -156,13 +156,15 @@ def test_csv_format(self): **{'format':'csv'} ) - # This is a hardcoded mess because of api.php. + # This is a hardcoded mess because of the field ordering constructed here: + # https://github.com/cmu-delphi/delphi-epidata/blob/f7da6598a810be8df5374e3a71512c631c3a14f1/src/server/endpoints/covidcast.py#L83-L93 column_order = [ - "geo_value", "signal", "time_value", "direction", "issue", "lag", "missing_value", + "geo_value", "signal", "source", "geo_type", "time_type", + "time_value", "direction", "issue", "lag", "missing_value", "missing_stderr", "missing_sample_size", "value", "stderr", "sample_size" ] expected = ( - row.as_api_compatibility_row_df() + row.as_api_row_df() .assign(direction = None) .to_csv(columns=column_order, index=False) ) @@ -179,7 +181,7 @@ def test_raw_json_format(self): # make the request response = self.request_based_on_row(row, **{'format':'json'}) - expected = [row.as_api_compatibility_row_dict()] + expected = [row.as_api_row_dict()] # assert that the right data came back self.assertEqual(response, expected) @@ -191,13 +193,13 @@ def test_fields(self): row = self._insert_placeholder_set_one() # limit fields - response = self.request_based_on_row(row, **{"fields":"time_value,geo_value"}) + response = self.request_based_on_row(row, **{"fields":"time_value,geo_value,geo_type,source,time_type"}) - expected = row.as_api_compatibility_row_dict() + expected = row.as_api_row_dict() expected_all = { 'result': 1, 'epidata': [{ - k: expected[k] for k in ['time_value', 'geo_value'] + k: expected[k] for k in ['time_value', 'geo_value', 'geo_type', 'source', 'time_type'] }], 'message': 'success', } @@ -206,7 +208,7 @@ def test_fields(self): self.assertEqual(response, expected_all) # limit using invalid fields - response = self.request_based_on_row(row, fields='time_value,geo_value,doesnt_exist') + response = self.request_based_on_row(row, fields='time_value,geo_value,geo_type,source,time_type,doesnt_exist') # assert that the right data came back (only valid fields) self.assertEqual(response, expected_all) @@ -226,7 +228,7 @@ def test_location_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], geo_value="*") @@ -243,7 +245,7 @@ def test_time_values_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_three() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], time_values="*") @@ -261,7 +263,7 @@ def test_issues_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_five() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], issues="*") @@ -279,7 +281,7 @@ def test_signal_wildcard(self): # insert placeholder data rows = self._insert_placeholder_set_four() - expected_signals = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected_signals = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], signals="*") @@ -297,7 +299,7 @@ def test_geo_value(self): # insert placeholder data rows = self._insert_placeholder_set_two() - expected = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected = [row.as_api_row_dict() for row in rows[:3]] def fetch(geo_value): # make the request @@ -335,7 +337,7 @@ def test_location_timeline(self): # insert placeholder data rows = self._insert_placeholder_set_three() - expected_timeseries = [row.as_api_compatibility_row_dict() for row in rows[:3]] + expected_timeseries = [row.as_api_row_dict() for row in rows[:3]] # make the request response = self.request_based_on_row(rows[0], time_values='20000101-20000105') @@ -372,7 +374,7 @@ def test_nullable_columns(self): # make the request response = self.request_based_on_row(row) - expected = row.as_api_compatibility_row_dict() + expected = row.as_api_row_dict() # assert that the right data came back self.assertEqual(response, { @@ -393,7 +395,7 @@ def test_temporal_partitioning(self): # make the request response = self.request_based_on_row(rows[1], time_values="*") - expected = [rows[1].as_api_compatibility_row_dict()] + expected = [rows[1].as_api_row_dict()] # assert that the right data came back self.assertEqual(response, { diff --git a/integrations/server/test_covidcast_endpoints.py b/integrations/server/test_covidcast_endpoints.py index 3ba0af039..206ed3c54 100644 --- a/integrations/server/test_covidcast_endpoints.py +++ b/integrations/server/test_covidcast_endpoints.py @@ -113,7 +113,34 @@ def test_compatibility(self): with self.subTest("simple"): out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) - self.assertEqual(len(out["epidata"]), len(rows)) + self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) + + with self.subTest("same contents sans excluded columns"): + compat = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + regular = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*") + # Remove keys from the regular row which are excluded in the compat rows + for row in regular['epidata']: + for key in ['source', 'geo_type', 'time_type']: + del row[key] + self.assertEqual(compat, regular) + + def test_compatibility_restricted_source(self): + """Restricted request at the /api.php endpoint.""" + rows = [CovidcastTestRow.make_default_row(time_value=2020_04_01 + i, value=i, source="quidel") for i in range(10)] + first = rows[0] + self._insert_rows(rows) + + with self.subTest("no_roles"): + out = self._fetch("/", signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertTrue("epidata" not in out) + + with self.subTest("no_api_key"): + out = self._fetch("/", auth=None, signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertTrue("epidata" not in out) + + with self.subTest("quidel_role"): + out = self._fetch("/", auth=("epidata", "quidel_key"), signal=first.signal_pair(), geo=first.geo_pair(), time="day:*", is_compatibility=True) + self.assertEqual(out["epidata"], [row.as_api_compatibility_row_dict() for row in rows]) def test_trend(self): """Request a signal from the /trend endpoint.""" diff --git a/integrations/server/test_covidcast_meta.py b/integrations/server/test_covidcast_meta.py index d03317c98..857422a41 100644 --- a/integrations/server/test_covidcast_meta.py +++ b/integrations/server/test_covidcast_meta.py @@ -14,7 +14,7 @@ import delphi.operations.secrets as secrets # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' AUTH = ('epidata', 'key') @@ -152,8 +152,7 @@ def _get_id(self): @staticmethod def _fetch(auth=AUTH, **kwargs): params = kwargs.copy() - params['endpoint'] = 'covidcast_meta' - response = requests.get(BASE_URL, params=params, auth=auth) + response = requests.get(f"{BASE_URL}/covidcast_meta", params=params, auth=auth) response.raise_for_status() return response.json() diff --git a/integrations/server/test_covidcast_nowcast.py b/integrations/server/test_covidcast_nowcast.py index 889d962dd..32445afdf 100644 --- a/integrations/server/test_covidcast_nowcast.py +++ b/integrations/server/test_covidcast_nowcast.py @@ -9,7 +9,7 @@ # use the local instance of the Epidata API -BASE_URL = 'http://delphi_web_epidata/epidata/api.php' +BASE_URL = 'http://delphi_web_epidata/epidata' AUTH = ('epidata', 'key') @@ -43,7 +43,7 @@ def tearDown(self): @staticmethod def _make_request(params: dict): - response = requests.get(BASE_URL, params=params, auth=AUTH) + response = requests.get(f"{BASE_URL}/covidcast_nowcast", params=params, auth=AUTH) response.raise_for_status() return response.json() @@ -59,7 +59,6 @@ def test_query(self): self.cnx.commit() # make the request with specified issue date params={ - 'source': 'covidcast_nowcast', 'data_source': 'src', 'signals': 'sig', 'sensor_names': 'sensor', diff --git a/integrations/server/test_fluview.py b/integrations/server/test_fluview.py index c192da637..48d9585fd 100644 --- a/integrations/server/test_fluview.py +++ b/integrations/server/test_fluview.py @@ -18,7 +18,7 @@ def setUpClass(cls): """Perform one-time setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def setUp(self): diff --git a/integrations/server/test_fluview_meta.py b/integrations/server/test_fluview_meta.py index 1e2cf73e3..6f81c1859 100644 --- a/integrations/server/test_fluview_meta.py +++ b/integrations/server/test_fluview_meta.py @@ -18,7 +18,7 @@ def setUpClass(cls): """Perform one-time setup.""" # use the local instance of the Epidata API - Epidata.BASE_URL = 'http://delphi_web_epidata/epidata/api.php' + Epidata.BASE_URL = 'http://delphi_web_epidata/epidata' Epidata.auth = ('epidata', 'key') def setUp(self): diff --git a/integrations/server/test_signal_dashboard.py b/integrations/server/test_signal_dashboard.py index e99a464fd..1aed3eec6 100644 --- a/integrations/server/test_signal_dashboard.py +++ b/integrations/server/test_signal_dashboard.py @@ -25,10 +25,7 @@ def setUp(self) -> None: def test_signal_dashboard_coverage(self): """Basic integration test for signal_dashboard_coverage endpoint""" - params = { - "endpoint": "signal_dashboard_coverage", - } - response = self.epidata_client._request(params=params) + response = self.epidata_client._request("signal_dashboard_coverage") self.assertEqual( response, { @@ -41,10 +38,7 @@ def test_signal_dashboard_coverage(self): def test_signal_dashboard_status(self): """Basic integration test for signal_dashboard_status endpoint""" - params = { - "endpoint": "signal_dashboard_status", - } - response = self.epidata_client._request(params=params) + response = self.epidata_client._request("signal_dashboard_status") self.assertEqual( response, { diff --git a/src/client/delphi_epidata.py b/src/client/delphi_epidata.py index fe8dbe51d..99d8f4abc 100644 --- a/src/client/delphi_epidata.py +++ b/src/client/delphi_epidata.py @@ -44,7 +44,7 @@ class Epidata: """An interface to DELPHI's Epidata API.""" # API base url - BASE_URL = "https://api.delphi.cmu.edu/epidata/api.php" + BASE_URL = "https://api.delphi.cmu.edu/epidata" auth = None client_version = _version @@ -68,17 +68,18 @@ def _list(values): @staticmethod @retry(reraise=True, stop=stop_after_attempt(2)) - def _request_with_retry(params): + def _request_with_retry(endpoint, params={}): """Make request with a retry if an exception is thrown.""" - req = requests.get(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) + request_url = f"{Epidata.BASE_URL}/{endpoint}" + req = requests.get(request_url, params, auth=Epidata.auth, headers=_HEADERS) if req.status_code == 414: - req = requests.post(Epidata.BASE_URL, params, auth=Epidata.auth, headers=_HEADERS) + req = requests.post(request_url, params, auth=Epidata.auth, headers=_HEADERS) # handle 401 and 429 req.raise_for_status() return req @staticmethod - def _request(params): + def _request(endpoint, params={}): """Request and parse epidata. We default to GET since it has better caching and logging @@ -86,7 +87,7 @@ def _request(params): long and returns a 414. """ try: - result = Epidata._request_with_retry(params) + result = Epidata._request_with_retry(endpoint, params) except Exception as e: return {"result": 0, "message": "error: " + str(e)} if params is not None and "format" in params and params["format"] == "csv": @@ -125,7 +126,6 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): raise EpidataBadRequestException(ISSUES_LAG_EXCLUSIVE) # Set up request params = { - "endpoint": "fluview", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -136,18 +136,13 @@ def fluview(regions, epiweeks, issues=None, lag=None, auth=None): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params) + return Epidata._request("fluview", params) # Fetch FluView metadata @staticmethod def fluview_meta(): """Fetch FluView metadata.""" - # Set up request - params = { - "endpoint": "fluview_meta", - } - # Make the API call - return Epidata._request(params) + return Epidata._request("fluview_meta") # Fetch FluView clinical data @staticmethod @@ -160,7 +155,6 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "fluview_clinical", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -169,7 +163,7 @@ def fluview_clinical(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request("fluview_clinical", params) # Fetch FluSurv data @staticmethod @@ -182,7 +176,6 @@ def flusurv(locations, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "flusurv", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } @@ -191,7 +184,7 @@ def flusurv(locations, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request("flusurv", params) # Fetch PAHO Dengue data @staticmethod @@ -204,7 +197,6 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "paho_dengue", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -213,7 +205,7 @@ def paho_dengue(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request("paho_dengue", params) # Fetch ECDC ILI data @staticmethod @@ -226,7 +218,6 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "ecdc_ili", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -235,7 +226,7 @@ def ecdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request("ecdc_ili", params) # Fetch KCDC ILI data @staticmethod @@ -248,7 +239,6 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "kcdc_ili", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -257,7 +247,7 @@ def kcdc_ili(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request("kcdc_ili", params) # Fetch Google Flu Trends data @staticmethod @@ -268,12 +258,11 @@ def gft(locations, epiweeks): raise EpidataBadRequestException(LOCATIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "gft", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request("gft", params) # Fetch Google Health Trends data @staticmethod @@ -286,14 +275,13 @@ def ght(auth, locations, epiweeks, query): ) # Set up request params = { - "endpoint": "ght", "auth": auth, "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), "query": query, } # Make the API call - return Epidata._request(params) + return Epidata._request("ght", params) # Fetch HealthTweets data @staticmethod @@ -306,7 +294,6 @@ def twitter(auth, locations, dates=None, epiweeks=None): raise EpidataBadRequestException("exactly one of `dates` and `epiweeks` is required") # Set up request params = { - "endpoint": "twitter", "auth": auth, "locations": Epidata._list(locations), } @@ -315,7 +302,7 @@ def twitter(auth, locations, dates=None, epiweeks=None): if epiweeks is not None: params["epiweeks"] = Epidata._list(epiweeks) # Make the API call - return Epidata._request(params) + return Epidata._request("twitter", params) # Fetch Wikipedia access data @staticmethod @@ -328,7 +315,6 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): raise EpidataBadRequestException("exactly one of `dates` and `epiweeks` is required") # Set up request params = { - "endpoint": "wiki", "articles": Epidata._list(articles), "language": language, } @@ -339,7 +325,7 @@ def wiki(articles, dates=None, epiweeks=None, hours=None, language="en"): if hours is not None: params["hours"] = Epidata._list(hours) # Make the API call - return Epidata._request(params) + return Epidata._request("wiki", params) # Fetch CDC page hits @staticmethod @@ -350,13 +336,12 @@ def cdc(auth, epiweeks, locations): raise EpidataBadRequestException("`auth`, `epiweeks`, and `locations` are all required") # Set up request params = { - "endpoint": "cdc", "auth": auth, "epiweeks": Epidata._list(epiweeks), "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params) + return Epidata._request("cdc", params) # Fetch Quidel data @staticmethod @@ -367,13 +352,12 @@ def quidel(auth, epiweeks, locations): raise EpidataBadRequestException("`auth`, `epiweeks`, and `locations` are all required") # Set up request params = { - "endpoint": "quidel", "auth": auth, "epiweeks": Epidata._list(epiweeks), "locations": Epidata._list(locations), } # Make the API call - return Epidata._request(params) + return Epidata._request("quidel", params) # Fetch NoroSTAT data (point data, no min/max) @staticmethod @@ -384,13 +368,12 @@ def norostat(auth, location, epiweeks): raise EpidataBadRequestException("`auth`, `location`, and `epiweeks` are all required") # Set up request params = { - "endpoint": "norostat", "auth": auth, "location": location, "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request("norostat", params) # Fetch NoroSTAT metadata @staticmethod @@ -401,11 +384,10 @@ def meta_norostat(auth): raise EpidataBadRequestException("`auth` is required") # Set up request params = { - "endpoint": "meta_norostat", "auth": auth, } # Make the API call - return Epidata._request(params) + return Epidata._request("meta_norostat", params) # Fetch NIDSS flu data @staticmethod @@ -418,7 +400,6 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nidss_flu", "regions": Epidata._list(regions), "epiweeks": Epidata._list(epiweeks), } @@ -427,7 +408,7 @@ def nidss_flu(regions, epiweeks, issues=None, lag=None): if lag is not None: params["lag"] = lag # Make the API call - return Epidata._request(params) + return Epidata._request("nidss_flu", params) # Fetch NIDSS dengue data @staticmethod @@ -438,12 +419,11 @@ def nidss_dengue(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nidss_dengue", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request("nidss_dengue", params) # Fetch Delphi's forecast @staticmethod @@ -454,12 +434,11 @@ def delphi(system, epiweek): raise EpidataBadRequestException("`system` and `epiweek` are both required") # Set up request params = { - "endpoint": "delphi", "system": system, "epiweek": epiweek, } # Make the API call - return Epidata._request(params) + return Epidata._request("delphi", params) # Fetch Delphi's digital surveillance sensors @staticmethod @@ -472,7 +451,6 @@ def sensors(auth, names, locations, epiweeks): ) # Set up request params = { - "endpoint": "sensors", "names": Epidata._list(names), "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), @@ -480,7 +458,7 @@ def sensors(auth, names, locations, epiweeks): if auth is not None: params["auth"] = auth # Make the API call - return Epidata._request(params) + return Epidata._request("sensors", params) # Fetch Delphi's dengue digital surveillance sensors @staticmethod @@ -493,14 +471,13 @@ def dengue_sensors(auth, names, locations, epiweeks): ) # Set up request params = { - "endpoint": "dengue_sensors", "auth": auth, "names": Epidata._list(names), "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request("dengue_sensors", params) # Fetch Delphi's wILI nowcast @staticmethod @@ -511,12 +488,11 @@ def nowcast(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "nowcast", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request("nowcast", params) # Fetch Delphi's dengue nowcast @staticmethod @@ -527,18 +503,17 @@ def dengue_nowcast(locations, epiweeks): raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "dengue_nowcast", "locations": Epidata._list(locations), "epiweeks": Epidata._list(epiweeks), } # Make the API call - return Epidata._request(params) + return Epidata._request("dengue_nowcast", params) # Fetch API metadata @staticmethod def meta(): """Fetch API metadata.""" - return Epidata._request({"endpoint": "meta"}) + return Epidata._request("meta") # Fetch Delphi's COVID-19 Surveillance Streams @staticmethod @@ -568,7 +543,6 @@ def covidcast( raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "endpoint": "covidcast", "data_source": data_source, "signals": Epidata._list(signals), "time_type": time_type, @@ -594,13 +568,13 @@ def covidcast( params["fields"] = kwargs["fields"] # Make the API call - return Epidata._request(params) + return Epidata._request("covidcast", params) # Fetch Delphi's COVID-19 Surveillance Streams metadata @staticmethod def covidcast_meta(): """Fetch Delphi's COVID-19 Surveillance Streams metadata""" - return Epidata._request({"endpoint": "covidcast_meta"}) + return Epidata._request("covidcast_meta") # Fetch COVID hospitalization data @staticmethod @@ -611,7 +585,6 @@ def covid_hosp(states, dates, issues=None, as_of=None): raise EpidataBadRequestException("`states` and `dates` are both required") # Set up request params = { - "endpoint": "covid_hosp", "states": Epidata._list(states), "dates": Epidata._list(dates), } @@ -620,7 +593,7 @@ def covid_hosp(states, dates, issues=None, as_of=None): if as_of is not None: params["as_of"] = as_of # Make the API call - return Epidata._request(params) + return Epidata._request("covid_hosp_state_timeseries", params) # Fetch COVID hospitalization data for specific facilities @staticmethod @@ -633,21 +606,20 @@ def covid_hosp_facility(hospital_pks, collection_weeks, publication_dates=None): ) # Set up request params = { - "source": "covid_hosp_facility", "hospital_pks": Epidata._list(hospital_pks), "collection_weeks": Epidata._list(collection_weeks), } if publication_dates is not None: params["publication_dates"] = Epidata._list(publication_dates) # Make the API call - return Epidata._request(params) + return Epidata._request("covid_hosp_facility", params) # Lookup COVID hospitalization facility identifiers @staticmethod def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_code=None): """Lookup COVID hospitalization facility identifiers.""" # Set up request - params = {"source": "covid_hosp_facility_lookup"} + params = {} if state is not None: params["state"] = state elif ccn is not None: @@ -663,7 +635,7 @@ def covid_hosp_facility_lookup(state=None, ccn=None, city=None, zip=None, fips_c "one of `state`, `ccn`, `city`, `zip`, or `fips_code` is required" ) # Make the API call - return Epidata._request(params) + return Epidata._request("covid_hosp_facility_lookup", params) # Fetch Delphi's COVID-19 Nowcast sensors @staticmethod @@ -693,7 +665,6 @@ def covidcast_nowcast( raise EpidataBadRequestException(REGIONS_EPIWEEKS_REQUIRED) # Set up request params = { - "source": "covidcast_nowcast", "data_source": data_source, "signals": Epidata._list(signals), "sensor_names": Epidata._list(sensor_names), @@ -717,15 +688,20 @@ def covidcast_nowcast( params["format"] = kwargs["format"] # Make the API call - return Epidata._request(params) + return Epidata._request("covidcast_nowcast", params) @staticmethod def async_epidata(param_list, batch_size=50): - """Make asynchronous Epidata calls for a list of parameters.""" + """[DEPRECATED] Make asynchronous Epidata calls for a list of parameters.""" + + import warnings + warnings.filterwarnings("once", category=DeprecationWarning, module="delphi_epidata") + warnings.warn("Method `Epidata.async_epidata()` is deprecated and will be removed in a future version.", + category=DeprecationWarning) async def async_get(params, session): """Helper function to make Epidata GET requests.""" - async with session.get(Epidata.BASE_URL, params=params) as response: + async with session.get(f"{Epidata.BASE_URL}/api.php", params=params) as response: response.raise_for_status() return await response.json(), params diff --git a/src/client/packaging/pypi/CHANGELOG.md b/src/client/packaging/pypi/CHANGELOG.md index 414909fe2..ee95128f7 100644 --- a/src/client/packaging/pypi/CHANGELOG.md +++ b/src/client/packaging/pypi/CHANGELOG.md @@ -3,6 +3,18 @@ All notable future changes to the `delphi_epidata` python client will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/). +## [4.1.11] - 2023-10-12 + +### Includes +- https://github.com/cmu-delphi/delphi-epidata/pull/1288 + +### Changed +- Internally uses newer endpoint-specific URLs instead of an older compatibility alias URL. +- Upper limit on the number of rows returned per request has been increased to 1M (was 3650). +- Method `Epidata.async_epidata()` is now deprecated and will be removed in a future version. +- The dict object returned from data request methods will now always have an entry for the "`epidata`" key, potentially with an empty list as its value. previously, if there were no results for the request, the "`epidata`" entry was not present. +- Results from the `Epidata.covidcast()` method (or potentially the `covidcast` endpoint via the `async_epidata()` method) will now include the keys "`source`", "`geo_type`", and "`time_type`" (and their associated values). + ## [4.1.10] - 2023-09-28 ### Added diff --git a/src/common/integration_test_base_class.py b/src/common/integration_test_base_class.py index 387c0f617..47c9f68e4 100644 --- a/src/common/integration_test_base_class.py +++ b/src/common/integration_test_base_class.py @@ -19,7 +19,7 @@ def __init__(self, methodName: str = "runTest") -> None: self.create_tables_list = [] self.role_name = None self.epidata_client = Epidata - self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata/api.php" + self.epidata_client.BASE_URL = "http://delphi_web_epidata/epidata" self.epidata_client.auth = ("epidata", "key") def create_key_with_role(self, cur, role_name: str): diff --git a/src/maintenance/signal_dash_data_generator.py b/src/maintenance/signal_dash_data_generator.py index 6eea06579..b7f1048f5 100644 --- a/src/maintenance/signal_dash_data_generator.py +++ b/src/maintenance/signal_dash_data_generator.py @@ -19,7 +19,7 @@ LOOKBACK_DAYS_FOR_COVERAGE = 56 -BASE_COVIDCAST = covidcast.covidcast.Epidata.BASE_URL[:-len("api.php")] + "covidcast" +BASE_COVIDCAST = covidcast.covidcast.Epidata.BASE_URL + "/covidcast" COVERAGE_URL = f"{BASE_COVIDCAST}/coverage?format=csv&signal={{source}}:{{signal}}&days={LOOKBACK_DAYS_FOR_COVERAGE}" @dataclass diff --git a/tests/server/dev_test_granular_sensor_authentication.py b/tests/server/dev_test_granular_sensor_authentication.py index bc742392a..2309c626e 100644 --- a/tests/server/dev_test_granular_sensor_authentication.py +++ b/tests/server/dev_test_granular_sensor_authentication.py @@ -1,4 +1,4 @@ -"""Unit tests for granular sensor authentication in api.php.""" +"""Unit tests for granular sensor authentication.""" # standard library import unittest diff --git a/tests/server/endpoints/test_nidss_flu.py b/tests/server/endpoints/test_nidss_flu.py index bc0723bf3..7ec50a6d2 100644 --- a/tests/server/endpoints/test_nidss_flu.py +++ b/tests/server/endpoints/test_nidss_flu.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest import base64 diff --git a/tests/server/test_exceptions.py b/tests/server/test_exceptions.py index 94cdc34f1..b07fe3ac7 100644 --- a/tests/server/test_exceptions.py +++ b/tests/server/test_exceptions.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest diff --git a/tests/server/test_query.py b/tests/server/test_query.py index ec07d3e8b..95b21a55a 100644 --- a/tests/server/test_query.py +++ b/tests/server/test_query.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest import base64 diff --git a/tests/server/test_security.py b/tests/server/test_security.py index e209d9342..eb8e32700 100644 --- a/tests/server/test_security.py +++ b/tests/server/test_security.py @@ -1,4 +1,4 @@ -"""Unit tests for granular sensor authentication in api.php.""" +"""Unit tests for granular sensor authentication.""" # standard library import unittest diff --git a/tests/server/test_validate.py b/tests/server/test_validate.py index 27ce28672..f06e9e997 100644 --- a/tests/server/test_validate.py +++ b/tests/server/test_validate.py @@ -1,5 +1,3 @@ -"""Unit tests for granular sensor authentication in api.php.""" - # standard library import unittest