From 7066b48ce84cdd785a4732448e71e2e2d64e12f0 Mon Sep 17 00:00:00 2001 From: dpb-bah Date: Thu, 12 Dec 2024 15:41:48 -0500 Subject: [PATCH 1/6] Removing max_login and all max.gov affiliated code --- APISubmissionGuide.md | 18 -- dataactbroker/handlers/account_handler.py | 147 +-------- dataactbroker/routes/login_routes.py | 8 - dataactcore/config_example.yml | 11 - doc/INSTALL.md | 2 +- doc/README.md | 1 - doc/api_docs/login/login.md | 2 +- doc/api_docs/login/max_login.md | 64 ---- .../dataactbroker/test_account_handler.py | 306 +----------------- tests/unit/dataactbroker/test_login_routes.py | 40 --- 10 files changed, 11 insertions(+), 588 deletions(-) delete mode 100644 doc/api_docs/login/max_login.md diff --git a/APISubmissionGuide.md b/APISubmissionGuide.md index eedc908e0..926e9f71c 100644 --- a/APISubmissionGuide.md +++ b/APISubmissionGuide.md @@ -6,24 +6,6 @@ While the Submission API has been designed to be as easy to understand as possib ## Login Process -### Login to Max -- Step 1: Authenticate with MAX directly to obtain the `ticket` value for Step 2 - - Please refer to documentation provided by MAX.gov [here](./Using_Digital_Certificates_for_MAX_Authentication.pdf). - - Information about requesting Data Broker permissions within MAX can be found [here](https://community.max.gov/x/fJwuRQ). - - While we do not control MAX's login process, for simplicity purposes, here is a sample CURL request to the MAX login endpoint: - ``` - curl -L -j -D - -b none - --cert max.crt - --key max.key - https://serviceauth.max.gov/cas-cert/login?service=https://broker-api.usaspending.gov - ``` - -- **NOTE**: Do **NOT** end the above service parameter url with a "/" -- You would locate the `ticket` value in the `Location` header in the first header block returned by this request, i.e., - `Location=https://broker-api.usaspending.gov?ticket=ST-123456-abcdefghijklmnopqrst-login.max.gov` -- Step 2: call `/v1/max_login/` (POST) current broker login endpoint for logging into broker using MAX login. For details on its use, click [here](./doc/api_docs/login/max_login.md) - - Be sure to use the provided ticket within 30 seconds to ensure it does not expire. - ### Login to the CAIA - Step 1: Get access to Treasury Mulesoft Exchange to view the Data Broker Experience API. diff --git a/dataactbroker/handlers/account_handler.py b/dataactbroker/handlers/account_handler.py index 9ea120fc9..e3712d49b 100644 --- a/dataactbroker/handlers/account_handler.py +++ b/dataactbroker/handlers/account_handler.py @@ -3,7 +3,6 @@ from operator import attrgetter import re import requests -import xmltodict from flask import g @@ -153,76 +152,6 @@ def proxy_login(self, session): # Return 500 return JsonResponse.error(e, StatusCode.INTERNAL_ERROR) - def max_login(self, session): - """ Logs a user in if their password matches using MAX - - Args: - session: Session object from flask - - Returns: - A JsonResponse containing the user information or details on which error occurred, such as whether a - type was wrong, something wasn't implemented, invalid keys were provided, login was denied, or a - different, unexpected error occurred. - """ - try: - safe_dictionary = RequestDictionary(self.request) - - ticket = safe_dictionary.get_value("ticket") - service = safe_dictionary.get_value('service') - - # Call MAX's serviceValidate endpoint and retrieve the response - max_dict = get_max_dict(ticket, service) - - if 'cas:authenticationSuccess' not in max_dict['cas:serviceResponse']: - raise ValueError("The Max CAS endpoint was unable to locate your session " - "using the ticket/service combination you provided.") - cas_attrs = max_dict['cas:serviceResponse']['cas:authenticationSuccess']['cas:attributes'] - - # Grab MAX ID to see if a service account is being logged in - max_id_components = cas_attrs['cas:MAX-ID'].split('_') - service_account_flag = (len(max_id_components) > 1 and max_id_components[0].lower() == 's') - - # Grab the email and list of groups from MAX's response - email = cas_attrs['cas:Email-Address'] - - try: - sess = GlobalDB.db().session - user = sess.query(User).filter(func.lower(User.email) == func.lower(email)).one_or_none() - - # If the user does not exist, create them since they are allowed to access the site because they got - # past the above group membership checks - if user is None: - user = User() - user.email = email - - first_name = cas_attrs['cas:First-Name'] - middle_name = cas_attrs['cas:Middle-Name'] - last_name = cas_attrs['cas:Last-Name'] - set_user_name(user, first_name, middle_name, last_name) - - set_max_perms(user, cas_attrs['cas:GroupList'], service_account_flag) - - sess.add(user) - sess.commit() - - except MultipleResultsFound: - raise ValueError("An error occurred during login.") - - return self.create_session_and_response(session, user) - - # Catch any specifically raised errors or any other errors that may have happened and return them cleanly. - # We add the error parameter here because this endpoint needs to provide better feedback, and to avoid changing - # the default behavior of the JsonResponse class globally. - except (TypeError, KeyError, NotImplementedError) as e: - # Return a 400 with appropriate message - return JsonResponse.error(e, StatusCode.CLIENT_ERROR, error=str(e)) - except ValueError as e: - # Return a 401 for login denied - return JsonResponse.error(e, StatusCode.LOGIN_REQUIRED, error=str(e)) - except Exception as e: - # Return 500 - return JsonResponse.error(e, StatusCode.INTERNAL_ERROR, error=str(e)) - def caia_login(self, session): """ Logs a user in if their CAIA validation succeeds @@ -376,8 +305,8 @@ def email_users(submission, system_email, template_type, user_ids): return JsonResponse.create(StatusCode.OK, {"message": "Emails successfully sent"}) -def perms_to_affiliations(perms, user_id, service_account_flag=False): - """ Convert a list of perms from MAX to a list of UserAffiliations. Filter out and log any malformed perms +def perms_to_affiliations(perms, user_id): + """ Convert a list of perms from CAIA to a list of UserAffiliations. Filter out and log any malformed perms Args: perms: list of permissions (as lists [code, perm]) for the user @@ -413,10 +342,7 @@ def perms_to_affiliations(perms, user_id, service_account_flag=False): perm_level = perm_level.lower() - if service_account_flag: - # Replace MAX Service Account permissions with Broker "write" and "editfabs" permissions - perm_level = 'we' - elif perm_level not in 'rwsef': + if perm_level not in 'rwsef': logger.warning(log_data) continue @@ -466,7 +392,7 @@ def best_affiliation(affiliations): def set_user_name(user, first_name, middle_name, last_name): - """ Update the name for the user based on the MAX attributes. + """ Update the name for the user based on the CAIA attributes. Args: user: the User object @@ -479,54 +405,6 @@ def set_user_name(user, first_name, middle_name, last_name): user.name = first_name + " " + middle_name[0] + ". " + last_name -def set_max_perms(user, max_group_list, service_account_flag=False): - """ Convert the user group lists present on MAX into a list of UserAffiliations and/or website_admin status. - - Permissions are encoded as a comma-separated list of: - {parent-group}-CGAC_{cgac-code}-PERM_{one-of-R-W-S-E-F} - {parent-group}-CGAC_{cgac-code}-FREC_{frec_code}-PERM_{one-of-R-W-S-E-F} - or - {parent-group}-CGAC_SYS to indicate website_admin - - Args: - user: the User object - max_group_list: list of all MAX groups the user has - service_account_flag: flag to indicate a service account - """ - prefix = CONFIG_BROKER['parent_group'] + '-CGAC_' - - # Each group name that we care about begins with the prefix, but once we have that list, we don't need the - # prefix anymore, so trim it off. - if max_group_list is not None: - group_names = [group_name[len(prefix):] - for group_name in max_group_list.split(',') - if group_name.startswith(prefix)] - elif service_account_flag: - raise ValueError("There are no Data Broker permissions assigned to this Service Account. You may request " - "permissions at https://community.max.gov/x/fJwuRQ") - else: - group_names = [] - - if 'SYS' in group_names: - user.website_admin = True - user.affiliations = [] - else: - user.website_admin = False - perms = [] - for group_name in group_names: - # Always starts with the 3-digit CGAC - code = group_name[:3] - if 'FREC' in group_name: - # If FREC, then its the [3-digit CGAC]-FREC_[4-digit FREC code] - code = group_name[9:13] - # Permission level is always the last character - perm = group_name[-1] - perms.append((code, perm)) - - if perms: - user.affiliations = best_affiliation(perms_to_affiliations(perms, user.user_id, service_account_flag)) - - def set_caia_perms(user, roles): """ Convert the user group list present on CAIA into a list of UserAffiliations and/or website_admin status. @@ -571,21 +449,6 @@ def json_for_user(user, session_id): } -def get_max_dict(ticket, service): - """ Get the result from MAX's serviceValidate functionality - - Args: - ticket: the ticket to send to MAX - service: the service to send to MAX - - Returns: - A dictionary of the response from MAX - """ - url = CONFIG_BROKER['cas_service_url'].format(ticket, service) - max_xml = requests.get(url).content - return xmltodict.parse(max_xml) - - def get_caia_tokens(code, redirect_uri): """ Verify the authorization code to get the logged in user's various tokens @@ -637,7 +500,7 @@ def refresh_tokens(refresh_token, redirect_uri): def get_caia_user_dict(accces_token): - """ Get the result from MAX's serviceValidate functionality + """ Get the result from CAIA's serviceValidate functionality Args: accces_token: the access token of the logged in user diff --git a/dataactbroker/routes/login_routes.py b/dataactbroker/routes/login_routes.py index c3768f4d3..70af07608 100644 --- a/dataactbroker/routes/login_routes.py +++ b/dataactbroker/routes/login_routes.py @@ -1,5 +1,4 @@ from flask import g, request, session -from flask_deprecate import deprecate_route from dataactcore.utils.jsonResponse import JsonResponse from dataactcore.utils.statusCode import StatusCode @@ -18,13 +17,6 @@ def proxy_login(): account_manager = AccountHandler(request) return account_manager.proxy_login(session) - @app.route("/v1/max_login/", methods=["POST"]) - @deprecate_route("MAX login will no longer be supported on January 1st, 2025." - " Instead, CAIA login is supported now (via /v1/caia_login) and will be the only option then.") - def max_login(): - account_manager = AccountHandler(request) - return account_manager.max_login(session) - @app.route("/v1/caia_login/", methods=["POST"]) def caia_login(): account_manager = AccountHandler(request) diff --git a/dataactcore/config_example.yml b/dataactcore/config_example.yml index 2a3ed1d41..ebbcb0262 100644 --- a/dataactcore/config_example.yml +++ b/dataactcore/config_example.yml @@ -94,17 +94,6 @@ broker: usas_public_reference_url: new-url.com usas_public_submissions_url: new-url.com - # link to help for failed MAX login - max_help_url: https://max_help_url.gov - - # MAX login - cas_service_url: https://cas.service.url - parent_group: sample - - max: - federal_hierarchy_api_url: https://federal.hierarchy.api.uri.gov/?api_key={} - federal_hierarchy_api_key: example - # Session timeout, in seconds session_timeout: 1800 diff --git a/doc/INSTALL.md b/doc/INSTALL.md index 90ecf5775..4e2f131e1 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -141,7 +141,7 @@ _:bulb: TIP: Many of the commands below use the format `docker exec -it dataact- ```bash $ docker exec -it dataact-broker-backend python dataactcore/scripts/initialize.py -a ``` -This creates a local admin user that you can use to log in. The Broker utilizes MAX.gov for login when using a remote server, but we cannot recieve their response locally so we use a username and password for local development login. The credentials for the user created are the values you have configured in the `db.admin_email` and `db.admin_password` config params in `config.yml` or overridden in `local_config.yml`. +This creates a local admin user that you can use to log in. The Broker utilizes CAIA for login when using a remote server, but we cannot recieve their response locally so we use a username and password for local development login. The credentials for the user created are the values you have configured in the `db.admin_email` and `db.admin_password` config params in `config.yml` or overridden in `local_config.yml`. Now try to browse to http://localhost:3002, and login with the configured credentials (`valid.email@domain.com` / `password`). You should get past the login screen to the home screen. diff --git a/doc/README.md b/doc/README.md index 8a4cf60c3..4f98b908d 100644 --- a/doc/README.md +++ b/doc/README.md @@ -135,7 +135,6 @@ Login routes are used to log a user in or out or check if the current session is - [login](./api_docs/login/login.md) - [logout](./api_docs/login/logout.md) -- [max\_login](./api_docs/login/max_login.md) (Deprecated, use `caia_login` instead) - [caia\_login](./api_docs/login/caia_login.md) - [session](./api_docs/login/session.md) diff --git a/doc/api_docs/login/login.md b/doc/api_docs/login/login.md index df38907f0..bacec0edf 100644 --- a/doc/api_docs/login/login.md +++ b/doc/api_docs/login/login.md @@ -2,7 +2,7 @@ **THIS ENDPOINT IS FOR LOCAL DEVELOPMENT ONLY AND CANNOT BE USED TO AUTHENTICATE INTO BROKER IN PRODUCTION** -This route checks the username and password against a credentials file. It is used solely as a workaround for developing on a local instance of the broker to bypass MAX.gov login. Accepts input as json or form-urlencoded, with keys "username" and "password". See `active_user` docs for details. +This route checks the username and password against a credentials file. It is used solely as a workaround for developing on a local instance of the broker to bypass CAIA login. Accepts input as json or form-urlencoded, with keys "username" and "password". See `active_user` docs for details. ## Body (JSON) diff --git a/doc/api_docs/login/max_login.md b/doc/api_docs/login/max_login.md deleted file mode 100644 index c8f8bba92..000000000 --- a/doc/api_docs/login/max_login.md +++ /dev/null @@ -1,64 +0,0 @@ -# POST "/v1/max\_login/" -This route sends a request to the backend with the ticket obtained from the MAX login endpoint in order to verify authentication and access to the Data Broker. If called by a service account, a certificate is required for authentication. **IMPORTANT**: The ticket has a 30 second expiration window so it must be used immediately after being received in order for it to be valid. - -### **NOTE**: This endpoint is deprecated, please use `caia_login` instead. - -## Body (JSON) - -``` -{ - "ticket": "ST-123456-abcdefghijklmnopqrst-login.max.gov", - "service": "https://broker-api.usaspending.gov" -} -``` - -## Body Description - -- `ticket`: (required, string) ticket string received from MAX from initial login request (pending validation) -- `service`: (required, string) URL encoded string that is the source of the initial login request. This may vary from the example based on the environment you are in. - -## Response (JSON) -More data will be added to the response depending on what we get back from MAX upon validating the ticket. - -``` -{ - "user_id": 42, - "name": "John", - "title": "Developer", - "skip_guide": false, - "website_admin": false, - "affiliations": [ - { - "agency_name": "Department of Labor (DOL)", - "permission": "writer" - } - ], - "session_id": "ABC123", - "message": "Login successful" -} -``` - -##### Response Description: -- `user_id`: (integer) database identifier of the logged in user, part of response only if login is successful -- `name`: (string) user's name, part of response only if login is successful -- `title`: (string) title of user, part of response only if login is successful -- `skip_guide`: (boolean) whether to show the DABS submission guide on the frontend or not (true = don't show), part of response only if login is successful -- `website_admin`: (boolean) describes a super-user status, part of response only if login is successful -- `affiliations`: ([dict]) dictionaries containing information about the user's agency affiliations (only used by the frontend if `website_admin` is false). Dictionaries contain the following information: - - `agency_name`: (string) the name of the agency the user is affiliated with - - `permission`: (string) the level of permissions the user has. For more information about what the levels mean, see the [permissions.md](../../permissions.md) file. Possible levels: - - `reader` - - `writer` - - `submitter` - - `editfabs` - - `fabs` -- `message`: (string) login error response "You have failed to login successfully with MAX", otherwise says "Login successful" -- `errorType`: (string) type of error, part of response only if login is unsuccessful -- `session_id`: (string) a hash the application uses to verify that user sending the request is logged in, part of response only if login is successful - - -## Errors -Possible HTTP Status Codes: - -- 400: Missing parameters -- 401: Login denied \ No newline at end of file diff --git a/tests/unit/dataactbroker/test_account_handler.py b/tests/unit/dataactbroker/test_account_handler.py index 47f70bae7..d4209e143 100644 --- a/tests/unit/dataactbroker/test_account_handler.py +++ b/tests/unit/dataactbroker/test_account_handler.py @@ -7,29 +7,10 @@ from dataactcore.models.userModel import UserAffiliation from dataactcore.utils.jsonResponse import JsonResponse from dataactcore.utils.statusCode import StatusCode -from tests.unit.mock_helpers import mock_response from tests.unit.dataactcore.factories.domain import CGACFactory, FRECFactory from tests.unit.dataactcore.factories.user import UserFactory -def make_max_dict(group_str): - """We need to create mock MAX data at multiple points in these tests""" - return { - 'cas:serviceResponse': { - 'cas:authenticationSuccess': { - 'cas:attributes': { - 'cas:Email-Address': 'test-user@email.com', - 'cas:GroupList': group_str, - 'cas:First-Name': 'test', - 'cas:Middle-Name': '', - 'cas:Last-Name': 'user', - 'cas:MAX-ID': 'id' - } - } - } - } - - def make_caia_token_dict(unique_id): """We need to create mock CAIA data at multiple points in these tests""" return { @@ -55,40 +36,6 @@ def make_caia_user_dict(role_str): } -@pytest.mark.usefixtures("user_constants") -def test_max_login_success_normal_login(monkeypatch): - ah = account_handler.AccountHandler(Mock()) - - mock_dict = Mock() - mock_dict.return_value.exists.return_value = False - mock_dict.return_value.safeDictionary.side_effect = {'ticket': '', 'service': ''} - monkeypatch.setattr(account_handler, 'RequestDictionary', mock_dict) - - max_dict = {'cas:serviceResponse': {}} - monkeypatch.setattr(account_handler, 'get_max_dict', Mock(return_value=max_dict)) - config = {'parent_group': 'parent-group'} - monkeypatch.setattr(account_handler, 'CONFIG_BROKER', config) - max_dict = make_max_dict('parent-group,parent-group-CGAC_SYS') - monkeypatch.setattr(account_handler, 'get_max_dict', Mock(return_value=max_dict)) - - # If it gets to this point, that means the user was in all the right groups aka successful login - monkeypatch.setattr(ah, 'create_session_and_response', - Mock(return_value=JsonResponse.create(StatusCode.OK, {"message": "Login successful"}))) - json_response = ah.max_login(Mock()) - - assert "Login successful" == json.loads(json_response.get_data().decode("utf-8"))['message'] - - max_dict = make_max_dict('') - monkeypatch.setattr(account_handler, 'get_max_dict', Mock(return_value=max_dict)) - - # If it gets to this point, that means the user was in all the right groups aka successful login - monkeypatch.setattr(ah, 'create_session_and_response', - Mock(return_value=JsonResponse.create(StatusCode.OK, {"message": "Login successful"}))) - json_response = ah.max_login(Mock()) - - assert "Login successful" == json.loads(json_response.get_data().decode("utf-8"))['message'] - - @pytest.mark.usefixtures("user_constants") def test_caia_login_success_normal_login(monkeypatch): ah = account_handler.AccountHandler(Mock()) @@ -127,26 +74,6 @@ def test_caia_login_success_normal_login(monkeypatch): assert "Login successful" == json.loads(json_response.get_data().decode("utf-8"))['message'] -def test_max_login_failure_normal_login(monkeypatch): - ah = account_handler.AccountHandler(Mock()) - config = {'parent_group': 'parent-group'} - monkeypatch.setattr(account_handler, 'CONFIG_BROKER', config) - - mock_dict = Mock() - mock_dict.return_value.exists.return_value = False - mock_dict.return_value.safeDictionary.side_effect = {'ticket': '', 'service': ''} - monkeypatch.setattr(account_handler, 'RequestDictionary', mock_dict) - - max_dict = {'cas:serviceResponse': {}} - monkeypatch.setattr(account_handler, 'get_max_dict', Mock(return_value=max_dict)) - json_response = ah.max_login(Mock()) - error_message = ("The Max CAS endpoint was unable to locate your session using " - "the ticket/service combination you provided.") - - # Did not get a successful response from MAX - assert error_message == json.loads(json_response.get_data().decode("utf-8"))['message'] - - def test_caia_login_failure_normal_login(monkeypatch): ah = account_handler.AccountHandler(Mock()) @@ -165,74 +92,6 @@ def test_caia_login_failure_normal_login(monkeypatch): assert error_message == json.loads(json_response.get_data().decode("utf-8"))['message'] -@pytest.mark.usefixtures("user_constants") -def test_max_login_success_cert_login(monkeypatch): - ah = account_handler.AccountHandler(Mock()) - - mock_dict = Mock() - mock_dict.return_value.exists.return_value = True - mock_dict.return_value.safeDictionary.side_effect = {'cert': ''} - monkeypatch.setattr(account_handler, 'RequestDictionary', mock_dict) - - max_dict = {'cas:serviceResponse': {}} - monkeypatch.setattr(account_handler, 'get_max_dict', Mock(return_value=max_dict)) - - config = {'parent_group': 'parent-group', 'full_url': 'full-url', 'max_cert_url': 'max-cert-url'} - monkeypatch.setattr(account_handler, 'CONFIG_BROKER', config) - - max_dict = make_max_dict('parent-group,parent-group-CGAC_SYS') - max_dict['cas:serviceResponse']['cas:authenticationSuccess']['cas:attributes']['cas:MAX-ID'] = 'S_id' - monkeypatch.setattr(account_handler, 'get_max_dict', Mock(return_value=max_dict)) - - mock_resp = Mock() - mock_resp.return_value = mock_response(url='ticket=12345') - monkeypatch.setattr('requests.get', mock_resp) - - # If it gets to this point, that means the user was in all the right groups aka successful login - monkeypatch.setattr(ah, 'create_session_and_response', - Mock(return_value=JsonResponse.create(StatusCode.OK, {"message": "Login successful"}))) - json_response = ah.max_login(Mock()) - - assert "Login successful" == json.loads(json_response.get_data().decode("utf-8"))['message'] - - max_dict = make_max_dict('') - monkeypatch.setattr(account_handler, 'get_max_dict', Mock(return_value=max_dict)) - - # If it gets to this point, that means the user was in all the right groups aka successful login - monkeypatch.setattr(ah, 'create_session_and_response', - Mock(return_value=JsonResponse.create(StatusCode.OK, {"message": "Login successful"}))) - json_response = ah.max_login(Mock()) - - assert "Login successful" == json.loads(json_response.get_data().decode("utf-8"))['message'] - - -def test_max_login_failure_cert_login(monkeypatch): - ah = account_handler.AccountHandler(Mock()) - config = {'parent_group': 'parent-group'} - monkeypatch.setattr(account_handler, 'CONFIG_BROKER', config) - - mock_dict = Mock() - mock_dict.return_value.exists.return_value = True - mock_dict.return_value.safeDictionary.side_effect = {'cert': ''} - monkeypatch.setattr(account_handler, 'RequestDictionary', mock_dict) - - mock_resp = Mock() - mock_resp.return_value = mock_response(url='ticket=12345') - monkeypatch.setattr('requests.get', mock_resp) - - config = {'full_url': 'full-url', 'max_cert_url': 'max-cert-url'} - monkeypatch.setattr(account_handler, 'CONFIG_BROKER', config) - - max_dict = {'cas:serviceResponse': {}} - monkeypatch.setattr(account_handler, 'get_max_dict', Mock(return_value=max_dict)) - json_response = ah.max_login(Mock()) - error_message = ("The Max CAS endpoint was unable to locate your session using " - "the ticket/service combination you provided.") - - # Did not get a successful response from MAX - assert error_message == json.loads(json_response.get_data().decode("utf-8"))['message'] - - def test_set_user_name_updated(): """ Tests set_user_name() updates a user's name """ @@ -272,163 +131,6 @@ def test_set_user_name_no_middle_name(): assert user.name == 'Test User' -@pytest.mark.usefixtures("user_constants") -def test_set_max_perms(database, monkeypatch): - """Verify that we get the _highest_ permission within our CGAC""" - cgac_abc = CGACFactory(cgac_code='ABC') - cgac_def = CGACFactory(cgac_code='DEF') - frec_abc = FRECFactory(frec_code='ABCD', cgac=cgac_abc) - frec_abc2 = FRECFactory(frec_code='EFGH', cgac=cgac_abc) - frec_def = FRECFactory(frec_code='IJKL', cgac=cgac_def) - user = UserFactory() - database.session.add_all([cgac_abc, cgac_def, frec_abc, frec_abc2, frec_def, user]) - database.session.commit() - - monkeypatch.setitem(account_handler.CONFIG_BROKER, 'parent_group', 'prefix') - - # test creating permission from string - account_handler.set_max_perms(user, 'prefix-CGAC_ABC-PERM_W') - database.session.commit() # populate ids - assert len(user.affiliations) == 1 - affil = user.affiliations[0] - assert affil.cgac_id == cgac_abc.cgac_id - assert affil.permission_type_id == PERMISSION_TYPE_DICT['writer'] - - # test creating max CGAC permission from two strings - account_handler.set_max_perms(user, 'prefix-CGAC_ABC-PERM_R,prefix-CGAC_ABC-PERM_S') - database.session.commit() # populate ids - assert len(user.affiliations) == 1 - affil = user.affiliations[0] - assert affil.cgac_id == cgac_abc.cgac_id - assert affil.permission_type_id == PERMISSION_TYPE_DICT['submitter'] - - # test creating two CGAC permissions with two strings - account_handler.set_max_perms(user, 'prefix-CGAC_ABC-PERM_R,prefix-CGAC_DEF-PERM_S') - database.session.commit() - assert len(user.affiliations) == 2 - affiliations = list(sorted(user.affiliations, key=lambda a: a.cgac.cgac_code)) - abc_aff, def_aff = affiliations - assert abc_aff.cgac.cgac_code == 'ABC' - assert abc_aff.frec is None - assert abc_aff.permission_type_id == PERMISSION_TYPE_DICT['reader'] - assert def_aff.cgac.cgac_code == 'DEF' - assert def_aff.frec is None - assert def_aff.permission_type_id == PERMISSION_TYPE_DICT['submitter'] - - # test creating max FREC permission from two strings - account_handler.set_max_perms(user, 'prefix-CGAC_ABC-FREC_ABCD-PERM_R,prefix-CGAC_ABC-FREC_ABCD-PERM_S') - database.session.commit() - assert len(user.affiliations) == 2 - frec_affils = [affil for affil in user.affiliations if affil.frec is not None] - assert frec_affils[0].cgac is None - assert frec_affils[0].frec.frec_code == 'ABCD' - assert frec_affils[0].permission_type_id == PERMISSION_TYPE_DICT['submitter'] - cgac_affils = [affil for affil in user.affiliations if affil.cgac is not None] - assert cgac_affils[0].cgac.cgac_code == 'ABC' - assert cgac_affils[0].frec is None - assert cgac_affils[0].permission_type_id == PERMISSION_TYPE_DICT['reader'] - - # test creating two FREC permissions from two strings - account_handler.set_max_perms(user, 'prefix-CGAC_ABC-FREC_ABCD-PERM_R,prefix-CGAC_ABC-FREC_EFGH-PERM_S') - database.session.commit() - assert len(user.affiliations) == 3 - frec_affils = [affil for affil in user.affiliations if affil.frec is not None] - frec_affiliations = list(sorted(frec_affils, key=lambda a: a.frec.frec_code)) - abcd_aff, efgh_aff = frec_affiliations - assert abcd_aff.cgac is None - assert abcd_aff.frec.frec_code == 'ABCD' - assert abcd_aff.permission_type_id == PERMISSION_TYPE_DICT['reader'] - assert efgh_aff.cgac is None - assert efgh_aff.frec.frec_code == 'EFGH' - assert efgh_aff.permission_type_id == PERMISSION_TYPE_DICT['submitter'] - cgac_affils = [affil for affil in user.affiliations if affil.cgac is not None] - assert cgac_affils[0].cgac.cgac_code == 'ABC' - assert cgac_affils[0].frec is None - assert cgac_affils[0].permission_type_id == PERMISSION_TYPE_DICT['reader'] - - # test creating one CGAC and one FREC permission from two strings - account_handler.set_max_perms(user, 'prefix-CGAC_ABC-PERM_S,prefix-CGAC_DEF-FREC_IJKL-PERM_R') - database.session.commit() - assert len(user.affiliations) == 3 - frec_affils = [affil for affil in user.affiliations if affil.frec is not None] - assert frec_affils[0].cgac is None - assert frec_affils[0].frec.frec_code == 'IJKL' - assert frec_affils[0].permission_type_id == PERMISSION_TYPE_DICT['reader'] - cgac_affils = [affil for affil in user.affiliations if affil.cgac is not None] - cgac_affiliations = list(sorted(cgac_affils, key=lambda a: a.cgac.cgac_code)) - abc_aff, def_aff = cgac_affiliations - assert abc_aff.cgac.cgac_code == 'ABC' - assert abc_aff.frec is None - assert abc_aff.permission_type_id == PERMISSION_TYPE_DICT['submitter'] - assert def_aff.cgac.cgac_code == 'DEF' - assert def_aff.frec is None - assert def_aff.permission_type_id == PERMISSION_TYPE_DICT['reader'] - - # test creating max DABS and FABS CGAC permissions from three strings - account_handler.set_max_perms(user, 'prefix-CGAC_ABC-PERM_R,prefix-CGAC_ABC-PERM_S,prefix-CGAC_ABC-PERM_F') - database.session.commit() - assert len(user.affiliations) == 2 - affiliations = list(sorted(user.affiliations, key=lambda a: a.permission_type_id)) - dabs_aff, fabs_aff = affiliations - assert dabs_aff.cgac.cgac_code == 'ABC' - assert dabs_aff.frec is None - assert dabs_aff.permission_type_id == PERMISSION_TYPE_DICT['submitter'] - assert fabs_aff.cgac.cgac_code == 'ABC' - assert fabs_aff.frec is None - assert fabs_aff.permission_type_id == PERMISSION_SHORT_DICT['f'] - - # test creating max DABS and FABS FREC permissions from three strings - perms_string = 'prefix-CGAC_ABC-FREC_ABCD-PERM_R,prefix-CGAC_ABC-FREC_ABCD-PERM_S,prefix-CGAC_ABC-FREC_ABCD-PERM_F' - account_handler.set_max_perms(user, perms_string) - database.session.commit() - assert len(user.affiliations) == 3 - frec_affils = [affil for affil in user.affiliations if affil.frec is not None] - frec_affiliations = list(sorted(frec_affils, key=lambda a: a.permission_type_id)) - dabs_aff, fabs_aff = frec_affiliations - assert dabs_aff.cgac is None - assert dabs_aff.frec.frec_code == 'ABCD' - assert dabs_aff.permission_type_id == PERMISSION_TYPE_DICT['submitter'] - assert fabs_aff.cgac is None - assert fabs_aff.frec.frec_code == 'ABCD' - assert fabs_aff.permission_type_id == PERMISSION_SHORT_DICT['f'] - cgac_affils = [affil for affil in user.affiliations if affil.cgac is not None] - assert cgac_affils[0].cgac.cgac_code == 'ABC' - assert cgac_affils[0].frec is None - assert cgac_affils[0].permission_type_id == PERMISSION_TYPE_DICT['reader'] - - # test creating DABS and FABS CGAC permissions from service accounts - perms_string = 'prefix-CGAC_ABC-PERM_W' - account_handler.set_max_perms(user, perms_string, service_account_flag=True) - database.session.commit() - assert len(user.affiliations) == 2 - abc_aff, def_aff = list(sorted(user.affiliations, key=lambda a: a.permission_type_id)) - assert abc_aff.cgac.cgac_code == 'ABC' - assert abc_aff.frec is None - assert abc_aff.permission_type_id == PERMISSION_TYPE_DICT['writer'] - assert def_aff.cgac.cgac_code == 'ABC' - assert def_aff.frec is None - assert def_aff.permission_type_id == PERMISSION_SHORT_DICT['e'] - - # test creating DABS and FABS FREC permissions from service accounts - perms_string = 'prefix-CGAC_ABC-FREC_ABCD-PERM_F' - account_handler.set_max_perms(user, perms_string, service_account_flag=True) - database.session.commit() - assert len(user.affiliations) == 3 - frec_affils = [affil for affil in user.affiliations if affil.frec is not None] - frec_affiliations = list(sorted(frec_affils, key=lambda a: a.permission_type_id)) - dabs_aff, fabs_aff = frec_affiliations - assert dabs_aff.cgac is None - assert dabs_aff.frec.frec_code == 'ABCD' - assert dabs_aff.permission_type_id == PERMISSION_TYPE_DICT['writer'] - assert fabs_aff.cgac is None - assert fabs_aff.frec.frec_code == 'ABCD' - assert fabs_aff.permission_type_id == PERMISSION_SHORT_DICT['e'] - cgac_affils = [affil for affil in user.affiliations if affil.cgac is not None] - assert cgac_affils[0].cgac.cgac_code == 'ABC' - assert cgac_affils[0].frec is None - assert cgac_affils[0].permission_type_id == PERMISSION_TYPE_DICT['reader'] - - @pytest.mark.usefixtures("user_constants") def test_set_caia_perms(database): """Verify that we get the _highest_ permission within our CGAC""" @@ -449,7 +151,7 @@ def test_set_caia_perms(database): assert affil.cgac_id == cgac_abc.cgac_id assert affil.permission_type_id == PERMISSION_TYPE_DICT['writer'] - # test creating max CGAC permission from two strings + # test creating CGAC permission from two strings account_handler.set_caia_perms(user, ['CGAC-ABC-R', 'CGAC-ABC-S']) database.session.commit() # populate ids assert len(user.affiliations) == 1 @@ -470,7 +172,7 @@ def test_set_caia_perms(database): assert def_aff.frec is None assert def_aff.permission_type_id == PERMISSION_TYPE_DICT['submitter'] - # test creating max FREC permission from two strings + # test creating FREC permission from two strings account_handler.set_caia_perms(user, ['FREC-ABCD-R', 'FREC-ABCD-S']) database.session.commit() assert len(user.affiliations) == 2 @@ -519,7 +221,7 @@ def test_set_caia_perms(database): assert def_aff.frec is None assert def_aff.permission_type_id == PERMISSION_TYPE_DICT['reader'] - # test creating max DABS and FABS CGAC permissions from three strings + # test creating DABS and FABS CGAC permissions from three strings account_handler.set_caia_perms(user, ['CGAC-ABC-R', 'CGAC-ABC-S', 'CGAC-ABC-F']) database.session.commit() assert len(user.affiliations) == 2 @@ -532,7 +234,7 @@ def test_set_caia_perms(database): assert fabs_aff.frec is None assert fabs_aff.permission_type_id == PERMISSION_SHORT_DICT['f'] - # test creating max DABS and FABS FREC permissions from three strings + # test creating DABS and FABS FREC permissions from three strings account_handler.set_caia_perms(user, ['FREC-ABCD-R', 'FREC-ABCD-S', 'FREC-ABCD-F']) database.session.commit() assert len(user.affiliations) == 3 diff --git a/tests/unit/dataactbroker/test_login_routes.py b/tests/unit/dataactbroker/test_login_routes.py index b5cc85c0c..a1a738d1b 100644 --- a/tests/unit/dataactbroker/test_login_routes.py +++ b/tests/unit/dataactbroker/test_login_routes.py @@ -55,21 +55,6 @@ } -@patch('dataactbroker.handlers.account_handler.get_max_dict') -@patch('dataactbroker.handlers.account_handler.AccountHandler.create_session_and_response') -def test_no_perms_broker_user_max(create_session_mock, max_dict_mock, database, monkeypatch): - ah = max_login_func(create_session_mock, max_dict_mock, monkeypatch, MAX_RESPONSE_NO_PERMS) - res = ah.max_login({}) - response = json.loads(res.get_data().decode("utf-8")) - sess = GlobalDB.db().session - # This is to prevent an integrity error with other tests that create users. - sess.query(User).filter(func.lower(User.email) == func.lower("something@test.com"))\ - .delete(synchronize_session=False) - sess.commit() - assert response['message'] == "There are no Data Broker permissions assigned to this Service Account. You " \ - "may request permissions at https://community.max.gov/x/fJwuRQ" - - @patch('dataactbroker.handlers.account_handler.get_caia_user_dict') @patch('dataactbroker.handlers.account_handler.get_caia_tokens') @patch('dataactbroker.handlers.account_handler.revoke_caia_access') @@ -89,19 +74,6 @@ def test_no_perms_broker_user_caia(create_session_mock, revoke_caia_mock, caia_t assert affiliations == [] -@patch('dataactbroker.handlers.account_handler.get_max_dict') -@patch('dataactbroker.handlers.account_handler.AccountHandler.create_session_and_response') -def test_w_perms_broker_user_max(create_session_mock, max_dict_mock, database, monkeypatch): - ah = max_login_func(create_session_mock, max_dict_mock, monkeypatch, MAX_RESPONSE_W_PERMS) - res = ah.max_login({}) - sess = GlobalDB.db().session - # This is to prevent an integrity error with other tests that create users. - sess.query(User).filter(func.lower(User.email) == func.lower("something@test.com"))\ - .delete(synchronize_session=False) - sess.commit() - assert res is True - - @patch('dataactbroker.handlers.account_handler.get_caia_user_dict') @patch('dataactbroker.handlers.account_handler.get_caia_tokens') @patch('dataactbroker.handlers.account_handler.revoke_caia_access') @@ -158,18 +130,6 @@ def test_proxy_login_invalid_token(create_session_mock, monkeypatch): assert response['message'] == "Invalid token" -def max_login_func(create_session_mock, max_dict_mock, monkeypatch, max_response): - def json_return(): - return {"ticket": "12345", "service": "https://some.url.gov"} - request = type('Request', (object,), {"is_json": True, "headers": {"Content-Type": "application/json"}, - "get_json": json_return}) - ah = account_handler.AccountHandler(request=request) - monkeypatch.setattr(account_handler, 'CONFIG_BROKER', {"parent_group": "test"}) - max_dict_mock.return_value = max_response - create_session_mock.return_value = True - return ah - - def caia_login_func(create_session_mock, revoke_caia_mock, caia_token_mock, caia_dict_mock, monkeypatch, caia_response): def json_return(): return {"code": "12345", "redirect_uri": "https://some.url.gov"} From 904902fbd6c7b997ebe0d6c32b75f56b30bbf870 Mon Sep 17 00:00:00 2001 From: dpb-bah Date: Thu, 12 Dec 2024 17:25:22 -0500 Subject: [PATCH 2/6] Adding extra troubleshooting error to API User Guide --- APISubmissionGuide.md | 1 + 1 file changed, 1 insertion(+) diff --git a/APISubmissionGuide.md b/APISubmissionGuide.md index eedc908e0..55d165d83 100644 --- a/APISubmissionGuide.md +++ b/APISubmissionGuide.md @@ -62,6 +62,7 @@ While the Submission API has been designed to be as easy to understand as possib - When finally logging into the Broker using the new credentials and you receive a message: - **"Authentication denied"**: The `client_id`/`client_secret` headers were not included. Make sure to include them. - **"Invalid Client"**: Your `client_id`/`client_secret` credentials were provided but incorrect. Ensure the values are correct. + - **"HTTP POST on resource '.../v1/proxy_login' failed: unauthorized (401)."**: Your system email was not initially entered into the database. Please reach out to the service desk with this error. ## DABS Submission Process From 569043de8f1e5037fb64914eb220b79e684ab48e Mon Sep 17 00:00:00 2001 From: Zach Flanders Date: Fri, 13 Dec 2024 12:18:30 -0600 Subject: [PATCH 3/6] [DEV-11811] - Add legacy city codes (PW, FM, MH) to teh city code loader --- dataactcore/scripts/setup/load_location_data.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dataactcore/scripts/setup/load_location_data.py b/dataactcore/scripts/setup/load_location_data.py index d6b353679..14f44585e 100644 --- a/dataactcore/scripts/setup/load_location_data.py +++ b/dataactcore/scripts/setup/load_location_data.py @@ -220,6 +220,16 @@ def load_city_data(force_reload): with RetrieveFileFromUri(city_file_url, 'r').get_file_object() as city_file: new_data = parse_city_file(city_file) + # parse the legacy city code data + legacy_city_filename = 'NationalFedCodes_LEGACY.txt' + legacy_city_url = f'{CONFIG_BROKER["usas_public_reference_url"]}/{legacy_city_filename}' + with RetrieveFileFromUri(legacy_city_url, 'r').get_file_object() as legacy_city_file: + legacy_data = parse_city_file(legacy_city_file) + + # only add legacy city code data if the state is not already in the new city data + add_legacy_data = legacy_data.loc[~legacy_data['state_code'].isin(new_data['state_code'].unique())] + new_data = pd.concat([new_data, add_legacy_data]) + diff_found = check_dataframe_diff(new_data, CityCode, ['city_code_id'], ['state_code', 'city_code']) if force_reload or diff_found: From 179d55f0ffb5cf3a5915c2abbf9ece1a92cdfc4b Mon Sep 17 00:00:00 2001 From: Zach Flanders Date: Fri, 13 Dec 2024 13:56:53 -0600 Subject: [PATCH 4/6] [DEV-11811] - Fix test with randomly failing condition --- tests/unit/dataactcore/utils/test_tracing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/dataactcore/utils/test_tracing.py b/tests/unit/dataactcore/utils/test_tracing.py index 91b95bf35..62acbc420 100644 --- a/tests/unit/dataactcore/utils/test_tracing.py +++ b/tests/unit/dataactcore/utils/test_tracing.py @@ -138,8 +138,8 @@ def test_logging_trace_spans(caplog, capsys): captured = capsys.readouterr() assert test_msg in caplog.text, "caplog.text did not seem to capture logging output during test" - assert hex(trace_id) in captured.out, "trace_id not found in logging output" - assert hex(span_id) in captured.out, "span_id not found in logging output" + assert f'{trace_id:x}' in captured.out, "trace_id not found in logging output" + assert f'{span_id:x}' in captured.out, "span_id not found in logging output" assert f"{test}_resource" in captured.out, "traced resource not found in logging output" From 724f7a83e50d43aa4fd9c96480d2e4f75e55b20d Mon Sep 17 00:00:00 2001 From: dpb-bah Date: Tue, 17 Dec 2024 18:21:36 -0500 Subject: [PATCH 5/6] Fixing production URL and adding notes for trailing slashes --- APISubmissionGuide.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/APISubmissionGuide.md b/APISubmissionGuide.md index 763bf3133..672da7c1b 100644 --- a/APISubmissionGuide.md +++ b/APISubmissionGuide.md @@ -35,7 +35,8 @@ While the Submission API has been designed to be as easy to understand as possib - Each call to the Broker below in this guide (or any other endpoint referenced in the documentation) will be called slightly different: - The root url will be replaced: - Before: `https://broker-api.usaspending.gov/` - - After: `https://api.fiscal.treasury.gov/ap/prod/exp/v1/data-act-broker/` + - After: `https://api.fiscal.treasury.gov/ap/exp/v1/data-act-broker/` + - **NOTE**: The API Proxy does **not** support endpoint URLs ending with a trailing slash (`/`). Please ensure that all your calls to the API do not include said trailing slash. - Two new headers must be added in every request: - `client_id`: The `Client ID` copied from earlier. - `client_secret`: The `Client Secret` copied from earlier. @@ -45,6 +46,7 @@ While the Submission API has been designed to be as easy to understand as possib - **"Authentication denied"**: The `client_id`/`client_secret` headers were not included. Make sure to include them. - **"Invalid Client"**: Your `client_id`/`client_secret` credentials were provided but incorrect. Ensure the values are correct. - **"HTTP POST on resource '.../v1/proxy_login' failed: unauthorized (401)."**: Your system email was not initially entered into the database. Please reach out to the service desk with this error. + - **"resource_not_found"**: Your endpoint URL is ending with a slash which is currently not supported with the API proxy specifically. To resolve, simply remove the final slash. ## DABS Submission Process From 85ebc61f95cfb015a1e65315dd7ce28cd5516069 Mon Sep 17 00:00:00 2001 From: Daniel Boos <29313570+dpb-bah@users.noreply.github.com> Date: Wed, 18 Dec 2024 15:54:25 -0500 Subject: [PATCH 6/6] Update APISubmissionGuide.md --- APISubmissionGuide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/APISubmissionGuide.md b/APISubmissionGuide.md index 672da7c1b..946fd1f8e 100644 --- a/APISubmissionGuide.md +++ b/APISubmissionGuide.md @@ -36,7 +36,7 @@ While the Submission API has been designed to be as easy to understand as possib - The root url will be replaced: - Before: `https://broker-api.usaspending.gov/` - After: `https://api.fiscal.treasury.gov/ap/exp/v1/data-act-broker/` - - **NOTE**: The API Proxy does **not** support endpoint URLs ending with a trailing slash (`/`). Please ensure that all your calls to the API do not include said trailing slash. + - **NOTE**: The API Proxy does **not** support endpoint URLs ending with a trailing slash (`/`). Please ensure that all your calls to the API do not include a trailing slash. - Two new headers must be added in every request: - `client_id`: The `Client ID` copied from earlier. - `client_secret`: The `Client Secret` copied from earlier.