Adds client class for KeystoneApi (#240)

* Start on building a usage table with info from keystone instead of crc-bank

* utility functions for interacting with keystone

* pull function for getting allocations by primary key

* add requests

* verify provided slurm account against keystone as well as slurm

* properly handle dates and pretty table rows

* split output into summary and usage tables, keep track of per cluster usage

* PEP fixes and comments

* Update crc_sus.py

Output remaining SUs on each clsuter using keystone requests and sreport

* adjustments from meeting

* filter on active and expiration date in request for allocations

* add and move slurm functions for gathering usage data into utils.system_info

* move slurm functions out of utils.keystone

* add usage table setup

* small fix with field_names

* fix imports, use account_name in checking for whether slurm account exists

* fix adding total to out_data

* clean up usage output formatting

* small formatting changes to crc_sus

* formatting adjustment

* add request primary key as query param for requests

* use cluster totals instead of allocation totals, other review items

* use request pk when gathering allocations

* use check_slurm_account_exists

* working state for crc-sus and crc-usage

* fix crc_usage test that wa failing

* pull duplicate functionality for running subprocesses

* small typo in crc_usage test string

* crc-proposal-end (#232)

* Update crc_proposal_end.py

Added authentication procedure

* Update crc_proposal_end.py

* Update crc_proposal_end.py

* Update crc_proposal_end.py

Updated requests with group_id

* Update crc_proposal_end.py

Update requests

* working version of crc_proposal_end

* remove unused imports

---------

Co-authored-by: Nickolas Comeau <[email protected]>

* fix expected output string

* codacy / PEP8 items

* pull crc-bank from dependencies

* add prettytable to deps

* needs to be an f string

* fix conversion to percent that was broken

* more broken division

* Adds API client class

* Minor doc updates

* Adds support for multiple content types

* Restores API class

* Defines default API client settings

---------

Co-authored-by: Comeani <[email protected]>
Co-authored-by: chnixi <[email protected]>

apps/utils/keystone.py

@@ -1,17 +1,225 @@
 """Utility functions used across various wrappers for interacting with keystone"""
 
 from datetime import date
+from typing import Any, Dict, Literal, Optional, Union
 
 import requests
 
+# Custom types
+ResponseContentType = Literal['json', 'text', 'content']
+ParsedResponseContent = Union[Dict[str, Any], str, bytes]
+
+# Default API configuratipn
 KEYSTONE_URL = "https://keystone.crc.pitt.edu"
+KEYSTONE_AUTH_ENDPOINT = 'authentication/new'
 RAWUSAGE_RESET_DATE = date.fromisoformat('2024-05-07')
 
 
+class KeystoneApi:
+    """API client for submitting requests to the Keystone API"""
+
+    def __init__(self, base_url: str = KEYSTONE_URL) -> None:
+        """Initializes the KeystoneApi class with the base URL of the API.
+
+        Args:
+            base_url: The base URL of the Keystone API
+        """
+
+        self.base_url = base_url
+        self._token: Optional[str] = None
+        self._timeout: int = 10
+
+    def login(self, username: str, password: str, endpoint: str = KEYSTONE_AUTH_ENDPOINT) -> None:
+        """Logs in to the Keystone API and caches the JWT token.
+
+        Args:
+            username: The username for authentication
+            password: The password for authentication
+            endpoint: The API endpoint to send the authentication request to
+
+        Raises:
+            requests.HTTPError: If the login request fails
+        """
+
+        response = requests.post(
+            f"{self.base_url}/{endpoint}",
+            json={"username": username, "password": password},
+            timeout=self._timeout
+        )
+        response.raise_for_status()
+        self._token = response.json().get("token")
+
+    def _get_headers(self) -> Dict[str, str]:
+        """Constructs the headers for an authenticated request.
+
+        Returns:
+            A dictionary of headers including the Authorization token
+
+        Raises:
+            ValueError: If the authentication token is not found
+        """
+
+        if not self._token:
+            raise ValueError("Authentication token not found. Please login first.")
+
+        return {
+            "Authorization": f"Bearer {self._token}",
+            "Content-Type": "application/json"
+        }
+
+    @staticmethod
+    def _process_response(response: requests.Response, response_type: ResponseContentType) -> ParsedResponseContent:
+        """Processes the response based on the expected response type.
+
+        Args:
+            response: The response object
+            response_type: The expected response type ('json', 'text', 'content')
+
+        Returns:
+            The response in the specified format
+
+        Raises:
+            ValueError: If the response type is invalid
+        """
+
+        if response_type == 'json':
+            return response.json()
+
+        elif response_type == 'text':
+            return response.text
+
+        elif response_type == 'content':
+            return response.content
+
+        else:
+            raise ValueError(f"Invalid response type: {response_type}")
+
+    def get(
+            self, endpoint: str, params: Optional[Dict[str, Any]] = None, response_type: ResponseContentType = 'json'
+    ) -> ParsedResponseContent:
+        """Makes a GET request to the specified endpoint.
+
+        Args:
+            endpoint: The API endpoint to send the GET request to
+            params: The query parameters to include in the request
+            response_type: The expected response type ('json', 'text', 'content')
+
+        Returns:
+            The response from the API in the specified format
+
+        Raises:
+            requests.HTTPError: If the GET request fails
+        """
+
+        response = requests.get(f"{self.base_url}/{endpoint}",
+                                headers=self._get_headers(),
+                                params=params,
+                                timeout=self._timeout
+                                )
+        response.raise_for_status()
+        return self._process_response(response, response_type)
+
+    def post(
+            self, endpoint: str, data: Optional[Dict[str, Any]] = None, response_type: ResponseContentType = 'json'
+    ) -> ParsedResponseContent:
+        """Makes a POST request to the specified endpoint.
+
+        Args:
+            endpoint: The API endpoint to send the POST request to
+            data: The JSON data to include in the POST request
+            response_type: The expected response type ('json', 'text', 'content')
+
+        Returns:
+            The response from the API in the specified format
+
+        Raises:
+            requests.HTTPError: If the POST request fails
+        """
+
+        response = requests.post(f"{self.base_url}/{endpoint}",
+                                 headers=self._get_headers(),
+                                 json=data,
+                                 timeout=self._timeout
+                                 )
+        response.raise_for_status()
+        return self._process_response(response, response_type)
+
+    def patch(
+            self, endpoint: str, data: Optional[Dict[str, Any]] = None, response_type: ResponseContentType = 'json'
+    ) -> ParsedResponseContent:
+        """Makes a PATCH request to the specified endpoint.
+
+        Args:
+            endpoint: The API endpoint to send the PATCH request to
+            data: The JSON data to include in the PATCH request
+            response_type: The expected response type ('json', 'text', 'content')
+
+        Returns:
+            The response from the API in the specified format
+
+        Raises:
+            requests.HTTPError: If the PATCH request fails
+        """
+
+        response = requests.patch(f"{self.base_url}/{endpoint}",
+                                  headers=self._get_headers(),
+                                  json=data,
+                                  timeout=self._timeout
+                                  )
+        response.raise_for_status()
+        return self._process_response(response, response_type)
+
+    def put(
+            self, endpoint: str, data: Optional[Dict[str, Any]] = None, response_type: ResponseContentType = 'json'
+    ) -> ParsedResponseContent:
+        """Makes a PUT request to the specified endpoint.
+
+        Args:
+            endpoint: The API endpoint to send the PUT request to
+            data: The JSON data to include in the PUT request
+            response_type: The expected response type ('json', 'text', 'content')
+
+        Returns:
+            The response from the API in the specified format
+
+        Raises:
+            requests.HTTPError: If the PUT request fails
+        """
+
+        response = requests.put(f"{self.base_url}/{endpoint}",
+                                headers=self._get_headers(),
+                                json=data,
+                                timeout=self._timeout
+                                )
+        response.raise_for_status()
+        return self._process_response(response, response_type)
+
+    def delete(self, endpoint: str, response_type: ResponseContentType = 'json') -> ParsedResponseContent:
+        """Makes a DELETE request to the specified endpoint.
+
+        Args:
+            endpoint: The API endpoint to send the DELETE request to
+            response_type: The expected response type ('json', 'text', 'content')
+
+        Returns:
+            The response from the API in the specified format
+
+        Raises:
+            requests.HTTPError: If the DELETE request fails
+        """
+
+        response = requests.delete(f"{self.base_url}/{endpoint}",
+                                   headers=self._get_headers(),
+                                   timeout=self._timeout
+                                   )
+        response.raise_for_status()
+        return self._process_response(response, response_type)
+
+
 def get_auth_header(keystone_url: str, auth_header: dict) -> dict:
     """ Generate an authorization header to be used for accessing information from keystone"""
 
-    response = requests.post(f"{keystone_url}/authentication/new/", json=auth_header)
+    response = requests.post(f"{keystone_url}/authentication/new/", json=auth_header, timeout=10)
     response.raise_for_status()
     tokens = response.json()
     return {"Authorization": f"Bearer {tokens['access']}"}
@@ -20,7 +228,10 @@ def get_auth_header(keystone_url: str, auth_header: dict) -> dict:
 def get_request_allocations(keystone_url: str, request_pk: int, auth_header: dict) -> dict:
     """Get All Allocation information from keystone for a given request"""
 
-    response = requests.get(f"{keystone_url}/allocations/allocations/?request={request_pk}", headers=auth_header)
+    response = requests.get(f"{keystone_url}/allocations/allocations/?request={request_pk}",
+                            headers=auth_header,
+                            timeout=10
+                            )
     response.raise_for_status()
     return response.json()
 
@@ -31,15 +242,20 @@ def get_active_requests(keystone_url: str, group_pk: int, auth_header: dict) ->
     today = date.today().isoformat()
     response = requests.get(
         f"{keystone_url}/allocations/requests/?group={group_pk}&status=AP&active__lte={today}&expire__gt={today}",
-        headers=auth_header)
+        headers=auth_header,
+        timeout=10
+    )
     response.raise_for_status()
     return [request for request in response.json()]
 
 
 def get_researchgroup_id(keystone_url: str, account_name: str, auth_header: dict) -> int:
     """Get the Researchgroup ID from keystone for the specified Slurm account"""
 
-    response = requests.get(f"{keystone_url}/users/researchgroups/?name={account_name}", headers=auth_header)
+    response = requests.get(f"{keystone_url}/users/researchgroups/?name={account_name}",
+                            headers=auth_header,
+                            timeout=10
+                            )
     response.raise_for_status()
 
     try:
@@ -71,7 +287,9 @@ def get_most_recent_expired_request(keystone_url: str, group_pk: int, auth_heade
     today = date.today().isoformat()
     response = requests.get(
         f"{keystone_url}/allocations/requests/?ordering=-expire&group={group_pk}&status=AP&expire__lte={today}",
-        headers=auth_header)
+        headers=auth_header,
+        timeout=10
+    )
     response.raise_for_status()
 
     return [response.json()[0]]
@@ -80,7 +298,7 @@ def get_most_recent_expired_request(keystone_url: str, group_pk: int, auth_heade
 def get_enabled_cluster_ids(keystone_url: str, auth_header: dict) -> dict():
     """Get the list of enabled clusters defined in Keystone along with their IDs"""
 
-    response = requests.get(f"{keystone_url}/allocations/clusters/?enabled=True", headers=auth_header)
+    response = requests.get(f"{keystone_url}/allocations/clusters/?enabled=True", headers=auth_header, timeout=10)
     response.raise_for_status()
     clusters = {}
     for cluster in response.json():