Merge pull request #224 from NASA-IMPACT/develop

Updates from Regression Testing and Minor Fixes

CHANGELOG.md

@@ -1,5 +1,10 @@
 # CHANGELOG
 
+## v1.2.1
+
+- Added support for automated regression testing
+- Revised output messages
+
 ## v1.2.0
 
 - Added support for ECHO10 Granule, UMM-G (UMM-JSON Granule) and UMM-C (UMM-JSON Collection) metadata

pyQuARC/__init__.py

@@ -2,6 +2,8 @@
 
 from pyQuARC.main import ARC
 
+from pyQuARC.code.constants import SUPPORTED_FORMATS as FORMATS
+
 from pyQuARC.code.checker import Checker
 from pyQuARC.code.downloader import Downloader
 
@@ -15,5 +17,7 @@
 with open(f"{ABS_PATH}/version.txt") as version_file:
     __version__ = version_file.read().strip()
 
-def get_version():
+def version():
+    """Returns the current version of pyQuARC.
+    """
     return __version__

pyQuARC/code/custom_validator.py

@@ -10,7 +10,6 @@ def __init__(self):
         super().__init__()
 
     @staticmethod
-    @if_arg
     def ends_at_present_flag_logic_check(
         ends_at_present_flag, ending_date_time, collection_state
     ):
@@ -62,7 +61,7 @@ def availability_check(field_value, parent_value):
     @if_arg
     def bounding_coordinate_logic_check(west, north, east, south):
         # Checks if the logic for coordinate values make sense
-        result = {"valid": False, "value": ""}
+        result = {"valid": False, "value": [west, north, east, south]}
         west = float(west)
         east = float(east)
         south = float(south)
@@ -138,8 +137,9 @@ def user_services_check(first_name, middle_name, last_name):
 
     @staticmethod
     def doi_missing_reason_explanation(explanation, missing_reason, doi):
+        validity = bool(doi or ((not doi) and missing_reason and explanation))
         return {
-            "valid": doi or not missing_reason or explanation,
+            "valid": validity,
             "value": explanation
         }
 
@@ -179,11 +179,10 @@ def uniqueness_check(list_of_objects, key):
         seen, duplicates = set(), set()
         if isinstance(list_of_objects, list):
             for url_obj in list_of_objects:
-                if description := url_obj.get(key) in seen:
+                if (description := url_obj.get(key)) in seen:
                     duplicates.add(description)
                 else:
                     seen.add(description)
-
         return {
             "valid": not bool(duplicates),
             "value": ', '.join(duplicates)

pyQuARC/code/downloader.py

@@ -1,9 +1,10 @@
+import os
 import re
-import requests
-
 from urllib.parse import urlparse
 
-from .utils import get_cmr_url
+import requests
+
+from .utils import get_cmr_url, get_headers
 
 
 class Downloader:
@@ -98,7 +99,8 @@ def download(self):
 
         # constructs url based on concept id
         url = self._construct_url()
-        response = requests.get(url)
+        headers = get_headers()
+        response = requests.get(url, headers=headers)
         # gets the response, makes sure it's 200, puts it in an object variable
         if response.status_code != 200:
             self.log_error(

pyQuARC/code/gcmd_validator.py

@@ -2,6 +2,8 @@
 import os
 import requests
 
+from .utils import get_headers
+
 from .constants import SCHEMA_PATHS, GCMD_LINKS, VERSION_FILE
 from datetime import datetime
 
@@ -123,6 +125,7 @@ def _download_files(force=False):
         Downloads and maintains a copy of the csv files from the gcmd server once a day
         """
         current_datetime = datetime.now()
+        headers = get_headers()
         date_str = current_datetime.strftime(DATE_FORMAT)
         if os.path.exists(VERSION_FILE):
             with open(VERSION_FILE) as file:
@@ -132,9 +135,9 @@ def _download_files(force=False):
             try:
                 for keyword, link in GCMD_LINKS.items():
                     # Downloading updated gcmd keyword files
-                    response = requests.get(link)
+                    response = requests.get(link, headers=headers)
                     data = response.text
-                    with open(SCHEMA_PATHS[keyword], "w") as download_file:
+                    with open(SCHEMA_PATHS[keyword], "w", encoding="utf-8") as download_file:
                         download_file.write(data)
                 with open(VERSION_FILE, "w") as version_file:
                     version_file.write(current_datetime.strftime(DATE_FORMAT))

pyQuARC/code/url_validator.py

@@ -3,7 +3,7 @@
 from urlextract import URLExtract
 
 from .string_validator import StringValidator
-from .utils import if_arg
+from .utils import get_headers, if_arg
 
 
 class UrlValidator(StringValidator):
@@ -45,8 +45,9 @@ def health_and_status_check(text_with_urls):
         """
 
         def status_code_from_request(url):
+            headers = get_headers()
             # timeout = 10 seconds, to allow for slow but not invalid connections
-            return requests.get(url, timeout=10).status_code
+            return requests.get(url, headers = headers, timeout=10).status_code
 
         results = []
 

pyQuARC/code/utils.py

@@ -19,6 +19,13 @@ def run_function_only_if_arg(*args):
             }
     return run_function_only_if_arg
 
+def get_headers():
+    token = os.environ.get("AUTH_TOKEN")
+    headers = None
+    if token:
+        headers = {"Authorization": f"Bearer {token}"}
+    return headers
+
 def _add_protocol(url):
     if not url.startswith("http"):
         url = f"https://{url}"
@@ -27,8 +34,9 @@ def _add_protocol(url):
 def is_valid_cmr_url(url):
     url = _add_protocol(url)
     valid = False
+    headers = get_headers()
     try: # some invalid url throw an exception
-        response = requests.get(url, timeout=5) # some invalid urls freeze
+        response = requests.get(url, headers=headers, timeout=5) # some invalid urls freeze
         valid = response.status_code == 200 and response.headers.get("CMR-Request-Id")
     except:
         valid = False
@@ -44,7 +52,8 @@ def set_cmr_prms(params, format='json', type="collections"):
     return f"{base_url}{urllib.parse.urlencode(params)}"
 
 def cmr_request(cmr_prms):
-    return requests.get(f'{get_cmr_url()}/search/{cmr_prms}').json()
+    headers = get_headers()
+    return requests.get(f'{get_cmr_url()}/search/{cmr_prms}', headers=headers).json()
 
 def collection_in_cmr(cmr_prms):
     return cmr_request(cmr_prms).get('hits', 0) > 0

pyQuARC/main.py

@@ -11,12 +11,13 @@
     from code.constants import COLOR, ECHO10_C, SUPPORTED_FORMATS
     from code.downloader import Downloader
     from code.utils import get_cmr_url, is_valid_cmr_url
+    from code.utils import get_headers
 else:
     from .code.checker import Checker
     from .code.constants import COLOR, ECHO10_C, SUPPORTED_FORMATS
     from .code.downloader import Downloader
     from .code.utils import get_cmr_url, is_valid_cmr_url
-
+    from .code.utils import get_headers
 
 ABS_PATH = os.path.abspath(os.path.dirname(__file__))
 END = COLOR["reset"]
@@ -104,9 +105,10 @@ def _cmr_query(self):
 
         orig_query = f"{self.query}&page_size={page_size}" if not already_selected else self.query
         query = orig_query
-
+        headers = get_headers()
+
         while True:
-            response = requests.get(query)
+            response = requests.get(query, headers=headers)
 
             if response.status_code != 200:
                 return {"error": "CMR Query failed"}

pyQuARC/schemas/MimeType.csv

@@ -1,4 +1,4 @@
-"Hits: 37","page_num: 1","page_size: 2000","Keyword Version: 13.8","Revision: 2022-06-03 10:18:29","Timestamp: 2022-06-28 10:40:09","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/MimeType/?format=xml","Case native"
+"Hits: 37","page_num: 1","page_size: 2000","Keyword Version: 14.3","Revision: 2022-08-26 10:35:56","Timestamp: 2022-09-15 12:28:30","Terms Of Use: https://cdn.earthdata.nasa.gov/conduit/upload/5182/KeywordsCommunityGuide_Baseline_v1_SIGNED_FINAL.pdf","The most up to date XML representations can be found here: https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/MimeType/?format=xml","Case native"
 MimeType,UUID
 "application/gml+xml","40bdf6e5-780c-43e2-ab8e-e5dfae4bd779"
 "application/gzip","a8ee535a-8bc8-46fd-8b97-917bd7ea7666"