From f6cd14ec219578eca504f8477016281034a83abf Mon Sep 17 00:00:00 2001 From: Eva Sebestova <73821679+esebesto@users.noreply.github.com> Date: Fri, 3 Nov 2023 19:38:30 +0100 Subject: [PATCH 1/2] Fix date format in Azure v1 reports (#472) --- nise/__init__.py | 2 +- nise/generators/azure/azure_generator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nise/__init__.py b/nise/__init__.py index 6c98004d..b6de30d0 100644 --- a/nise/__init__.py +++ b/nise/__init__.py @@ -1,3 +1,3 @@ -__version__ = "4.4.4" +__version__ = "4.4.5" VERSION = __version__.split(".") diff --git a/nise/generators/azure/azure_generator.py b/nise/generators/azure/azure_generator.py index 146b7de2..38ace7fc 100644 --- a/nise/generators/azure/azure_generator.py +++ b/nise/generators/azure/azure_generator.py @@ -312,7 +312,7 @@ def _add_common_usage_info(self, row, start, end, **kwargs): row["BillingPeriodEndDate"] = self.last_day_of_month(start).strftime(DATE_FMT) else: row["SubscriptionGuid"] = self.subscription_guid - row["UsageDateTime"] = start + row["UsageDateTime"] = start.strftime("%Y-%m-%d %H:%M:%S") return row def _add_tag_data(self, row): From 7ae475b3967c09fe4ff592d69c623644578a6ff7 Mon Sep 17 00:00:00 2001 From: Chris Hambridge Date: Fri, 3 Nov 2023 12:31:10 -0400 Subject: [PATCH 2/2] [COST-4407] Add service account support * Add service account support as basic auth is becoming EOL --- .env.example | 5 +++++ README.md | 2 +- nise/__main__.py | 10 +++++++--- nise/report.py | 27 ++++++++++++++++++++++++++- tests/test_report.py | 17 ++++++++++++++++- 5 files changed, 55 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 171486fb..c4e3fe70 100644 --- a/.env.example +++ b/.env.example @@ -9,3 +9,8 @@ OCI_CREDENTIALS=OCI_PRIVATE_KEY OCI_REGION=OCI_REGION OCI_BUCKET_NAME=OCI_BUCKET_NAME OCI_NAMESPACE=OCI_NAMESPACE + +HCC_SERVICE_ACCOUNT_ID=SERVICE_ACCOUNT_ID +HCC_SERVICE_ACCOUNT_SECRET=SERVICE_ACCOUNT_SECRET +HCC_TOKEN_URL=https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token +HCC_TOKEN_SCOPE=api.console diff --git a/README.md b/README.md index b03d9238..0b2970ce 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ The `make run-iqe` command by default will run the smoke tests. However, if you - `overwrite` will generate a regular report with the invoice id populated. 1. If `--insights-upload` is specified and pointing to a URL endpoint, - you must have `INSIGHTS_USER` and `INSIGHTS_PASSWORD` set in your + you must have `HCC_SERVICE_ACCOUNT_ID` and `HCC_SERVICE_ACCOUNT_SECRET` set in your environment. Payloads for insights uploads will be split on a per-file basis. 1. If `--static-report-file` is used start_date will default to first diff --git a/nise/__main__.py b/nise/__main__.py index dbd25f24..30143748 100644 --- a/nise/__main__.py +++ b/nise/__main__.py @@ -558,15 +558,19 @@ def _validate_ocp_arguments(parser, options): elif insights_upload is not None and not os.path.isdir(insights_upload): insights_user = os.environ.get("INSIGHTS_USER") insights_password = os.environ.get("INSIGHTS_PASSWORD") + hcc_service_account_id = os.environ.get("HCC_SERVICE_ACCOUNT_ID") + hcc_service_account_secret = os.environ.get("HCC_SERVICE_ACCOUNT_SECRET") insights_account_id = os.environ.get("INSIGHTS_ACCOUNT_ID") insights_org_id = os.environ.get("INSIGHTS_ORG_ID") - if (insights_account_id is None or insights_org_id is None) and ( - insights_user is None or insights_password is None + if ( + (insights_account_id is None or insights_org_id is None) + and (insights_user is None or insights_password is None) + and (hcc_service_account_id is None or hcc_service_account_secret is None) ): msg = ( f"\n\t--insights-upload {insights_upload} was supplied as an argument\n" "\tbut this directory does not exist locally. Attempting to upload to Ingress instead, but\n" - "\tthe environment must have \n\t\tINSIGHTS_USER and INSIGHTS_PASSWORD\n\tor\n" + "\tthe environment must have \n\t\\HCC_SERVICE_ACCOUNT_ID and HCC_SERVICE_ACCOUNT_SECRET\n\tor\n" "\t\tINSIGHTS_ACCOUNT_ID and INSIGHTS_ORG_ID\n\tdefined when attempting an upload to Ingress.\n" ) msg = msg.format("--insights-upload", insights_upload) diff --git a/nise/report.py b/nise/report.py index 89bb4881..cb69005a 100644 --- a/nise/report.py +++ b/nise/report.py @@ -283,6 +283,12 @@ def post_payload_to_ingest_service(insights_upload, local_path): insights_org_id = os.environ.get("INSIGHTS_ORG_ID") insights_user = os.environ.get("INSIGHTS_USER") insights_password = os.environ.get("INSIGHTS_PASSWORD") + hcc_service_account_id = os.environ.get("HCC_SERVICE_ACCOUNT_ID") + hcc_service_account_secret = os.environ.get("HCC_SERVICE_ACCOUNT_SECRET") + hcc_token_url = os.environ.get( + "HCC_TOKEN_URL", "https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token" + ) + hcc_token_scope = os.environ.get("HCC_TOKEN_SCOPE", "api.console") content_type = "application/vnd.redhat.hccm.tar+tgz" if os.path.isfile(local_path): file_info = os.stat(local_path) @@ -306,11 +312,30 @@ def post_payload_to_ingest_service(insights_upload, local_path): headers=headers, ) + if insights_user and insights_password: + return requests.post( + insights_upload, + data={}, + files={"file": ("payload.tar.gz", upload_file, content_type)}, + auth=(insights_user, insights_password), + verify=False, + ) + + headers = {"Content-Type": "application/x-www-form-urlencoded"} + data = f"client_id={hcc_service_account_id}&client_secret={hcc_service_account_secret}" + data += f"&grant_type=client_credentials&scope={hcc_token_scope}" + token_resp = requests.post(hcc_token_url, data=data, headers=headers) + token = None + if token_resp.ok: + token_json = token_resp.json() + token = token_json.get("access_token") + + headers = {"Authorization": f"Bearer {token}"} return requests.post( insights_upload, data={}, files={"file": ("payload.tar.gz", upload_file, content_type)}, - auth=(insights_user, insights_password), + headers=headers, verify=False, ) diff --git a/tests/test_report.py b/tests/test_report.py index 5913d3aa..eae39723 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -337,7 +337,7 @@ def test_post_payload_to_ingest_service_with_identity_header(self, mock_post): @patch.dict(os.environ, {"INSIGHTS_USER": "12345", "INSIGHTS_PASSWORD": "54321"}) @patch("nise.report.requests.post") def test_post_payload_to_ingest_service_with_basic_auth(self, mock_post): - """Test that the identity header path is taken.""" + """Test that the basic auth path is taken.""" insights_user = os.environ.get("INSIGHTS_USER") insights_password = os.environ.get("INSIGHTS_PASSWORD") @@ -354,6 +354,21 @@ def test_post_payload_to_ingest_service_with_basic_auth(self, mock_post): self.assertEqual(mock_post.call_args[1].get("auth"), auth) self.assertNotIn("headers", mock_post.call_args[1]) + @patch.dict(os.environ, {"HCC_SERVICE_ACCOUNT_ID": "12345", "HCC_SERVICE_ACCOUNT_SECRET": "54321"}) + @patch("nise.report.requests.post") + def test_post_payload_to_ingest_service_with_service_account(self, mock_post): + """Test that the service account path is taken.""" + temp_file = NamedTemporaryFile(mode="w", delete=False) + headers = ["col1", "col2"] + data = [{"col1": "r1c1", "col2": "r1c2"}, {"col1": "r2c1", "col2": "r2c2"}] + _write_csv(temp_file.name, data, headers) + + insights_upload = {} + data = {} + + post_payload_to_ingest_service(insights_upload, temp_file.name) + self.assertEqual(mock_post.call_args[1].get("data"), data) + def test_defaulting_currency(self): """Test that if no currency is provide in options or static it defaults to USD.""" currency = None