Add support for private GCS buckets

.pre-commit-config.yaml

@@ -18,7 +18,6 @@ repos:
   hooks:
   - id: mypy
     exclude: ^(tests/)
-    additional_dependencies: [types-all]
 - repo: https://github.com/PyCQA/flake8
   rev: "7.1.1"
   hooks:

docs/cli_usage_guide.md

@@ -73,6 +73,7 @@ Options:
                                   code if a test failure is found.
   --jira-config-path PATH         The path to the jira configuration file
   --firewatch-config-path PATH    The path to the firewatch configuration file
+  --gcs-creds-file PATH           The path to the GCS credentials file
   --gcs-bucket TEXT               The name of the GCS bucket that holds
                                   OpenShift CI logs
   --pr-id TEXT                    The pull request number that the rehearsal

docs/configuration_guide.md

@@ -1,24 +1,27 @@
-# Configuring Firewatch
+# Configuring Firewatch<!-- omit in toc -->
 
-## Table of Contents
+## Table of Contents<!-- omit in toc -->
 
 - [Jira Issue Creation Configuration](#jira-issue-creation-configuration)
-  - [Getting Started](#getting-started)
-    - [Usage in OpenShift CI](#usage-in-openshift-ci)
-  - [Rule Configuration Value Definitions](#rule-configuration-value-definitions)
-    - [`jira_project`](#jiraproject)
-    - [`step`](#step)
-    - [`failure_type`](#failuretype)
-    - [`classification`](#classification)
-    - [`jira_epic`](#jiraepic)
-    - [`jira_component`](#jiracomponent)
-    - [`jira_affects_version`](#jiraaffectsversion)
-    - [`jira_additional_labels`](#jiraadditionallabels)
-    - [`jira_assignee`](#jiraassignee)
-    - [`jira_priority`](#jirapriority)
-    - [`jira_security_level`](#jirasecuritylevel)
-    - [`ignore`](#ignore)
-    - [`group`](#group)
+  - [`failure_rules`](#failure_rules)
+  - [`success_rules` (OPTIONAL)](#success_rules-optional)
+- [Rule Configuration Value Definitions](#rule-configuration-value-definitions)
+  - [`jira_project`](#jira_project)
+  - [`step`](#step)
+  - [`failure_type`](#failure_type)
+  - [`classification`](#classification)
+  - [`jira_epic`](#jira_epic)
+  - [`jira_component`](#jira_component)
+  - [`jira_affects_version`](#jira_affects_version)
+  - [`jira_additional_labels`](#jira_additional_labels)
+  - [`jira_assignee`](#jira_assignee)
+  - [`jira_priority`](#jira_priority)
+  - [`jira_security_level`](#jira_security_level)
+  - [`ignore`](#ignore)
+  - [`group`](#group)
+  - [Using a base config file](#using-a-base-config-file)
+- [Configuring Use With Private GCS Bucket](#configuring-use-with-private-gcs-bucket)
+
 
 ## Jira Issue Creation Configuration
 
@@ -64,6 +67,7 @@ The firewatch configuration is a list of rules, each rule is defined using the f
 
 **Optional Values**:
 
+<!-- no toc -->
 - [`jira_epic`](#jiraepic)
 - [`jira_component`](#jiracomponent)
 - [`jira_affects_version`](#jiraaffectsversion)
@@ -358,3 +362,26 @@ By setting the env var, the user may override the base config with:
 - The base file configures a rule for the step `exact-step-name`
   - A specific scenario requires to apply this rule on a set of similar steps
   - In this case, we will extend it and use `*-step-*` to override the step name by a pattern
+
+## Configuring Use With Private GCS Bucket
+
+Firewatch allows for the use of a private GCS bucket if needed. In order to use this, you will need to receive [service account credentials](https://developers.google.com/workspace/guides/create-credentials) from an administator of the GCS bucket. The credentials should be in JSON format. Example:
+
+```json
+{
+  "type": "service_account",
+  "project_id": "some-project",
+  "private_key_id": "0000000000000000000000000",
+  "private_key": "<PRIVATE KEY>",
+  "client_email": "[email protected]",
+  "client_id": "12345678",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://oauth2.googleapis.com/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "<Cert URL>",
+  "universe_domain": "googleapis.com"
+}
+
+```
+
+You will need to provide the name of the bucket and the path to the service account credentials in the [CLI](./cli_usage_guide.md#report). Exmaple: `firewatch report --gcs-bucket "my-bucket-name" --gcs-creds-file ./some/creds/file.json`

src/commands/report.py

@@ -77,6 +77,11 @@ def validate_verbose_test_failure_reporting_ticket_limit(
     default="test-platform-results",
     type=click.STRING,
 )
+@click.option(
+    "--gcs-creds-file",
+    help="The path to the GCS credentials file",
+    type=click.Path(exists=True),
+)
 @click.option(
     "--firewatch-config-path",
     help="The path to the firewatch configuration file",
@@ -139,6 +144,7 @@ def report(
     build_id: str,
     pr_id: str,
     gcs_bucket: str,
+    gcs_creds_file: Optional[str],
     firewatch_config_path: Optional[str],
     jira_config_path: str,
     fail_with_test_failures: bool,
@@ -168,6 +174,7 @@ def report(
         name_safe=job_name_safe,
         build_id=build_id,
         gcs_bucket=gcs_bucket,
+        gcs_creds_file=gcs_creds_file,
         firewatch_config=config,
         pr_id=pr_id,
     )

src/objects/job.py

@@ -6,6 +6,7 @@
 
 import junitparser
 from google.cloud import storage
+from google.oauth2 import service_account
 from simple_logger.logger import get_logger
 
 from src.objects.configuration import Configuration
@@ -19,6 +20,7 @@ def __init__(
         name_safe: str,
         build_id: Optional[str],
         gcs_bucket: str,
+        gcs_creds_file: Optional[str],
         firewatch_config: Configuration,
         pr_id: Optional[str] = "",
     ) -> None:
@@ -58,8 +60,11 @@ def __init__(
 
         # Set GCS bucket values
         self.gcs_bucket = gcs_bucket
-        self.storage_client = storage.Client.create_anonymous_client()
-        self.bucket = self.storage_client.bucket(gcs_bucket)
+        self.storage_client = (
+            storage.Client(credentials=service_account.Credentials.from_service_account_file(gcs_creds_file))
+            if gcs_creds_file
+            else storage.Client.create_anonymous_client()
+        )
 
         # Get a list of steps
         self.steps = self._get_steps(

tests/unittests/conftest.py

@@ -221,6 +221,7 @@ def job(firewatch_config, build_id):
         name_safe="openshift-pipelines-interop-aws",
         build_id=build_id,
         gcs_bucket="test-platform-results",
+        gcs_creds_file=None,
         firewatch_config=firewatch_config,
     )
 

tests/unittests/functions/report/report_base_test.py

@@ -38,7 +38,14 @@ def setUp(self, mock_jira):
             "src.objects.job.storage.Client.create_anonymous_client",
         )
         self.mock_storage_client.start().return_value = MagicMock()
-        self.job = Job("job1", "job1_safe", "123", "bucket1", self.config)
+        self.job = Job(
+            name="job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         self.report = Report(self.config, self.job)
 
     def tearDown(self):

tests/unittests/functions/report/test_firewatch_functions_report.py

@@ -5,14 +5,28 @@
 
 class TestReport(ReportBaseTest):
     def test_report_initialization_with_job_rehearsal(self):
-        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
+        job = Job(
+            name="rehearse-1234-job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         with self.assertRaises(SystemExit) as rc:
             Report(self.config, job)
         error = rc.exception
         self.assertEqual(error.code, 0)
 
     def test_report_initialization_with_job_rehearsal_flag_false_test_failures(self):
-        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
+        job = Job(
+            name="rehearse-1234-job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         job.has_test_failures = True
         with self.assertRaises(SystemExit) as rc:
             Report(self.config, job)
@@ -21,15 +35,29 @@ def test_report_initialization_with_job_rehearsal_flag_false_test_failures(self)
 
     def test_report_initialization_with_job_rehearsal_flag_true_no_failures(self):
         self.config.fail_with_test_failures = True
-        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
+        job = job = Job(
+            name="rehearse-1234-job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         with self.assertRaises(SystemExit) as rc:
             Report(self.config, job)
         error = rc.exception
         self.assertEqual(error.code, 0)
 
     def test_report_initialization_with_job_rehearsal_flag_true_test_failures(self):
         self.config.fail_with_test_failures = True
-        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
+        job = job = Job(
+            name="rehearse-1234-job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         job.has_test_failures = True
         with self.assertRaises(SystemExit) as rc:
             Report(self.config, job)
@@ -38,7 +66,14 @@ def test_report_initialization_with_job_rehearsal_flag_true_test_failures(self):
 
     def test_report_initialization_with_job_rehearsal_flag_true_pod_failures(self):
         self.config.fail_with_pod_failures = True
-        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
+        job = job = Job(
+            name="rehearse-1234-job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         job.has_pod_failures = True
         with self.assertRaises(SystemExit) as rc:
             Report(self.config, job)

tests/unittests/objects/job/test_fail_with_test_failures_should_not_cause_failure_for_ignored_step.py

@@ -81,6 +81,7 @@ def job(firewatch_config):
         name_safe="amq-broker-interop-aws",
         build_id="1769607768542547968",
         gcs_bucket="test-platform-results",
+        gcs_creds_file=None,
         firewatch_config=firewatch_config,
     )
 

tests/unittests/objects/job/test_firewatch_objects_job.py

@@ -4,7 +4,14 @@
 
 class TestJob(JobBaseTest):
     def test_initialization_with_valid_parameters(self):
-        job = Job("job1", "job1_safe", "123", "bucket1", self.config)
+        job = Job(
+            name="job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         self.assertEqual(job.name, "job1")
         self.assertEqual(job.name_safe, "job1_safe")
         self.assertEqual(job.build_id, "123")

tests/unittests/objects/job/test_firewatch_objects_job_check_is_rehearsal.py

@@ -4,9 +4,23 @@
 
 class TestCheckIsRehearsal(JobBaseTest):
     def test_rehearsal_job_true(self):
-        job = Job("rehearse-1234-job1", "job1_safe", "123", "bucket1", self.config)
+        job = Job(
+            name="rehearse-1234-job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         self.assertTrue(job.is_rehearsal)
 
     def test_rehearsal_job_false(self):
-        job = Job("job1", "job1_safe", "123", "bucket1", self.config)
+        job = Job(
+            name="job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         self.assertFalse(job.is_rehearsal)

tests/unittests/objects/job/test_firewatch_objects_job_find_failures.py

@@ -10,7 +10,14 @@ def test_find_test_failures_no_pod_failures(self):
         temp_dir = tempfile.TemporaryDirectory()
         junit_dir = helpers._get_tmp_junit_dir(tmp_path=temp_dir.name)
         logs_dir = helpers._get_tmp_logs_dir(tmp_path=temp_dir.name)
-        job = Job("job1", "job1_safe", "123", "bucket1", self.config)
+        job = Job(
+            name="rehearse-1234-job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         helpers._create_failed_step_junit(junit_dir=junit_dir)
         failures = job._find_failures(
             junit_dir=junit_dir,
@@ -22,7 +29,14 @@ def test_find_pod_failures_no_test_failures(self):
         temp_dir = tempfile.TemporaryDirectory()
         junit_dir = helpers._get_tmp_junit_dir(tmp_path=temp_dir.name)
         logs_dir = helpers._get_tmp_logs_dir(tmp_path=temp_dir.name)
-        job = Job("job1", "job1_safe", "123", "bucket1", self.config)
+        job = Job(
+            name="rehearse-1234-job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         helpers._create_failed_step_pod(logs_dir=logs_dir)
         failures = job._find_failures(
             junit_dir=junit_dir,
@@ -34,7 +48,14 @@ def test_find_all_failures(self):
         temp_dir = tempfile.TemporaryDirectory()
         junit_dir = helpers._get_tmp_junit_dir(tmp_path=temp_dir.name)
         logs_dir = helpers._get_tmp_logs_dir(tmp_path=temp_dir.name)
-        job = Job("job1", "job1_safe", "123", "bucket1", self.config)
+        job = Job(
+            name="rehearse-1234-job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         helpers._create_failed_step_pod(logs_dir=logs_dir)
         helpers._create_failed_step_junit(junit_dir=junit_dir)
         failures = job._find_failures(
@@ -47,7 +68,14 @@ def test_failures_no_failures(self):
         temp_dir = tempfile.TemporaryDirectory()
         junit_dir = helpers._get_tmp_junit_dir(tmp_path=temp_dir.name)
         logs_dir = helpers._get_tmp_logs_dir(tmp_path=temp_dir.name)
-        job = Job("job1", "job1_safe", "123", "bucket1", self.config)
+        job = Job(
+            name="rehearse-1234-job1",
+            name_safe="job1_safe",
+            build_id="123",
+            gcs_bucket="bucket1",
+            gcs_creds_file=None,
+            firewatch_config=self.config,
+        )
         helpers._create_successful_step_pod(logs_dir=logs_dir)
         helpers._create_successful_step_junit(junit_dir=junit_dir)
         failures = job._find_failures(