Merge branch 'main' into feat-statuscake-webhook

docs/deployment/kubernetes/installation.mdx

@@ -8,23 +8,28 @@ The recommended way to install Keep on Kubernetes is via Helm Chart. <br></br>
 Follow these steps to set it up.
 </Tip>
 
-## Prerequisites
+# Prerequisites
 
-### Helm CLI
+## Helm CLI
 See the [Helm documentation](https://helm.sh/docs/intro/install/) for instructions about installing helm.
 
-### Ingress Controller (Optional)
+## Ingress Controller (Optional)
 <Info>
 You can skip this step if:
 1. You already have **ingress-nginx** installed.
 2. You don't need to expose Keep to the internet/network.
 </Info>
 
-#### Overview
+### Overview
 An ingress controller is essential for managing external access to services in your Kubernetes cluster. It acts as a smart router and load balancer, allowing you to expose multiple services through a single entry point while handling SSL termination and routing rules.
-**Keep works the best with** [ingress-nginx](https://github.com/kubernetes/ingress-nginx) **but you can customize the helm chart for other ingress controllers too.**
 
 
+
+**Keep works best with both** [ingress-nginx](https://github.com/kubernetes/ingress-nginx) **and** [HAProxy Ingress](https://haproxy-ingress.github.io/) **controllers, but you can customize the helm chart for other ingress controllers too.**
+
+
+### Nginx Ingress Controller
+
 #### Check ingress-nginx Installed
 You check if you already have ingress-nginx installed:
 ```bash
@@ -67,6 +72,34 @@ kubectl get configmap -n ingress-nginx ingress-nginx-controller -o yaml | grep a
 allow-snippet-annotations: "true"
 ```
 
+### HAProxy Ingress Controller
+
+#### Install ingress-haproxy
+<Info>
+To read about more installation options, see [haproxy-ingress installation docs](https://haproxy-ingress.github.io/docs/getting-started/).
+</Info>
+
+```bash
+# simplest way to install
+helm upgrade --install haproxy-ingress haproxy-ingress \
+  --repo https://haproxy-ingress.github.io/charts \
+  --namespace ingress-haproxy --create-namespace
+```
+
+Verify installation:
+```bash
+kubectl get ingressclass
+NAME      CONTROLLER                      PARAMETERS   AGE
+haproxy   haproxy-ingress.github.io/controller        <none>        4h19m
+```
+
+Verify if controller is running:
+```bash
+kubectl get pods -n ingress-haproxy -l app.kubernetes.io/instance=haproxy-ingress
+NAME                                READY   STATUS    RESTARTS   AGE
+haproxy-ingress-controller-x4n2z   1/1     Running   0          4h19m
+```
+
 ## Installation
 
 ### With Ingress-NGINX (Recommended)
@@ -79,7 +112,17 @@ helm repo add keephq https://keephq.github.io/helm-charts
 helm install keep keephq/keep -n keep --create-namespace
 ```
 
-### Without Ingress-NGINX (Not Recommended)
+### With Ingress-HAProxy (Recommended)
+
+```bash
+# Add the Helm repository
+helm repo add keephq https://keephq.github.io/helm-charts
+
+# Install Keep with ingress enabled
+helm install keep keephq/keep -n keep --create-namespace --set global.ingress.className=haproxy
+```
+
+### Without Ingress (Not Recommended)
 
 ```bash
 # Add the Helm repository

keep-ui/package.json

@@ -80,7 +80,7 @@
     "convert-source-map": "^1.9.0",
     "cookie": "^0.7.0",
     "cosmiconfig": "^7.1.0",
-    "cross-spawn": "^7.0.3",
+    "cross-spawn": "^7.0.5",
     "crypto-js": "^4.2.0",
     "css-unit-converter": "^1.1.2",
     "cssesc": "^3.0.0",

keep/api/utils/email_utils.py

@@ -33,17 +33,22 @@ class EmailTemplates(enum.Enum):
 FROM_EMAIL = config("SENDGRID_FROM_EMAIL", default="[email protected]")
 API_KEY = config("SENDGRID_API_KEY", default=None)
 CC = config("SENDGRID_CC", default="[email protected]")
+KEEP_EMAILS_ENABLED = config("KEEP_EMAILS_ENABLED", default=False, cast=bool)
 
 
 def send_email(
     to_email: str,
     template_id: EmailTemplates,
     **kwargs,
-):
+) -> bool:
+    if not KEEP_EMAILS_ENABLED:
+        logger.debug("Emails are disabled, skipping sending email")
+        return False
+
     # that's ok on OSS
     if not API_KEY:
         logger.debug("No SendGrid API key, skipping sending email")
-        return
+        return False
 
     message = Mail(from_email=FROM_EMAIL, to_emails=to_email)
     message.template_id = template_id.value
@@ -55,6 +60,7 @@ def send_email(
         sg = SendGridAPIClient(API_KEY)
         sg.send(message)
         logger.info(f"Email sent to {to_email} with template {template_id}")
+        return True
     except Exception as e:
         logger.error(
             f"Failed to send email to {to_email} with template {template_id}: {e}"

keep/functions/__init__.py

@@ -115,7 +115,7 @@ def to_timestamp(dt: datetime.datetime | str = "") -> int:
 
 
 def datetime_compare(t1: datetime = None, t2: datetime = None) -> float:
-    if t1 is None or t2 is None:
+    if not t1 or not t2:
         return 0
     diff = (t1 - t2).total_seconds() / 3600
     return diff

keep/iohandler/iohandler.py

@@ -330,7 +330,17 @@ def _parse(self, tree):
                         .replace("\n", "\\n")
                     )
                     t = self._encode_single_quotes_in_double_quotes(t)
-                    tree = ast.parse(t)
+                    try:
+                        tree = ast.parse(t)
+                    except Exception:
+                        # For strings where ' is used as the delimeter and we failed to escape all ' in the string
+                        # @tb: again, this is not ideal but it's best effort...
+                        t = (
+                            t.replace("('", '("')
+                            .replace("')", '")')
+                            .replace("',", '",')
+                        )
+                        tree = ast.parse(t)
             else:
                 # for strings such as "45%\n", we need to escape
                 tree = ast.parse(token.encode("unicode_escape"))

keep/providers/servicenow_provider/servicenow_provider.py

@@ -45,6 +45,25 @@ class ServicenowProviderAuthConfig:
         }
     )
 
+    # @tb: based on this https://www.servicenow.com/community/developer-blog/oauth-2-0-with-inbound-rest/ba-p/2278926
+    client_id: str = dataclasses.field(
+        metadata={
+            "required": False,
+            "description": "The client ID to use OAuth 2.0 based authentication",
+            "sensitive": False,
+        },
+        default="",
+    )
+
+    client_secret: str = dataclasses.field(
+        metadata={
+            "required": False,
+            "description": "The client secret to use OAuth 2.0 based authentication",
+            "sensitive": True,
+        },
+        default="",
+    )
+
 
 class ServicenowProvider(BaseTopologyProvider):
     """Manage ServiceNow tickets."""
@@ -65,6 +84,33 @@ def __init__(
         self, context_manager: ContextManager, provider_id: str, config: ProviderConfig
     ):
         super().__init__(context_manager, provider_id, config)
+        self._access_token = None
+        if (
+            self.authentication_config.client_id
+            and self.authentication_config.client_secret
+        ):
+            url = f"{self.authentication_config.service_now_base_url}/oauth_token.do"
+            payload = {
+                "grant_type": "password",
+                "username": self.authentication_config.username,
+                "password": self.authentication_config.password,
+                "client_id": self.authentication_config.client_id,
+                "client_secret": self.authentication_config.client_secret,
+            }
+            response = requests.post(
+                url,
+                json=payload,
+            )
+            if response.ok:
+                self._access_token = response.json().get("access_token")
+            else:
+                self.logger.error(
+                    "Failed to get access token",
+                    extra={
+                        "response": response.text,
+                        "status_code": response.status_code,
+                    },
+                )
 
     @property
     def service_now_base_url(self):
@@ -80,15 +126,23 @@ def validate_scopes(self):
         try:
             self.logger.info("Validating ServiceNow scopes")
             url = f"{self.authentication_config.service_now_base_url}/api/now/table/sys_user_role?sysparm_query=user_name={self.authentication_config.username}"
-            response = requests.get(
-                url,
-                auth=HTTPBasicAuth(
-                    self.authentication_config.username,
-                    self.authentication_config.password,
-                ),
-                verify=False,
-                timeout=10,
-            )
+            if self._access_token:
+                response = requests.get(
+                    url,
+                    headers={"Authorization": f"Bearer {self._access_token}"},
+                    verify=False,
+                    timeout=10,
+                )
+            else:
+                response = requests.get(
+                    url,
+                    auth=HTTPBasicAuth(
+                        self.authentication_config.username,
+                        self.authentication_config.password,
+                    ),
+                    verify=False,
+                    timeout=10,
+                )
 
             try:
                 response.raise_for_status()
@@ -131,13 +185,57 @@ def validate_config(self):
             **self.config.authentication
         )
 
+    def _query(
+        self,
+        table_name: str,
+        get_incidents: bool = False,
+        incident_id: str = None,
+        **kwargs: dict,
+    ):
+        request_url = f"{self.authentication_config.service_now_base_url}/api/now/table/{table_name}"
+        headers = {"Content-Type": "application/json", "Accept": "application/json"}
+        auth = (
+            (
+                self.authentication_config.username,
+                self.authentication_config.password,
+            )
+            if not self._access_token
+            else None
+        )
+        if self._access_token:
+            headers["Authorization"] = f"Bearer {self._access_token}"
+
+        response = requests.get(
+            request_url,
+            headers=headers,
+            auth=auth,
+            params=kwargs,
+            verify=False,
+            timeout=10,
+        )
+
+        if not response.ok:
+            self.logger.error(
+                f"Failed to query {table_name}",
+                extra={"status_code": response.status_code, "response": response.text},
+            )
+            return []
+
+        return response.json().get("result", [])
+
     def pull_topology(self) -> list[TopologyServiceInDto]:
-        # TODO: in scable, we'll need to use pagination around here
+        # TODO: in scale, we'll need to use pagination around here
         headers = {"Content-Type": "application/json", "Accept": "application/json"}
         auth = (
-            self.authentication_config.username,
-            self.authentication_config.password,
+            (
+                self.authentication_config.username,
+                self.authentication_config.password,
+            )
+            if not self._access_token
+            else None
         )
+        if self._access_token:
+            headers["Authorization"] = f"Bearer {self._access_token}"
         topology = []
         self.logger.info(
             "Pulling topology", extra={"tenant_id": self.context_manager.tenant_id}
@@ -253,7 +351,16 @@ def dispose(self):
     def _notify(self, table_name: str, payload: dict = {}, **kwargs: dict):
         # Create ticket
         headers = {"Content-Type": "application/json", "Accept": "application/json"}
-
+        auth = (
+            (
+                self.authentication_config.username,
+                self.authentication_config.password,
+            )
+            if not self._access_token
+            else None
+        )
+        if self._access_token:
+            headers["Authorization"] = f"Bearer {self._access_token}"
         # otherwise, create the ticket
         if not table_name:
             raise ProviderException("Table name is required")
@@ -271,10 +378,7 @@ def _notify(self, table_name: str, payload: dict = {}, **kwargs: dict):
         # HTTP request
         response = requests.post(
             url,
-            auth=(
-                self.authentication_config.username,
-                self.authentication_config.password,
-            ),
+            auth=auth,
             headers=headers,
             data=json.dumps(payload),
             verify=False,
@@ -302,12 +406,20 @@ def _notify(self, table_name: str, payload: dict = {}, **kwargs: dict):
     def _notify_update(self, table_name: str, ticket_id: str, fingerprint: str):
         url = f"{self.authentication_config.service_now_base_url}/api/now/table/{table_name}/{ticket_id}"
         headers = {"Content-Type": "application/json", "Accept": "application/json"}
-        response = requests.get(
-            url,
-            auth=(
+        auth = (
+            (
                 self.authentication_config.username,
                 self.authentication_config.password,
-            ),
+            )
+            if self._access_token
+            else None
+        )
+        if self._access_token:
+            headers["Authorization"] = f"Bearer {self._access_token}"
+
+        response = requests.get(
+            url,
+            auth=auth,
             headers=headers,
             verify=False,
         )