From b3b1e066eb09a95c4e8d5610e9cfa3760ed9f15a Mon Sep 17 00:00:00 2001 From: Tal Date: Sat, 16 Nov 2024 18:26:35 +0200 Subject: [PATCH 1/3] chore: servicenow should support oauth to obtain token (#2502) --- keep/api/utils/email_utils.py | 10 +- keep/functions/__init__.py | 2 +- keep/iohandler/iohandler.py | 12 +- .../servicenow_provider.py | 154 +++++++++++++++--- keep/workflowmanager/workflowscheduler.py | 48 +++--- 5 files changed, 176 insertions(+), 50 deletions(-) diff --git a/keep/api/utils/email_utils.py b/keep/api/utils/email_utils.py index d76001495..34e708fba 100644 --- a/keep/api/utils/email_utils.py +++ b/keep/api/utils/email_utils.py @@ -33,17 +33,22 @@ class EmailTemplates(enum.Enum): FROM_EMAIL = config("SENDGRID_FROM_EMAIL", default="platform@keephq.dev") API_KEY = config("SENDGRID_API_KEY", default=None) CC = config("SENDGRID_CC", default="founders@keephq.dev") +KEEP_EMAILS_ENABLED = config("KEEP_EMAILS_ENABLED", default=False, cast=bool) def send_email( to_email: str, template_id: EmailTemplates, **kwargs, -): +) -> bool: + if not KEEP_EMAILS_ENABLED: + logger.debug("Emails are disabled, skipping sending email") + return False + # that's ok on OSS if not API_KEY: logger.debug("No SendGrid API key, skipping sending email") - return + return False message = Mail(from_email=FROM_EMAIL, to_emails=to_email) message.template_id = template_id.value @@ -55,6 +60,7 @@ def send_email( sg = SendGridAPIClient(API_KEY) sg.send(message) logger.info(f"Email sent to {to_email} with template {template_id}") + return True except Exception as e: logger.error( f"Failed to send email to {to_email} with template {template_id}: {e}" diff --git a/keep/functions/__init__.py b/keep/functions/__init__.py index 6a04469c9..503021a8e 100644 --- a/keep/functions/__init__.py +++ b/keep/functions/__init__.py @@ -115,7 +115,7 @@ def to_timestamp(dt: datetime.datetime | str = "") -> int: def datetime_compare(t1: datetime = None, t2: datetime = None) -> float: - if t1 is None or t2 is None: + if not t1 or not t2: return 0 diff = (t1 - t2).total_seconds() / 3600 return diff diff --git a/keep/iohandler/iohandler.py b/keep/iohandler/iohandler.py index 88a857aff..7f0f22fec 100644 --- a/keep/iohandler/iohandler.py +++ b/keep/iohandler/iohandler.py @@ -330,7 +330,17 @@ def _parse(self, tree): .replace("\n", "\\n") ) t = self._encode_single_quotes_in_double_quotes(t) - tree = ast.parse(t) + try: + tree = ast.parse(t) + except Exception: + # For strings where ' is used as the delimeter and we failed to escape all ' in the string + # @tb: again, this is not ideal but it's best effort... + t = ( + t.replace("('", '("') + .replace("')", '")') + .replace("',", '",') + ) + tree = ast.parse(t) else: # for strings such as "45%\n", we need to escape tree = ast.parse(token.encode("unicode_escape")) diff --git a/keep/providers/servicenow_provider/servicenow_provider.py b/keep/providers/servicenow_provider/servicenow_provider.py index c9c1ee967..f1afd1776 100644 --- a/keep/providers/servicenow_provider/servicenow_provider.py +++ b/keep/providers/servicenow_provider/servicenow_provider.py @@ -45,6 +45,25 @@ class ServicenowProviderAuthConfig: } ) + # @tb: based on this https://www.servicenow.com/community/developer-blog/oauth-2-0-with-inbound-rest/ba-p/2278926 + client_id: str = dataclasses.field( + metadata={ + "required": False, + "description": "The client ID to use OAuth 2.0 based authentication", + "sensitive": False, + }, + default="", + ) + + client_secret: str = dataclasses.field( + metadata={ + "required": False, + "description": "The client secret to use OAuth 2.0 based authentication", + "sensitive": True, + }, + default="", + ) + class ServicenowProvider(BaseTopologyProvider): """Manage ServiceNow tickets.""" @@ -65,6 +84,33 @@ def __init__( self, context_manager: ContextManager, provider_id: str, config: ProviderConfig ): super().__init__(context_manager, provider_id, config) + self._access_token = None + if ( + self.authentication_config.client_id + and self.authentication_config.client_secret + ): + url = f"{self.authentication_config.service_now_base_url}/oauth_token.do" + payload = { + "grant_type": "password", + "username": self.authentication_config.username, + "password": self.authentication_config.password, + "client_id": self.authentication_config.client_id, + "client_secret": self.authentication_config.client_secret, + } + response = requests.post( + url, + json=payload, + ) + if response.ok: + self._access_token = response.json().get("access_token") + else: + self.logger.error( + "Failed to get access token", + extra={ + "response": response.text, + "status_code": response.status_code, + }, + ) @property def service_now_base_url(self): @@ -80,15 +126,23 @@ def validate_scopes(self): try: self.logger.info("Validating ServiceNow scopes") url = f"{self.authentication_config.service_now_base_url}/api/now/table/sys_user_role?sysparm_query=user_name={self.authentication_config.username}" - response = requests.get( - url, - auth=HTTPBasicAuth( - self.authentication_config.username, - self.authentication_config.password, - ), - verify=False, - timeout=10, - ) + if self._access_token: + response = requests.get( + url, + headers={"Authorization": f"Bearer {self._access_token}"}, + verify=False, + timeout=10, + ) + else: + response = requests.get( + url, + auth=HTTPBasicAuth( + self.authentication_config.username, + self.authentication_config.password, + ), + verify=False, + timeout=10, + ) try: response.raise_for_status() @@ -131,13 +185,57 @@ def validate_config(self): **self.config.authentication ) + def _query( + self, + table_name: str, + get_incidents: bool = False, + incident_id: str = None, + **kwargs: dict, + ): + request_url = f"{self.authentication_config.service_now_base_url}/api/now/table/{table_name}" + headers = {"Content-Type": "application/json", "Accept": "application/json"} + auth = ( + ( + self.authentication_config.username, + self.authentication_config.password, + ) + if not self._access_token + else None + ) + if self._access_token: + headers["Authorization"] = f"Bearer {self._access_token}" + + response = requests.get( + request_url, + headers=headers, + auth=auth, + params=kwargs, + verify=False, + timeout=10, + ) + + if not response.ok: + self.logger.error( + f"Failed to query {table_name}", + extra={"status_code": response.status_code, "response": response.text}, + ) + return [] + + return response.json().get("result", []) + def pull_topology(self) -> list[TopologyServiceInDto]: - # TODO: in scable, we'll need to use pagination around here + # TODO: in scale, we'll need to use pagination around here headers = {"Content-Type": "application/json", "Accept": "application/json"} auth = ( - self.authentication_config.username, - self.authentication_config.password, + ( + self.authentication_config.username, + self.authentication_config.password, + ) + if not self._access_token + else None ) + if self._access_token: + headers["Authorization"] = f"Bearer {self._access_token}" topology = [] self.logger.info( "Pulling topology", extra={"tenant_id": self.context_manager.tenant_id} @@ -253,7 +351,16 @@ def dispose(self): def _notify(self, table_name: str, payload: dict = {}, **kwargs: dict): # Create ticket headers = {"Content-Type": "application/json", "Accept": "application/json"} - + auth = ( + ( + self.authentication_config.username, + self.authentication_config.password, + ) + if not self._access_token + else None + ) + if self._access_token: + headers["Authorization"] = f"Bearer {self._access_token}" # otherwise, create the ticket if not table_name: raise ProviderException("Table name is required") @@ -271,10 +378,7 @@ def _notify(self, table_name: str, payload: dict = {}, **kwargs: dict): # HTTP request response = requests.post( url, - auth=( - self.authentication_config.username, - self.authentication_config.password, - ), + auth=auth, headers=headers, data=json.dumps(payload), verify=False, @@ -302,12 +406,20 @@ def _notify(self, table_name: str, payload: dict = {}, **kwargs: dict): def _notify_update(self, table_name: str, ticket_id: str, fingerprint: str): url = f"{self.authentication_config.service_now_base_url}/api/now/table/{table_name}/{ticket_id}" headers = {"Content-Type": "application/json", "Accept": "application/json"} - response = requests.get( - url, - auth=( + auth = ( + ( self.authentication_config.username, self.authentication_config.password, - ), + ) + if self._access_token + else None + ) + if self._access_token: + headers["Authorization"] = f"Bearer {self._access_token}" + + response = requests.get( + url, + auth=auth, headers=headers, verify=False, ) diff --git a/keep/workflowmanager/workflowscheduler.py b/keep/workflowmanager/workflowscheduler.py index 5ed3d139c..fe402a32e 100644 --- a/keep/workflowmanager/workflowscheduler.py +++ b/keep/workflowmanager/workflowscheduler.py @@ -583,32 +583,30 @@ def _finish_workflow_execution( or previous_execution.status != WorkflowStatus.ERROR.value ): workflow = get_workflow_db(tenant_id=tenant_id, workflow_id=workflow_id) - self.logger.info( - f"Sending email to {workflow.created_by} for failed workflow {workflow_id}" - ) - - # send the email (commented out) try: - # from keep.api.core.config import config - # from keep.api.utils.email_utils import EmailTemplates, send_email - # TODO - should be handled - # keep_platform_url = config( - # "KEEP_PLATFORM_URL", default="https://platform.keephq.dev" - # ) - # error_logs_url = f"{keep_platform_url}/workflows/{workflow_id}/runs/{workflow_execution_id}" - # send_email( - # to_email=workflow.created_by, - # template_id=EmailTemplates.WORKFLOW_RUN_FAILED, - # workflow_id=workflow_id, - # workflow_name=workflow.name, - # workflow_execution_id=workflow_execution_id, - # error=error, - # url=error_logs_url, - # ) - # self.logger.info( - # f"Email sent to {workflow.created_by} for failed workflow {workflow_id}" - # ) - pass + from keep.api.core.config import config + from keep.api.utils.email_utils import EmailTemplates, send_email + + keep_platform_url = config( + "KEEP_PLATFORM_URL", default="https://platform.keephq.dev" + ) + error_logs_url = f"{keep_platform_url}/workflows/{workflow_id}/runs/{workflow_execution_id}" + self.logger.debug( + f"Sending email to {workflow.created_by} for failed workflow {workflow_id}" + ) + email_sent = send_email( + to_email=workflow.created_by, + template_id=EmailTemplates.WORKFLOW_RUN_FAILED, + workflow_id=workflow_id, + workflow_name=workflow.name, + workflow_execution_id=workflow_execution_id, + error=error, + url=error_logs_url, + ) + if email_sent: + self.logger.info( + f"Email sent to {workflow.created_by} for failed workflow {workflow_id}" + ) except Exception as e: self.logger.error( f"Failed to send email to {workflow.created_by} for failed workflow {workflow_id}: {e}" From efacd907e1bdee9282613e7411127addfd5234ca Mon Sep 17 00:00:00 2001 From: Shahar Glazner Date: Sat, 16 Nov 2024 18:47:28 +0200 Subject: [PATCH 2/3] feat(docs): add docs for haproxy (#2504) --- docs/deployment/kubernetes/installation.mdx | 55 ++++++++++++++++++--- pyproject.toml | 2 +- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/docs/deployment/kubernetes/installation.mdx b/docs/deployment/kubernetes/installation.mdx index 9680df532..594df7c3b 100644 --- a/docs/deployment/kubernetes/installation.mdx +++ b/docs/deployment/kubernetes/installation.mdx @@ -8,23 +8,28 @@ The recommended way to install Keep on Kubernetes is via Helm Chart.

Follow these steps to set it up. -## Prerequisites +# Prerequisites -### Helm CLI +## Helm CLI See the [Helm documentation](https://helm.sh/docs/intro/install/) for instructions about installing helm. -### Ingress Controller (Optional) +## Ingress Controller (Optional) You can skip this step if: 1. You already have **ingress-nginx** installed. 2. You don't need to expose Keep to the internet/network. -#### Overview +### Overview An ingress controller is essential for managing external access to services in your Kubernetes cluster. It acts as a smart router and load balancer, allowing you to expose multiple services through a single entry point while handling SSL termination and routing rules. -**Keep works the best with** [ingress-nginx](https://github.com/kubernetes/ingress-nginx) **but you can customize the helm chart for other ingress controllers too.** + +**Keep works best with both** [ingress-nginx](https://github.com/kubernetes/ingress-nginx) **and** [HAProxy Ingress](https://haproxy-ingress.github.io/) **controllers, but you can customize the helm chart for other ingress controllers too.** + + +### Nginx Ingress Controller + #### Check ingress-nginx Installed You check if you already have ingress-nginx installed: ```bash @@ -67,6 +72,34 @@ kubectl get configmap -n ingress-nginx ingress-nginx-controller -o yaml | grep a allow-snippet-annotations: "true" ``` +### HAProxy Ingress Controller + +#### Install ingress-haproxy + +To read about more installation options, see [haproxy-ingress installation docs](https://haproxy-ingress.github.io/docs/getting-started/). + + +```bash +# simplest way to install +helm upgrade --install haproxy-ingress haproxy-ingress \ + --repo https://haproxy-ingress.github.io/charts \ + --namespace ingress-haproxy --create-namespace +``` + +Verify installation: +```bash +kubectl get ingressclass +NAME CONTROLLER PARAMETERS AGE +haproxy haproxy-ingress.github.io/controller 4h19m +``` + +Verify if controller is running: +```bash +kubectl get pods -n ingress-haproxy -l app.kubernetes.io/instance=haproxy-ingress +NAME READY STATUS RESTARTS AGE +haproxy-ingress-controller-x4n2z 1/1 Running 0 4h19m +``` + ## Installation ### With Ingress-NGINX (Recommended) @@ -79,7 +112,17 @@ helm repo add keephq https://keephq.github.io/helm-charts helm install keep keephq/keep -n keep --create-namespace ``` -### Without Ingress-NGINX (Not Recommended) +### With Ingress-HAProxy (Recommended) + +```bash +# Add the Helm repository +helm repo add keephq https://keephq.github.io/helm-charts + +# Install Keep with ingress enabled +helm install keep keephq/keep -n keep --create-namespace --set global.ingress.className=haproxy +``` + +### Without Ingress (Not Recommended) ```bash # Add the Helm repository diff --git a/pyproject.toml b/pyproject.toml index 79267d83d..6ed27766c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "keep" -version = "0.28.10" +version = "0.29.0" description = "Alerting. for developers, by developers." authors = ["Keep Alerting LTD"] readme = "README.md" From 92ca1f59236ad422346ad98c8f1131a1e0328f93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Nov 2024 19:09:29 +0200 Subject: [PATCH 3/3] chore(deps): bump cross-spawn from 7.0.3 to 7.0.5 in /keep-ui (#2505) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- keep-ui/package-lock.json | 7 ++++--- keep-ui/package.json | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/keep-ui/package-lock.json b/keep-ui/package-lock.json index 6e3563b45..295bfb0fe 100644 --- a/keep-ui/package-lock.json +++ b/keep-ui/package-lock.json @@ -79,7 +79,7 @@ "convert-source-map": "^1.9.0", "cookie": "^0.7.0", "cosmiconfig": "^7.1.0", - "cross-spawn": "^7.0.3", + "cross-spawn": "^7.0.5", "crypto-js": "^4.2.0", "css-unit-converter": "^1.1.2", "cssesc": "^3.0.0", @@ -10222,8 +10222,9 @@ } }, "node_modules/cross-spawn": { - "version": "7.0.3", - "license": "MIT", + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.5.tgz", + "integrity": "sha512-ZVJrKKYunU38/76t0RMOulHOnUcbU9GbpWKAOZ0mhjr7CX6FVrH+4FrAapSOekrgFQ3f/8gwMEuIft0aKq6Hug==", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", diff --git a/keep-ui/package.json b/keep-ui/package.json index 62d468ef5..09dedf974 100644 --- a/keep-ui/package.json +++ b/keep-ui/package.json @@ -80,7 +80,7 @@ "convert-source-map": "^1.9.0", "cookie": "^0.7.0", "cosmiconfig": "^7.1.0", - "cross-spawn": "^7.0.3", + "cross-spawn": "^7.0.5", "crypto-js": "^4.2.0", "css-unit-converter": "^1.1.2", "cssesc": "^3.0.0",