diff --git a/README.md b/README.md index 32bbd1f9..9b7dfad1 100644 --- a/README.md +++ b/README.md @@ -52,16 +52,32 @@ The add-on uses Splunk encrypted secrets storage, so admins require `admin_all_o 4. Under Log Access Keys, click Add Key > Add Write Key (required for alert action). 5. Optionally, click the pencil icon to rename the keys. +### SentinelOne Platform with Singularity Data Lake +To get the AuthN API token follow the below mentioned details: +1. Login into the SentinelOne console, click on your User Name > My User +![My User Page](README_images/my_user.png) +2. Click on Actions > API Token Operations > Generate / Regenrate API token. + - If you are generating the API Token for the first time then you will have the Generate API Token option. Otherwise you will find the generate API Token. +![Token generation](README_images/generate_token.png) +3. Copy the API Token and save it for configuration. + ### Splunk 1. In Splunk, open the Add-on -![Configuring DataSet Account](README_images/setup_account.png) +![Configuring DataSet Account](README_images/acc_details_new.png) 2. On the configuration > account tab: - Click Add - Enter a user-friendly account name. For multiple accounts, the account name can be used in queries (more details below). - Enter the full URL noted above (e.g.: `https://app.scalyr.com`, `https://xdr.us1.sentinelone.net` or `https://xdr.eu1.sentinelone.net`). -- Enter the DataSet read key from above (required for searching) +- Enter Tenant value, it can be True/False/Blank. If set to True, the queries will run for the entire Tenant or if set to False, provide Account IDs as a comma separated values to run searches in those specific accounts. Leave it blank if you are not trying use the Tenant level searches. +- Provide the comma seperated Account Ids, if Tenant is False. eg: 1234567890,9876543210. +- Enter the AuthN API Token First part which includes first 220 characters. +- Enter the AuthN API Token Second part which includes remaining characters. + - Use this command to prepare both parts of AuthN API token: + `read -p "Enter Token: " input_string && echo "Part1: $(echo $input_string | cut -c 1-220)"; echo "Part2: $(echo $input_string | cut -c 221-)"` + - Reason for creating 2 parts of AuthN Token: Splunk Storage Manager has a limitation of storing only 256 characters of encrypted data from inputs. And the AuthN Token can have length <256, hence its split into 2 parts, the first one is encrypted (first 220 chars) and the second one is not. As we are encrypting most of the Token, its use is safe. +- Enter the DataSet read key from above (required for searching), please ignore this if AuthN token value is provided. - Enter the DataSet write key from above (only required for alert actions). - Click Save diff --git a/README_images/acc_details_new.png b/README_images/acc_details_new.png new file mode 100644 index 00000000..c603efcb Binary files /dev/null and b/README_images/acc_details_new.png differ diff --git a/README_images/generate_token.png b/README_images/generate_token.png new file mode 100644 index 00000000..45442ddb Binary files /dev/null and b/README_images/generate_token.png differ diff --git a/README_images/my_user.png b/README_images/my_user.png new file mode 100644 index 00000000..98641f2d Binary files /dev/null and b/README_images/my_user.png differ diff --git a/TA_dataset/README/ta_dataset_settings.conf.spec b/TA_dataset/README/ta_dataset_settings.conf.spec index 2404f880..28bfbf31 100644 --- a/TA_dataset/README/ta_dataset_settings.conf.spec +++ b/TA_dataset/README/ta_dataset_settings.conf.spec @@ -1,5 +1,9 @@ [account] url = +tenant = +account_ids = +authn_token_part_one = +authn_token_part_two = dataset_log_read_access_key = dataset_log_write_access_key = diff --git a/TA_dataset/bin/dataset_alerts.py b/TA_dataset/bin/dataset_alerts.py index 79262891..a97f03f5 100644 --- a/TA_dataset/bin/dataset_alerts.py +++ b/TA_dataset/bin/dataset_alerts.py @@ -12,6 +12,7 @@ get_acct_info, get_log_level, get_proxy, + get_tenant_related_payload, get_url, relative_to_epoch, ) @@ -110,7 +111,13 @@ def stream_events(self, inputs, ew): ds_headers = { "Authorization": "Bearer " + acct_dict[ds_acct]["ds_api_key"] } - + tenant_related_payload = get_tenant_related_payload( + acct_dict.get(ds_acct) + ) + ds_payload.update(tenant_related_payload) + logger.debug( + "ds payload in power query stream events = {}".format(ds_payload) + ) # Create checkpointer checkpoint = checkpointer.KVStoreCheckpointer( input_name, session_key, APP_NAME diff --git a/TA_dataset/bin/dataset_api.py b/TA_dataset/bin/dataset_api.py index b57a1ea8..0af8fb5f 100644 --- a/TA_dataset/bin/dataset_api.py +++ b/TA_dataset/bin/dataset_api.py @@ -53,6 +53,8 @@ def ds_lrq_log_query( filter_expr, limit, proxy, + logger, + tenant_related_payload, ): client = AuthenticatedClient( base_url=base_url, token=api_key, proxy=convert_proxy(proxy) @@ -63,11 +65,21 @@ def ds_lrq_log_query( end_time=end_time, log=LogAttributes(filter_=filter_expr, limit=limit), ) - return ds_lrq_run_loop(client=client, body=body) + body.additional_properties = tenant_related_payload + return ds_lrq_run_loop(logger, client=client, body=body) # Executes Dataset LongRunningQuery using PowerQuery language -def ds_lrq_power_query(base_url, api_key, start_time, end_time, query, proxy): +def ds_lrq_power_query( + base_url, + api_key, + start_time, + end_time, + query, + proxy, + logger, + tenant_related_payload, +): client = AuthenticatedClient( base_url=base_url, token=api_key, proxy=convert_proxy(proxy) ) @@ -77,7 +89,8 @@ def ds_lrq_power_query(base_url, api_key, start_time, end_time, query, proxy): end_time=end_time, pq=PQAttributes(query=query), ) - return ds_lrq_run_loop(client=client, body=body) + body.additional_properties = tenant_related_payload + return ds_lrq_run_loop(logger, client=client, body=body) # Executes Dataset LongRunningQuery to fetch facet values @@ -90,6 +103,8 @@ def ds_lrq_facet_values( name, max_values, proxy, + logger, + tenant_related_payload, ): client = AuthenticatedClient( base_url=base_url, token=api_key, proxy=convert_proxy(proxy) @@ -102,17 +117,24 @@ def ds_lrq_facet_values( filter_=filter, name=name, max_values=max_values ), ) - return ds_lrq_run_loop(client=client, body=body) + body.additional_properties = tenant_related_payload + return ds_lrq_run_loop(logger, client=client, body=body) # Executes LRQ run loop of launch-ping-remove API requests until the query completes # with a result # Returns tuple - value, error message def ds_lrq_run_loop( - client: AuthenticatedClient, body: PostQueriesLaunchQueryRequestBody + log, + client: AuthenticatedClient, + body: PostQueriesLaunchQueryRequestBody, ): + """ + The log parameter assists in generating logs with the chosen level + in the index="_internal". This helps in debugging. + """ body.query_priority = PostQueriesLaunchQueryRequestBodyQueryPriority.HIGH - response = post_queries.sync_detailed(client=client, json_body=body) + response = post_queries.sync_detailed(client=client, json_body=body, logger=log) logger().debug(response) result = response.parsed if result: @@ -242,7 +264,7 @@ def parse_query(ds_columns, match_list, sessions): if ds_columns is None: session_key = match_list["session"] - for session_entry, session_dict in sessions.items(): + for session_entry, session_dict in list(sessions.items()): if session_entry == session_key: for key in session_dict: ds_event_dict[key] = session_dict[key] diff --git a/TA_dataset/bin/dataset_common.py b/TA_dataset/bin/dataset_common.py index 0543029f..87b842c3 100644 --- a/TA_dataset/bin/dataset_common.py +++ b/TA_dataset/bin/dataset_common.py @@ -167,9 +167,16 @@ def get_acct_info(self, logger, account=None): for conf in confs: acct_dict[conf.name] = {} acct_dict[conf.name]["base_url"] = conf.url - acct_dict[conf.name]["ds_api_key"] = get_token( - self, conf.name, "read", logger + acct_dict[conf.name]["ds_api_key"] = get_token_from_config( + self, conf, conf.name, logger ) + if hasattr(conf, "tenant"): + tenant_value = get_tenant_value(conf, logger) + acct_dict[conf.name]["tenant"] = tenant_value + if not tenant_value: + acct_dict[conf.name]["account_ids"] = get_account_ids( + conf, logger + ) except Exception as e: msg = "Error retrieving add-on settings, error = {}".format(e) logger.error(msg + " - %s", e, exc_info=True) @@ -182,9 +189,16 @@ def get_acct_info(self, logger, account=None): conf = self.service.confs[conf_name][entry] acct_dict[entry] = {} acct_dict[entry]["base_url"] = conf.url - acct_dict[entry]["ds_api_key"] = get_token( - self, entry, "read", logger + acct_dict[entry]["ds_api_key"] = get_token_from_config( + self, conf, entry, logger ) + if hasattr(conf, "tenant"): + tenant_value = get_tenant_value(conf, logger) + acct_dict[entry]["tenant"] = tenant_value + if not tenant_value: + acct_dict[entry]["account_ids"] = get_account_ids( + conf, logger + ) except Exception as e: msg = "Error retrieving account settings, error = {}".format(e) logger.error(msg + " - %s", e, exc_info=True) @@ -197,9 +211,16 @@ def get_acct_info(self, logger, account=None): for conf in confs: acct_dict[conf.name] = {} acct_dict[conf.name]["base_url"] = conf.url - acct_dict[conf.name]["ds_api_key"] = get_token( - self, conf.name, "read", logger + acct_dict[conf.name]["ds_api_key"] = get_token_from_config( + self, conf, conf.name, logger ) + if hasattr(conf, "tenant"): + tenant_value = get_tenant_value(conf, logger) + acct_dict[conf.name]["tenant"] = tenant_value + if not tenant_value: + acct_dict[conf.name]["account_ids"] = get_account_ids( + conf, logger + ) break except Exception as e: msg = ( @@ -212,7 +233,63 @@ def get_acct_info(self, logger, account=None): return acct_dict -def get_token(self, account, rw, logger): +def get_tenant_value(conf, logger): + tenant_value = conf.tenant + tenant_value = tenant_value.strip() + logger.debug("The provided tenant value in config is {}".format(tenant_value)) + if tenant_value.lower() == "false" or tenant_value.lower() == "0": + return False + return True + + +def get_account_ids(conf, logger): + account_ids_array = [] + if hasattr(conf, "account_ids"): + account_ids_conf = conf.account_ids + account_ids_conf = account_ids_conf.strip() + if account_ids_conf: + account_ids_array = account_ids_conf.split(",") + logger.debug(f"the provided account ids in config: {account_ids_array}") + if not account_ids_array: + raise Exception( + "Tenant is false, so please provide the valid comma-separated account IDs" + " in the account configuration page." + ) + return account_ids_array + + +def get_token_from_config(self, conf, name, logger): + authn_token = "" + if hasattr(conf, "authn_token_part_one"): + logger.debug("The AuthN api token first part was available") + first_half = get_token(self, name, "authn", logger, "authn_token_part_one") + authn_token += first_half + if hasattr(conf, "authn_token_part_two"): + logger.debug("The AuthN api token second part was available") + second_part = conf.authn_token_part_two + authn_token += second_part + if not hasattr(conf, "authn_token_part_one") and not hasattr( + conf, "authn_token_part_two" + ): + logger.debug("The AuthN api token was not available") + return get_token(self, name, "read", logger) + + return authn_token + + +def get_tenant_related_payload(ds_acct): + if ds_acct.get("tenant") is not None: + tenant_value = ds_acct.get("tenant") + if tenant_value: + return {"tenant": True} + return { + "tenant": False, + "accountIds": ds_acct["account_ids"], + } + return {} + + +def get_token(self, account, token_type, logger, config_key=None): try: # use Python SDK secrets retrieval for credential in self.service.storage_passwords: @@ -224,15 +301,21 @@ def get_token(self, account, rw, logger): and credential.username.startswith(account) ): cred = credential.content.get("clear_password") - if rw == "read": + if token_type == "authn": + if config_key in cred: + cred_json = json.loads(cred) + token = cred_json[config_key] + if token_type == "read": if "dataset_log_read_access_key" in cred: cred_json = json.loads(cred) token = cred_json["dataset_log_read_access_key"] - elif rw == "write": + elif token_type == "write": if "dataset_log_write_access_key" in cred: cred_json = json.loads(cred) token = cred_json["dataset_log_write_access_key"] return token + else: + logger.debug("the credentials were not retireived") except Exception as e: logger.error( self, diff --git a/TA_dataset/bin/dataset_powerquery.py b/TA_dataset/bin/dataset_powerquery.py index d078f9e0..734d64fc 100644 --- a/TA_dataset/bin/dataset_powerquery.py +++ b/TA_dataset/bin/dataset_powerquery.py @@ -8,7 +8,13 @@ import import_declare_test # noqa: F401 import requests from dataset_api import build_payload, parse_powerquery -from dataset_common import get_acct_info, get_log_level, get_proxy, get_url +from dataset_common import ( + get_acct_info, + get_log_level, + get_proxy, + get_tenant_related_payload, + get_url, +) from solnlib import log from solnlib.modular_input import checkpointer from splunklib import modularinput as smi @@ -117,7 +123,13 @@ def stream_events(self, inputs, ew): ds_headers = { "Authorization": "Bearer " + acct_dict[ds_acct]["ds_api_key"] } - + tenant_related_payload = get_tenant_related_payload( + acct_dict.get(ds_acct) + ) + ds_payload.update(tenant_related_payload) + logger.info( + "ds payload in power query stream events = {}".format(ds_payload) + ) # Create checkpointer checkpoint = checkpointer.KVStoreCheckpointer( input_name, session_key, APP_NAME diff --git a/TA_dataset/bin/dataset_query.py b/TA_dataset/bin/dataset_query.py index 93b2da78..0f1b51c0 100755 --- a/TA_dataset/bin/dataset_query.py +++ b/TA_dataset/bin/dataset_query.py @@ -16,6 +16,7 @@ get_acct_info, get_logger, get_proxy, + get_tenant_related_payload, get_url, relative_to_epoch, ) @@ -144,7 +145,12 @@ def stream_events(self, inputs, ew): proxy = get_proxy(session_key, logger) acct_dict = get_acct_info(self, logger, ds_account) for ds_acct in acct_dict.keys(): + tenant_related_payload = get_tenant_related_payload( + acct_dict.get(ds_acct) + ) + ds_payload.update(tenant_related_payload) curr_payload = copy.deepcopy(ds_payload) + logger.info("query api account curr payload {}".format(curr_payload)) curr_maxcount = copy.copy(ds_maxcount) ds_url = get_url(acct_dict[ds_acct]["base_url"], "query") ds_headers = { diff --git a/TA_dataset/bin/dataset_search_command.py b/TA_dataset/bin/dataset_search_command.py index b28fe517..c38e106f 100755 --- a/TA_dataset/bin/dataset_search_command.py +++ b/TA_dataset/bin/dataset_search_command.py @@ -26,6 +26,7 @@ get_acct_info, get_logger, get_proxy, + get_tenant_related_payload, get_url, logger, relative_to_epoch, @@ -317,6 +318,9 @@ def generate(self): ) try: + tenant_related_payload = get_tenant_related_payload( + acct_dict.get(ds_acct) + ) if ds_method == "query": result = ds_lrq_log_query( base_url=ds_base_url, @@ -326,6 +330,8 @@ def generate(self): filter_expr=ds_search, limit=ds_maxcount, proxy=proxy, + logger=logger, + tenant_related_payload=tenant_related_payload, ) logger.debug("QUERY RESULT, result={}".format(result)) @@ -368,6 +374,8 @@ def generate(self): end_time=ds_end, query=pq, proxy=proxy, + logger=logger, + tenant_related_payload=tenant_related_payload, ) logger.debug("QUERY RESULT, result={}".format(result)) data = result.data # TableResultData @@ -404,6 +412,8 @@ def generate(self): name=f_field, max_values=ds_maxcount, proxy=proxy, + logger=logger, + tenant_related_payload=tenant_related_payload, ) logger.debug("QUERY RESULT, result={}".format(result)) facet = result.data.facet # FacetValuesResultData.data -> FacetData @@ -436,6 +446,10 @@ def generate(self): "DataSetFunction=makeRequest, destination={}, startTime={}" .format(ds_url, time.time()) ) + ds_payload.update(tenant_related_payload) + logger.debug( + "The paylaod for the timeseries api {}".format(ds_payload) + ) r = requests.post( url=ds_url, headers=ds_headers, json=ds_payload, proxies=proxy ) diff --git a/TA_dataset/lib/dataset_query_api_client/api/default/post_queries.py b/TA_dataset/lib/dataset_query_api_client/api/default/post_queries.py index cfde546d..26f4dbd0 100644 --- a/TA_dataset/lib/dataset_query_api_client/api/default/post_queries.py +++ b/TA_dataset/lib/dataset_query_api_client/api/default/post_queries.py @@ -29,7 +29,6 @@ def _get_kwargs( proxies: Dict[str, str] = client.get_proxy() json_json_body = json_body.to_dict() - return { "method": "post", "url": url, @@ -87,6 +86,7 @@ def sync_detailed( *, client: Client, json_body: PostQueriesLaunchQueryRequestBody, + logger=None, ) -> Response[QueryResult]: """Launch a query @@ -107,11 +107,10 @@ def sync_detailed( Response[QueryResult] """ - kwargs = _get_kwargs( - client=client, - json_body=json_body, + kwargs = _get_kwargs(client=client, json_body=json_body) + logger.debug( + "Making api call to the queries with payload{}".format(kwargs.get("json")) ) - response = httpx.request( verify=client.verify_ssl, **kwargs, diff --git a/e2e/global.setup.ts b/e2e/global.setup.ts index a96d560d..64788ff0 100644 --- a/e2e/global.setup.ts +++ b/e2e/global.setup.ts @@ -58,7 +58,7 @@ setup('login and create account', async ({ page }) => { // Setup locators const locAccount = page.locator('div').filter({ hasText: /^\*?Account nameEnter a unique name for this account\.$/ }).locator('[data-test="textbox"]') const locUrl = page.locator('div').filter({ hasText: /^\*?URLEnter DataSet URL\.$/ }).locator('[data-test="textbox"]') - const locReadKey = page.locator('[data-test="body"] form div').filter({ hasText: 'DataSet Log Read Access KeyRequired to enable inputs and SPL comand. Include tra' }).locator('[data-test="textbox"]'); + const locReadKey = page.locator('[data-test="body"] form div').filter({ hasText: 'DataSet Log Read Access KeyRequired (if no AuthN token provided) to enable' }).locator('[data-test="textbox"]'); const locWriteKey = page.locator('[data-test="body"] form div').filter({ hasText: 'DataSet Log Write Access KeyRequired to enable alert action. Include trailing hy' }).locator('[data-test="textbox"]'); // Read env with values diff --git a/globalConfig.json b/globalConfig.json index 61a006ca..0d5aa1a5 100644 --- a/globalConfig.json +++ b/globalConfig.json @@ -31,686 +31,749 @@ "clone" ] }, - "entity": [ - { - "field": "name", - "label": "Account name", - "type": "text", - "required": true, - "help": "Enter a unique name for this account.", - "validators": [ - { - "type": "string", - "minLength": 1, - "maxLength": 25, - "errorMsg": "Length of Account name should be between 1 and 25" - }, - { - "type": "regex", - "pattern": "^[a-zA-Z]\\w*$", - "errorMsg": "Account name must start with a letter and followed by alphabetic letters, digits or underscores." - } - ] - }, - { - "field": "url", - "label": "URL", - "type": "text", - "help": "Enter DataSet URL.", - "required": true, - "defaultValue": "https://app.scalyr.com", - "validators": [ - { - "type": "regex", - "pattern": "^https:\\/{2}(?:\\S)+\\.\\w{2,}$", - "errorMsg": "URL must begin with https:// and end with top-level domain. E.g.: https://app.us1.dataset.com" - } - ] - }, - { - "field": "dataset_log_read_access_key", - "label": "DataSet Log Read Access Key", - "type": "text", - "help": "Required to enable inputs and SPL comand. Include trailing hyphens if applicable.", - "required": false, - "defaultValue": "", - "encrypted": true, - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 100, - "errorMsg": "Max length of password is 100" - } - ] - }, - { - "field": "dataset_log_write_access_key", - "label": "DataSet Log Write Access Key", - "type": "text", - "help": "Required to enable alert action. Include trailing hyphens if applicable.", - "required": false, - "defaultValue": "", - "encrypted": true, - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 100, - "errorMsg": "Max length of password is 100" - } - ] - } - ] - }, - { - "name": "logging", - "title": "Logging", - "entity": [ - { - "field": "loglevel", - "label": "Log level", - "type": "singleSelect", - "options": { - "disableSearch": true, - "autoCompleteFields": [ - { - "label": "DEBUG", - "value": "DEBUG" - }, - { - "label": "INFO", - "value": "INFO" - }, - { - "label": "WARNING", - "value": "WARNING" - }, + "entity": [ + { + "field": "name", + "label": "Account name", + "type": "text", + "required": true, + "help": "Enter a unique name for this account.", + "validators": [ { - "label": "ERROR", - "value": "ERROR" + "type": "string", + "minLength": 1, + "maxLength": 25, + "errorMsg": "Length of Account name should be between 1 and 25" }, { - "label": "CRITICAL", - "value": "CRITICAL" + "type": "regex", + "pattern": "^[a-zA-Z]\\w*$", + "errorMsg": "Account name must start with a letter and followed by alphabetic letters, digits or underscores." } ] }, - "defaultValue": "INFO" - } - ] - }, - { - "name": "proxy", - "title": "Proxy", - "entity": [ - { - "field": "proxy_enabled", - "label": "Enable", - "type": "checkbox" - }, - { - "type": "singleSelect", - "label": "Proxy Type", - "options": { - "disableSearch": true, - "autoCompleteFields": [ - { - "value": "http", - "label": "http" + { + "field": "url", + "label": "URL", + "type": "text", + "help": "Enter DataSet URL.", + "required": true, + "defaultValue": "https://app.scalyr.com", + "validators": [ + { + "type": "regex", + "pattern": "^https:\\/{2}(?:\\S)+\\.\\w{2,}$", + "errorMsg": "URL must begin with https:// and end with top-level domain. E.g.: https://app.us1.dataset.com" } ] }, - "defaultValue": "http", - "field": "proxy_type" + { + "field": "tenant", + "label": "Tenant", + "type": "text", + "help": "Tenant field can be True/False/Blank. If True, entire Tenant will be considered or if False, only the Account IDs will be considered for queries. If Blank neither consider", + "required": false, + "defaultValue": "", + "validators": [ + { + "type": "regex", + "pattern": "^(true|True|TRUE|false|False|FALSE)$", + "errorMsg": "Tenant must be a value of True or False or empty" + } + ] }, { - "field": "proxy_url", - "label": "Host", - "type": "text", - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 4096, - "errorMsg": "Max host length is 4096" - } - ] + "field": "account_ids", + "label": "Account IDs", + "type": "text", + "help": "Provide comma separated values of Account IDs on which the queries should run, when Tenant is set to False.", + "required": false, + "defaultValue": "", + "validators": [ + { + "type": "regex", + "pattern": "^\\d+(,\\d+)*$", + "errorMsg": "Account IDs must be a comma separated ids" + } + ] }, { - "field": "proxy_port", - "label": "Port", - "type": "text", - "validators": [ - { - "type": "number", - "range": [ - 1, - 65535 - ] - } - ] + "field": "authn_token_part_one", + "label": "AuthN API Token First Part", + "type": "text", + "help": "Encrypted part of the Token required to enable inputs and SPL command.", + "required": false, + "defaultValue": "", + "encrypted": true, + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 220, + "errorMsg": "Max length of password is 220" + } + ] }, { - "field": "proxy_username", - "label": "Username", - "type": "text", - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 50, - "errorMsg": "Max length of username is 50" - } - ] + "field": "authn_token_part_two", + "label": "AuthN API Token Second Part", + "type": "text", + "help": "Remaining part of the Token required to enable inputs and SPL command.", + "required": false, + "defaultValue": "", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 250, + "errorMsg": "Max length of password is 250" + } + ] }, { - "field": "proxy_password", - "label": "Password", - "type": "text", - "encrypted": true, - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 8192, - "errorMsg": "Max length of password is 8192" - } - ] - } - ], - "options": { - "saveValidator": "function(formData) { if(!formData.proxy_enabled || formData.proxy_enabled === '0') {return true; } if(!formData.proxy_url) { return 'Proxy Host can not be empty'; } if(!formData.proxy_port) { return 'Proxy Port can not be empty'; } return true; }" - } - } - ] - }, - "inputs": { - "title": "Inputs", - "description": "Manage your data inputs", - "table": { - "header": [ - { - "field": "name", - "label": "Name" - }, - { - "field": "interval", - "label": "Interval" - }, - { - "field": "index", - "label": "Index" - }, - { - "field": "account", - "label": "DataSet Account" - }, - { - "field": "disabled", - "label": "Status" - } - ], - "moreInfo": [ - { - "field": "name", - "label": "Name" - }, - { - "field": "interval", - "label": "Interval" - }, - { - "field": "index", - "label": "Index" + "field": "dataset_log_read_access_key", + "label": "DataSet Log Read Access Key", + "type": "text", + "help": "Required (if no AuthN token provided) to enable inputs and SPL command. Include trailing hyphens if applicable.", + "required": false, + "defaultValue": "", + "encrypted": true, + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 100, + "errorMsg": "Max length of password is 100" + } + ] + }, + { + "field": "dataset_log_write_access_key", + "label": "DataSet Log Write Access Key", + "type": "text", + "help": "Required to enable alert action. Include trailing hyphens if applicable.", + "required": false, + "defaultValue": "", + "encrypted": true, + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 100, + "errorMsg": "Max length of password is 100" + } + ] + } + ] }, { - "field": "account", - "label": "DataSet Account" + "name": "logging", + "title": "Logging", + "entity": [ + { + "field": "loglevel", + "label": "Log level", + "type": "singleSelect", + "options": { + "disableSearch": true, + "autoCompleteFields": [ + { + "label": "DEBUG", + "value": "DEBUG" + }, + { + "label": "INFO", + "value": "INFO" + }, + { + "label": "WARNING", + "value": "WARNING" + }, + { + "label": "ERROR", + "value": "ERROR" + }, + { + "label": "CRITICAL", + "value": "CRITICAL" + } + ] + }, + "defaultValue": "INFO" + } + ] }, { - "field": "disabled", - "label": "Status" + "name": "proxy", + "title": "Proxy", + "entity": [ + { + "field": "proxy_enabled", + "label": "Enable", + "type": "checkbox" + }, + { + "type": "singleSelect", + "label": "Proxy Type", + "options": { + "disableSearch": true, + "autoCompleteFields": [ + { + "value": "http", + "label": "http" + } + ] + }, + "defaultValue": "http", + "field": "proxy_type" + }, + { + "field": "proxy_url", + "label": "Host", + "type": "text", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 4096, + "errorMsg": "Max host length is 4096" + } + ] + }, + { + "field": "proxy_port", + "label": "Port", + "type": "text", + "validators": [ + { + "type": "number", + "range": [ + 1, + 65535 + ] + } + ] + }, + { + "field": "proxy_username", + "label": "Username", + "type": "text", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 50, + "errorMsg": "Max length of username is 50" + } + ] + }, + { + "field": "proxy_password", + "label": "Password", + "type": "text", + "encrypted": true, + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 8192, + "errorMsg": "Max length of password is 8192" + } + ] + } + ], + "options": { + "saveValidator": "function(formData) { if(!formData.proxy_enabled || formData.proxy_enabled === '0') {return true; } if(!formData.proxy_url) { return 'Proxy Host can not be empty'; } if(!formData.proxy_port) { return 'Proxy Port can not be empty'; } return true; }" + } } - ], - "actions": [ - "edit", - "enable", - "delete", - "clone" ] }, - "services": [ - { - "name": "dataset_query", - "title": "DataSet Query", - "entity": [ + "inputs": { + "title": "Inputs", + "description": "Manage your data inputs", + "table": { + "header": [ { "field": "name", - "label": "Name", - "type": "text", - "help": "Enter a unique name for the data input", - "required": true, - "validators": [ - { - "type": "regex", - "pattern": "^[a-zA-Z]\\w*$", - "errorMsg": "Input Name must start with a letter and followed by alphabetic letters, digits or underscores." - }, - { - "type": "string", - "minLength": 1, - "maxLength": 100, - "errorMsg": "Length of input name should be between 1 and 100" - } - ] - }, - { - "field": "account", - "label": "Account", - "help": "DataSet account", - "required": true, - "type": "singleSelect", - "options": { - "referenceName": "account" - } + "label": "Name" }, { "field": "interval", - "label": "Interval", - "type": "text", - "required": true, - "help": "Time interval of input in seconds.", - "validators": [ - { - "type": "regex", - "pattern": "^\\-[1-9]\\d*$|^\\d*$", - "errorMsg": "Interval must be an integer." - } - ] + "label": "Interval" }, { "field": "index", - "label": "Index", - "type": "singleSelect", - "defaultValue": "default", - "options": { - "endpointUrl": "data/indexes", - "createSearchChoice": true, - "denyList": "^_.*$" - }, - "required": true, - "validators": [ - { - "type": "string", - "minLength": 1, - "maxLength": 80, - "errorMsg": "Length of index name should be between 1 and 80." - } - ] - }, - { - "field": "start_time", - "label": "Start Time", - "help": "Relative time to query back. Use short form relative time, e.g.: 24h or 30d. Reference https://app.scalyr.com/help/time-reference.", - "required": true, - "type": "text", - "defaultValue": "5m", - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 8192, - "errorMsg": "Max length of text input is 8192" - }, - { - "type": "regex", - "pattern": "^\\d+(d|h|m|s)$", - "errorMsg": "Start time must be a digit follow by one of: d, h, m, s." - } - ] - }, - { - "field": "end_time", - "label": "End Time", - "help": "If left blank, present time at query execution is used. If defined, use short form relative time.", - "required": false, - "type": "text", - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 8192, - "errorMsg": "Max length of text input is 8192" - }, - { - "type": "regex", - "pattern": "^\\d+(d|h|m|s)$", - "errorMsg": "End time must be a digit follow by one of: d, h, m, s." - } - ] - }, - { - "field": "dataset_query_string", - "label": "DataSet Query String", - "help": "If left blank, all records (limited by max count) are retrieved.", - "required": false, - "type": "text", - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 8192, - "errorMsg": "Max length of text input is 8192" - } - ] + "label": "Index" }, { - "field": "dataset_query_columns", - "label": "Columns", - "help": "If left blank, all columns are returned.", - "required": false, - "type": "text", - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 8192, - "errorMsg": "Max length of text input is 8192" - }, - { - "type": "regex", - "pattern": "^(\\w+,\\s*)*\\w+$", - "errorMsg": "Column names must be comma separated." - } - ] + "field": "account", + "label": "DataSet Account" }, { - "field": "max_count", - "label": "Max Count", - "help": "Specifies the maximum number of records to return. If left blank, the default is 100.", - "required": false, - "type": "text", - "validators": [ - { - "type": "number", - "range": [ - 1, - 9999999 - ], - "errorMsg": "Max Count must be a number" - } - ] + "field": "disabled", + "label": "Status" } - ] - }, - { - "name": "dataset_powerquery", - "title": "DataSet PowerQuery", - "entity": [ + ], + "moreInfo": [ { "field": "name", - "label": "Name", - "type": "text", - "help": "Enter a unique name for the data input", - "required": true, - "validators": [ - { - "type": "regex", - "pattern": "^[a-zA-Z]\\w*$", - "errorMsg": "Input Name must start with a letter and followed by alphabetic letters, digits or underscores." - }, - { - "type": "string", - "minLength": 1, - "maxLength": 100, - "errorMsg": "Length of input name should be between 1 and 100" - } - ] - }, - { - "field": "account", - "label": "Account", - "help": "", - "required": true, - "type": "singleSelect", - "options": { - "referenceName": "account" - } + "label": "Name" }, { "field": "interval", - "label": "Interval", - "type": "text", - "required": true, - "help": "Time interval of input in seconds.", - "validators": [ - { - "type": "regex", - "pattern": "^\\-[1-9]\\d*$|^\\d*$", - "errorMsg": "Interval must be an integer." - } - ] + "label": "Interval" }, { "field": "index", - "label": "Index", - "type": "singleSelect", - "defaultValue": "default", - "options": { - "endpointUrl": "data/indexes", - "createSearchChoice": true, - "denyList": "^_.*$" - }, - "required": true, - "validators": [ - { - "type": "string", - "minLength": 1, - "maxLength": 80, - "errorMsg": "Length of index name should be between 1 and 80." - } - ] - }, - { - "field": "start_time", - "label": "Start Time", - "help": "Relative time to query back. Use short form relative time, e.g.: 24h or 30d. Reference https://app.scalyr.com/help/time-reference.", - "required": true, - "type": "text", - "defaultValue": "5m", - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 8192, - "errorMsg": "Max length of text input is 8192" - }, - { - "type": "regex", - "pattern": "^\\d+(d|h|m|s)$", - "errorMsg": "Start time must be a digit follow by one of: d, h, m, s." - } - ] + "label": "Index" }, { - "field": "end_time", - "label": "End Time", - "help": "If left blank, present time at query execution is used. If defined, use short form relative time.", - "required": false, - "type": "text", - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 8192, - "errorMsg": "Max length of text input is 8192" - }, - { - "type": "regex", - "pattern": "^\\d+(d|h|m|s)$", - "errorMsg": "End time must be a digit follow by one of: d, h, m, s." - } - ] + "field": "account", + "label": "DataSet Account" }, { - "field": "dataset_query_string", - "label": "DataSet PowerQuery String", - "help": "DataSet PowerQuery to return results.", - "required": true, - "type": "text", - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 8192, - "errorMsg": "Max length of text input is 8192" - } - ] + "field": "disabled", + "label": "Status" } + ], + "actions": [ + "edit", + "enable", + "delete", + "clone" ] }, - { - "name": "dataset_alerts", - "title": "DataSet Alerts", - "entity": [ - { - "field": "name", - "label": "Name", - "type": "text", - "help": "Enter a unique name for the data input", - "required": true, - "validators": [ - { - "type": "regex", - "pattern": "^[a-zA-Z]\\w*$", - "errorMsg": "Input Name must start with a letter and followed by alphabetic letters, digits or underscores." - }, - { - "type": "string", - "minLength": 1, - "maxLength": 100, - "errorMsg": "Length of input name should be between 1 and 100" + "services": [ + { + "name": "dataset_query", + "title": "DataSet Query", + "entity": [ + { + "field": "name", + "label": "Name", + "type": "text", + "help": "Enter a unique name for the data input", + "required": true, + "validators": [ + { + "type": "regex", + "pattern": "^[a-zA-Z]\\w*$", + "errorMsg": "Input Name must start with a letter and followed by alphabetic letters, digits or underscores." + }, + { + "type": "string", + "minLength": 1, + "maxLength": 100, + "errorMsg": "Length of input name should be between 1 and 100" + } + ] + }, + { + "field": "account", + "label": "Account", + "help": "DataSet account", + "required": true, + "type": "singleSelect", + "options": { + "referenceName": "account" } - ] - }, - { - "field": "account", - "label": "Account", - "help": "", - "required": true, - "type": "singleSelect", - "options": { - "referenceName": "account" + }, + { + "field": "interval", + "label": "Interval", + "type": "text", + "required": true, + "help": "Time interval of input in seconds.", + "validators": [ + { + "type": "regex", + "pattern": "^\\-[1-9]\\d*$|^\\d*$", + "errorMsg": "Interval must be an integer." + } + ] + }, + { + "field": "index", + "label": "Index", + "type": "singleSelect", + "defaultValue": "default", + "options": { + "endpointUrl": "data/indexes", + "createSearchChoice": true, + "denyList": "^_.*$" + }, + "required": true, + "validators": [ + { + "type": "string", + "minLength": 1, + "maxLength": 80, + "errorMsg": "Length of index name should be between 1 and 80." + } + ] + }, + { + "field": "start_time", + "label": "Start Time", + "help": "Relative time to query back. Use short form relative time, e.g.: 24h or 30d. Reference https://app.scalyr.com/help/time-reference.", + "required": true, + "type": "text", + "defaultValue": "5m", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 8192, + "errorMsg": "Max length of text input is 8192" + }, + { + "type": "regex", + "pattern": "^\\d+(d|h|m|s)$", + "errorMsg": "Start time must be a digit follow by one of: d, h, m, s." + } + ] + }, + { + "field": "end_time", + "label": "End Time", + "help": "If left blank, present time at query execution is used. If defined, use short form relative time.", + "required": false, + "type": "text", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 8192, + "errorMsg": "Max length of text input is 8192" + }, + { + "type": "regex", + "pattern": "^\\d+(d|h|m|s)$", + "errorMsg": "End time must be a digit follow by one of: d, h, m, s." + } + ] + }, + { + "field": "dataset_query_string", + "label": "DataSet Query String", + "help": "If left blank, all records (limited by max count) are retrieved.", + "required": false, + "type": "text", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 8192, + "errorMsg": "Max length of text input is 8192" + } + ] + }, + { + "field": "dataset_query_columns", + "label": "Columns", + "help": "If left blank, all columns are returned.", + "required": false, + "type": "text", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 8192, + "errorMsg": "Max length of text input is 8192" + }, + { + "type": "regex", + "pattern": "^(\\w+,\\s*)*\\w+$", + "errorMsg": "Column names must be comma separated." + } + ] + }, + { + "field": "max_count", + "label": "Max Count", + "help": "Specifies the maximum number of records to return. If left blank, the default is 100.", + "required": false, + "type": "text", + "validators": [ + { + "type": "number", + "range": [ + 1, + 9999999 + ], + "errorMsg": "Max Count must be a number" + } + ] } - }, - { - "field": "interval", - "label": "Interval", - "type": "text", - "required": true, - "help": "Time interval of input in seconds.", - "validators": [ - { - "type": "regex", - "pattern": "^\\-[1-9]\\d*$|^\\d*$", - "errorMsg": "Interval must be an integer." + ] + }, + { + "name": "dataset_powerquery", + "title": "DataSet PowerQuery", + "entity": [ + { + "field": "name", + "label": "Name", + "type": "text", + "help": "Enter a unique name for the data input", + "required": true, + "validators": [ + { + "type": "regex", + "pattern": "^[a-zA-Z]\\w*$", + "errorMsg": "Input Name must start with a letter and followed by alphabetic letters, digits or underscores." + }, + { + "type": "string", + "minLength": 1, + "maxLength": 100, + "errorMsg": "Length of input name should be between 1 and 100" + } + ] + }, + { + "field": "account", + "label": "Account", + "help": "", + "required": true, + "type": "singleSelect", + "options": { + "referenceName": "account" } - ] - }, - { - "field": "index", - "label": "Index", - "type": "singleSelect", - "defaultValue": "default", - "options": { - "endpointUrl": "data/indexes", - "createSearchChoice": true, - "denyList": "^_.*$" }, - "required": true, - "validators": [ - { - "type": "string", - "minLength": 1, - "maxLength": 80, - "errorMsg": "Length of index name should be between 1 and 80." + { + "field": "interval", + "label": "Interval", + "type": "text", + "required": true, + "help": "Time interval of input in seconds.", + "validators": [ + { + "type": "regex", + "pattern": "^\\-[1-9]\\d*$|^\\d*$", + "errorMsg": "Interval must be an integer." + } + ] + }, + { + "field": "index", + "label": "Index", + "type": "singleSelect", + "defaultValue": "default", + "options": { + "endpointUrl": "data/indexes", + "createSearchChoice": true, + "denyList": "^_.*$" + }, + "required": true, + "validators": [ + { + "type": "string", + "minLength": 1, + "maxLength": 80, + "errorMsg": "Length of index name should be between 1 and 80." + } + ] + }, + { + "field": "start_time", + "label": "Start Time", + "help": "Relative time to query back. Use short form relative time, e.g.: 24h or 30d. Reference https://app.scalyr.com/help/time-reference.", + "required": true, + "type": "text", + "defaultValue": "5m", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 8192, + "errorMsg": "Max length of text input is 8192" + }, + { + "type": "regex", + "pattern": "^\\d+(d|h|m|s)$", + "errorMsg": "Start time must be a digit follow by one of: d, h, m, s." + } + ] + }, + { + "field": "end_time", + "label": "End Time", + "help": "If left blank, present time at query execution is used. If defined, use short form relative time.", + "required": false, + "type": "text", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 8192, + "errorMsg": "Max length of text input is 8192" + }, + { + "type": "regex", + "pattern": "^\\d+(d|h|m|s)$", + "errorMsg": "End time must be a digit follow by one of: d, h, m, s." + } + ] + }, + { + "field": "dataset_query_string", + "label": "DataSet PowerQuery String", + "help": "DataSet PowerQuery to return results.", + "required": true, + "type": "text", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 8192, + "errorMsg": "Max length of text input is 8192" + } + ] + } + ] + }, + { + "name": "dataset_alerts", + "title": "DataSet Alerts", + "entity": [ + { + "field": "name", + "label": "Name", + "type": "text", + "help": "Enter a unique name for the data input", + "required": true, + "validators": [ + { + "type": "regex", + "pattern": "^[a-zA-Z]\\w*$", + "errorMsg": "Input Name must start with a letter and followed by alphabetic letters, digits or underscores." + }, + { + "type": "string", + "minLength": 1, + "maxLength": 100, + "errorMsg": "Length of input name should be between 1 and 100" + } + ] + }, + { + "field": "account", + "label": "Account", + "help": "", + "required": true, + "type": "singleSelect", + "options": { + "referenceName": "account" } - ] - }, - { - "field": "start_time", - "label": "Start Time", - "help": "Relative time to query back. Use short form relative time, e.g.: 24h or 30d. Reference https://app.scalyr.com/help/time-reference", - "required": true, - "type": "text", - "defaultValue": "24h", - "validators": [ - { - "type": "string", - "minLength": 0, - "maxLength": 8192, - "errorMsg": "Max length of text input is 8192" + }, + { + "field": "interval", + "label": "Interval", + "type": "text", + "required": true, + "help": "Time interval of input in seconds.", + "validators": [ + { + "type": "regex", + "pattern": "^\\-[1-9]\\d*$|^\\d*$", + "errorMsg": "Interval must be an integer." + } + ] + }, + { + "field": "index", + "label": "Index", + "type": "singleSelect", + "defaultValue": "default", + "options": { + "endpointUrl": "data/indexes", + "createSearchChoice": true, + "denyList": "^_.*$" + }, + "required": true, + "validators": [ + { + "type": "string", + "minLength": 1, + "maxLength": 80, + "errorMsg": "Length of index name should be between 1 and 80." + } + ] + }, + { + "field": "start_time", + "label": "Start Time", + "help": "Relative time to query back. Use short form relative time, e.g.: 24h or 30d. Reference https://app.scalyr.com/help/time-reference", + "required": true, + "type": "text", + "defaultValue": "24h", + "validators": [ + { + "type": "string", + "minLength": 0, + "maxLength": 8192, + "errorMsg": "Max length of text input is 8192" + } + ], + "options": { + "placeholder": "24h" } - ], - "options": { - "placeholder": "24h" } - } - ] - } - ] - } - }, - "alerts": [ - { - "name": "dataset_event", - "label": "DataSet Event", - "description": "Send log to DataSet based on the search result", - "entity": [ - { - "type": "singleSelectSplunkSearch", - "label": "Select Account", - "field": "account", - "search": "| rest servicesNS/nobody/TA_dataset/admin/TA_dataset_account splunk_server=local | dedup title", - "valueField": "title", - "labelField": "title", - "required": true, - "help": "Select the DataSet Account" - }, - { - "type": "text", - "label": "ServerHost", - "field": "dataset_serverhost", - "defaultValue": "splunk", - "required": true, - "help": "DataSet serverHost value." - }, - { - "type": "text", - "label": "DataSet Message", - "field": "dataset_message", - "defaultValue": "$name$", - "required": true, - "help": "DataSet message body" - }, - { - "type": "text", - "label": "Severity", - "field": "dataset_severity", - "defaultValue": "3", - "required": false, - "help": "DataSet Severity (1 - 6)" - }, - { - "type": "text", - "label": "Parser", - "field": "dataset_parser", - "defaultValue": "splunk", - "required": true, - "help": "DataSet parser" - } - ] - } - ] -} + ] + } + ] + } + }, + "alerts": [ + { + "name": "dataset_event", + "label": "DataSet Event", + "description": "Send log to DataSet based on the search result", + "entity": [ + { + "type": "singleSelectSplunkSearch", + "label": "Select Account", + "field": "account", + "search": "| rest servicesNS/nobody/TA_dataset/admin/TA_dataset_account splunk_server=local | dedup title", + "valueField": "title", + "labelField": "title", + "required": true, + "help": "Select the DataSet Account" + }, + { + "type": "text", + "label": "ServerHost", + "field": "dataset_serverhost", + "defaultValue": "splunk", + "required": true, + "help": "DataSet serverHost value." + }, + { + "type": "text", + "label": "DataSet Message", + "field": "dataset_message", + "defaultValue": "$name$", + "required": true, + "help": "DataSet message body" + }, + { + "type": "text", + "label": "Severity", + "field": "dataset_severity", + "defaultValue": "3", + "required": false, + "help": "DataSet Severity (1 - 6)" + }, + { + "type": "text", + "label": "Parser", + "field": "dataset_parser", + "defaultValue": "splunk", + "required": true, + "help": "DataSet parser" + } + ] + } + ] + }