diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py index d0f907d..2847d46 100644 --- a/bin/modules/UlsArgsParser.py +++ b/bin/modules/UlsArgsParser.py @@ -30,6 +30,31 @@ def init(): default=(os.environ.get('ULS_LOGLEVEL') or uls_config.log_level_default), choices=uls_config.log_levels_available, help=f'Adjust the loglevel Default: {uls_config.log_level_default}') + + parser.add_argument('--json-log', + action='store', + dest='jsonlog', + type=bool, + nargs='?', + default=(os.environ.get('ULS_JSONLOG') or uls_config.log_jsonlog), + const=True, + help=f"Should ULS write its own logdata in JSON format instead of plain text output ? (Default: {uls_config.log_jsonlog})") + + parser.add_argument('--ulslogformat', + action='store', + dest='logformat', + type=str, + default=(os.environ.get('ULS_LOGFORMAT') or False), + help=f"Custom logging format (ULS internal logs) see additional features documentation for more information - (Default: False)") + + parser.add_argument('--ulslogdatefmt', + action='store', + dest='logdatefmt', + type=str, + default=(os.environ.get('ULS_LOG_DATEFORMAT') or uls_config.log_datefmt), + help=f"Adjust the logging date/time format to your needs, (Default: {uls_config.log_datefmt.replace("%", "%%")})") + # Added double %% to have argsparser display proper string as it tries do to % replacement :D + # put loglines into debug log parser.add_argument('--debugloglines', action='store', @@ -48,6 +73,15 @@ def init(): const=True, help=f'Display {uls_config.__tool_name_short__} version and operational information') + parser.add_argument('--nocallhome', + action='store', + type=bool, + default=os.environ.get('ULS_NOCALLHOME') or not uls_config.callhome_enabled, + nargs='?', + const=True, + help=f"Disable the ULS CallHome feature that helps the ULS developers to continue improving ULS. Default: {not uls_config.callhome_enabled}") + + # ---------------------- # Input GROUP input_group = parser.add_argument_group(title="Input", @@ -347,6 +381,47 @@ def init(): default=(os.environ.get('ULS_AUTORESUME_WRITEAFTER') or uls_config.autoresume_write_after), help=f'Specify after how many loglines a checkpoint should be written [Default: {uls_config.autoresume_write_after}]') + + #------------------------- + prometheus_group = parser.add_argument_group(title="Prometheus", + description="Define Prometheus Monitoring Settings") + # Prometheues switch + prometheus_group.add_argument('--prometheus', + action='store', + type=bool, + dest='prometheus_enabled', + default=(os.environ.get('ULS_PROMETHEUS') or uls_config.prometheus_enabled), + nargs='?', + const=True, + help=f'Enable prometheues monitoring support - Default: {uls_config.prometheus_enabled}') + + prometheus_group.add_argument('--promport', '--prometheus-port', + action='store', + dest='prometheus_port', + type=int, + default=(os.environ.get('ULS_PROMETHEUS_PORT') or uls_config.prometheus_port), + help=f'Prometheues port to listen on [Default: {uls_config.prometheus_port}]') + + prometheus_group.add_argument('--promaddr', '--prometheus-addr', + action='store', + dest='prometheus_addr', + type=str, + default=(os.environ.get('ULS_PROMETHEUS_ADDR') or uls_config.prometheus_addr), + help=f'Prometheues bind address to listen on [Default: {uls_config.prometheus_addr}]') + + prometheus_group.add_argument('--promcert', '--prometheus-certfile', + action='store', + dest='prometheus_certfile', + type=str, + default=(os.environ.get('ULS_PROMETHEUS_CERTFILE') or uls_config.prometheus_certfile), + help=f'Prometheues certificate file (required alongside a keyfile) [Default: {uls_config.prometheus_certfile}]') + + prometheus_group.add_argument('--promkey', '--prometheus-keyfile', + action='store', + dest='prometheus_keyfile', + type=str, + default=(os.environ.get('ULS_PROMETHEUS_KEYFILE') or uls_config.prometheus_keyfile), + help=f'Prometheues key file (required alongside a certfile) [Default: {uls_config.prometheus_keyfile}]') return parser.parse_args() diff --git a/bin/modules/UlsMonitoring.py b/bin/modules/UlsMonitoring.py index a486c0c..5a1f4f3 100644 --- a/bin/modules/UlsMonitoring.py +++ b/bin/modules/UlsMonitoring.py @@ -24,7 +24,12 @@ class UlsMonitoring: - def __init__(self, stopEvent, product, feed, output): + def __init__(self, stopEvent, product, feed, output, + prom_enabled: bool = False, + prom_port: int = 8000, + prom_host: str = '127.0.0.1', + prom_certfile: str = None, + prom_keyfile: str = None): """ Hanlde ULS self monitoring, spills out performance counter on stdout. @@ -34,11 +39,24 @@ def __init__(self, stopEvent, product, feed, output): feed (string): specific data feed being consumed by ULS """ + # Core monitoring stuff self._stopEvent = stopEvent self._product = product self._feed = feed self._output = output + # Prometheues + self.prometheues_enabled = prom_enabled + self.promeuteheus_port = prom_port + self.promeuteheus_host = prom_host + self.promeuteheus_cert = prom_certfile + self.promeuteheus_key = prom_keyfile + + + + self.prom_overall_messages = None + self.prom_overall_bytes = None + self.prom_overall_messages_ingested = None # Prevent other thread interact with the performance counters self._metricLock = threading.Lock() @@ -47,6 +65,7 @@ def __init__(self, stopEvent, product, feed, output): self.monitoring_interval = uls_config.monitoring_interval # Monitoring interval self._version = uls_config.__version__ + # Definitions self.name = "UlsMonitoring" # Class Human readable name self.overall_messages_handled = 0 # Define overall number of messages handled @@ -67,6 +86,42 @@ def start(self): else: aka_log.log.debug(f"{self.name} monitoring was disabled - not starting.") + if self.prometheues_enabled: + aka_log.log.debug(f"{self.name} Prometheus monitoring started...") + self.start_prometheus(port=self.promeuteheus_port, host=self.promeuteheus_host, cert=self.promeuteheus_cert, key=self.promeuteheus_key) + + def start_prometheus(self, port, host="127.0.0.1", cert=None, key=None): + from prometheus_client import start_http_server + from prometheus_client import Info, Counter, Gauge + from prometheus_client import REGISTRY, PROCESS_COLLECTOR, PLATFORM_COLLECTOR + + # Disable unwanted collectors + REGISTRY.unregister(PROCESS_COLLECTOR) + REGISTRY.unregister(PLATFORM_COLLECTOR) + REGISTRY.unregister(REGISTRY._names_to_collectors['python_gc_objects_collected_total']) + + # Start the Server + server, t = start_http_server(port=port, addr=host, certfile=cert, keyfile=key) + server.base_environ.clear() + + # Show the version + version = Info('uls_version', 'The current ULS Version') + version.info({'version': uls_config.__version__}) + + uls_stream_info = Info('uls_stream_info', "The selected ULS input product") + uls_stream_info.info({'product': self._product, 'feed': self._feed, 'output': self._output}) + + starttime = Info('uls_starttime', "The time, the uls process was started") + starttime.info({'starttime': f'{self.init_time}'}) + + + + # Counters + self.prom_overall_messages = Counter('uls_overall_messages_incoming', 'Number of all handled incoming log lines') + self.prom_overall_bytes = Counter('uls_overall_bytes_incoming', 'Size of all handled incoming log lines') + self.prom_overall_messages_ingested = Counter('uls_overall_messages_ingested', 'Number of all handled outgoing log lines') + + def display(self): """ Entry point for the monitoring thread @@ -109,9 +164,14 @@ def increase_message_count(self, bytes=0): self.window_messages_handled = self.window_messages_handled + 1 self.window_messages_bytes += bytes + # Also increase the prom counters + self.prom_overall_messages.inc() + self.prom_overall_bytes.inc(bytes) + def increase_message_ingested(self): with self._metricLock: self.window_messages_ingested += 1 + self.prom_overall_messages_ingested.inc() def get_message_count(self): @@ -124,5 +184,6 @@ def get_stats(self): def _runtime(self): return int(time.time() - self.init_time) + # EOF diff --git a/bin/modules/UlsTools.py b/bin/modules/UlsTools.py index 53afa6c..3556378 100644 --- a/bin/modules/UlsTools.py +++ b/bin/modules/UlsTools.py @@ -23,6 +23,7 @@ # ULS modules import modules.aka_log as aka_log +import requests import uls_config.global_config as uls_config @@ -340,3 +341,14 @@ def get_install_id(install_id_file=str(root_path()) + "/var/uls_install_id"): return data +def callhome(nocallhome_state: bool, input: str = "n/a", feed: str = "n/a", output: str = "n/a", position: str = "n/a"): + if not nocallhome_state: + try: + url = f"/{position}?version={uls_config.__version__}&input={input}&feed={feed}&output={output}&install_id={get_install_id()['install_id']}&os_platform={platform.platform()}&pyhton={sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}&container={check_container()}" + aka_log.log.debug(f"Sending a CallHome request containing the following data: {url}") + result = requests.get(uls_config.callhome_url + url, timeout=int(uls_config.callhome_timeout)) + aka_log.log.debug(f"Callhome response code: {result.status_code}") + except: + aka_log.log.debug(f"Callhome went wrong ...") + else: + aka_log.log.debug(f"Callhome functionality has been disabled - not sending any data") diff --git a/bin/modules/UlsTransformation.py b/bin/modules/UlsTransformation.py index d371c68..c5c048a 100644 --- a/bin/modules/UlsTransformation.py +++ b/bin/modules/UlsTransformation.py @@ -213,7 +213,17 @@ def _jmespath_transformation(self, log_line, expression): data = json.loads(log_line.decode()) my_output = expression.search(data) - - return str(my_output) + try: + # Attempt to convert the variable to a JSON string + json_data = json.dumps(my_output) + except (TypeError, ValueError) as e: + # This block executes if a TypeError or ValueError occurs during conversion + aka_log.log.debug(f'{self.name} - Transformation ({self.transformation}) ' + f'transformation triggered but ' + f'final json conversion failed ... ') + return str(my_output) + else: + # This block executes if no exceptions are raised + return json_data # EOF diff --git a/bin/modules/aka_log.py b/bin/modules/aka_log.py index dedf1fe..4ade3d8 100644 --- a/bin/modules/aka_log.py +++ b/bin/modules/aka_log.py @@ -13,13 +13,25 @@ # limitations under the License. import logging +import uls_config.global_config as uls_config - -def init(loglevel='WARNING', loggername=None): +def init(loglevel='WARNING', loggername=None, jsonlogs: bool = False, logformat=None, logdatefmt=uls_config.log_datefmt): global log log = logging.getLogger(loggername) console_handler = logging.StreamHandler() - formatter = logging.Formatter('%(asctime)s %(name)s %(levelname).1s %(message)s') + + + if jsonlogs: + if not logformat: + formatter = logging.Formatter('{"timestamp": "%(asctime)s", "log_level": "%(levelname)s", "component": "%(name)s", "message": "%(message)s"}', datefmt=logdatefmt) + else: + formatter = logging.Formatter(logformat, datefmt=logdatefmt) + else: + if not logformat: + formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s', datefmt=logdatefmt) + else: + formatter = logging.Formatter(logformat, datefmt=logdatefmt) + console_handler.setFormatter(formatter) log.addHandler(console_handler) log.setLevel(loglevel) diff --git a/bin/requirements.txt b/bin/requirements.txt index 1a83dcb..d342bf1 100644 --- a/bin/requirements.txt +++ b/bin/requirements.txt @@ -1,3 +1,5 @@ -jmespath>=0.10.0 -requests>=2.25.1 -pytz>=2021.1 +jmespath>=1.0.1 +requests>=2.32.3 +pytz>=2024.1 +prometheus_client>=0.20.0 +setuptools>=72.1.0 diff --git a/bin/uls.py b/bin/uls.py index d21dc6e..10c79ce 100755 --- a/bin/uls.py +++ b/bin/uls.py @@ -61,7 +61,7 @@ def main(): uls_args = ArgsParser.init() # Load the LOG system - aka_log.init(uls_args.loglevel, uls_config.__tool_name_short__) + aka_log.init(uls_args.loglevel, uls_config.__tool_name_short__, jsonlogs=uls_args.jsonlog, logformat=uls_args.logformat, logdatefmt=uls_args.logdatefmt) # Determine root directory root_path = str(UlsTools.root_path()) @@ -94,7 +94,12 @@ def main(): my_monitor = UlsMonitoring.UlsMonitoring(stopEvent=stopEvent, product=uls_args.input, feed=uls_args.feed, - output=uls_args.output) + output=uls_args.output, + prom_enabled=uls_args.prometheus_enabled, + prom_port=uls_args.prometheus_port, + prom_host=uls_args.prometheus_addr, + prom_certfile=uls_args.prometheus_certfile, + prom_keyfile=uls_args.prometheus_keyfile) my_monitor.start() # Connect to an Input Handler UlsInputCli @@ -170,12 +175,18 @@ def main(): # Connect the output handler my_output.connect() + + # Send CallHome Request, if not opted_out + UlsTools.callhome(nocallhome_state=uls_args.nocallhome, position="uls_start", input=uls_args.input, feed=uls_args.feed, output=uls_args.output) + # New ULS/1.5: the input module is ingesting messages # into a thread safe queue. The function call will immediately # return event_q = queue.Queue(uls_args.input_queue_size) my_input.ingest(stopEvent, event_q, my_monitor) + + # Now we are back to the main thread to process the message while not stopEvent.is_set(): try: diff --git a/bin/uls_config/global_config.py b/bin/uls_config/global_config.py index d0ba71e..ad59977 100644 --- a/bin/uls_config/global_config.py +++ b/bin/uls_config/global_config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # Common global variables / constants -__version__ = "1.7.5" +__version__ = "1.8.0-alpha" __tool_name_long__ = "Akamai Unified Log Streamer" __tool_name_short__ = "ULS" @@ -50,10 +50,12 @@ transformation_choices = ['MCAS', 'JMESPATH'] - # LogLevels + # ULS Logging & LogLevels log_levels_available = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] log_level_default = 'WARNING' log_debugloglines_default = False +log_jsonlog = False +log_datefmt = "%Y-%m-%d %H:%M:%S%z" # INPUT Configuration input_rerun_retries = 3 # Number of rerun attempts before giving up @@ -110,3 +112,15 @@ autoresume_supported_inputs = ['ETP', 'EAA', 'GC', 'SIA', 'ACC'] # Internal Var only, to adjust supported inputs autoresume_write_after = 1000 # Write checkpoint only every ${autoresume_write_every} loglines +# CAllHome Configuration +callhome_enabled = "True" # CallHome Functionality is enabled / disabled +callhome_url = "https://uls-beacon.akamaized.net" # CallHome URL Target +callhome_timeout = "2" # Callhome Timeout in seconds + +# Prometheus Monitoring basics +prometheus_enabled = "False" # Do not eneable prometheues by default +prometheus_port = 8000 # Default Prometheus port +prometheus_addr = "127.0.0.1" # Default Prometheus bind address +prometheus_certfile = None # Prometheus Cert file +prometheus_keyfile = None # Prometheus Key file +# prometheus_client_cafile \ No newline at end of file diff --git a/docs/ADDITIONAL_FEATURES.md b/docs/ADDITIONAL_FEATURES.md index ae76e40..8980faf 100644 --- a/docs/ADDITIONAL_FEATURES.md +++ b/docs/ADDITIONAL_FEATURES.md @@ -7,6 +7,7 @@ This document handles the usage of features not explained anywhere else. - [ULS TRANSFORMATIONS](#uls-transformations) - [AUTORESUME / RESUME](#autoresume--resume) - [POST PROCESSING OF FILES (FileOutput only)](#post-processing-of-files-fileoutput-only) +- [ULS LOGFORMAT](ADDITIONAL_FEATURES.md#uls-logformat) ## FILTER (--filter) Feature @@ -120,3 +121,34 @@ Here's a recommendation on how to use this feature to avoid any "glitches": Use the --fileaction handler to move the file into an observed queue directory and start a new process from there. This will ensure a "fast handling" within ULS and provide even more flexibility/stability towards the worklfow and its error handling. +# ULS Logformat +We had customers asking for a JSON logformat (ULS logs) and the additional capability of modifying the logformat. +To accomplish an easy (consumeable way) for json, we have introduced a new command line argument in ULS v1.8.0 `--json-log` in order to make ULS log in a standard JSON format. +The default ULS JSON logformat looks like the following: +```json +{ + "timestamp": "%(asctime)s", + "log_level": "%(levelname)s", + "component": "%(name)s", + "message": "%(message)s" +} +``` + +If this is not sufficient for your use-case - the format can actually be configured as well by using the following argument (or ENV var): `--ulslogformat ""` +As default attributes, the following variables can be used: https://docs.python.org/3/library/logging.html#logrecord-attributes + +Here are some examples: + +**Replicate the original "format"** +```bash +--ulslogformat '{"timestamp": "%(asctime)s", "log_level": "%(levelname)s", "component": "%(name)s", "message": "%(message)s"}' +``` + +**Additional logfields"** +```bash +--ulslogformat '{"timestamp": "%(asctime)s", "log_level": "%(levelname)s", "component": "%(name)s", "message": "%(message)s", "pathname": "%(pathname)s"}' +``` +You can freely adjust the format to your needs.There is no need to use JSON format, you can also simply use the `%(message)s` format to just show the log message. + + + diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md index f793646..88fa23d 100644 --- a/docs/ARGUMENTS_ENV_VARS.md +++ b/docs/ARGUMENTS_ENV_VARS.md @@ -12,12 +12,16 @@ The following tables list all available command line parameters and their corres ## Global -| Parameter | Env - Var | Options | Default | Description | -|--------------------|--------------|-------------------------------------------------|---------|-----------------------------------------------------------| -| -h
--help | n/a | n/a | None | Display help / usage information | -| -l
--loglevel | ULS_LOGLEVEL | 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' | WARNING | Adjust the overall loglevel | -| -v
--version | n/a | n/a | None | Display ULS version information (incl. CLI & OS versions) | -| --debugloglines | ULS_DEBUGLOGLINES | 'True', 'False' | False | Should the debug log contain Loglines (useful to debug transformations) | +| Parameter | Env - Var | Options | Default | Description | +|--------------------|---------------------|----------------------------------------------------------------------------------------------|------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| -h
--help | n/a | n/a | None | Display help / usage information | +| -l
--loglevel | ULS_LOGLEVEL | 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' | WARNING | Adjust the overall loglevel | +| --json-log | ULS_JSONLOG | 'True', 'False' | False | Should ULS write its own logdata in JSON format instead of plain text output ? | +| --ulslogformat | ULS_LOGFORMAT | 'yourlogformatstring' | False | Custom logging format (ULS internal logs) see [additional features documentation](ADDITIONAL_FEATURES.md#uls-logformat) for more information - (Default: False) | +| --ulslogdatefmt | ULS_LOG_DATEFORMAT | All [STRFTIME](https://docs.python.org/3/library/time.html#time.strftime) option | "%Y-%m-%d %H:%M:%S%z" | djust the logging date/time format to your needs, | +| -v
--version | n/a | n/a | None | Display ULS version information (incl. CLI & OS versions) | +| --debugloglines | ULS_DEBUGLOGLINES | 'True', 'False' | False | Should the debug log contain Loglines (useful to debug transformations) | +| --nocallhome | ULS_NOCALLHOME | 'True', 'False' | False | Disable the ULS CallHome feature that helps the ULS developers to continue improving ULS. (enabled by default) | ## INPUT @@ -79,6 +83,14 @@ The following tables list all available command line parameters and their corres | --autoresumepath | ULS_AUTORESUME_PATH | '/path/to/store/checkpoints/' | var/ | Specify the path where checkpoint files should be written to. (Trailing /) | | --autoresumewriteafter | ULS_AUTORESUME_WRITEAFTER | | 1000 | Specify after how many loglines a checkpoint should be written. | +## Prometheus +| Parameter | Env - Var | Options | Default | Description | +|---------------------------------------|-------------------------|---------------------------|-----------|-------------------------------------------------------------| +| --prometheus | ULS_PROMETHEUS | [True, False] | False | Enable prometheues monitoring support | +| --prometheus-port
--promport | ULS_PROMETHEUS_PORT | | 8000 | Prometheues port to listen on | +| --prometheus-addr
--promaddr | ULS_PROMETHEUS_ADDR | xxx.xxx.xxx.xxx | 127.0.0.1 | Prometheues bind address to listen on | +| --prometheus-certfile
--promcert | ULS_PROMETHEUS_CERTFILE | '/path/to/store/promcert' | None | Prometheues certificate file (required alongside a keyfile) | +| --prometheus-keyfile
--promkey | ULS_PROMETHEUS_KEYFILE | '/path/to/store/promkey' | None | Prometheues key file (required alongside a certfile) | ## Customizing HTTP & TCPUDP Formatting diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0ca0db2..0076cf9 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,4 +1,30 @@ # Version History + +## v1.8.0 +||| +|---|---| +|Date|2024-XX-XX +|Kind| MINOR release +|Author|mschiess@akamai.com + +- **Features** + - Prometheus monitoring support added to allow smoother monitoring into third party (prometheus compatible) monitoring sytems. More information [here](MONITORING.md#prometheus) + - CallHome (opt-out) function to enable the ULS team to collect anonymous statistics & usage information - more information [here](./FAQ.md#i-do-not-want-to-send-any-data-to-akamai) + - Added the option to toggle Log Output towards "JSON" format ([see feature request](https://github.com/akamai/uls/issues/66)) + - Added an option to manipulate the ULS internal logging format ([see feature request](https://github.com/akamai/uls/issues/66) and the [Additional Features section](ADDITIONAL_FEATURES.md#uls-logformat)) + - Added an option to manipulate the ULS internal logging date/time format ([see feature request](https://github.com/akamai/uls/issues/66) + + +- **Minor improvements** + - Updated all required packages to the latest version(s) + - [docker] Bumped Python version to 3.12.5 + + +- **Bugfixes** + - issue when using jmespath transformation the result was not proper json - big thanks to @bart-parka for coming up with a PR to fix this + +--- + ## v1.7.5 ||| |---|---| diff --git a/docs/FAQ.md b/docs/FAQ.md index 9c24982..a3d1287 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -15,6 +15,7 @@ - [What is HTTP FORMATTYPE](#what-is-http-formattype) - [Error: "Capacity exceeded, too many incoming data vs. slow output"](#error-capacity-exceeded-too-many-incoming-data-vs-slow-output) - [Error: "Invalid timestamp" on API call](#error-invalid-timestamp-on-api-call) +- [I do not want to send any data to Akamai](#i-do-not-want-to-send-any-data-to-akamai) ---- ## FAQ @@ -186,3 +187,27 @@ clientIp": "10.9.9.9",\n "requestId": "ALC1234",\n "requestTime": "2023-10-24T ``` This error points towards a potential issue with the time configuration on the ULS host. The time of the host ULS runs on, should be synced with some NTP service(s). As you can see in the above example, the host timestamp is `2023-10-24 07:00:17,315` but the request timestamp (returned from the API) is more than 1 minute ahaed `"2023-10-24T07:01:40Z"`. + +--- +### I do not want to send any data to Akamai +With ULS Version 1.8.0 we introduced a call home functionality that once sends data to AKAMAI upon starting ULS. +This data helps us continue the ULS development in the future. So if possible, please allow ULS to send this data. +We are not sending any sensitive or PII data. The Debug logs show the exact data that has been sent. + +The data includes: +- current ULS version +- ULS input +- ULS feed +- ULS output +- ULS installation ID +- OS platform type & version +- Python version +- container status + +Example data: +```curl +/uls_start?version=1.8.0-alpha&input=EAA&feed=ACCESS&output=RAW&install_id=OU5UR0RHLTIwMjIxMTI4LTEuNi4y&os_platform=macOS-14.5-arm64-arm-64bit&pyhton=3.12.4&container=False +``` + +If you still want to disable the CallHome functionality within ULS, +you can do so by setting the `--nocallhome` command line parameter OR by using the ENV VAR: `export ULS_NOCALLHOME=TRUE` \ No newline at end of file diff --git a/docs/MONITORING.md b/docs/MONITORING.md index b699ca6..2a9f66a 100644 --- a/docs/MONITORING.md +++ b/docs/MONITORING.md @@ -1,8 +1,49 @@ # ULS Monitoring -This document describes the ULS monitoring produced on the standard output. +This document describes the ULS monitoring options + +## Prometheus +Since version 1.8.0 ULS supports Prometheus monitoring. +By enabling prometheus monitoring (enabled by using the `--prometheus` arugment or by setting the `ULS_PROMETHEUS=True` env var, +ULS will start a listening port on the configured addr & port from which you can use prometheus to collect the data. + +Configuraiton options for prometheus can be found [here](ARGUMENTS_ENV_VARS.md#prometheus). + +### Prometheues sample output +```bash +# HELP uls_version_info The current ULS Version +# TYPE uls_version_info gauge +uls_version_info{version="1.8.0-alpha"} 1.0 +# HELP uls_stream_info_info The selected ULS input product +# TYPE uls_stream_info_info gauge +uls_stream_info_info{feed="DNS",output="RAW",product="ETP"} 1.0 +# HELP uls_starttime_info The time, the uls process was started +# TYPE uls_starttime_info gauge +uls_starttime_info{starttime="1721909590.092048"} 1.0 +# HELP uls_overall_messages_incoming_total Number of all handled incoming log lines +# TYPE uls_overall_messages_incoming_total counter +uls_overall_messages_incoming_total 0.0 +# HELP uls_overall_messages_incoming_created Number of all handled incoming log lines +# TYPE uls_overall_messages_incoming_created gauge +uls_overall_messages_incoming_created 1.721909590112399e+09 +# HELP uls_overall_bytes_incoming_total Size of all handled incoming log lines +# TYPE uls_overall_bytes_incoming_total counter +uls_overall_bytes_incoming_total 0.0 +# HELP uls_overall_bytes_incoming_created Size of all handled incoming log lines +# TYPE uls_overall_bytes_incoming_created gauge +uls_overall_bytes_incoming_created 1.721909590112406e+09 +# HELP uls_overall_messages_ingested_total Number of all handled outgoing log lines +# TYPE uls_overall_messages_ingested_total counter +uls_overall_messages_ingested_total 0.0 +# HELP uls_overall_messages_ingested_created Number of all handled outgoing log lines +# TYPE uls_overall_messages_ingested_created gauge +uls_overall_messages_ingested_created 1.72190959011241e+09 +``` + + +## STDOUT The output will be sent every 5 minutes to `stdout` by default. -## Field description +### Field description | Field | Example | Description | |-------------------------|----------------------------|-----------------------------------------------------------------| | dt | 2021-06-09T08:15:35.092889 | Date & Time (local OS Timezone) | @@ -19,17 +60,17 @@ The output will be sent every 5 minutes to `stdout` by default. | mon_interval | 300 | Monitoring interval in seconds | -## Example Output +### Example Output The output is delivered in JSON format ```json {"dt": "2022-07-11T20:59:48.511534", "uls_product": "EAA", "uls_feed": "ACCESS", "uls_output": "HTTP", "uls_version": "1.5.0", "uls_runtime": 300, "event_count": 635450, "event_count_interval": 635450, "event_ingested_interval": 635451, "event_bytes_interval": 602300990, "event_rate": 2118.17, "mon_interval": 300} {"dt": "2022-07-11T21:04:48.511982", "uls_product": "EAA", "uls_feed": "ACCESS", "uls_output": "HTTP", "uls_version": "1.5.0", "uls_runtime": 600, "event_count": 1099860, "event_count_interval": 464410, "event_ingested_interval": 464409, "event_bytes_interval": 440221417, "event_rate": 1548.03, "mon_interval": 300} ``` -## Send Docker logs to Splunk +### Send Docker logs to Splunk For this we're using the embedded docker - Splunk logging module. -### Docker-compose +#### Docker-compose Example (add to every service in your `docker-compose.yml`) ```yaml version: "3.0" @@ -46,7 +87,7 @@ services: ... ``` -### Docker.compose template +#### Docker.compose template https://github.com/compose-spec/compose-spec/blob/master/spec.md#using-extensions-as-fragments ```yaml version: "3.0" @@ -75,5 +116,5 @@ Sidenote: you will still receive logs on the CLI running `docker-compose logs -f ![Docker logs in splunk](images/uls_docker_logs_to_splunk.png) -## Docker logs to other SIEMs +### Docker logs to other SIEMs If you have other siem, please see dockers logging capabilities [here](https://docs.docker.com/config/containers/logging/configure/). \ No newline at end of file