Merge pull request #2 from akamai/development

Bugfixes and minor improvements (v0.0.2)

Dockerfile

@@ -10,7 +10,6 @@ ARG             EXT_DIR="$ULS_DIR/ext"
 ARG             ETP_CLI_VERSION="0.3.4"
 ARG             EAA_CLI_VERSION="0.3.8"
 ARG             MFA_CLI_VERSION="0.0.4"
-ARG             ULS_VERSION="0.0.1"
 
 # ENV VARS
 ENV             ULS_DIR=$ULS_DIR
@@ -37,8 +36,7 @@ RUN             mkdir -p ${HOMEDIR}/uls
 
 
 # Install ULS
-ENV             ULS_VERSION=$ULS_VERSION
-RUN             git clone --depth 1 -b "${ULS_VERSION}" --single-branch https://github.com/akamai/uls.git ${ULS_DIR}
+COPY            bin/ ${ULS_DIR}/bin
 WORKDIR         ${ULS_DIR}
 
 # Install external CLI'S

README.md

@@ -7,6 +7,7 @@ The Unified Log Streamer (ULS) is designed to simplify SIEM integrations for Aka
 - [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp)
 
 Thanks to its modular design, ULS allows the connection of many SIEM solutions out-of-the-box.  
+ULS can send data into any SIEM that supports either TCP, UDP or HTTP ingestion.  
 It can be run directly as Python code, as a provided Docker container or through `docker compose` scripts.
 
 

bin/config/global_config.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 
 # Common global variables / constants
-__version__ = "0.0.1"
+__version__ = "0.0.2"
 __tool_name_long__ = "Akamai Unified Log Streamer"
 __tool_name_short__ = "ULS"
 

bin/modules/UlsArgsParser.py

@@ -126,7 +126,7 @@ def init():
     output_group.add_argument('--httpinsecure',
                               action='store',
                               type=bool,
-                              default=(os.environ.get('ULS_HTTP_NO_VERIFY_TLS') or False),
+                              default=(os.environ.get('ULS_HTTP_INSECURE') or False),
                               nargs='?',
                               const=True,
                               help=f'Disable TLS CA Certificate verification. Default: False')

bin/modules/UlsInputCli.py

@@ -116,7 +116,7 @@ def proc_create(self, product=None,
                 if not rawcmd:
                     feed = self._feed_selector(feed, product_feeds)
                     cli_command = [self.bin_python, product_path, 'log', feed.lower(), '-f']
-                    cli_command[2:2] = self._uls_useragent(product_path, product, feed)
+                    cli_command[2:2] = self._uls_useragent(product, feed)
                     cli_command[2:2] = edgegrid_auth
                     cli_command[2:2] = self._prep_proxy(inproxy)
                     if self._format_selector(cliformat) == "JSON":
@@ -183,8 +183,8 @@ def proc_create(self, product=None,
                 aka_log.log.error(f'{self.name} - {my_error} -> {self.proc.stderr.read()}')
 
             if self.running is False and rerun_counter > self.rerun_retries:
-                aka_log.log.critical(f'Not able to start the CLI for {product}. See above errors'
-                                     f'giving up after {rerun_counter - 1} retries.')
+                aka_log.log.critical(f'Not able to start the CLI for {product}. See above errors. '
+                                     f'Giving up after {rerun_counter - 1} retries.')
                 sys.exit(1)
 
     def check_proc(self):

bin/modules/UlsMonitoring.py

@@ -4,6 +4,7 @@
 import threading
 import json
 import datetime
+import sys
 
 import modules.aka_log as aka_log
 import config.global_config as uls_config
@@ -65,13 +66,16 @@ def display(self):
                        'dt': datetime.datetime.utcnow().isoformat(),
                        'uls_product': self._product,
                        'uls_feed': self._feed,
-                       'uls_outpout': self._output,
+                       'uls_output': self._output,
                        'uls_runtime': self._runtime(),
                        'event_count': self.overall_messages_handled,
                        'event_rate': round(self.window_messages_handled / self.monitoring_interval, 2),
                        'mon_interval': self.monitoring_interval
                     }
-                    print(json.dumps(mon_msg))
+                    #print(json.dumps(mon_msg))
+                    sys.stdout.write(json.dumps(mon_msg) + "\n")
+                    sys.stdout.flush()
+
                     # Reset window based vars
                     with self._metricLock:
                         self.window_messages_handled = 0

docker-compose/README.md

@@ -0,0 +1,9 @@
+# Docker-Compose Configurations
+Within this directory, we provide some `docker compose` examples including example files.
+
+The docker-compose.yml controls the docker - relevant attributes like mounting the `.edgerc` file into the container.
+The `.env` files control the ULS via dedicated [ENVIRONMENTAL VARIABLES](../docs/ARGUMENTS_ENV_VARS.md).
+
+The [simple](./simple/README.md) directory provides a simple example running ULS via `docker compose`  
+The [complex](./complex/README.md) directory provides a more "real world" example combining multiple feeds and different outputs.  
+The [example](examples/README.md) directory provides different configuration snippets.

docker-compose/complex/README.md

@@ -0,0 +1,24 @@
+# ULS Complex Example
+
+This example provides a real world usage example for ULS within a `docker compose` setup.
+It combines three different AKAMAI Secure Enterprise Access feeds.
+All docker related controls can be found in [docker-compose.yml](docker-compose.yml)
+
+## EAA-ACCESS
+|IN|OUT|FILE|
+|---|---|---|
+|EAA-ACCESS|TCP|[eaa-access.env](eaa-access.env)|
+
+## ETP-THREAT
+|IN|OUT|FILE|
+|---|---|---|
+|ETP-THREAT|HTTP|[etp-threat.env](etp-threat.env)|
+
+## MFA-AUTH
+|IN|OUT|FILE|
+|---|---|---|
+|MFA-AUTH|UDP|[mfa-auth.env](mfa-auth.env)|
+
+
+
+

docker-compose/complex/docker-compose.yml

@@ -6,21 +6,21 @@ services:
     env_file: etp-threat.env
     volumes:
       - type: bind
-        source: /Users/mschiess/.edgerc
+        source: /patch/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
   eaa-access:
     image: akamai/uls:latest
     restart: always
     env_file: eaa-access.env
     volumes:
       - type: bind
-        source: /Users/mschiess/.edgerc
+        source: /patch/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc
   eaa-admin:
     image: akamai/uls:latest
     restart: always
-    env_file: eaa-admin.env
+    env_file: mfa-auth.env
     volumes:
       - type: bind
-        source: /Users/mschiess/.edgerc
+        source: /patch/to/your/.edgerc
         target: /opt/akamai-uls/.edgerc

docker-compose/complex/eaa-access.env

@@ -1,28 +1,15 @@
-# This is a sample ENV file for EAA - Threat logs (via ULS)
-
+# This is a sample ENV file for ULS EAA - ACCESS logs (output via TCP)
 # GENERIC Config
-ULS_LOGLEVEL=DEBUG
+#ULS_LOGLEVEL=DEBUG
 
 # INPUT CONFIGURATION
-    # THE INPUT PRODUCT [EAA | ETP | MFA]
     ULS_INPUT=EAA
-    # THE INPUT FEED
-    #   EAA: [ ADMIN | ACCESS]
-    #   ETP: [ THREAT | AUP ]
-    #   MFA: [ POLICY | AUTH ]
     ULS_FEED=ACCESS
-    # INPUT FORRMAT
     ULS_FORMAT=JSON
-    # LOCATION OF THE AKAMAI .EDGERC FILE
-    ULS_EDGERC='~/.edgerc'
-    # RELEVANT SECTION WITHIN THE EDGERC FILE
-    ULS_SECTION=akamaidemo
+    ULS_EDGERC='/opt/akamai-uls/.edgerc'
 
 
 # OUTPUT CONFIGURATION
-    # OUTPUT PATH [ TCP / UDP / HTTP ]
-    ULS_OUTPUT=UDP
-    # OUTPUT TARGET HOST
-    ULS_OUTPUT_HOST=192.168.86.34
-    # OUTPUT TARGET PORT
-    ULS_OUTPUT_PORT=9091
+    ULS_OUTPUT=TCP
+    ULS_OUTPUT_HOST=127.0.0.1
+    ULS_OUTPUT_PORT=9051

docker-compose/complex/etp-threat.env

@@ -1,28 +1,17 @@
-# This is a sample ENV file for EAA - Threat logs (via ULS)
+# This is a sample ENV file for ULS ETP - Threat logs (output via HTTP to SPLUNK)
 
 # GENERIC Config
-ULS_LOGLEVEL=DEBUG
+ULS_LOGLEVEL=INFO
 
 # INPUT CONFIGURATION
-    # THE INPUT PRODUCT [EAA | ETP | MFA]
     ULS_INPUT=ETP
-    # THE INPUT FEED
-    #   EAA: [ ADMIN | ACCESS]
-    #   ETP: [ THREAT | AUP ]
-    #   MFA: [ POLICY | AUTH ]
     ULS_FEED=THREAT
-    # INPUT FORRMAT
     ULS_FORMAT=JSON
-    # LOCATION OF THE AKAMAI .EDGERC FILE
-    ULS_EDGERC='~/.edgerc'
-    # RELEVANT SECTION WITHIN THE EDGERC FILE
-    ULS_SECTION=akamaidemo
-
+    ULS_EDGERC='/opt/akamai-uls/.edgerc'
 
 # OUTPUT CONFIGURATION
-    # OUTPUT PATH [ TCP / UDP / HTTP ]
-    ULS_OUTPUT=TCP
-    # OUTPUT TARGET HOST
-    ULS_OUTPUT_HOST=192.168.86.34
-    # OUTPUT TARGET PORT
-    ULS_OUTPUT_PORT=9091
+    ULS_OUTPUT=HTTP
+    ULS_HTTP_URL='https://127.0.0.1:8088/services/collector/event'
+    ULS_HTTP_AUTH_HEADER='{"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}'
+    ULS_HTTP_INSECURE="True"
+    ULS_HTTP_FORMAT='{"event": %s}'

docker-compose/complex/mfa-auth.env

@@ -0,0 +1,16 @@
+# This is a sample ENV file for ULS MFA - AUTH logs (output via UDP)
+
+# GENERIC Config
+ULS_LOGLEVEL=INFO
+
+# INPUT CONFIGURATION
+    ULS_INPUT=MFA
+    ULS_FEED=AUTH
+    ULS_FORMAT=JSON
+    ULS_EDGERC='/opt/akamai-uls/.edgerc'
+
+
+# OUTPUT CONFIGURATION
+    ULS_OUTPUT=UDP
+    ULS_OUTPUT_HOST=127.0.0.1
+    ULS_OUTPUT_PORT=9053

docker-compose/examples/README.md

@@ -0,0 +1,9 @@
+# ULS DOCKER COMPOSE examples
+This directory contains configuration examples (for simple copy & paste usage)
+
+## docker-compose.yml
+This file contains all currently supported feeds: [all_services_docker-compose.yml](all_services_docker-compose.yml)
+
+## ENV file
+This file contains all available ENV VARS explained in a single file.
+[example_env_file.env](example_env_file.env)

docker-compose/examples/all_services_docker-compose.yml

@@ -0,0 +1,58 @@
+version: "3.0"
+services:
+  # THREAT
+  etp-threat:
+    image: akamai/uls:latest
+    restart: always
+    env_file: etp-threat.env
+    volumes:
+      - type: bind
+        source: /patch/to/your/.edgerc
+        target: /opt/akamai-uls/.edgerc
+  # AUP
+  etp-aup:
+    image: akamai/uls:latest
+    restart: always
+    env_file: etp-aup.env
+    volumes:
+      - type: bind
+        source: /patch/to/your/.edgerc
+        target: /opt/akamai-uls/.edgerc
+# EAA
+  # ACCESS
+  eaa-access:
+    image: akamai/uls:latest
+    restart: always
+    env_file: eaa-access.env
+    volumes:
+      - type: bind
+        source: /patch/to/your/.edgerc
+        target: /opt/akamai-uls/.edgerc
+  # ADMIN
+  eaa-admin:
+    image: akamai/uls:latest
+    restart: always
+    env_file: eaa-admin.env
+    volumes:
+      - type: bind
+        source: /patch/to/your/.edgerc
+        target: /opt/akamai-uls/.edgerc
+# MFA
+  # AUTH
+  mfa-auth:
+    image: akamai/uls:latest
+    restart: always
+    env_file: mfa-auth.env
+    volumes:
+      - type: bind
+        source: /patch/to/your/.edgerc
+        target: /opt/akamai-uls/.edgerc
+  # POLICY
+  mfa-policy:
+    image: akamai/uls:latest
+    restart: always
+    env_file: mfa-policy.env
+    volumes:
+      - type: bind
+        source: /patch/to/your/.edgerc
+        target: /opt/akamai-uls/.edgerc

docker-compose/examples/example_env_file.env

@@ -0,0 +1,40 @@
+# This is a FULL example of all possible ENV FILE OPTIONS
+# Be aware that mixing TCP/HTTP can lead to unwanted behaviour !!
+# Rather use them exclusively
+
+# GENERIC Config
+ULS_LOGLEVEL=DEBUG
+
+# INPUT CONFIGURATION
+    # THE INPUT PRODUCT [EAA | ETP | MFA]
+    ULS_INPUT=ETP
+    # THE INPUT FEED
+    #   EAA: [ ADMIN | ACCESS]
+    #   ETP: [ THREAT | AUP ]
+    #   MFA: [ POLICY | AUTH ]
+    ULS_FEED=THREAT
+    # INPUT FORRMAT
+    ULS_FORMAT=JSON
+    # LOCATION OF THE AKAMAI .EDGERC FILE
+    ULS_EDGERC='/opt/akamai-uls/.edgerc'
+    # RELEVANT SECTION WITHIN THE EDGERC FILE
+    ULS_SECTION=default
+    # PROXY TO ACCESS AKAMAI API'S WHILE FETCHING THE LOGS
+    #ULS_INPUT_PROXY='None'
+
+# OUTPUT CONFIGURATION
+    # OUTPUT PATH [ TCP / UDP / HTTP ]
+    ULS_OUTPUT=TCP
+    # OUTPUT TARGET HOST (FOR TCP/UDP only)
+    ULS_OUTPUT_HOST=127.0.0.1
+    # OUTPUT TARGET PORT (FOR TCP/UDP only)
+    ULS_OUTPUT_PORT=9091
+
+    # HTTP URL (FOR HTTP only)
+    ULS_HTTP_URL='https://SPLUNK:8088/services/collector/event''
+    # HTTP AUTH HEADER (required for i.e. SPLUNK) (FOR HTTP only)
+    ULS_HTTP_AUTH_HEADER='{"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}'
+    # DISABLE HTTP CA CERT VERIFICATION - INSECURE !!! (FOR HTTP only)
+    ULS_HTTP_INSECURE="True"
+    # HTTP OUTPUT FORMAT - %s will be replaced by the data feed (Default value works for splunk) (FOR HTTP only)
+    ULS_HTTP_FORMAT='{"event": %s}'