diff --git a/Dockerfile b/Dockerfile
index 65b1355..3877ed6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,7 +10,6 @@ ARG EXT_DIR="$ULS_DIR/ext"
ARG ETP_CLI_VERSION="0.3.4"
ARG EAA_CLI_VERSION="0.3.8"
ARG MFA_CLI_VERSION="0.0.4"
-ARG ULS_VERSION="0.0.1"
# ENV VARS
ENV ULS_DIR=$ULS_DIR
@@ -37,8 +36,7 @@ RUN mkdir -p ${HOMEDIR}/uls
# Install ULS
-ENV ULS_VERSION=$ULS_VERSION
-RUN git clone --depth 1 -b "${ULS_VERSION}" --single-branch https://github.com/akamai/uls.git ${ULS_DIR}
+COPY bin/ ${ULS_DIR}/bin
WORKDIR ${ULS_DIR}
# Install external CLI'S
diff --git a/README.md b/README.md
index b0b7ee9..5f6bb52 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,7 @@ The Unified Log Streamer (ULS) is designed to simplify SIEM integrations for Aka
- [Akamai Phish-proof Multi Factor Authenticator (AKAMAI-MFA)](https://www.akamai.com/us/en/products/security/akamai-mfa.jsp)
Thanks to its modular design, ULS allows the connection of many SIEM solutions out-of-the-box.
+ULS can send data into any SIEM that supports either TCP, UDP or HTTP ingestion.
It can be run directly as Python code, as a provided Docker container or through `docker compose` scripts.
diff --git a/bin/config/global_config.py b/bin/config/global_config.py
index f0b6316..b6cd6c0 100644
--- a/bin/config/global_config.py
+++ b/bin/config/global_config.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# Common global variables / constants
-__version__ = "0.0.1"
+__version__ = "0.0.2"
__tool_name_long__ = "Akamai Unified Log Streamer"
__tool_name_short__ = "ULS"
diff --git a/bin/modules/UlsArgsParser.py b/bin/modules/UlsArgsParser.py
index 8495a7f..40cc159 100644
--- a/bin/modules/UlsArgsParser.py
+++ b/bin/modules/UlsArgsParser.py
@@ -126,7 +126,7 @@ def init():
output_group.add_argument('--httpinsecure',
action='store',
type=bool,
- default=(os.environ.get('ULS_HTTP_NO_VERIFY_TLS') or False),
+ default=(os.environ.get('ULS_HTTP_INSECURE') or False),
nargs='?',
const=True,
help=f'Disable TLS CA Certificate verification. Default: False')
diff --git a/bin/modules/UlsInputCli.py b/bin/modules/UlsInputCli.py
index 9055d4d..6b1b90e 100644
--- a/bin/modules/UlsInputCli.py
+++ b/bin/modules/UlsInputCli.py
@@ -116,7 +116,7 @@ def proc_create(self, product=None,
if not rawcmd:
feed = self._feed_selector(feed, product_feeds)
cli_command = [self.bin_python, product_path, 'log', feed.lower(), '-f']
- cli_command[2:2] = self._uls_useragent(product_path, product, feed)
+ cli_command[2:2] = self._uls_useragent(product, feed)
cli_command[2:2] = edgegrid_auth
cli_command[2:2] = self._prep_proxy(inproxy)
if self._format_selector(cliformat) == "JSON":
@@ -183,8 +183,8 @@ def proc_create(self, product=None,
aka_log.log.error(f'{self.name} - {my_error} -> {self.proc.stderr.read()}')
if self.running is False and rerun_counter > self.rerun_retries:
- aka_log.log.critical(f'Not able to start the CLI for {product}. See above errors'
- f'giving up after {rerun_counter - 1} retries.')
+ aka_log.log.critical(f'Not able to start the CLI for {product}. See above errors. '
+ f'Giving up after {rerun_counter - 1} retries.')
sys.exit(1)
def check_proc(self):
diff --git a/bin/modules/UlsMonitoring.py b/bin/modules/UlsMonitoring.py
index cfcafe0..3daec77 100644
--- a/bin/modules/UlsMonitoring.py
+++ b/bin/modules/UlsMonitoring.py
@@ -4,6 +4,7 @@
import threading
import json
import datetime
+import sys
import modules.aka_log as aka_log
import config.global_config as uls_config
@@ -65,13 +66,16 @@ def display(self):
'dt': datetime.datetime.utcnow().isoformat(),
'uls_product': self._product,
'uls_feed': self._feed,
- 'uls_outpout': self._output,
+ 'uls_output': self._output,
'uls_runtime': self._runtime(),
'event_count': self.overall_messages_handled,
'event_rate': round(self.window_messages_handled / self.monitoring_interval, 2),
'mon_interval': self.monitoring_interval
}
- print(json.dumps(mon_msg))
+ #print(json.dumps(mon_msg))
+ sys.stdout.write(json.dumps(mon_msg) + "\n")
+ sys.stdout.flush()
+
# Reset window based vars
with self._metricLock:
self.window_messages_handled = 0
diff --git a/docker-compose/README.md b/docker-compose/README.md
new file mode 100644
index 0000000..fc4b153
--- /dev/null
+++ b/docker-compose/README.md
@@ -0,0 +1,9 @@
+# Docker-Compose Configurations
+Within this directory, we provide some `docker compose` examples including example files.
+
+The docker-compose.yml controls the docker - relevant attributes like mounting the `.edgerc` file into the container.
+The `.env` files control the ULS via dedicated [ENVIRONMENTAL VARIABLES](../docs/ARGUMENTS_ENV_VARS.md).
+
+The [simple](./simple/README.md) directory provides a simple example running ULS via `docker compose`
+The [complex](./complex/README.md) directory provides a more "real world" example combining multiple feeds and different outputs.
+The [example](examples/README.md) directory provides different configuration snippets.
diff --git a/docker-compose/complex/README.md b/docker-compose/complex/README.md
new file mode 100644
index 0000000..c7af2c5
--- /dev/null
+++ b/docker-compose/complex/README.md
@@ -0,0 +1,24 @@
+# ULS Complex Example
+
+This example provides a real world usage example for ULS within a `docker compose` setup.
+It combines three different AKAMAI Secure Enterprise Access feeds.
+All docker related controls can be found in [docker-compose.yml](docker-compose.yml)
+
+## EAA-ACCESS
+|IN|OUT|FILE|
+|---|---|---|
+|EAA-ACCESS|TCP|[eaa-access.env](eaa-access.env)|
+
+## ETP-THREAT
+|IN|OUT|FILE|
+|---|---|---|
+|ETP-THREAT|HTTP|[etp-threat.env](etp-threat.env)|
+
+## MFA-AUTH
+|IN|OUT|FILE|
+|---|---|---|
+|MFA-AUTH|UDP|[mfa-auth.env](mfa-auth.env)|
+
+
+
+
diff --git a/docker-compose/complex/docker-compose.yml b/docker-compose/complex/docker-compose.yml
index 642d598..b18de7f 100644
--- a/docker-compose/complex/docker-compose.yml
+++ b/docker-compose/complex/docker-compose.yml
@@ -6,7 +6,7 @@ services:
env_file: etp-threat.env
volumes:
- type: bind
- source: /Users/mschiess/.edgerc
+ source: /patch/to/your/.edgerc
target: /opt/akamai-uls/.edgerc
eaa-access:
image: akamai/uls:latest
@@ -14,13 +14,13 @@ services:
env_file: eaa-access.env
volumes:
- type: bind
- source: /Users/mschiess/.edgerc
+ source: /patch/to/your/.edgerc
target: /opt/akamai-uls/.edgerc
eaa-admin:
image: akamai/uls:latest
restart: always
- env_file: eaa-admin.env
+ env_file: mfa-auth.env
volumes:
- type: bind
- source: /Users/mschiess/.edgerc
+ source: /patch/to/your/.edgerc
target: /opt/akamai-uls/.edgerc
\ No newline at end of file
diff --git a/docker-compose/complex/eaa-access.env b/docker-compose/complex/eaa-access.env
index 91de9a2..7c5d1c2 100644
--- a/docker-compose/complex/eaa-access.env
+++ b/docker-compose/complex/eaa-access.env
@@ -1,28 +1,15 @@
-# This is a sample ENV file for EAA - Threat logs (via ULS)
-
+# This is a sample ENV file for ULS EAA - ACCESS logs (output via TCP)
# GENERIC Config
-ULS_LOGLEVEL=DEBUG
+#ULS_LOGLEVEL=DEBUG
# INPUT CONFIGURATION
- # THE INPUT PRODUCT [EAA | ETP | MFA]
ULS_INPUT=EAA
- # THE INPUT FEED
- # EAA: [ ADMIN | ACCESS]
- # ETP: [ THREAT | AUP ]
- # MFA: [ POLICY | AUTH ]
ULS_FEED=ACCESS
- # INPUT FORRMAT
ULS_FORMAT=JSON
- # LOCATION OF THE AKAMAI .EDGERC FILE
- ULS_EDGERC='~/.edgerc'
- # RELEVANT SECTION WITHIN THE EDGERC FILE
- ULS_SECTION=akamaidemo
+ ULS_EDGERC='/opt/akamai-uls/.edgerc'
# OUTPUT CONFIGURATION
- # OUTPUT PATH [ TCP / UDP / HTTP ]
- ULS_OUTPUT=UDP
- # OUTPUT TARGET HOST
- ULS_OUTPUT_HOST=192.168.86.34
- # OUTPUT TARGET PORT
- ULS_OUTPUT_PORT=9091
\ No newline at end of file
+ ULS_OUTPUT=TCP
+ ULS_OUTPUT_HOST=127.0.0.1
+ ULS_OUTPUT_PORT=9051
\ No newline at end of file
diff --git a/docker-compose/complex/eaa-admin.env b/docker-compose/complex/eaa-admin.env
deleted file mode 100644
index 8276b1c..0000000
--- a/docker-compose/complex/eaa-admin.env
+++ /dev/null
@@ -1,28 +0,0 @@
-# This is a sample ENV file for EAA - Threat logs (via ULS)
-
-# GENERIC Config
-ULS_LOGLEVEL=DEBUG
-
-# INPUT CONFIGURATION
- # THE INPUT PRODUCT [EAA | ETP | MFA]
- ULS_INPUT=EAA
- # THE INPUT FEED
- # EAA: [ ADMIN | ACCESS]
- # ETP: [ THREAT | AUP ]
- # MFA: [ POLICY | AUTH ]
- ULS_FEED=ADMIN
- # INPUT FORRMAT
- ULS_FORMAT=JSON
- # LOCATION OF THE AKAMAI .EDGERC FILE
- ULS_EDGERC='~/.edgerc'
- # RELEVANT SECTION WITHIN THE EDGERC FILE
- ULS_SECTION=akamaidemo
-
-
-# OUTPUT CONFIGURATION
- # OUTPUT PATH [ TCP / UDP / HTTP ]
- ULS_OUTPUT=TCP
- # OUTPUT TARGET HOST
- ULS_OUTPUT_HOST=192.168.86.34
- # OUTPUT TARGET PORT
- ULS_OUTPUT_PORT=9091
\ No newline at end of file
diff --git a/docker-compose/complex/etp-threat.env b/docker-compose/complex/etp-threat.env
index c1331af..b315624 100644
--- a/docker-compose/complex/etp-threat.env
+++ b/docker-compose/complex/etp-threat.env
@@ -1,28 +1,17 @@
-# This is a sample ENV file for EAA - Threat logs (via ULS)
+# This is a sample ENV file for ULS ETP - Threat logs (output via HTTP to SPLUNK)
# GENERIC Config
-ULS_LOGLEVEL=DEBUG
+ULS_LOGLEVEL=INFO
# INPUT CONFIGURATION
- # THE INPUT PRODUCT [EAA | ETP | MFA]
ULS_INPUT=ETP
- # THE INPUT FEED
- # EAA: [ ADMIN | ACCESS]
- # ETP: [ THREAT | AUP ]
- # MFA: [ POLICY | AUTH ]
ULS_FEED=THREAT
- # INPUT FORRMAT
ULS_FORMAT=JSON
- # LOCATION OF THE AKAMAI .EDGERC FILE
- ULS_EDGERC='~/.edgerc'
- # RELEVANT SECTION WITHIN THE EDGERC FILE
- ULS_SECTION=akamaidemo
-
+ ULS_EDGERC='/opt/akamai-uls/.edgerc'
# OUTPUT CONFIGURATION
- # OUTPUT PATH [ TCP / UDP / HTTP ]
- ULS_OUTPUT=TCP
- # OUTPUT TARGET HOST
- ULS_OUTPUT_HOST=192.168.86.34
- # OUTPUT TARGET PORT
- ULS_OUTPUT_PORT=9091
\ No newline at end of file
+ ULS_OUTPUT=HTTP
+ ULS_HTTP_URL='https://127.0.0.1:8088/services/collector/event'
+ ULS_HTTP_AUTH_HEADER='{"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}'
+ ULS_HTTP_INSECURE="True"
+ ULS_HTTP_FORMAT='{"event": %s}'
\ No newline at end of file
diff --git a/docker-compose/complex/mfa-auth.env b/docker-compose/complex/mfa-auth.env
new file mode 100644
index 0000000..722a678
--- /dev/null
+++ b/docker-compose/complex/mfa-auth.env
@@ -0,0 +1,16 @@
+# This is a sample ENV file for ULS MFA - AUTH logs (output via UDP)
+
+# GENERIC Config
+ULS_LOGLEVEL=INFO
+
+# INPUT CONFIGURATION
+ ULS_INPUT=MFA
+ ULS_FEED=AUTH
+ ULS_FORMAT=JSON
+ ULS_EDGERC='/opt/akamai-uls/.edgerc'
+
+
+# OUTPUT CONFIGURATION
+ ULS_OUTPUT=UDP
+ ULS_OUTPUT_HOST=127.0.0.1
+ ULS_OUTPUT_PORT=9053
\ No newline at end of file
diff --git a/docker-compose/examples/README.md b/docker-compose/examples/README.md
new file mode 100644
index 0000000..50debca
--- /dev/null
+++ b/docker-compose/examples/README.md
@@ -0,0 +1,9 @@
+# ULS DOCKER COMPOSE examples
+This directory contains configuration examples (for simple copy & paste usage)
+
+## docker-compose.yml
+This file contains all currently supported feeds: [all_services_docker-compose.yml](all_services_docker-compose.yml)
+
+## ENV file
+This file contains all available ENV VARS explained in a single file.
+[example_env_file.env](example_env_file.env)
\ No newline at end of file
diff --git a/docker-compose/examples/all_services_docker-compose.yml b/docker-compose/examples/all_services_docker-compose.yml
new file mode 100644
index 0000000..fb7dbc1
--- /dev/null
+++ b/docker-compose/examples/all_services_docker-compose.yml
@@ -0,0 +1,58 @@
+version: "3.0"
+services:
+ # THREAT
+ etp-threat:
+ image: akamai/uls:latest
+ restart: always
+ env_file: etp-threat.env
+ volumes:
+ - type: bind
+ source: /patch/to/your/.edgerc
+ target: /opt/akamai-uls/.edgerc
+ # AUP
+ etp-aup:
+ image: akamai/uls:latest
+ restart: always
+ env_file: etp-aup.env
+ volumes:
+ - type: bind
+ source: /patch/to/your/.edgerc
+ target: /opt/akamai-uls/.edgerc
+# EAA
+ # ACCESS
+ eaa-access:
+ image: akamai/uls:latest
+ restart: always
+ env_file: eaa-access.env
+ volumes:
+ - type: bind
+ source: /patch/to/your/.edgerc
+ target: /opt/akamai-uls/.edgerc
+ # ADMIN
+ eaa-admin:
+ image: akamai/uls:latest
+ restart: always
+ env_file: eaa-admin.env
+ volumes:
+ - type: bind
+ source: /patch/to/your/.edgerc
+ target: /opt/akamai-uls/.edgerc
+# MFA
+ # AUTH
+ mfa-auth:
+ image: akamai/uls:latest
+ restart: always
+ env_file: mfa-auth.env
+ volumes:
+ - type: bind
+ source: /patch/to/your/.edgerc
+ target: /opt/akamai-uls/.edgerc
+ # POLICY
+ mfa-policy:
+ image: akamai/uls:latest
+ restart: always
+ env_file: mfa-policy.env
+ volumes:
+ - type: bind
+ source: /patch/to/your/.edgerc
+ target: /opt/akamai-uls/.edgerc
\ No newline at end of file
diff --git a/docker-compose/examples/example_env_file.env b/docker-compose/examples/example_env_file.env
new file mode 100644
index 0000000..2c7bc45
--- /dev/null
+++ b/docker-compose/examples/example_env_file.env
@@ -0,0 +1,40 @@
+# This is a FULL example of all possible ENV FILE OPTIONS
+# Be aware that mixing TCP/HTTP can lead to unwanted behaviour !!
+# Rather use them exclusively
+
+# GENERIC Config
+ULS_LOGLEVEL=DEBUG
+
+# INPUT CONFIGURATION
+ # THE INPUT PRODUCT [EAA | ETP | MFA]
+ ULS_INPUT=ETP
+ # THE INPUT FEED
+ # EAA: [ ADMIN | ACCESS]
+ # ETP: [ THREAT | AUP ]
+ # MFA: [ POLICY | AUTH ]
+ ULS_FEED=THREAT
+ # INPUT FORRMAT
+ ULS_FORMAT=JSON
+ # LOCATION OF THE AKAMAI .EDGERC FILE
+ ULS_EDGERC='/opt/akamai-uls/.edgerc'
+ # RELEVANT SECTION WITHIN THE EDGERC FILE
+ ULS_SECTION=default
+ # PROXY TO ACCESS AKAMAI API'S WHILE FETCHING THE LOGS
+ #ULS_INPUT_PROXY='None'
+
+# OUTPUT CONFIGURATION
+ # OUTPUT PATH [ TCP / UDP / HTTP ]
+ ULS_OUTPUT=TCP
+ # OUTPUT TARGET HOST (FOR TCP/UDP only)
+ ULS_OUTPUT_HOST=127.0.0.1
+ # OUTPUT TARGET PORT (FOR TCP/UDP only)
+ ULS_OUTPUT_PORT=9091
+
+ # HTTP URL (FOR HTTP only)
+ ULS_HTTP_URL='https://SPLUNK:8088/services/collector/event''
+ # HTTP AUTH HEADER (required for i.e. SPLUNK) (FOR HTTP only)
+ ULS_HTTP_AUTH_HEADER='{"Authorization": "Splunk xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"}'
+ # DISABLE HTTP CA CERT VERIFICATION - INSECURE !!! (FOR HTTP only)
+ ULS_HTTP_INSECURE="True"
+ # HTTP OUTPUT FORMAT - %s will be replaced by the data feed (Default value works for splunk) (FOR HTTP only)
+ ULS_HTTP_FORMAT='{"event": %s}'
\ No newline at end of file
diff --git a/docker-compose/simple/README.md b/docker-compose/simple/README.md
new file mode 100644
index 0000000..bab6f97
--- /dev/null
+++ b/docker-compose/simple/README.md
@@ -0,0 +1,16 @@
+# ULS SIMPLE Example
+
+This example provides demo usage example for ULS within a `docker compose` setup.
+It shows a single configured feed for visibility.
+You might want to look over to a [real world example.](../complex/README.md)
+All docker related controls can be found in [docker-compose.yml](docker-compose.yml)
+
+## ETP-THREAT
+|IN|OUT|FILE|
+|---|---|---|
+|ETP-THREAT|TCP|[etp-threat.env](etp-threat.env)|
+
+
+
+
+
diff --git a/docker-compose/simple/docker-compose.yml b/docker-compose/simple/docker-compose.yml
index f0a5383..2b8bf6a 100644
--- a/docker-compose/simple/docker-compose.yml
+++ b/docker-compose/simple/docker-compose.yml
@@ -6,5 +6,5 @@ services:
env_file: etp-threat.env
volumes:
- type: bind
- source: /Users/mschiess/.edgerc
+ source: /patch/to/your/.edgerc
target: /opt/akamai-uls/.edgerc
\ No newline at end of file
diff --git a/docs/ARGUMENTS_ENV_VARS.md b/docs/ARGUMENTS_ENV_VARS.md
index 992cc73..9cd570e 100644
--- a/docs/ARGUMENTS_ENV_VARS.md
+++ b/docs/ARGUMENTS_ENV_VARS.md
@@ -14,7 +14,7 @@ The following tables list all available command line parameters and their corres
|Parameter|Env - Var|Options|Default|Description|
|---|---|---|---|---|
|-i
--input | ULS_INPUT | 'EAA', 'ETP', 'MFA' | None | Specify the desired INPUT source |
-|--feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN'
ETP: 'THREAT', 'AUP'
MFA: 'AUTH' | None | Specify the desired INPUT feed |
+|--feed | ULS_FEED | EAA: 'ACCESS', 'ADMIN'
ETP: 'THREAT', 'AUP'
MFA: 'AUTH','POLICY' | None | Specify the desired INPUT feed |
|--format | ULS_FORMAT | 'JSON', 'TEXT' | JSON | Specify the desired INPUT (=OUTPUT) format |
|--inproxy
--inputproxy | ULS_INPUT_PROXY | HOST:PORT| None | Adjust proxy usage for INPUT data collection (cli) |
|--rawcmd | ULS_RAWCMD | \ | None | USE with caution /!\
This is meant only to be used when told by AKAMAI|
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 672e619..9d57195 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -1,5 +1,14 @@
# Version History
+## v0.0.2
+|||
+|---|---|
+|Date|2021-06-10
+|Kind|Bugfix
+|Author|mschiess@akamai.com
adrocho@akamai.com
+- fixed monitoring output bug in docker-compose
+- fixed bug in Dockerfile that prevented development builds
+- fixed a bug in EAA CLI handler
## v0.0.1 (Initial Commit)
|version|v0.0.1|
diff --git a/docs/DOCKER-COMPOSE_USAGE.md b/docs/DOCKER-COMPOSE_USAGE.md
index 62557e9..f622bcb 100644
--- a/docs/DOCKER-COMPOSE_USAGE.md
+++ b/docs/DOCKER-COMPOSE_USAGE.md
@@ -55,5 +55,5 @@ docker compose up -d
**Files:**
- [docker-compose.yml](../docker-compose/complex/docker-compose.yml)
- [etp-threat.env](../docker-compose/complex/etp-threat.env)
- - [eaa-admin.env](../docker-compose/complex/eaa-admin.env)
+ - [eaa-admin.env](../docker-compose/complex/eaa-access.env)
- [eaa-access.env](../docker-compose/complex/eaa-access.env)
\ No newline at end of file