diff --git a/README.md b/README.md index 5c48bf3e7..1a1c37a55 100644 --- a/README.md +++ b/README.md @@ -139,21 +139,21 @@ You can then observe that the images have been retrieved by running `docker imag ``` $ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/arkime 3.1.1 xxxxxxxxxxxx 39 hours ago 683MB -malcolmnetsec/elasticsearch-od 3.1.1 xxxxxxxxxxxx 40 hours ago 690MB -malcolmnetsec/file-monitor 3.1.1 xxxxxxxxxxxx 39 hours ago 470MB -malcolmnetsec/file-upload 3.1.1 xxxxxxxxxxxx 39 hours ago 199MB -malcolmnetsec/filebeat-oss 3.1.1 xxxxxxxxxxxx 39 hours ago 555MB -malcolmnetsec/freq 3.1.1 xxxxxxxxxxxx 39 hours ago 390MB -malcolmnetsec/htadmin 3.1.1 xxxxxxxxxxxx 39 hours ago 180MB -malcolmnetsec/kibana-helper 3.1.1 xxxxxxxxxxxx 40 hours ago 141MB -malcolmnetsec/kibana-od 3.1.1 xxxxxxxxxxxx 40 hours ago 1.16GB -malcolmnetsec/logstash-oss 3.1.1 xxxxxxxxxxxx 39 hours ago 1.41GB -malcolmnetsec/name-map-ui 3.1.1 xxxxxxxxxxxx 39 hours ago 137MB -malcolmnetsec/nginx-proxy 3.1.1 xxxxxxxxxxxx 39 hours ago 120MB -malcolmnetsec/pcap-capture 3.1.1 xxxxxxxxxxxx 39 hours ago 111MB -malcolmnetsec/pcap-monitor 3.1.1 xxxxxxxxxxxx 39 hours ago 157MB -malcolmnetsec/zeek 3.1.1 xxxxxxxxxxxx 39 hours ago 887MB +malcolmnetsec/arkime 3.2.0 xxxxxxxxxxxx 39 hours ago 683MB +malcolmnetsec/elasticsearch-od 3.2.0 xxxxxxxxxxxx 40 hours ago 690MB +malcolmnetsec/file-monitor 3.2.0 xxxxxxxxxxxx 39 hours ago 470MB +malcolmnetsec/file-upload 3.2.0 xxxxxxxxxxxx 39 hours ago 199MB +malcolmnetsec/filebeat-oss 3.2.0 xxxxxxxxxxxx 39 hours ago 555MB +malcolmnetsec/freq 3.2.0 xxxxxxxxxxxx 39 hours ago 390MB +malcolmnetsec/htadmin 3.2.0 xxxxxxxxxxxx 39 hours ago 180MB +malcolmnetsec/kibana-helper 3.2.0 xxxxxxxxxxxx 40 hours ago 141MB +malcolmnetsec/kibana-od 3.2.0 xxxxxxxxxxxx 40 hours ago 1.16GB +malcolmnetsec/logstash-oss 3.2.0 xxxxxxxxxxxx 39 hours ago 1.41GB +malcolmnetsec/name-map-ui 3.2.0 xxxxxxxxxxxx 39 hours ago 137MB +malcolmnetsec/nginx-proxy 3.2.0 xxxxxxxxxxxx 39 hours ago 120MB +malcolmnetsec/pcap-capture 3.2.0 xxxxxxxxxxxx 39 hours ago 111MB +malcolmnetsec/pcap-monitor 3.2.0 xxxxxxxxxxxx 39 hours ago 157MB +malcolmnetsec/zeek 3.2.0 xxxxxxxxxxxx 39 hours ago 887MB ``` #### Import from pre-packaged tarballs @@ -1440,7 +1440,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu ``` … -Finished, created "/malcolm-build/malcolm-iso/malcolm-3.1.1.iso" +Finished, created "/malcolm-build/malcolm-iso/malcolm-3.2.0.iso" … ``` @@ -1823,21 +1823,21 @@ Pulling zeek ... done user@host:~/Malcolm$ docker images REPOSITORY TAG IMAGE ID CREATED SIZE -malcolmnetsec/arkime 3.1.1 xxxxxxxxxxxx 39 hours ago 683MB -malcolmnetsec/elasticsearch-od 3.1.1 xxxxxxxxxxxx 40 hours ago 690MB -malcolmnetsec/file-monitor 3.1.1 xxxxxxxxxxxx 39 hours ago 470MB -malcolmnetsec/file-upload 3.1.1 xxxxxxxxxxxx 39 hours ago 199MB -malcolmnetsec/filebeat-oss 3.1.1 xxxxxxxxxxxx 39 hours ago 555MB -malcolmnetsec/freq 3.1.1 xxxxxxxxxxxx 39 hours ago 390MB -malcolmnetsec/htadmin 3.1.1 xxxxxxxxxxxx 39 hours ago 180MB -malcolmnetsec/kibana-helper 3.1.1 xxxxxxxxxxxx 40 hours ago 141MB -malcolmnetsec/kibana-od 3.1.1 xxxxxxxxxxxx 40 hours ago 1.16GB -malcolmnetsec/logstash-oss 3.1.1 xxxxxxxxxxxx 39 hours ago 1.41GB -malcolmnetsec/name-map-ui 3.1.1 xxxxxxxxxxxx 39 hours ago 137MB -malcolmnetsec/nginx-proxy 3.1.1 xxxxxxxxxxxx 39 hours ago 120MB -malcolmnetsec/pcap-capture 3.1.1 xxxxxxxxxxxx 39 hours ago 111MB -malcolmnetsec/pcap-monitor 3.1.1 xxxxxxxxxxxx 39 hours ago 157MB -malcolmnetsec/zeek 3.1.1 xxxxxxxxxxxx 39 hours ago 887MB +malcolmnetsec/arkime 3.2.0 xxxxxxxxxxxx 39 hours ago 683MB +malcolmnetsec/elasticsearch-od 3.2.0 xxxxxxxxxxxx 40 hours ago 690MB +malcolmnetsec/file-monitor 3.2.0 xxxxxxxxxxxx 39 hours ago 470MB +malcolmnetsec/file-upload 3.2.0 xxxxxxxxxxxx 39 hours ago 199MB +malcolmnetsec/filebeat-oss 3.2.0 xxxxxxxxxxxx 39 hours ago 555MB +malcolmnetsec/freq 3.2.0 xxxxxxxxxxxx 39 hours ago 390MB +malcolmnetsec/htadmin 3.2.0 xxxxxxxxxxxx 39 hours ago 180MB +malcolmnetsec/kibana-helper 3.2.0 xxxxxxxxxxxx 40 hours ago 141MB +malcolmnetsec/kibana-od 3.2.0 xxxxxxxxxxxx 40 hours ago 1.16GB +malcolmnetsec/logstash-oss 3.2.0 xxxxxxxxxxxx 39 hours ago 1.41GB +malcolmnetsec/name-map-ui 3.2.0 xxxxxxxxxxxx 39 hours ago 137MB +malcolmnetsec/nginx-proxy 3.2.0 xxxxxxxxxxxx 39 hours ago 120MB +malcolmnetsec/pcap-capture 3.2.0 xxxxxxxxxxxx 39 hours ago 111MB +malcolmnetsec/pcap-monitor 3.2.0 xxxxxxxxxxxx 39 hours ago 157MB +malcolmnetsec/zeek 3.2.0 xxxxxxxxxxxx 39 hours ago 887MB ``` Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background. diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 3615c6941..0cf052844 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -119,7 +119,7 @@ x-pcap-capture-variables: &pcap-capture-variables services: elasticsearch: - image: malcolmnetsec/elasticsearch-od:3.1.1 + image: malcolmnetsec/elasticsearch-od:3.2.0 restart: "no" stdin_open: false tty: true @@ -156,7 +156,7 @@ services: retries: 3 start_period: 180s kibana-helper: - image: malcolmnetsec/kibana-helper:3.1.1 + image: malcolmnetsec/kibana-helper:3.2.0 restart: "no" stdin_open: false tty: true @@ -184,7 +184,7 @@ services: retries: 3 start_period: 30s kibana: - image: malcolmnetsec/kibana-od:3.1.1 + image: malcolmnetsec/kibana-od:3.2.0 restart: "no" stdin_open: false tty: true @@ -205,7 +205,7 @@ services: retries: 3 start_period: 210s logstash: - image: malcolmnetsec/logstash-oss:3.1.1 + image: malcolmnetsec/logstash-oss:3.2.0 restart: "no" stdin_open: false tty: true @@ -239,7 +239,7 @@ services: retries: 3 start_period: 600s filebeat: - image: malcolmnetsec/filebeat-oss:3.1.1 + image: malcolmnetsec/filebeat-oss:3.2.0 restart: "no" stdin_open: false tty: true @@ -276,7 +276,7 @@ services: retries: 3 start_period: 60s arkime: - image: malcolmnetsec/arkime:3.1.1 + image: malcolmnetsec/arkime:3.2.0 restart: "no" stdin_open: false tty: true @@ -315,7 +315,7 @@ services: retries: 3 start_period: 210s zeek: - image: malcolmnetsec/zeek:3.1.1 + image: malcolmnetsec/zeek:3.2.0 restart: "no" stdin_open: false tty: true @@ -341,7 +341,7 @@ services: retries: 3 start_period: 60s file-monitor: - image: malcolmnetsec/file-monitor:3.1.1 + image: malcolmnetsec/file-monitor:3.2.0 restart: "no" stdin_open: false tty: true @@ -364,7 +364,7 @@ services: retries: 3 start_period: 60s pcap-capture: - image: malcolmnetsec/pcap-capture:3.1.1 + image: malcolmnetsec/pcap-capture:3.2.0 restart: "no" stdin_open: false tty: true @@ -390,7 +390,7 @@ services: retries: 3 start_period: 60s pcap-monitor: - image: malcolmnetsec/pcap-monitor:3.1.1 + image: malcolmnetsec/pcap-monitor:3.2.0 restart: "no" stdin_open: false tty: true @@ -413,7 +413,7 @@ services: retries: 3 start_period: 90s upload: - image: malcolmnetsec/file-upload:3.1.1 + image: malcolmnetsec/file-upload:3.2.0 restart: "no" stdin_open: false tty: true @@ -439,7 +439,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:3.1.1 + image: malcolmnetsec/htadmin:3.2.0 restart: "no" stdin_open: false tty: true @@ -461,7 +461,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:3.1.1 + image: malcolmnetsec/freq:3.2.0 restart: "no" stdin_open: false tty: true @@ -479,7 +479,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:3.1.1 + image: malcolmnetsec/name-map-ui:3.2.0 restart: "no" stdin_open: false tty: true @@ -500,7 +500,7 @@ services: retries: 3 start_period: 60s nginx-proxy: - image: malcolmnetsec/nginx-proxy:3.1.1 + image: malcolmnetsec/nginx-proxy:3.2.0 restart: "no" stdin_open: false tty: true diff --git a/docker-compose.yml b/docker-compose.yml index 175e4a17b..ddfb27627 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -122,7 +122,7 @@ services: build: context: . dockerfile: Dockerfiles/elasticsearch.Dockerfile - image: malcolmnetsec/elasticsearch-od:3.1.1 + image: malcolmnetsec/elasticsearch-od:3.2.0 restart: "no" stdin_open: false tty: true @@ -162,7 +162,7 @@ services: build: context: . dockerfile: Dockerfiles/kibana-helper.Dockerfile - image: malcolmnetsec/kibana-helper:3.1.1 + image: malcolmnetsec/kibana-helper:3.2.0 restart: "no" stdin_open: false tty: true @@ -193,7 +193,7 @@ services: build: context: . dockerfile: Dockerfiles/kibana.Dockerfile - image: malcolmnetsec/kibana-od:3.1.1 + image: malcolmnetsec/kibana-od:3.2.0 restart: "no" stdin_open: false tty: true @@ -217,7 +217,7 @@ services: build: context: . dockerfile: Dockerfiles/logstash.Dockerfile - image: malcolmnetsec/logstash-oss:3.1.1 + image: malcolmnetsec/logstash-oss:3.2.0 restart: "no" stdin_open: false tty: true @@ -256,7 +256,7 @@ services: build: context: . dockerfile: Dockerfiles/filebeat.Dockerfile - image: malcolmnetsec/filebeat-oss:3.1.1 + image: malcolmnetsec/filebeat-oss:3.2.0 restart: "no" stdin_open: false tty: true @@ -297,7 +297,7 @@ services: build: context: . dockerfile: Dockerfiles/arkime.Dockerfile - image: malcolmnetsec/arkime:3.1.1 + image: malcolmnetsec/arkime:3.2.0 restart: "no" stdin_open: false tty: true @@ -342,7 +342,7 @@ services: build: context: . dockerfile: Dockerfiles/zeek.Dockerfile - image: malcolmnetsec/zeek:3.1.1 + image: malcolmnetsec/zeek:3.2.0 restart: "no" stdin_open: false tty: true @@ -372,7 +372,7 @@ services: build: context: . dockerfile: Dockerfiles/file-monitor.Dockerfile - image: malcolmnetsec/file-monitor:3.1.1 + image: malcolmnetsec/file-monitor:3.2.0 restart: "no" stdin_open: false tty: true @@ -398,7 +398,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-capture.Dockerfile - image: malcolmnetsec/pcap-capture:3.1.1 + image: malcolmnetsec/pcap-capture:3.2.0 restart: "no" stdin_open: false tty: true @@ -427,7 +427,7 @@ services: build: context: . dockerfile: Dockerfiles/pcap-monitor.Dockerfile - image: malcolmnetsec/pcap-monitor:3.1.1 + image: malcolmnetsec/pcap-monitor:3.2.0 restart: "no" stdin_open: false tty: true @@ -453,7 +453,7 @@ services: build: context: . dockerfile: Dockerfiles/file-upload.Dockerfile - image: malcolmnetsec/file-upload:3.1.1 + image: malcolmnetsec/file-upload:3.2.0 restart: "no" stdin_open: false tty: true @@ -479,7 +479,7 @@ services: retries: 3 start_period: 60s htadmin: - image: malcolmnetsec/htadmin:3.1.1 + image: malcolmnetsec/htadmin:3.2.0 build: context: . dockerfile: Dockerfiles/htadmin.Dockerfile @@ -504,7 +504,7 @@ services: retries: 3 start_period: 60s freq: - image: malcolmnetsec/freq:3.1.1 + image: malcolmnetsec/freq:3.2.0 build: context: . dockerfile: Dockerfiles/freq.Dockerfile @@ -525,7 +525,7 @@ services: retries: 3 start_period: 60s name-map-ui: - image: malcolmnetsec/name-map-ui:3.1.1 + image: malcolmnetsec/name-map-ui:3.2.0 build: context: . dockerfile: Dockerfiles/name-map-ui.Dockerfile @@ -552,7 +552,7 @@ services: build: context: . dockerfile: Dockerfiles/nginx.Dockerfile - image: malcolmnetsec/nginx-proxy:3.1.1 + image: malcolmnetsec/nginx-proxy:3.2.0 restart: "no" stdin_open: false tty: true diff --git a/kibana/scripts/kibana-create-moloch-sessions-index.sh b/kibana/scripts/kibana-create-moloch-sessions-index.sh index 965abd3a4..b9a053790 100755 --- a/kibana/scripts/kibana-create-moloch-sessions-index.sh +++ b/kibana/scripts/kibana-create-moloch-sessions-index.sh @@ -49,7 +49,7 @@ if [[ "$CREATE_ES_ARKIME_SESSION_INDEX" = "true" ]] ; then /data/register-elasticsearch-snapshot-repo.sh # tweak the sessions template (sessions2-* zeek template file) to use the index management policy - if [[ -r "$INDEX_POLICY_FILE_HOST" ]] && (( $(jq length "$INDEX_POLICY_FILE_HOST") > 0 )); then + if [[ -f "$INDEX_POLICY_FILE_HOST" ]] && (( $(jq length "$INDEX_POLICY_FILE_HOST") > 0 )); then # user has provided a file for index management, use it cp "$INDEX_POLICY_FILE_HOST" "$INDEX_POLICY_FILE" INDEX_POLICY_NAME="$(cat "$INDEX_POLICY_FILE" | jq '..|objects|.policy_id//empty' | tr -d '"')" @@ -67,34 +67,45 @@ if [[ "$CREATE_ES_ARKIME_SESSION_INDEX" = "true" ]] ; then > "$INDEX_POLICY_FILE" fi - if [[ -r "$INDEX_POLICY_FILE" ]]; then + if [[ -f "$INDEX_POLICY_FILE" ]]; then # make API call to define index management policy # https://opendistro.github.io/for-elasticsearch-docs/docs/ism/api/#create-policy - curl -L --silent --output /dev/null --show-error -XPUT -H "Content-Type: application/json" "$ES_URL/_opendistro/_ism/policies/$INDEX_POLICY_NAME" -d "@$INDEX_POLICY_FILE" + curl -w "\n" -L --silent --output /dev/null --show-error -XPUT -H "Content-Type: application/json" "$ES_URL/_opendistro/_ism/policies/$INDEX_POLICY_NAME" -d "@$INDEX_POLICY_FILE" - if [[ -r "$ZEEK_TEMPLATE_FILE_ORIG" ]]; then + if [[ -f "$ZEEK_TEMPLATE_FILE_ORIG" ]]; then # insert opendistro.index_state_management.policy_id into index template settings: will be # imported by kibana-create-moloch-sessions-index.sh cat "$ZEEK_TEMPLATE_FILE_ORIG" | jq ".settings += {\"opendistro.index_state_management.policy_id\": \"$INDEX_POLICY_NAME\"}" > "$ZEEK_TEMPLATE_FILE" fi fi - echo "Importing Kibana saved objects..." + echo "Importing zeek_template..." + + if [[ -f "$ZEEK_TEMPLATE_FILE_ORIG" ]] && [[ ! -f "$ZEEK_TEMPLATE_FILE" ]]; then + cp "$ZEEK_TEMPLATE_FILE_ORIG" "$ZEEK_TEMPLATE_FILE" + fi # load zeek_template containing zeek field type mappings (merged from /data/zeek_template.json to /data/init/zeek_template.json in kibana_helpers.sh on startup) - curl -L --silent --output /dev/null --show-error -XPOST -H "Content-Type: application/json" "$ES_URL/_template/zeek_template?include_type_name=true" -d "@$ZEEK_TEMPLATE_FILE" + curl -w "\n" -sSL --fail -XPOST -H "Content-Type: application/json" \ + "$ES_URL/_template/zeek_template?include_type_name=true" -d "@$ZEEK_TEMPLATE_FILE" 2>&1 + + echo "Importing index pattern..." # From https://github.com/elastic/kibana/issues/3709 # Create index pattern - curl -L --silent --output /dev/null --show-error --fail -XPOST -H "Content-Type: application/json" -H "kbn-xsrf: anything" \ + curl -w "\n" -sSL --fail -XPOST -H "Content-Type: application/json" -H "kbn-xsrf: anything" \ "$KIB_URL/api/saved_objects/index-pattern/$INDEX_PATTERN_ID" \ - -d"{\"attributes\":{\"title\":\"$INDEX_PATTERN\",\"timeFieldName\":\"$INDEX_TIME_FIELD\"}}" + -d"{\"attributes\":{\"title\":\"$INDEX_PATTERN\",\"timeFieldName\":\"$INDEX_TIME_FIELD\"}}" 2>&1 + + echo "Setting default index pattern..." # Make it the default index - curl -L --silent --output /dev/null --show-error -XPOST -H "Content-Type: application/json" -H "kbn-xsrf: anything" \ + curl -w "\n" -sSL -XPOST -H "Content-Type: application/json" -H "kbn-xsrf: anything" \ "$KIB_URL/api/kibana/settings/defaultIndex" \ -d"{\"value\":\"$INDEX_PATTERN_ID\"}" + echo "Importing Kibana saved objects..." + # install default dashboards, index patterns, etc. for i in /opt/kibana/dashboards/*.json; do curl -L --silent --output /dev/null --show-error -XPOST "$KIB_URL/api/kibana/dashboards/import?force=true" -H 'kbn-xsrf:true' -H 'Content-type:application/json' -d "@$i" diff --git a/kibana/scripts/register-elasticsearch-snapshot-repo.sh b/kibana/scripts/register-elasticsearch-snapshot-repo.sh index 1a4360de2..4b63fb4aa 100755 --- a/kibana/scripts/register-elasticsearch-snapshot-repo.sh +++ b/kibana/scripts/register-elasticsearch-snapshot-repo.sh @@ -13,7 +13,7 @@ else fi [[ -n $ISM_SNAPSHOT_REPO ]] && \ - curl -H "Accept: application/json" \ + curl -w "\n" -H "Accept: application/json" \ -H "Content-type: application/json" \ -XPUT -fsSL "$ES_URL/_snapshot/$ISM_SNAPSHOT_REPO" \ -d "{ \"type\": \"fs\", \"settings\": { \"location\": \"$ISM_SNAPSHOT_REPO\", \"compress\": ${ISM_SNAPSHOT_COMPRESSED:-false} } }" \ No newline at end of file diff --git a/kibana/supervisord.conf b/kibana/supervisord.conf index 2f4ed6db0..3f6a1edc2 100644 --- a/kibana/supervisord.conf +++ b/kibana/supervisord.conf @@ -30,7 +30,7 @@ stdout_logfile_maxbytes=0 redirect_stderr=true [program:maps] -command=/usr/bin/http-server /opt/maps --cors='*' -d false -i false --no-dotfiles -p %(ENV_KIBANA_OFFLINE_REGION_MAPS_PORT)s +command=/usr/local/bin/http-server /opt/maps --cors='*' -d false -i false --no-dotfiles -p %(ENV_KIBANA_OFFLINE_REGION_MAPS_PORT)s autostart=true autorestart=true startsecs=0 diff --git a/sensor-iso/README.md b/sensor-iso/README.md index b28aa3d49..0e287797a 100644 --- a/sensor-iso/README.md +++ b/sensor-iso/README.md @@ -403,7 +403,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu ``` … -Finished, created "/sensor-build/hedgehog-3.1.1.iso" +Finished, created "/sensor-build/hedgehog-3.2.0.iso" … ```