v1.8.1 development (#99)

- Update to Elastic stack 7.5.1 (and fixed hopefully all the compatibility issues that arose) - Moloch version 2.1.2 - fix issues with initial build and download of maxmind geoip database files - documentation updates and fixes - some improvements to help with higher bitrate capture (increasing ring buffer sizes) - improvements to ISO for Malcolm (aggregator) and Hedgehog (sensor)
cisagov · Jan 10, 2020 · 2d09b51 · 2d09b51
1 parent 33f7598
commit 2d09b51
Show file tree

Hide file tree

Showing 148 changed files with 11,956 additions and 3,884 deletions.
diff --git a/Dockerfiles/elastalert.Dockerfile b/Dockerfiles/elastalert.Dockerfile
@@ -1,4 +1,4 @@
-FROM bitsensor/elastalert:2.0.0
+FROM mmguero/elastalert:2.0.2
 
 # Copyright (c) 2019 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.elastic.co/beats/filebeat-oss:6.8.5
+FROM docker.elastic.co/beats/filebeat-oss:7.5.1
 
 # Copyright (c) 2019 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"
@@ -55,7 +55,7 @@ ENV FILEBEAT_CLEAN_REMOVED $FILEBEAT_CLEAN_REMOVED
 ENV FILEBEAT_LOG_PATH $FILEBEAT_LOG_PATH
 ENV AUTO_TAG $AUTO_TAG
 
-ENV FILEBEAT_REGISTRY_FILE "/usr/share/filebeat/data/registry"
+ENV FILEBEAT_REGISTRY_FILE "/usr/share/filebeat/data/registry/filebeat/data.json"
 ENV FILEBEAT_ZEEK_DIR "/data/zeek/"
 ENV PATH="/data:${PATH}"
 

diff --git a/Dockerfiles/kibana.Dockerfile b/Dockerfiles/kibana.Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.elastic.co/kibana/kibana-oss:6.8.5
+FROM docker.elastic.co/kibana/kibana-oss:7.5.1
 
 # Copyright (c) 2019 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"
@@ -47,10 +47,15 @@ ADD kibana/kibana-offline-maps.yml /opt/kibana/config/kibana-offline-maps.yml
 ADD kibana/supervisord.conf /etc/supervisord.conf
 ADD kibana/dashboards /opt/kibana/dashboards
 ADD kibana/maps /opt/maps
-ADD https://github.com/bitsensor/elastalert-kibana-plugin/releases/download/1.0.3/elastalert-kibana-plugin-1.0.3-6.8.0.zip /tmp/elastalert-kibana-plugin.zip
-ADD https://github.com/prelert/kibana-swimlane-vis/releases/download/v6.8.1/prelert_swimlane_vis-6.8.1.zip /tmp/kibana-swimlane.zip
-ADD https://github.com/gwintzer/kibana-comments-app-plugin/releases/download/6.7.1/kibana-comments-app-plugin-6.7.1-1.zip /tmp/kibana-comments.zip
-ADD https://github.com/walterra/kibana-milestones-vis/releases/download/v6.8.2/kibana-milestones-vis-6.8.2.zip /tmp/kibana-milestones.zip
+ADD https://github.com/bitsensor/elastalert-kibana-plugin/releases/download/1.1.0/elastalert-kibana-plugin-1.1.0-7.5.0.zip /tmp/elastalert-kibana-plugin.zip
+
+# todo: these extra plugins are kind of gutted right now with 7.5.x, need to fix
+
+# ADD https://github.com/prelert/kibana-swimlane-vis/releases/download/v7.4.2/prelert_swimlane_vis-7.4.2.zip /tmp/kibana-swimlane.zip
+# ADD https://github.com/gwintzer/kibana-comments-app-plugin/releases/download/7.4.0/kibana-comments-app-plugin-7.4.0-latest.zip /tmp/kibana-comments.zip
+
+# see https://github.com/walterra/kibana-milestones-vis/issues/9
+#ADD https://github.com/walterra/kibana-milestones-vis/releases/download/v7.1.1/kibana-milestones-vis-7.1.1.zip /tmp/kibana-milestones.zip
 
 # TODO: commented out because it's not optimizing in 6.6+ correctly
 # put these back in here and below in the build section:
@@ -60,7 +65,7 @@ ADD https://github.com/walterra/kibana-milestones-vis/releases/download/v6.8.2/k
 #    unzip kibana-calendar.zip kibana/kibana_calendar_vis/package.json && \
 #    sed -i "s/6\.4\.0/6\.6\.0/g" kibana/kibana_calendar_vis/package.json && \
 #    zip kibana-calendar.zip kibana/kibana_calendar_vis/package.json && \
-#    /usr/share/kibana/bin/kibana-plugin install file:///tmp/kibana-calendar.zip && \
+#    /usr/share/kibana/bin/kibana-plugin install file:///tmp/kibana-calendar.zip --allow-root && \
 #    rm -rf /tmp/kibana-calendar.zip /tmp/kibana && \
 
 RUN chmod 755 /data/*.sh /data/*.py && \
@@ -71,28 +76,33 @@ RUN chmod 755 /data/*.sh /data/*.py && \
     cd /tmp && \
     echo "Installing ElastAlert plugin..." && \
       unzip elastalert-kibana-plugin.zip kibana/elastalert-kibana-plugin/package.json && \
-      sed -i "s/6\.8\.0/6\.8\.5/g" kibana/elastalert-kibana-plugin/package.json && \
+      sed -i "s/7\.5\.0/7\.5\.1/g" kibana/elastalert-kibana-plugin/package.json && \
       zip elastalert-kibana-plugin.zip kibana/elastalert-kibana-plugin/package.json && \
-      /usr/share/kibana/bin/kibana-plugin install file:///tmp/elastalert-kibana-plugin.zip && \
-      rm -f /tmp/elastalert-kibana-plugin.zip && \
-    echo "Installing Swimlanes visualization..." && \
-      unzip kibana-swimlane.zip kibana/prelert_swimlane_vis-6.8.1/package.json && \
-      sed -i "s/6\.8\.1/6\.8\.5/g" kibana/prelert_swimlane_vis-6.8.1/package.json && \
-      zip kibana-swimlane.zip kibana/prelert_swimlane_vis-6.8.1/package.json && \
-      /usr/share/kibana/bin/kibana-plugin install file:///tmp/kibana-swimlane.zip && \
-      rm -f /tmp/kibana-swimlane.zip && \
-    echo "Installing Comments visualization..." && \
-      unzip kibana-comments.zip kibana/kibana-comments-app-plugin/package.json && \
-      sed -i "s/6\.7\.1/6\.8\.5/g" kibana/kibana-comments-app-plugin/package.json && \
-      zip kibana-comments.zip kibana/kibana-comments-app-plugin/package.json && \
-      /usr/share/kibana/bin/kibana-plugin install file:///tmp/kibana-comments.zip && \
-      rm -rf /tmp/kibana-comments.zip /tmp/kibana && \
-    echo "Installing Milestones visualization..." && \
-      unzip kibana-milestones.zip kibana/kibana-milestones-vis/package.json && \
-      sed -i "s/6\.8\.2/6\.8\.5/g" kibana/kibana-milestones-vis/package.json && \
-      zip kibana-milestones.zip kibana/kibana-milestones-vis/package.json && \
-      /usr/share/kibana/bin/kibana-plugin install file:///tmp/kibana-milestones.zip && \
-      rm -rf /tmp/kibana-milestones.zip /tmp/kibana
+      /usr/share/kibana/bin/kibana-plugin install file:///tmp/elastalert-kibana-plugin.zip --allow-root && \
+      rm -f /tmp/elastalert-kibana-plugin.zip
+
+    ## && \
+    ## echo "Installing Swimlanes visualization..." && \
+    ##  unzip kibana-swimlane.zip kibana/prelert_swimlane_vis-7.4.2/package.json && \
+    ##  sed -i "s/7\.4\.2/7\.5\.1/g" kibana/prelert_swimlane_vis-7.4.2/package.json && \
+    ##  zip kibana-swimlane.zip kibana/prelert_swimlane_vis-7.4.2/package.json && \
+    ##  /usr/share/kibana/bin/kibana-plugin install file:///tmp/kibana-swimlane.zip --allow-root && \
+    ##  bash -c "find /usr/share/kibana/plugins/prelert_swimlane_vis/ -type f -exec chmod 644 '{}' \;" && \
+    ##  rm -f /tmp/kibana-swimlane.zip && \
+    ## echo "Installing Comments visualization..." && \
+    ##   unzip kibana-comments.zip kibana/kibana-comments-app-plugin/package.json && \
+    ##   sed -i "s/7\.4\.0/7\.5\.1/g" kibana/kibana-comments-app-plugin/package.json && \
+    ##   zip kibana-comments.zip kibana/kibana-comments-app-plugin/package.json && \
+    ##   /usr/share/kibana/bin/kibana-plugin install file:///tmp/kibana-comments.zip --allow-root && \
+    ##   rm -rf /tmp/kibana-comments.zip /tmp/kibana && \
+    ## https://github.com/walterra/kibana-milestones-vis/issues/9
+    ## && \
+    ## echo "Installing Milestones visualization..." && \
+    ##  unzip kibana-milestones.zip kibana/kibana-milestones-vis/package.json && \
+    ##  sed -i "s/7\.1\.1/7\.5\.1/g" kibana/kibana-milestones-vis/package.json && \
+    ##  zip kibana-milestones.zip kibana/kibana-milestones-vis/package.json && \
+    ##  /usr/share/kibana/bin/kibana-plugin install file:///tmp/kibana-milestones.zip --allow-root && \
+    ##  rm -rf /tmp/kibana-milestones.zip /tmp/kibana
 
 ADD docs/images/kibana/ebdca7741674eca4e1fadeca157f3ae6.svg /usr/share/kibana/optimize/bundles/ebdca7741674eca4e1fadeca157f3ae6.svg
 

diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile
@@ -42,7 +42,7 @@ RUN /bin/bash -lc "command curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -
     git clone --depth 1 https://github.com/mmguero/logstash-filter-ieee_oui.git /opt/logstash-filter-ieee_oui && \
     /bin/bash -lc "cd /opt/logstash-filter-ieee_oui && bundle install && gem build logstash-filter-ieee_oui.gemspec && bundle info logstash-filter-ieee_oui"
 
-FROM docker.elastic.co/logstash/logstash-oss:6.8.5
+FROM docker.elastic.co/logstash/logstash-oss:7.5.1
 USER root
 
 COPY --from=build /opt/logstash-filter-ieee_oui /opt/logstash-filter-ieee_oui

diff --git a/Dockerfiles/moloch.Dockerfile b/Dockerfiles/moloch.Dockerfile
@@ -4,7 +4,7 @@ FROM debian:buster-slim AS build
 
 ENV DEBIAN_FRONTEND noninteractive
 
-ENV MOLOCH_VERSION "2.1.1"
+ENV MOLOCH_VERSION "2.1.2"
 ENV MOLOCHDIR "/data/moloch"
 
 ADD moloch/scripts/bs4_remove_div.py /data/
@@ -110,6 +110,7 @@ ARG AUTO_TAG=true
 ARG PCAP_PIPELINE_DEBUG=false
 ARG PCAP_PIPELINE_DEBUG_EXTRA=false
 ARG PCAP_MONITOR_HOST=pcap-monitor
+ARG MAXMIND_GEOIP_DB_LICENSE_KEY=""
 
 # Declare envs vars for each arg
 ENV ES_HOST $ES_HOST
@@ -179,22 +180,30 @@ ADD shared/bin/pcap_moloch_and_zeek_processor.py /data/
 ADD shared/bin/pcap_utils.py /data/
 ADD shared/bin/elastic_search_status.sh /data/
 ADD moloch/etc $MOLOCHDIR/etc/
-ADD https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv $MOLOCHDIR/etc/ipv4-address-space.csv
-ADD https://raw.githubusercontent.com/wireshark/wireshark/master/manuf $MOLOCHDIR/etc/oui.txt
-ADD https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-Country /tmp/GeoLite2-Country.mmdb.gz
-ADD https://updates.maxmind.com/app/update_secure?edition_id=GeoLite2-ASN /tmp/GeoLite2-ASN.mmdb.gz
 ADD moloch/wise/source.*.js $MOLOCHDIR/wiseService/
 ADD moloch/supervisord.conf /etc/supervisord.conf
 
+# MaxMind now requires a (free) license key to download the free versions of
+# their GeoIP databases. This should be provided as a build argument.
+#   see https://dev.maxmind.com/geoip/geoipupdate/#Direct_Downloads
+#   see https://github.com/aol/moloch/issues/1350
+#   see https://github.com/aol/moloch/issues/1352
+RUN [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do \
+      cd /tmp && \
+      curl -s -S -L -o "GeoLite2-$DB.mmdb.tar.gz" "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-$DB&license_key=$MAXMIND_GEOIP_DB_LICENSE_KEY&suffix=tar.gz" && \
+      tar xf "GeoLite2-$DB.mmdb.tar.gz" --wildcards --no-anchored '*.mmdb' --strip=1 && \
+      mkdir -p $MOLOCHDIR/etc/ && \
+      mv -v "GeoLite2-$DB.mmdb" $MOLOCHDIR/etc/; \
+      rm -f "GeoLite2-$DB*"; \
+    done; \
+  curl -s -S -L -o $MOLOCHDIR/etc/ipv4-address-space.csv "https://www.iana.org/assignments/ipv4-address-space/ipv4-address-space.csv" && \
+  curl -s -S -L -o $MOLOCHDIR/etc/oui.txt "https://raw.githubusercontent.com/wireshark/wireshark/master/manuf"
+
 RUN groupadd --gid 1000 $MOLOCHUSER && \
     useradd -M --uid 1000 --gid 1000 --home $MOLOCHDIR $MOLOCHUSER && \
     chmod 755 /data/*.sh && \
     ln -sfr /data/pcap_moloch_and_zeek_processor.py /data/pcap_moloch_processor.py && \
     cp -f /data/moloch_update_geo.sh $MOLOCHDIR/bin/moloch_update_geo.sh && \
-    bash -c "zcat /tmp/GeoLite2-Country.mmdb.gz > $MOLOCHDIR/etc/GeoLite2-Country.mmdb" && \
-    rm -f /tmp/GeoLite2-Country.mmdb.gz && \
-    bash -c "zcat /tmp/GeoLite2-ASN.mmdb.gz > $MOLOCHDIR/etc/GeoLite2-ASN.mmdb" && \
-    rm -f /tmp/GeoLite2-ASN.mmdb.gz && \
     sed -i "s/^\(MOLOCH_LOCALELASTICSEARCH=\).*/\1"$MOLOCH_LOCALELASTICSEARCH"/" $MOLOCHDIR/bin/Configure && \
     sed -i "s/^\(MOLOCH_INET=\).*/\1"$MOLOCH_INET"/" $MOLOCHDIR/bin/Configure && \
     chmod u+s $MOLOCHDIR/bin/moloch-capture && \