v3.0.1 development (#161)

Malcolm v3.0.1

- Version bumps
  - Open Distro for Elastic ([v1.13.0](https://github.com/opendistro-for-elasticsearch/opendistro-build/blob/main/release-notes/opendistro-for-elasticsearch-release-notes-1.13.0.md)), which adds the following functionality over the previous release
    - [Reporting](https://opendistro.github.io/for-elasticsearch-docs/docs/kibana/reporting/)
    - [Historical data anomaly detection](https://opendistro.github.io/for-elasticsearch-docs/docs/ad/#step-6-analyze-historical-data)
  - ODFE v1.13.0 is based on the Elastic components 7.10.2 ([elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/release-notes-7.10.2.html), [kibana](https://www.elastic.co/guide/en/kibana/current/release-notes-7.10.2.html), [logstash](https://www.elastic.co/guide/en/logstash/current/logstash-7-10-2.html), [beats](https://www.elastic.co/guide/en/beats/libbeat/master/release-notes-7.10.2.html))
  - Zeek [3.0.13](https://github.com/zeek/zeek/releases/tag/v3.0.13)
  - NGINX [1.19.7](https://nginx.org/en/CHANGES)
  - Alpine Linux [3.13](https://alpinelinux.org/posts/Alpine-3.13.0-released.html) Docker base layer
  - docker-compose [1.28.5](https://docs.docker.com/compose/release-notes/) in Malcolm installable ISO version
- Restored the [sankey visualization](https://github.com/uniberg/kbn_sankey_vis) which was temporarily removed in Malcolm v3.0.0 (although there are still a few minor cosmetic [issues](uniberg/kbn_sankey_vis#15) with it)
- Removed port 8443 for upload (now just use /upload over the regular HTTPS port)
- Fixed issue with ODFE email alerts not being able to use self-signed SMTP certificates by importing CA certs in `nginx/ca-trust` into the JDK trust store for Elasticsearch and Logstash (see idaholab#37)
- Don't expose the Elasticsearch 9200 by default, it must now be explicitly be enabled during `install.py -c` (see idaholab#38)
- For ISO-installed versions of Malcolm and Hedgehog Linux, populate `/etc/os-release` with information about the build/release version
- Populate user-agent for a few clients ([Arkime's moloch-capture](arkime/arkime#1615), some hedgehog test connection processes) so they're not just sent as blank when communicating with Malcolm
- Added Arkime link to Kibana dashboards' navigation pane
- Fix some issues in control script with older python3 versions (3.6.x) with `contextlib.nullcontext` not being available
- Fix suggestion for yum-based distributions to install python 3 requests via pip

Dockerfiles/elasticsearch.Dockerfile

@@ -1,4 +1,4 @@
-FROM amazon/opendistro-for-elasticsearch:1.12.0
+FROM amazon/opendistro-for-elasticsearch:1.13.0
 
 # Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"
@@ -23,19 +23,22 @@ ENV TERM xterm
 ARG GITHUB_OAUTH_TOKEN=""
 ARG DISABLE_INSTALL_DEMO_CONFIG=true
 ENV DISABLE_INSTALL_DEMO_CONFIG $DISABLE_INSTALL_DEMO_CONFIG
+ENV JAVA_HOME=/usr/share/elasticsearch/jdk
 
 # Malcolm manages authentication and encryption via NGINX reverse proxy
 # https://opendistro.github.io/for-elasticsearch-docs/docs/security/configuration/disable/
 # https://opendistro.github.io/for-elasticsearch-docs/docs/install/docker/#customize-the-docker-image
 # https://github.com/opendistro-for-elasticsearch/opendistro-build/issues/613
-RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security && \
-    /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_performance_analyzer && \
+RUN yum install -y openssl && \
+  /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security && \
   echo -e 'cluster.name: "docker-cluster"\nnetwork.host: 0.0.0.0' > /usr/share/elasticsearch/config/elasticsearch.yml && \
   chown -R $PUSER:$PGROUP /usr/share/elasticsearch/config/elasticsearch.yml && \
-  sed -i "s/\b1000\b/\${PUID:-${DEFAULT_UID}}/g" /usr/local/bin/docker-entrypoint.sh
+  sed -i "s/\b1000\b/\${PUID:-${DEFAULT_UID}}/g" /usr/local/bin/docker-entrypoint.sh && \
+  sed -i '/[^#].*\/usr\/share\/elasticsearch\/bin\/elasticsearch.*/i /usr/local/bin/jdk-cacerts-auto-import.sh || true' /usr/local/bin/docker-entrypoint.sh
 
 # just used for initial keystore creation
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
+ADD shared/bin/jdk-cacerts-auto-import.sh /usr/local/bin/
 
 # to be populated at build-time:
 ARG BUILD_DATE

Dockerfiles/filebeat.Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.elastic.co/beats/filebeat-oss:7.10.0
+FROM docker.elastic.co/beats/filebeat-oss:7.10.2
 
 # Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

Dockerfiles/kibana-helper.Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.12
+FROM alpine:3.13
 
 # Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

Dockerfiles/kibana.Dockerfile

@@ -1,6 +1,72 @@
-FROM amazon/opendistro-for-elasticsearch-kibana:1.12.0
+FROM centos:7 AS build
 
 # Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
+
+# set up build environment for kibana plugins built from source
+
+ARG DEFAULT_UID=1000
+ARG DEFAULT_GID=1000
+ENV DEFAULT_UID $DEFAULT_UID
+ENV DEFAULT_GID $DEFAULT_GID
+ENV PUSER "kibana"
+ENV PGROUP "kibana"
+
+ENV TERM xterm
+
+ARG ELASTIC_VERSION="7.10.2"
+ENV ELASTIC_VERSION $ELASTIC_VERSION
+
+# base system dependencies for checking out and building elastic/kibana-based plugins
+
+USER root
+
+RUN yum install -y epel-release && \
+    yum update -y && \
+    yum install -y curl patch psmisc zip unzip gcc-c++ make && \
+    yum install -y https://packages.endpoint.com/rhel/7/os/x86_64/endpoint-repo-1.7-1.x86_64.rpm && \
+    yum install -y git && \
+    yum clean all && \
+    groupadd -g ${DEFAULT_GID} ${PGROUP} && \
+    adduser -u ${DEFAULT_UID} -d /home/kibana -s /bin/bash -G ${PGROUP} -g ${PUSER} ${PUSER} && \
+    mkdir -p /usr/share && \
+    git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch --branch "v${ELASTIC_VERSION}" https://github.com/elastic/elasticsearch /usr/share/elastic && \
+    git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch --branch "v${ELASTIC_VERSION}" https://github.com/elastic/kibana /usr/share/kibana && \
+    chown -R ${DEFAULT_UID}:${DEFAULT_GID} /usr/share/kibana /usr/share/elastic
+
+# build plugins as non-root
+
+USER ${PUSER}
+
+# use nodenv (https://github.com/nodenv/nodenv) to manage nodejs/yarn
+
+ENV PATH "/home/kibana/.nodenv/bin:${PATH}"
+
+RUN git clone --single-branch --depth=1 --recurse-submodules --shallow-submodules https://github.com/nodenv/nodenv.git /home/kibana/.nodenv && \
+    cd /home/kibana/.nodenv && \
+    ./src/configure && \
+    make -C src && \
+    cd /tmp && \
+    eval "$(nodenv init -)" && \
+    mkdir -p "$(nodenv root)"/plugins && \
+    git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch https://github.com/nodenv/node-build.git "$(nodenv root)"/plugins/node-build && \
+    git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch https://github.com/nodenv/nodenv-update.git "$(nodenv root)"/plugins/nodenv-update && \
+    git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch https://github.com/pine/nodenv-yarn-install.git "$(nodenv root)"/plugins/nodenv-yarn-install && \
+    nodenv install "$(cat /usr/share/kibana/.node-version)" && \
+    nodenv global "$(cat /usr/share/kibana/.node-version)"
+
+# check out and build plugins
+
+RUN eval "$(nodenv init -)" && \
+    mkdir -p /usr/share/kibana/plugins && \
+    git clone --depth 1 --recurse-submodules --shallow-submodules --single-branch --branch feature/update_7.10.1 https://github.com/mmguero-dev/kbn_sankey_vis.git /usr/share/kibana/plugins/sankey_vis && \
+    cd /usr/share/kibana/plugins/sankey_vis && \
+    yarn kbn bootstrap && \
+    yarn install && \
+    yarn build --kibana-version "${ELASTIC_VERSION}" && \
+    mv ./build/kbnSankeyVis-7.10.2.zip ./build/kbnSankeyVis.zip
+
+FROM amazon/opendistro-for-elasticsearch-kibana:1.13.0
+
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors='[email protected]'
 LABEL org.opencontainers.image.url='https://github.com/cisagov/Malcolm'
@@ -40,23 +106,6 @@ ENV KIBANA_DEFAULT_DASHBOARD $KIBANA_DEFAULT_DASHBOARD
 
 USER root
 
-# TODO:
-# see https://github.com/uniberg/kbn_sankey_vis/issues/15#issuecomment-720700879
-# curl -sSL -o /tmp/kibana-sankey.zip "https://codeload.github.com/mmguero-dev/kbn_sankey_vis/zip/master" && \
-# cd /tmp && \
-#   echo "Installing Sankey visualization..." && \
-#   unzip /tmp/kibana-sankey.zip && \
-#   mkdir ./kibana &&\
-#   mv ./kbn_sankey_vis-* ./kibana/sankey_vis && \
-#   cd ./kibana/sankey_vis && \
-#   sed -i "s/7\.6\.3/7\.10\.0/g" ./package.json && \
-#   npm install && \
-#   cd /tmp && \
-#   zip -r sankey_vis.zip kibana --exclude ./kibana/sankey_vis/.git\* && \
-#   cd /usr/share/kibana/plugins && \
-#   /usr/share/kibana/bin/kibana-plugin install file:///tmp/sankey_vis.zip --allow-root && \
-#   rm -rf /tmp/kibana /tmp/*sankey* && \
-
 # curl -sSL -o /tmp/kibana-drilldown.zip "https://codeload.github.com/mmguero-dev/kibana-plugin-drilldownmenu/zip/master" && \
 # cd /tmp && \
 #   echo "Installing Drilldown menu plugin..." && \
@@ -74,15 +123,24 @@ USER root
 # cd /tmp && \
 #     rm -rf /tmp/npm-*
 
-RUN yum install -y epel-release && \
-      yum update -y && \
-      yum install -y curl git npm patch psmisc zip unzip && \
-      yum clean all && \
-      usermod -a -G tty ${PUSER} && \
-      # Malcolm manages authentication and encryption via NGINX reverse proxy
-      /usr/share/kibana/bin/kibana-plugin remove opendistroSecurityKibana --allow-root && \
-      # https://github.com/opendistro-for-elasticsearch/kibana-reports/issues/259
-      /usr/share/kibana/bin/kibana-plugin remove opendistroReportsKibana --allow-root
+COPY --from=build /usr/share/kibana/plugins/sankey_vis/build/kbnSankeyVis.zip /tmp/kbnSankeyVis.zip
+ADD "https://github.com/dlumbrer/kbn_network/releases/download/7.10.0-1/kbn_network-7.10.0.zip" /tmp/kibana-network.zip
+
+RUN yum install -y curl psmisc zip unzip && \
+    yum clean all && \
+    usermod -a -G tty ${PUSER} && \
+    # Malcolm manages authentication and encryption via NGINX reverse proxy
+    /usr/share/kibana/bin/kibana-plugin remove opendistroSecurityKibana --allow-root && \
+    cd /usr/share/kibana/plugins && \
+      /usr/share/kibana/bin/kibana-plugin install file:///tmp/kbnSankeyVis.zip --allow-root && \
+    cd /tmp && \
+      unzip kibana-network.zip kibana/kbn_network/kibana.json kibana/kbn_network/package.json && \
+      sed -i "s/7\.10\.0/7\.10\.2/g" kibana/kbn_network/kibana.json && \
+      sed -i "s/7\.10\.0/7\.10\.2/g" kibana/kbn_network/package.json && \
+      zip kibana-network.zip kibana/kbn_network/kibana.json kibana/kbn_network/package.json && \
+      cd /usr/share/kibana/plugins && \
+      /usr/share/kibana/bin/kibana-plugin install file:///tmp/kibana-network.zip --allow-root && \
+      rm -rf /tmp/kibana-comments.zip /tmp/kibana
 
 ADD kibana/kibana.yml /usr/share/kibana/config/kibana.yml
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/

Dockerfiles/logstash.Dockerfile

@@ -22,7 +22,7 @@ RUN /bin/bash -lc "command curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -
       curl -sSL "$OUIFILTER_URL" | tar xzvf - -C ./logstash-filter-ieee_oui --strip-components 1 && \
       /bin/bash -lc "cd /opt/logstash-filter-ieee_oui && bundle install && gem build logstash-filter-ieee_oui.gemspec && bundle info logstash-filter-ieee_oui"
 
-FROM docker.elastic.co/logstash/logstash-oss:7.10.0
+FROM docker.elastic.co/logstash/logstash-oss:7.10.2
 
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors='[email protected]'
@@ -54,14 +54,15 @@ ENV LOGSTASH_PARSE_PIPELINE_ADDRESSES $LOGSTASH_PARSE_PIPELINE_ADDRESSES
 ENV LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_INTERNAL $LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_INTERNAL
 ENV LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_EXTERNAL $LOGSTASH_ELASTICSEARCH_PIPELINE_ADDRESS_EXTERNAL
 ENV LOGSTASH_ELASTICSEARCH_OUTPUT_PIPELINE_ADDRESSES $LOGSTASH_ELASTICSEARCH_OUTPUT_PIPELINE_ADDRESSES
+ENV JAVA_HOME=/usr/share/logstash/jdk
 
 USER root
 
 COPY --from=build /opt/logstash-filter-ieee_oui /opt/logstash-filter-ieee_oui
 
 RUN yum install -y epel-release && \
     yum update -y && \
-    yum install -y curl gettext python-setuptools python-pip python-requests python-yaml && \
+    yum install -y curl gettext python-setuptools python-pip python-requests python-yaml openssl && \
     yum clean all && \
     pip install py2-ipaddress supervisor && \
     logstash-plugin install logstash-filter-translate logstash-filter-cidr logstash-filter-dns \
@@ -73,6 +74,7 @@ RUN yum install -y epel-release && \
     rm -rf /opt/logstash-filter-ieee_oui /root/.cache /root/.gem /root/.bundle
 
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
+ADD shared/bin/jdk-cacerts-auto-import.sh /usr/local/bin/
 ADD logstash/maps/*.yaml /etc/
 ADD logstash/config/log4j2.properties /usr/share/logstash/config/
 ADD logstash/config/logstash.yml /usr/share/logstash/config/

Dockerfiles/name-map-ui.Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.12
+FROM alpine:3.13
 
 # Copyright (c) 2021 Battelle Energy Alliance, LLC.  All rights reserved.
 LABEL maintainer="[email protected]"

Dockerfiles/nginx.Dockerfile

@@ -10,7 +10,7 @@
 # build a patched APK of stunnel supporting ldap StartTLS (patched protocols.c)
 # (based on https://www.stunnel.org/pipermail/stunnel-users/2013-November/004437.html)
 
-FROM alpine:3.12 as stunnel_build
+FROM alpine:3.13 as stunnel_build
 
 ARG DEFAULT_UID=1000
 ARG DEFAULT_GID=300
@@ -33,20 +33,24 @@ RUN set -x ; \
 
 USER ${PUSER}
 
+# todo: when aports updates stunnel to 5.58, this will need to be updated
+
 RUN set -x ; \
     cd /apkbuild ; \
     tar xvf /aports-master.tar.gz aports-master/community/stunnel ; \
     cp /usr/src/patches/stunnel-5.56-open-ldap.patch /apkbuild/aports-master/community/stunnel/ ; \
     cd /apkbuild/aports-master/community/stunnel ; \
+    sed -i 's@https://www.stunnel.org/downloads/stunnel.*gz@ftp://ftp.stunnel.org/stunnel/archive/5.x/stunnel-5.57.tar.gz@' APKBUILD ; \
     sed -i 's/\(^makedepends="\)/\1patchutils /' APKBUILD ; \
     sed -i '/^source=/a \ \ \ \ \ \ \ \ stunnel-5.56-open-ldap.patch' APKBUILD ; \
-    sed -i "/^sha512sums=/a $(sha512sum stunnel-5.56-open-ldap.patch)" APKBUILD ; \
+    sed -i '/^sha512sums/,$d' APKBUILD ; \
     abuild-keygen -a -i -n ; \
+    abuild checksum ; \
     abuild -R
 
 ####################################################################################
 
-FROM alpine:3.12
+FROM alpine:3.13
 
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors='[email protected]'
@@ -100,7 +104,7 @@ ENV NGINX_LDAP_TLS_STUNNEL_CHECK_IP $NGINX_LDAP_TLS_STUNNEL_CHECK_IP
 ENV NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL $NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL
 
 # build latest nginx with nginx-auth-ldap
-ENV NGINX_VERSION=1.19.6
+ENV NGINX_VERSION=1.19.7
 ENV NGINX_AUTH_LDAP_BRANCH=master
 
 ADD https://codeload.github.com/mmguero-dev/nginx-auth-ldap/tar.gz/$NGINX_AUTH_LDAP_BRANCH /nginx-auth-ldap.tar.gz

Dockerfiles/zeek.Dockerfile

@@ -13,7 +13,7 @@ ENV SRC_BASE_DIR "/usr/local/src"
 ENV ZEEK_DIR "/opt/zeek"
 ENV ZEEK_PATCH_DIR "${SRC_BASE_DIR}/zeek-patches"
 ENV ZEEK_SRC_DIR "${SRC_BASE_DIR}/zeek-${ZEEK_VERSION}"
-ENV ZEEK_VERSION "3.0.12"
+ENV ZEEK_VERSION "3.0.13"
 
 ENV PATH "${ZEEK_DIR}/bin:${PATH}"