Merge pull request #188 from cisagov/v524_merge

v5.2.4 development

- New features
  - idaholab#74 (automatically generate Zeek intelligence indicators from STIX/TAXII)

- Improvements
  - group MAC addresses and OUI (vendors) into `related.mac` and `related.oui` for easier searching across all fields
  - improvements to default anomaly detectors

- Bug fixes
  - Fix idaholab#75 (OpenSearch Dashboards loads slowly without network connectivity)
  - Fix idaholab#76 (directory creation race condition starting up zeek on sensor which may cause zeekctl to fail)

Dockerfiles/arkime.Dockerfile

@@ -13,7 +13,7 @@ ENV ARKIME_INET yes
 ADD arkime/scripts/bs4_remove_div.py /opt/
 ADD arkime/patch/* /opt/patches/
 ADD README.md $ARKIMEDIR/doc/
-ADD doc.css $ARKIMEDIR/doc/
+ADD docs/doc.css $ARKIMEDIR/doc/
 ADD docs/images $ARKIMEDIR/doc/images/
 
 RUN apt-get -q update && \

Dockerfiles/dashboards.Dockerfile

@@ -121,6 +121,18 @@ RUN yum install -y curl psmisc util-linux zip unzip && \
 ADD dashboards/opensearch_dashboards.yml /usr/share/opensearch-dashboards/config/opensearch_dashboards.yml
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 
+# Yeah, I know about https://opensearch.org/docs/latest/dashboards/branding ... but I can't figure out a way
+# to specify the entries in the opensearch_dashboards.yml such that they are valid BOTH from the
+# internal opensearch code validating them AND the web browser retrieving them. So we're going scorched earth instead.
+ADD docs/images/logo/malcolm_logo.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/default_branding/opensearch_logo.svg
+ADD docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/default_branding/opensearch_mark_dark_mode.svg
+ADD docs/images/icon/malcolm_mark_dashboards.svg /usr/share/opensearch-dashboards/src/core/server/core_app/assets/default_branding/opensearch_mark_default_mode.svg
+ADD docs/images/favicon/favicon.ico /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon.ico
+ADD docs/images/favicon/favicon16.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon-16x16.png
+ADD docs/images/favicon/favicon32.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/favicon-32x32.png
+ADD docs/images/favicon/apple-touch-icon-precomposed.png /usr/share/opensearch-dashboards/src/core/server/core_app/assets/favicons/apple-touch-icon.png
+
+
 ENTRYPOINT ["/usr/local/bin/docker-uid-gid-setup.sh"]
 
 CMD ["/usr/share/opensearch-dashboards/opensearch-dashboards-docker-entrypoint.sh"]

Dockerfiles/nginx.Dockerfile

@@ -195,7 +195,7 @@ ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD nginx/scripts /usr/local/bin/
 ADD nginx/*.conf /etc/nginx/
 ADD nginx/supervisord.conf /etc/
-ADD docs/images/icon/favicon.ico /etc/nginx/favicon.ico
+ADD docs/images/icon/favicon.ico /usr/share/nginx/html/favicon.ico
 
 EXPOSE 80
 

Dockerfiles/zeek.Dockerfile

@@ -32,6 +32,12 @@ ENV ZEEK_LTS $ZEEK_LTS
 ENV ZEEK_VERSION $ZEEK_VERSION
 ENV SPICY_VERSION $SPICY_VERSION
 
+ENV SUPERCRONIC_VERSION "0.1.12"
+ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
+ENV SUPERCRONIC "supercronic-linux-amd64"
+ENV SUPERCRONIC_SHA1SUM "048b95b48b708983effb2e5c935a1ef8483d9e3e"
+ENV SUPERCRONIC_CRONTAB "/etc/crontab"
+
 # for build
 ENV CCACHE_DIR "/var/spool/ccache"
 ENV CCACHE_COMPRESS 1
@@ -81,12 +87,16 @@ RUN apt-get -q update && \
       psmisc \
       python3 \
       python3-git \
+      python3-pip \
       python3-semantic-version \
+      python3-setuptools \
+      python3-wheel \
       python3-zmq \
       supervisor \
       swig \
       vim-tiny \
       zlib1g-dev && \
+    pip3 install --no-cache-dir stix2 taxii2-client && \
     mkdir -p /tmp/zeek-packages && \
       cd /tmp/zeek-packages && \
       if [ -n "${ZEEK_LTS}" ]; then ZEEK_LTS="-lts"; fi && export ZEEK_LTS && \
@@ -106,6 +116,11 @@ RUN apt-get -q update && \
     curl -sSL --remote-name-all \
       "https://github.com/zeek/spicy/releases/download/v${SPICY_VERSION}/spicy_linux_debian11.deb" && \
       dpkg -i ./*.deb && \
+    curl -fsSLO "$SUPERCRONIC_URL" && \
+      echo "${SUPERCRONIC_SHA1SUM}  ${SUPERCRONIC}" | sha1sum -c - && \
+      chmod +x "$SUPERCRONIC" && \
+      mv "$SUPERCRONIC" "/usr/local/bin/${SUPERCRONIC}" && \
+      ln -s "/usr/local/bin/${SUPERCRONIC}" /usr/local/bin/supercronic && \
     cd /tmp && \
     mkdir -p "${CCACHE_DIR}" && \
     zkg autoconfig --force && \
@@ -122,7 +137,7 @@ RUN apt-get -q update && \
     mkdir -p "${ZEEK_DIR}"/var/lib/zkg/clones/package/spicy-plugin/plugin/lib/ && \
       ln -s -r "${ZEEK_DIR}"/lib/zeek/plugins/packages/spicy-plugin/lib/bif \
                "${ZEEK_DIR}"/var/lib/zkg/clones/package/spicy-plugin/plugin/lib/bif && \
-    mkdir -p "${ZEEK_DIR}"/share/zeek/site/intel && \
+    mkdir -p "${ZEEK_DIR}"/share/zeek/site/intel/STIX && \
         touch "${ZEEK_DIR}"/share/zeek/site/intel/__load__.zeek && \
     cd /usr/lib/locale && \
       ( ls | grep -Piv "^(en|en_US|en_US\.utf-?8|C\.utf-?8)$" | xargs -l -r rm -rf ) && \
@@ -134,6 +149,7 @@ RUN apt-get -q update && \
 ADD shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 ADD shared/bin/pcap_arkime_and_zeek_processor.py /usr/local/bin/
 ADD shared/bin/pcap_utils.py /usr/local/bin/
+ADD shared/bin/stix*.py ${ZEEK_DIR}/bin/
 ADD shared/pcaps /tmp/pcaps
 ADD zeek/supervisord.conf /etc/supervisord.conf
 ADD zeek/config/*.zeek ${ZEEK_DIR}/share/zeek/site/
@@ -159,14 +175,17 @@ RUN mkdir -p /tmp/logs && \
 RUN groupadd --gid ${DEFAULT_GID} ${PUSER} && \
     useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} --home /nonexistant ${PUSER} && \
     usermod -a -G tty ${PUSER} && \
-    chown -R ${DEFAULT_UID}:${DEFAULT_GID} "${ZEEK_DIR}"/share/zeek/site/intel && \
+    touch "${SUPERCRONIC_CRONTAB}" && \
+    chown -R ${DEFAULT_UID}:${DEFAULT_GID} "${ZEEK_DIR}"/share/zeek/site/intel "${SUPERCRONIC_CRONTAB}" && \
     ln -sfr /usr/local/bin/pcap_arkime_and_zeek_processor.py /usr/local/bin/pcap_zeek_processor.py
 
 #Whether or not to auto-tag logs based on filename
 ARG AUTO_TAG=true
 #Whether or not to run "zeek -r XXXXX.pcap local" on each pcap file
 ARG ZEEK_AUTO_ANALYZE_PCAP_FILES=false
 ARG ZEEK_AUTO_ANALYZE_PCAP_THREADS=1
+ARG ZEEK_INTEL_ITEM_EXPIRATION=-1min
+ARG ZEEK_INTEL_REFRESH_CRON_EXPRESSION=
 ARG ZEEK_EXTRACTOR_MODE=none
 ARG ZEEK_EXTRACTOR_PATH=/zeek/extract_files
 ARG PCAP_PIPELINE_DEBUG=false
@@ -176,6 +195,8 @@ ARG PCAP_MONITOR_HOST=pcap-monitor
 ENV AUTO_TAG $AUTO_TAG
 ENV ZEEK_AUTO_ANALYZE_PCAP_FILES $ZEEK_AUTO_ANALYZE_PCAP_FILES
 ENV ZEEK_AUTO_ANALYZE_PCAP_THREADS $ZEEK_AUTO_ANALYZE_PCAP_THREADS
+ENV ZEEK_INTEL_ITEM_EXPIRATION $ZEEK_INTEL_ITEM_EXPIRATION
+ENV ZEEK_INTEL_REFRESH_CRON_EXPRESSION $ZEEK_INTEL_REFRESH_CRON_EXPRESSION
 ENV ZEEK_EXTRACTOR_MODE $ZEEK_EXTRACTOR_MODE
 ENV ZEEK_EXTRACTOR_PATH $ZEEK_EXTRACTOR_PATH
 ENV PCAP_PIPELINE_DEBUG $PCAP_PIPELINE_DEBUG