Update docker-zeek to use zeek 6.2.1 (#41)

* Update docker-zeek to use zeek 6.2.1

* Disable crashing hostnames.zeek

* Disable mailing hourly report

* Do not load misc/scan, no longer included.

* Remove misc/scan with a large hammer

* Remove misc/scan another place with a large hammer

* kill bug with AF_Packet load

* Update default to latest

* Add zeek-open-connections

.github/workflows/docker-build.yml

@@ -24,6 +24,7 @@ jobs:
           # - "3.2.4"
           - "4.0.5"
           - "4.2.0"
+          - "6.2.1"
         include:
           # Extra data for arch
           - platform: linux/amd64
@@ -47,11 +48,17 @@ jobs:
           - version: "4.0.5"
             af-packet: "3.0.2"
             zkg: "2.12.0"
-            release-tag: lts
+            release-tag: v4-lts
           - version: "4.2.0"
             af-packet: "3.0.2"
             zkg: "2.12.0"
+            release-tag: v4-latest
+          - version: "6.2.1"
+            zkg: "3.0.1"
             release-tag: latest
+          - version: "6.2.1"
+            zkg: "3.0.1"
+            release-tag: lts
 
     steps:
       - 
@@ -127,16 +134,21 @@ jobs:
           # - "3.2.4"
           - "4.0.5"
           - "4.2.0"
+          - "6.2.1"
         include:
           # Extra data for versions
           # - version: "3.0.12"
           #   release-tag: v3-lts
           # - version: "3.2.4"
           #   release-tag: v3-latest
           - version: "4.0.5"
-            release-tag: lts
+            release-tag: v4-lts
           - version: "4.2.0"
+            release-tag: v4-latest
+          - version: "6.2.1"
             release-tag: latest
+          - version: "6.2.1"
+            release-tag: lts
 
     steps:
       -

Dockerfile

@@ -1,7 +1,7 @@
-FROM alpine:3.12 as builder
+FROM alpine AS builder
 
-ARG ZEEK_VERSION=4.2.0
-ARG AF_PACKET_VERSION=3.0.2
+ARG ZEEK_VERSION=6.2.1
+#ARG AF_PACKET_VERSION=3.0.2
 
 ARG BUILD_PROCS=2
 
@@ -14,41 +14,46 @@ RUN apk add --no-cache -t .build-deps \
     libpcap-dev \
     python3-dev \
     zlib-dev \
+    flex-dev \
     binutils \
     fts-dev \
     cmake \
-    clang \
     bison \
     bash \
     swig \
     perl \
     make \
     flex \
     git \
+    gcc \
     g++ \
     fts \
     krb5-dev
 
+    #Removed clang, nodejs-dev, nodejs (the nodejs ones since we now disable javascript in configure)
+
 RUN echo "===> Cloning zeek..." \
     && cd /tmp \
     && git clone --recursive --branch v$ZEEK_VERSION https://github.com/zeek/zeek.git
 
 RUN echo "===> Compiling zeek..." \
     && cd /tmp/zeek \
-    && CC=clang ./configure --prefix=/usr/local/zeek \
+    && CC=gcc ./configure --prefix=/usr/local/zeek \
     --build-type=Release \
     --disable-broker-tests \
     --disable-auxtools \
+    --disable-javascript \
     && make -j $BUILD_PROCS \
     && make install
 
-RUN echo "===> Compiling af_packet plugin..." \
-    && git clone https://github.com/J-Gras/zeek-af_packet-plugin.git --branch ${AF_PACKET_VERSION} /tmp/zeek-af_packet-plugin \
-    && cd /tmp/zeek-af_packet-plugin \
-    && CC=clang ./configure --with-kernel=/usr --zeek-dist=/tmp/zeek \
-    && make -j $BUILD_PROCS \
-    && make install \
-    && /usr/local/zeek/bin/zeek -NN Zeek::AF_Packet
+#As of Zeek 5.2.0 af_packet is included with zeek.
+#RUN echo "===> Compiling af_packet plugin..." \
+#    && git clone https://github.com/J-Gras/zeek-af_packet-plugin.git --branch ${AF_PACKET_VERSION} /tmp/zeek-af_packet-plugin \
+#    && cd /tmp/zeek-af_packet-plugin \
+#    && CC=gcc ./configure --with-kernel=/usr --zeek-dist=/tmp/zeek \
+#    && make -j $BUILD_PROCS \
+#    && make install \
+#    && /usr/local/zeek/bin/zeek -NN Zeek::AF_Packet
 
 RUN echo "===> Shrinking image..." \
     && strip -s /usr/local/zeek/bin/zeek
@@ -57,7 +62,7 @@ RUN echo "===> Size of the Zeek install..." \
     && du -sh /usr/local/zeek
 
 ####################################################################################################
-FROM alpine:3.12
+FROM alpine
 
 # python3 & bash are needed for zeekctl scripts
 # ethtool is needed to manage interface features
@@ -74,20 +79,20 @@ RUN ln -s $(which ethtool) /sbin/ethtool
 
 COPY --from=builder /usr/local/zeek /usr/local/zeek
 
-ENV ZEEKPATH .:/usr/local/zeek/share/zeek:/usr/local/zeek/share/zeek/policy:/usr/local/zeek/share/zeek/site
-ENV PATH $PATH:/usr/local/zeek/bin
+ENV ZEEKPATH=.:/usr/local/zeek/share/zeek:/usr/local/zeek/share/zeek/policy:/usr/local/zeek/share/zeek/site
+ENV PATH=$PATH:/usr/local/zeek/bin
 
 # Install Zeek package manager
 # In Zeek v4, zkg is bundled with Zeek. However, the configuration of zkg when bundled with Zeek
 # differs from the configuration when installed via pip. The state directory is
 # /usr/local/zeek/var/lib/zkg when using v4's bundled zkg. When zkg is installed via pip
 # or the --user flag is supplied to the bundled zkg, .root/zkg is used as the state directory.
 # In order to re-use the same configuration across v3 and v4, we manually install zkg from pip.
-ARG ZKG_VERSION=2.12.0
+ARG ZKG_VERSION=3.0.1
 
-ARG ZEEK_DEFAULT_PACKAGES="bro-interface-setup bro-doctor ja3"
+ARG ZEEK_DEFAULT_PACKAGES="bro-interface-setup bro-doctor ja3 zeek-open-connections"
 
-RUN pip install zkg==$ZKG_VERSION \
+RUN pip install --break-system-packages zkg==$ZKG_VERSION \
     && zkg autoconfig \
     && zkg refresh \
     && zkg install --force $ZEEK_DEFAULT_PACKAGES

Readme.md

@@ -18,8 +18,10 @@ The docker tags correspond with the version of [Zeek](https://zeek.org/get-zeek/
 
 * `v3-latest`, `3.2`, `3.2.3`
 * `v3-lts`, `3`, `3.0`, `3.0.12`
-* `latest`, `4.2`, `4.2.0`
-* `lts`, `4.0`, `4.0.5`
+* `v4-latest`, `4.2`, `4.2.0`
+* `v4-lts`, `4.0`, `4.0.5`
+* `latest`, `6.2`, `6.2.1`
+* `lts`, `6.2`, `6.2.1`
 
 ## Quickstart
 
@@ -59,7 +61,7 @@ source /etc/profile.d/zeek.sh
 
 ### Zeek Version
 
-The default version tag is `4.2.0` which will correspond to the latest release in the 4.2.0 Zeek release channel. You can customize this with the `zeek_release` environment variable. Set this variable to your desired Docker image tag. For example, to use the latest feature release:
+The default version tag is `6.2.1` which will correspond to the latest release in the 6.2.1 Zeek release channel. You can customize this with the `zeek_release` environment variable. Set this variable to your desired Docker image tag. For example, to use the latest feature release:
 
 ```bash
 echo "export zeek_release=latest" | sudo tee -a /etc/profile.d/zeek.sh

docker-entrypoint.sh

@@ -36,7 +36,8 @@ trap 'diag' ERR
 # ensure Zeek has a valid, updated config, and then start Zeek
 echo "Checking your Zeek configuration..."
 # generate a single local.zeek from a bunch of partials
-cat /usr/local/zeek/share/zeek/site/autoload/* | grep -v '^#' > /usr/local/zeek/share/zeek/site/local.zeek
+#We specifically strip out the line for misc/scan as it's no longer part of zeek and it's darn near impossible to find.
+cat /usr/local/zeek/share/zeek/site/autoload/* | grep -v '^#' | grep -v 'misc/scan' >/usr/local/zeek/share/zeek/site/local.zeek
 zeekctl check >/dev/null
 zeekctl install
 zeekctl start

docs/build.md

@@ -5,7 +5,7 @@ The steps to take vary based on what has changed in the new version.
 If the Zeek version changes it needs to be updated in the following places:
 - Default value for the `ZEEK_VERSION` build arg in the `Dockerfile`
 - List of available tags in `Readme.md`
-- Version specified in the Github workflow (`.github/workflows/docker.yml`)
+- Version specified in the Github workflow (`.github/workflows/docker-build.yml`)
 
 If the `Readme.md` changes the contents need to be copied to the Dockerhub project manually. This is due to using Github Actions to push up multiple images (vs. using Dockerhub to pull the code and build a single image). Dockerhub does not automatically update the project with the readme when using the push model. An API is not currently available to do this programmatically.
 

etc/zeekctl.cfg

@@ -10,7 +10,8 @@ MailTo = root@localhost
 # means mail connection summaries, and a value of 0 means do not mail
 # connection summaries.  This option has no effect if the trace-summary
 # script is not available.
-MailConnectionSummary = 1
+# We're disabling this because sendmail is not commonly set up and it provides an error very hour.
+MailConnectionSummary = 0
 
 # Lower threshold (in percentage of disk space) for space available on the
 # disk that holds SpoolDir. If less space is available, "zeekctl cron" starts

share/zeek/site/autoload/100-default.zeek

@@ -14,8 +14,8 @@
 # Enable logging of memory, packet and lag statistics.
 @load misc/stats
 
-# Load the scan detection script.
-@load misc/scan
+# DO NOT Load the scan detection script, no longer included.
+# @load misc/scan
 
 # Detect traceroute being run on the network. This could possibly cause
 # performance trouble when there are a lot of traceroutes on your network.
@@ -85,8 +85,10 @@
 # Detect SHA1 sums in Team Cymru's Malware Hash Registry.
 @load frameworks/files/detect-MHR
 
-# Extend email alerting to include hostnames
-@load policy/frameworks/notice/extend-email/hostnames
+# DO NOT Extend email alerting to include hostnames
+# This module causes errors in docker-zeek:
+# timestamp expression error in /usr/local/zeek/share/zeek/policy/frameworks/notice/extend-email/hostnames.zeek, line 39: no such index (Notice::tmp_notice_storage[Notice::uid])
+# @load policy/frameworks/notice/extend-email/hostnames
 
 # Uncomment the following line to enable detection of the heartbleed attack. Enabling
 # this might impact performance a bit.

zeek

@@ -26,7 +26,7 @@ fi
 
 #The user can set the top level directory that holds all zeek content by setting it in "zeek_top_dir" (default "/opt/zeek")
 HOST_ZEEK=${zeek_top_dir:-/opt/zeek}
-IMAGE_NAME="activecm/zeek:${zeek_release:-4.2.0}"
+IMAGE_NAME="activecm/zeek:${zeek_release:-latest}"
 
 # initilizes Zeek directories and config files on the host
 init_zeek_cfg() {
@@ -251,7 +251,7 @@ main() {
 		docker_cmd+=("--entrypoint" "/bin/bash")										#Running /bin/bash -c "series ; of ; shell ; commands" lets use effectively run a shell script inside the container.
 		docker_cmd+=("$IMAGE_NAME")
 		#If you want to output diags before running, add "  ; /usr/local/zeek/bin/zeekctl diag    just before running zeek in the following.
-		docker_cmd+=("-c" "/bin/cat /usr/local/zeek/share/zeek/site/autoload/* | /bin/grep -v '^#' >/usr/local/zeek/share/zeek/site/local.zeek ; /usr/local/zeek/bin/zeek -C -r /incoming.pcap local 'Site::local_nets += { 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 }' 'Notice::sendmail = ' 2>&1 | grep -v 'Node names are not added to logs (not in cluster mode'")
+		docker_cmd+=("-c" "/bin/cat /usr/local/zeek/share/zeek/site/autoload/* | /bin/grep -v '^#' | /bin/grep -v 'misc/scan' >/usr/local/zeek/share/zeek/site/local.zeek ; /bin/mv -f /usr/local/zeek/share/zeek/builtin-plugins/Zeek_AF_Packet/{__load__.zeek,init.zeek} /usr/local/zeek/share/zeek/builtin-plugins/ || /bin/true ; /usr/local/zeek/bin/zeek -C -r /incoming.pcap local 'Site::local_nets += { 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16 }' 'Notice::sendmail = ' 2>&1 | grep -v 'Node names are not added to logs (not in cluster mode'")
 		echo "Starting the Zeek docker container" >&2
 		echo "Zeek logs will be saved to $MANUAL_LOG_DIR" >&2
 		#Show the command, useful for debugging