Merge pull request #442 from simsong/fix-scan_find

Fix scan find

.github/workflows/continuous-integration-pip.yml

@@ -12,34 +12,16 @@ name: BE2 Unit Tests on MacOS and Ubuntu (c++17)
 on: [push, pull_request]
 
 jobs:
-  # https://github.com/marketplace/actions/skip-duplicate-actions
-  pre_job:
-    runs-on: ${{ matrix.os }}
+  build:
     strategy:
       matrix:
-        os: ['macos-latest', 'ubuntu-latest']
-    outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip }}
-    steps:
-      - id: skip_check
-        uses: fkirc/[email protected]
-        with:
-          # All of these options are optional, so you can remove them if you are happy with the defaults
-          concurrent_skipping: 'never'
-          skip_after_successful_duplicate: 'true'
-          paths_ignore: '["**/README.md", "**/docs/**"]'
-          do_not_skip: '["pull_request", "workflow_dispatch", "schedule"]'
-
-  main_job:
-    needs: pre_job
-    if: ${{ needs.pre_job.outputs.should_skip != 'true' }}
+        #os: ['macos-13', 'macos-13-arm64', 'ubuntu-22.04']
+        os: ['ubuntu-22.04']
     runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: ['macos-latest', 'ubuntu-latest']
     env:
       DEBUG_5G: FALSE
       DEBUG_FAST: TRUE
+      MAKE_OPTS: -j2
 
     steps:
       # https://github.com/actions/checkout
@@ -53,50 +35,68 @@ jobs:
         run: |
           echo "" | bash etc/CONFIGURE_MACOS.bash
 
-      - name: "Ubuntu: run CONFIGURE_UBUNTU20LTS.bash"
+      - name: "Ubuntu: run CONFIGURE_UBUNTU22LTS.bash"
         if: startsWith(matrix.os, 'ubuntu')
         run: |
-          echo "" | bash etc/CONFIGURE_UBUNTU20LTS.bash
+          echo "" | bash etc/CONFIGURE_UBUNTU22LTS.bash
+
 
-      - name: C++ checks with address-sanitizer (Mac and Linux)
+      - name: Make configure script
         run: |
-          echo GCC VERSION:
+          autoconf --version
+          automake --version
+          aclocal --version
           gcc --version
-          echo G++ VERSION:
           g++ --version
           bash bootstrap.sh
-          echo === Try Address Sanitizer Without Optimization ===
-          ./configure -q --disable-opt --enable-address-sanitizer
-          make clean
-          make all
-          (cd src; make check || (cat test-suite.log; exit 1))
+
+      - name: Dump configure script
+        run: |
+          cat configure
+
+      - name: C++ checks not optimizaiton with address-sanitizer (Mac and Linux)
+        run: |
+          ./configure --disable-opt --enable-address-sanitizer
+          make ${{MAKE_OPTS}} all
+          cd src
+          make ${{MAKE_OPTS}} bulk_extractor
+          make check || (cat test-suite.log; exit 1)
+          cd ..
           make distclean
+
+      - name: C++ checks optimization with address-sanitizer (Mac and Linux)
+        run: |
           echo === Try Address Sanitizer Optimization ===
-          ./configure -q --enable-address-sanitizer
-          make clean
-          make all
-          (cd src; make check || (cat test-suite.log; exit 1))
+          ./configure --enable-address-sanitizer
+          make ${{ MAKE_OPTS }} all
+          cd src
+          make ${{ MAKE_OPTS }} bulk_extractor
+          make check || (cat test-suite.log; exit 1)
+          cd ..
           make distclean
 
       - name: C++ checks with thread-sanitizer on ubuntu are disabled
         if: startsWith(matrix.os, 'ubuntu-DISABLED')
         run: |
           bash bootstrap.sh
-          ./configure -q --enable-silent-rules --enable-thread-sanitizer
+          ./configure --enable-silent-rules --enable-thread-sanitizer
           make clean
-          make all
-          (cd src; make check || (cat test-suite.log; exit 1))
+          make ${{ MAKE_OPTS }}  all
+          cd src
+          make ${{ MAKE_OPTS }}  bulk_extractor
+          make check || (cat test-suite.log; exit 1)
+          cd ..
           make distclean
 
       - name: C++ checks with codecov
         if: startsWith(matrix.os, 'ubuntu')
         run: |
           bash bootstrap.sh
-          ./configure -q --disable-opt --enable-silent-rules CFLAGS='-g -O0 -fprofile-arcs -ftest-coverage' CXXFLAGS='-g -O0 -fprofile-arcs -ftest-coverage' LIBS='-lgcov'
+          ./configure --disable-opt CFLAGS='-g -O0 -fprofile-arcs -ftest-coverage' CXXFLAGS='-g -O0 -fprofile-arcs -ftest-coverage' LIBS='-lgcov -lre2'
           make clean
-          make all
+          make ${{ MAKE_OPTS }}  all
           cd src
-          make check || (echo ==error== ; cat test-suite.log; exit 1)
+          make ${{ MAKE_OPTS }}  check || (echo ==error== ; cat test-suite.log; exit 1)
 
       - name: upload codecov report
         if: startsWith(matrix.os, 'ubuntu')
@@ -107,24 +107,17 @@ jobs:
           gcov-9 -o . *.cpp be20_api/*.cpp
           bash <(curl -s https://codecov.io/bash)
 
-          #
-          # set up and run codecov
-          #
-          ## sudo apt install -y gpg
-          ## curl https://keybase.io/codecovsecurity/pgp_keys.asc | \
-          ## gpg --no-default-keyring --keyring trustedkeys.gpg --import
-          ## curl -Os https://uploader.codecov.io/latest/linux/codecov
-          ## curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
-          ## curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
-          ## gpgv codecov.SHA256SUM.sig codecov.SHA256SUM
-          ## shasum -a 256 -c codecov.SHA256SUM
-          ## chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}
-
       - uses: ammaraskar/[email protected]
         name: GCC Problem Matcher
 
       - name: distcheck
         run: |
-          ./configure -q
-          make clean
-          make distcheck
+          ./configure
+          make ${{ MAKE_OPTS }}  clean
+          #make distcheck
+          make ${{ MAKE_OPTS }}  dist
+          ls -l
+          tar xfvz *.tar.gz
+          cd $(basename *gz .tar.gz)
+          ./configure
+          make ${{ MAKE_OPTS }} check

Makefile.am

@@ -31,12 +31,13 @@ EXTRA_DIST = \
 	m4/ac_prog_javah.m4 \
 	m4/ac_try_compile_java.m4 \
 	m4/ac_try_run_javac.m4 \
-	m4/ax_pthread.m4 \
+	m4/ax_cxx_compile_stdcxx.m4 \
+	m4/ax_cxx_compile_stdcxx_17.m4 \
 	bootstrap.sh \
 	README.md \
 	$(SRC_WIN_DIST)
 
-ACLOCAL_AMFLAGS = -I m4
+ACLOCAL_AMFLAGS = ${ACLOCAL_FLAGS} -I m4
 
 distclean2:
 	@echo Deleting:

configure.ac

@@ -8,23 +8,22 @@
 # and http://www.bioinf.uni-freiburg.de/~mmann/HowTo/automake.html
 
 AC_PREREQ([2.69])
-AC_INIT([BULK_EXTRACTOR],[2.0.6],[[email protected]])
+AC_INIT([BULK_EXTRACTOR],[2.1.0],[[email protected]])
 AC_CONFIG_MACRO_DIR(m4)
-AC_CONFIG_AUX_DIR([build-aux])
+
+AC_MSG_NOTICE([at start CPPFLAGS are $CPPFLAGS])
 ################################################################
 ## Includes
 
-m4_include([m4/slg_mingw_support.m4])
-m4_include([m4/slg_searchdirs.m4])
-m4_include([src/be20_api/be20_configure.m4])
-m4_include([src/be20_api/dfxml_cpp/src/dfxml_configure.m4])
-
 AC_CONFIG_FILES([Makefile doc/Makefile doc/latex_manuals/Makefile src/Makefile src/tests/Makefile man/Makefile \
                  python/Makefile specfiles/Makefile specfiles/bulk_extractor.spec.m4 tests/Makefile ])
 
 AC_CONFIG_HEADERS([config.h])
+AC_CONFIG_AUX_DIR([build-aux])
+
 AM_INIT_AUTOMAKE
 AM_MAINTAINER_MODE
+AC_PROG_RANLIB
 AC_PREFIX_PROGRAM(bulk_extractor) dnl build for same location
 
 # Programs we will be using
@@ -33,20 +32,33 @@ AC_PROG_CXX
 AM_PROG_CC_C_O                    dnl allow per-produce flags
 AC_PROG_INSTALL
 AC_PROG_LEX([noyywrap])
-# LT_INIT([disable-shared])
 
 # Must use C++17 mode.
-# Validate that we have <filesystem>
-
 m4_include([m4/ax_cxx_compile_stdcxx.m4])
 AC_LANG_PUSH(C++)
 AX_CXX_COMPILE_STDCXX([17], [noext], [mandatory])
-AC_CHECK_HEADER([filesystem], [], AC_MSG_ERROR([<filesystem> header not installed; bulk_extractor requires a C++ compiler with a full C++17 implementation]))
 AC_LANG_POP()
 
 # Turn on all warnings
 m4_include([src/be20_api/m4/slg_gcc_all_warnings.m4])
 
+################################################################
+# Specify our headers
+#
+# Check for headers used by bulk Extractor
+# do not put pthread here
+#
+# Check dfxml and be20_api headers
+
+m4_include([src/be20_api/dfxml_cpp/src/dfxml_configure.m4])
+m4_include([src/be20_api/m4/slg_address_sanitizer.m4])
+m4_include([src/be20_api/m4/slg_noopt.m4])
+m4_include([src/be20_api/be20_configure.m4])
+
+m4_include([m4/slg_mingw_support.m4])
+m4_include([m4/slg_searchdirs.m4])
+
+
 # These are needed for bulk_extractor. They are not needed for tcpflow
 AC_DEFINE([BULK_EXTRACTOR],1,[We are compiling bulk_extractor])
 
@@ -66,8 +78,6 @@ else
   AC_MSG_ERROR([$LEX does not support -R.  Please get a modern version of gnu flex])
 fi
 
-
-
 ################################################################
 ## rar support
 AC_ARG_ENABLE([rar],
@@ -294,10 +304,6 @@ else
   CXXFLAGS="$CXXFLAGS -fPIC"
 fi
 
-m4_include([src/be20_api/m4/slg_address_sanitizer.m4])
-m4_include([src/be20_api/m4/slg_noopt.m4])
-
-
 
 ################################################################
 ## Check on two annoying warnings
@@ -307,12 +313,23 @@ m4_include([src/be20_api/m4/slg_noopt.m4])
 # ---------------------------------------------------------------
 # Try to compile PROGRAM.
 AC_DEFUN([MY_COMPILE_CLEAN_IFELSE],
-[AC_REQUIRE([AC_PROG_EGREP])
-AC_COMPILE_IFELSE([$1],[retval=0
-if $EGREP -i -c -E 'fatal|error|unrecognized|not found|not exist' conftest.err >/dev/null; then retval=1; fi
-],[retval=1])
+  [AC_REQUIRE([AC_PROG_EGREP])
+   AC_COMPILE_IFELSE([$1],[retval=0
+   if $EGREP -i -c -E 'fatal|error|unrecognized|not found|not exist' conftest.err >/dev/null; then retval=1; fi
+  ],[retval=1])
 AS_IF([test $retval = 0],[$2],[$3])])
 
+################################################################
+# Take out duplicate flags
+CFLAGS=$(echo $CFLAGS | tr ' ' '\n' | sort -u | tr '\n' ' ')
+CPPFLAGS=$(echo $CPPFLAGS | tr ' ' '\n' | sort -u | tr '\n' ' ')
+CXXFLAGS=$(echo $CXXFLAGS | tr ' ' '\n' | sort -u | tr '\n' ' ')
+
+################################################################
+AC_MSG_NOTICE([CFLAGS are now $CFLAGS])
+AC_MSG_NOTICE([CPPFLAGS are now $CPPFLAGS])
+AC_MSG_NOTICE([CXXFLAGS are now $CXXFLAGS])
+
 
 AC_MSG_NOTICE([*************************************])
 AC_MSG_NOTICE([*************************************])

etc/CONFIGURE_MACOS.bash

@@ -22,7 +22,7 @@ read
 
 # Note: openssl no longer required
 # Apple's provided flex is 2.6.4, which is the same that is provided by brew
-PKGS+="wget libtool autoconf automake libtool libxml2 libewf json-c"
+PKGS+="wget libtool autoconf automake libtool libxml2 libewf json-c re2 abseil pkg-config"
 
 $WHICH install $PKGS || (echo installation install failed; exit 1)
 exit 0

etc/CONFIGURE_UBUNTU20LTS.bash

@@ -4,7 +4,7 @@ RELEASE=20
 CONFIGURE="./configure -q --enable-silent-rules"
 AUTOCONF_DIST=https://ftp.gnu.org/gnu/autoconf/autoconf-2.71.tar.gz
 AUTOMAKE_DIST=https://ftp.gnu.org/gnu/automake/automake-1.16.3.tar.gz
-MKPGS="build-essential flex libexpat1-dev libxml2-utils libssl-dev libtool make pkg-config zlib1g-dev"
+MKPGS="build-essential flex libabsl-dev libexpat1-dev libre2-dev libssl-dev libtool libxml2-utils make pkg-config zlib1g-dev"
 # libxml2-utils needed for xmllint
 WGET="wget -nv --no-check-certificate"
 CONFIGURE="./configure -q --enable-silent-rules"

etc/CONFIGURE_UBUNTU20_win64.bash

@@ -3,3 +3,4 @@ sudo apt-get install -y mingw-w64
 sudo apt-get install -y wine
 sudo apt install libz-mingw-w64-dev
 sudo apt install libgcrypt-mingw-w64-dev
+sudo apt install libxml2-utils

etc/CONFIGURE_UBUNTU22LTS.bash

@@ -0,0 +1,68 @@
+#!/bin/bash
+SCRIPT_DIR="$(readlink -f $(dirname "${BASH_SOURCE[0]}"))"
+RELEASE=22
+CONFIGURE="./configure -q --enable-silent-rules"
+MKPGS="autoconf automake g++ flex libabsl-dev libexpat1-dev libre2-dev libssl-dev libtool libssl-dev libxml2-utils make pkg-config zlib1g-dev"
+WGET="wget -nv --no-check-certificate"
+CONFIGURE="./configure -q --enable-silent-rules"
+MAKE="make -j2"
+trap "exit 1" TERM
+export TOP_PID=$$
+cat <<EOF
+*******************************************************************
+        Configuring Ubuntu $RELEASE.04 LTS to compile bulk_extractor.
+*******************************************************************
+
+Install Ubuntu $RELEASE.04 and follow these commands:
+
+# sudo apt-get install git
+# git clone --recursive https://github.com/simsong/bulk_extractor.git
+# bash bulk_extractor/etc/CONFIGURE_UBUNTU22LTS.bash
+# (cd bulk_extractor; sudo make install)
+
+press any key to continue...
+EOF
+read IGNORE
+
+source $SCRIPT_DIR/paths.bash
+
+function fail() {
+    echo FAIL: $@
+    kill -s TERM $TOP_PID
+}
+
+if [ ! -r /etc/os-release ]; then
+    echo == This requires an /etc/os-release file.
+    exit 1
+fi
+
+source /etc/os-release
+
+if [ x$ID != xubuntu ]; then
+    echo == This really requires ubuntu. You have $ID
+    exit 1
+fi
+
+MAJOR_VERSION=`echo $VERSION_ID|sed s/[.].*//`
+if [ $MAJOR_VERSION -lt $RELEASE ]; then
+    echo == This requires at least Ubuntu $RELEASE Linux.
+    exit 1
+fi
+
+echo == Will now try apt update
+sudo apt update --fix-missing -y || fail could not apt update
+
+echo == Will now try to install
+sudo apt install -y $MKPGS || fail could not apt install $MKPGS
+
+echo == manually installing a modern libewf from $LIBEWF_URL
+cd /tmp
+/bin/rm -rf src
+mkdir src
+cd src
+
+$WGET $LIBEWF_URL || (echo could not download $LIBEWF_URL; exit 1)
+tar xfz libewf*gz   && (cd libewf*/   && $CONFIGURE && $MAKE >/dev/null && sudo make install)
+ls -l /etc/ld.so.conf.d/
+sudo ldconfig
+ewfinfo -h >/dev/null || (echo could not install libewf; exit 1)