Merge pull request #160 from CodePhase/scrublogsupdate

Updating log utilities
the-modem-distro · Nov 27, 2022 · 70985d0 · 70985d0
2 parents 748048f + df38cc2
commit 70985d0
Show file tree

Hide file tree

Showing 2 changed files with 128 additions and 54 deletions.
diff --git a/tools/helpers/collect_logs b/tools/helpers/collect_logs
@@ -6,21 +6,21 @@ get_persistent_logs(){
 
 echo "  -> In case of persistent logging get logs of previous boot"
 
-adb pull /persist/openqti.log.1
-mv openqti.log.1 openqti-${TIMESTAMP}.log.1
+sudo sh -c "adb pull /persist/openqti-1.log"
+[ -f "openqti-1.log" ] && sudo mv openqti-1.log openqti-1-${TIMESTAMP}.log
 
-journalctl -u eg25-manager.service -e --boot -1 > eg25-manager-${TIMESTAMP}.log.1
-journalctl -u ModemManager.service -e --boot -1 > modemmanager-${TIMESTAMP}.log.1
-journalctl -u NetworkManager.service -e --boot -1 > networkmanager-${TIMESTAMP}.log.1
+journalctl -u eg25-manager.service -e --boot -1 > eg25-manager-1-${TIMESTAMP}.log
+journalctl -u ModemManager.service -e --boot -1 > modemmanager-1-${TIMESTAMP}.log
+journalctl -u NetworkManager.service -e --boot -1 > networkmanager-1-${TIMESTAMP}.log
 }
 
-adb shell 'echo Collecting dmesg... > /dev/kmsg'
-adb shell dmesg > dmesg-modem-${TIMESTAMP}.txt
-adb pull /var/log/openqti.log
-mv openqti.log openqti-${TIMESTAMP}.log
+sudo sh -c "adb shell 'echo Collecting dmesg... > /dev/kmsg'"
+sudo sh -c "adb shell dmesg > dmesg-modem-${TIMESTAMP}.txt"
+sudo sh -c "adb pull /var/log/openqti.log"
+[ -f "openqti.log" ] && sudo mv openqti.log openqti-${TIMESTAMP}.log
 
 sudo sh -c 'echo Collecting dmesg... > /dev/kmsg'
-dmesg > dmesg-pinephone-${TIMESTAMP}.txt
+sudo sh -c "dmesg > dmesg-pinephone-${TIMESTAMP}.txt"
 
 journalctl -u eg25-manager.service -e > eg25-manager-${TIMESTAMP}.log
 journalctl -u ModemManager.service -e > modemmanager-${TIMESTAMP}.log
@@ -29,6 +29,6 @@ journalctl -u NetworkManager.service -e > networkmanager-${TIMESTAMP}.log
 get_persistent_logs
 
 SCRIPT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
-/usr/bin/bash ${SCRIPT_DIR}/scrub_logs
+/usr/bin/bash ${SCRIPT_DIR}/scrub_logs -t ${TIMESTAMP}
 
 exit 0
diff --git a/tools/helpers/scrub_logs b/tools/helpers/scrub_logs
@@ -1,77 +1,151 @@
 #!/usr/bin/env bash
 
-declare -a scrubFiles=( "dmesg-pinephone.txt" "modemmanager.log" "networkmanager.log" "openqti.log" )
 subText="<redacted>"
-modemNum="015550199999"
+modemNum="015550199999\|223344556677"
 
-function now {
-  ts=$(date +%Y%m%d%H%M%S)
-  echo "Backup files using timestamp ${ts}"
+function getArgs {
+  while [ $# -gt 0 ]
+  do
+    case "$1" in
+      "-t"|"--timestamp")
+        shift
+        TIMESTAMP="$1"
+        shift
+        ;;
+      "-h"|"--help")
+        showHelp
+        ;;
+      *)
+        echo "Argument not recognized: $1"
+        showHelp
+        ;;
+    esac
+  done
+}
+
+function showHelp {
+  cat << EOS
+  Scrub sensitive information from phone logs after backup of original unmodified files:
+   - phone numbers
+   - MAC addresses
+   - IP addresses (ipv4/6)
+   - Wifi SSID's
+
+  Usage: scrub_logs [-t|--timestamp]
+         scrub_logs --help
+
+    -t
+    --timestamp <TIMESTAMP>
+                  label backed up log files with the current timestamp
+    -h
+    --help
+                  Show this help and exit
+
+  NOTE: This utilty is designed to be used with 'collect_logs' and will run that first
+        if this is run directly
+EOS
+
+ exit 0
 }
 
 function backupFiles {
-  echo "Backing up files"
+  echo "Backing up files before redaction with timestamp ${TIMESTAMP}"
   for logFile in "${scrubFiles[@]}"; do
-    cp -p "${logFile}" "${logFile}-${ts}"
+    if [ -f "$logFile" ]; then
+      origLogFile="${logFile/%.log/-orig.log}"
+      origLogFile="${origLogFile/%.txt/-orig.txt}"
+      cp -a "${logFile}" "${origLogFile}"
+    fi
+    lastBootLogFile="${logFile/-${TIMESTAMP}/-1-${TIMESTAMP}}"
+    if [ -f "$lastBootLogFile" ]; then
+      origLogFile="${lastBootLogFile/%\.log/-orig.log}"
+      origLogFile="${origLogFile/%\.txt/-orig.txt}"
+      cp -a "${lastBootLogFile}" "${origLogFile}"
+    fi
   done
 }
 
 function scrubDmesgPp {
-  echo "Scrubbing ${scrubFiles[0]}"
-  sed -i 's/\([a-f0-9]\{2\}:\)\{5\}[a-f0-9]\{2\}/'${subText}'/' "${scrubFiles[0]}"
+  if [ -f "${scrubFiles[0]}" ]; then
+    echo "Scrubbing ${scrubFiles[0]}"
+    sed -i 's/\([a-f0-9]\{2\}:\)\{5\}[a-f0-9]\{2\}/'${subText}'/' "${scrubFiles[0]}"
+  fi
 }
 
 function scrubModemManager {
-  echo "Scrubbing ${scrubFiles[1]}"
-  sed -i -e 's/address: .*/address: '${subText}'/' -e 's/gateway: .*/gateway: '${subText}'/' -e 's/DNS #\([1-9]\): .*/DNS #\1: '${subText}'/' "${scrubFiles[1]}"
+  for MdmMgrFile in "${scrubFiles[1]}" "${scrubFiles[1]/-${TIMESTAMP}/-1-${TIMESTAMP}}"; do
+    if [ -f "${MdmMgrFile}" ]; then
+      echo "Scrubbing ${MdmMgrFile}"
+      sed -i -e 's/address: .*/address: '${subText}'/' -e 's/gateway: .*/gateway: '${subText}'/' -e 's/DNS #\([1-9]\): .*/DNS #\1: '${subText}'/' "${MdmMgrFile}"
+    fi
+  done
 }
 
 function scrubNetworkManager {
-  echo "Scrubbing ${scrubFiles[2]}"
-  unset redacts
-  declare -a redacts
+  for netMgrFile in "${scrubFiles[2]}" "${scrubFiles[2]/-${TIMESTAMP}/-1-${TIMESTAMP}}"; do
+    if [ -f "${netMgrFile}" ]; then
+      echo "Scrubbing ${netMgrFile}"
+      unset redacts
+      declare -a redacts
 
-  unset netIds
-  declare -a netIds
-  readarray -t netIds <<< $(grep 'starting connection' "${scrubFiles[2]}" | awk $'{ match($0,/\'[a-zA-Z0-9 ]+\'/,netname); gsub(/\'/, "", netname[0]); match($0,/\([a-z0-9-]+\)$/,netid); gsub(/[()]/, "", netid[0]); printf "%s\\n%s\\n", netname[0], netid[0] }' | sort | uniq)
+      unset netIds
+      declare -a netIds
+      readarray -t netIds <<< $(grep 'starting connection' "${netMgrFile}" | awk $'{ match($0,/\'[a-zA-Z0-9 ]+\'/,netname); gsub(/\'/, "", netname[0]); match($0,/\([a-z0-9-]+\)$/,netid); gsub(/[()]/, "", netid[0]); printf "%s\\n%s\\n", netname[0], netid[0] }' | sort | uniq)
 
-  unset macs; declare -a macs
-  readarray -t macs <<<$(grep -Eo '([a-fA-F0-9]{2}:){5}[a-fA-F0-9]{2}' "${scrubFiles[2]}" | sort | uniq)
+      unset macs; declare -a macs
+      readarray -t macs <<<$(grep -Eo '([a-fA-F0-9]{2}:){5}[a-fA-F0-9]{2}' "${netMgrFile}" | sort | uniq)
 
-  unset ipv4s
-  declare -a ipv4s
-  readarray -t ipv4s <<< $(grep -Eo '([1-9][0-9]{0,2}\.){3}[1-9][0-9]{0,2}(/[1-9][0-9]?)?' "${scrubFiles[2]}" | sort | uniq)
+      unset ipv4s
+      declare -a ipv4s
+      readarray -t ipv4s <<< $(grep -Eo '([1-9][0-9]{0,2}\.){3}[1-9][0-9]{0,2}(/[1-9][0-9]?)?' "${netMgrFile}" | sort | uniq)
 
-  unset ipv6s
-  declare -a ipv6s
-  readarray -t ipv6s <<< $(grep -Eo '([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}(/[1-9][0-9]?)?' "${scrubFiles[2]}" | while read line; do   unset valid; numCols=$(echo "$line" | awk -F":" '{print NF-1}'); if [ $numCols -eq 7 ]; then valid=1; elif [ $numCols -lt 7 ] && [ $numCols -ge 2 ]; then if echo "${line}" | grep -q '::'; then valid=1; fi; fi; if [ "$valid" ]; then echo "$line"; fi; done | sort | uniq)
+      unset ipv6s
+      declare -a ipv6s
+      readarray -t ipv6s <<< $(grep -Eo '([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}(/[1-9][0-9]?)?' "${netMgrFile}" | while read line; do   unset valid; numCols=$(echo "$line" | awk -F":" '{print NF-1}'); if [ $numCols -eq 7 ]; then valid=1; elif [ $numCols -lt 7 ] && [ $numCols -ge 2 ]; then if echo "${line}" | grep -q '::'; then valid=1; fi; fi; if [ "$valid" ]; then echo "$line"; fi; done | sort | uniq)
 
-  redacts=( "${netIds[@]}" "${macs[@]}" "${ipv4s[@]}" "${ipv6s[@]}")
+      redacts=( "${netIds[@]}" "${macs[@]}" "${ipv4s[@]}" "${ipv6s[@]}")
 
-  sedExpr=""
-  for val in "${redacts[@]}"; do
-    # Make sure the value isn't blank
-    [ -z "${val}" ] && continue
-    sedExpr+="-e 's,${val},${subText},g' "
+      sedExpr=""
+      for val in "${redacts[@]}"; do
+        # Make sure the value isn't blank
+        [ -z "${val}" ] && continue
+        sedExpr+="-e 's,${val},${subText},g' "
+      done
+      [ "${sedExpr}" ] && eval sed -i $sedExpr "${netMgrFile}"
+    fi
   done
-  [ "${sedExpr}" ] && eval sed -i $sedExpr "${scrubFiles[2]}"
 }
 
 function scrubOpenQti {
-  echo "Scrubbing ${scrubFiles[3]}"
-  unset phNums
-  declare -a phNums
-  readarray -t phNums <<< $(grep -E 'Call request .*[0-9]+$|Call status.*[0-9]+$' "${scrubFiles[3]}" | grep -v ${modemNum} | grep -Eo '[0-9]+$' | sort | uniq)
-  sedExpr=""
-  for phNum in "${phNums[@]}"; do
-    # Make sure the value isn't blank
-    [ -z "${phNum}" ] && continue
-    sedExpr+="-e 's/${phNum}/${subText}/g' "
+  for openQtiFile in "${scrubFiles[3]}" "${scrubFiles[3]/-${TIMESTAMP}/-1-${TIMESTAMP}}"; do
+    if [ -f "${openQtiFile}" ]; then
+      echo "Scrubbing ${openQtiFile}"
+      unset phNums
+      declare -a phNums
+      readarray -t phNums <<< $(grep -E 'Call request .*[0-9]+$|Call status.*[0-9]+$' "${openQtiFile}" | grep -v "${modemNum}" | grep -Eo '[0-9]+$' | sort | uniq)
+      sedExpr=""
+      for phNum in "${phNums[@]}"; do
+        # Make sure the value isn't blank
+        [ -z "${phNum}" ] && continue
+        sedExpr+="-e 's/${phNum}/${subText}/g' "
+      done
+      [ "${sedExpr}" ] &&  eval sed -i $sedExpr "${openQtiFile}"
+    fi
   done
-  [ "${sedExpr}" ] &&  eval sed -i $sedExpr "${scrubFiles[3]}"
 }
 
-now
+getArgs "$@"
+
+if [ -z "$TIMESTAMP" ]; then
+  SCRIPT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd)
+  /bin/bash ${SCRIPT_DIR}/collect_logs
+  exit $?
+fi
+
+# Note, there doesn't seem to be a need to redact anything in dmesg-modem or eg25-manager
+# so don't process them
+declare -a scrubFiles=( "dmesg-pinephone-${TIMESTAMP}.txt" "modemmanager-${TIMESTAMP}.log" "networkmanager-${TIMESTAMP}.log" "openqti-${TIMESTAMP}.log" )
+
 backupFiles
 scrubDmesgPp
 scrubModemManager