From 70af10439ec71e100feb1a9f321849f6097989c6 Mon Sep 17 00:00:00 2001
From: DerSkythe <skif@skif.ws>
Date: Fri, 7 Apr 2023 22:29:16 +0400
Subject: [PATCH 1/6] feat: made normally output to console and write to log
 file

---
 scripts/backup.sh         | 124 +++++++++++++++++++++++++++-----------
 scripts/backup_restore.py |  20 +++---
 2 files changed, 100 insertions(+), 44 deletions(-)

diff --git a/scripts/backup.sh b/scripts/backup.sh
index 6856c1d78..0e2bd73d7 100755
--- a/scripts/backup.sh
+++ b/scripts/backup.sh
@@ -48,6 +48,21 @@ else
   USER=$(whoami)
 fi
 
+# Print to log file or to log file and console?
+# 0 = only log
+# 1 = console and file
+VERBOSE=true
+# But we can't do it here thus too many positional args
+# changing will broke compartibility
+#while getopts "v" OPTION
+#do
+#  case $OPTION in
+#    v) VERBOSE=true
+#       ;;
+#  esac
+#done
+#echo "VERBOSE LEVEL: $VERBOSE"
+
 BASEDIR=./backups
 TMPDIR=./.tmp
 DOW=$(date +%u)
@@ -66,26 +81,60 @@ ROLLING="$BASEDIR/rolling/backup_$DOW.tar.gz"
 [ -d ./.tmp/backup ] || mkdir -p ./.tmp/backup
 [ -d ./.tmp/databases_backup ] || mkdir -p ./.tmp/databases_backup
 
+#------------------------------------------------------------------------------
+# echo pass params and log to file LOGFILE
+# depend to global var VERBOSE
+# Example:
+# doLog "Something"
+#------------------------------------------------------------------------------
+doLog(){
+  # also we can log every command with timestamp
+  MSG=''
+  # Check if string is empty print empty line without timestamp
+  if [ -z "$*" ]
+  then
+    MSG=$(echo "$*")
+  else
+    MSG=$(echo "`date '+%Y.%m.%dT%H:%M:%S'` - ""$*")
+  fi
+  # with colors code it's looks very ugly
+  # taken from: https://stackoverflow.com/a/51141872/660753
+  MSG=$(echo "$MSG" | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g')
+
+  # write to console if VERBOSE is set
+  if [ $VERBOSE ]
+  then
+    # also on the screen lines not looking cool, remove \r symbol
+    echo "$MSG" | sed 's/\r/\n/g'
+  fi
+
+  # write to LOGFILE
+  echo "$MSG" >> $LOGFILE
+}
+#eof func doLog
+
 touch $LOGFILE
-echo ""  > $LOGFILE
-echo "### IOTstack backup generator log ###" >> $LOGFILE
-echo "Started At: $(date +"%Y-%m-%dT%H-%M-%S")" >> $LOGFILE
-echo "Current Directory: $(pwd)" >> $LOGFILE
-echo "Backup Type: $BACKUPTYPE" >> $LOGFILE
+doLog ""  > $LOGFILE
+doLog "### IOTstack backup generator log ###"
+doLog "Started At: $(date +"%Y-%m-%dT%H-%M-%S")"
+doLog "Current Directory: $(pwd)"
+doLog "Backup Type: $BACKUPTYPE"
 
 if [[ "$BACKUPTYPE" -eq "1" || "$BACKUPTYPE" -eq "3" ]]; then
-  echo "Backup File: $BACKUPFILE" >> $LOGFILE
+  doLog "Backup File: $BACKUPFILE"
 fi
 
 if [[ "$BACKUPTYPE" -eq "2" || "$BACKUPTYPE" -eq "3" ]]; then
-  echo "Rolling File: $ROLLING" >> $LOGFILE
+  doLog "Rolling File: $ROLLING"
 fi
 
 echo "" >> $BACKUPLIST
 
-echo "" >> $LOGFILE
-echo "Executing prebackup scripts" >> $LOGFILE
-bash ./scripts/backup_restore/pre_backup_complete.sh >> $LOGFILE 2>&1
+doLog ""
+doLog "Executing prebackup scripts"
+# Output to log file
+docker-compose logs --no-color >> $LOGFILE
+doLog $(bash ./scripts/backup_restore/pre_backup_complete.sh)
 
 echo "./services/" >> $BACKUPLIST
 echo "./volumes/" >> $BACKUPLIST
@@ -97,12 +146,14 @@ echo "./volumes/" >> $BACKUPLIST
 [ -f "./post_backup.sh" ] && echo "./post_backup.sh" >> $BACKUPLIST
 [ -f "./pre_backup.sh" ] && echo "./pre_backup.sh" >> $BACKUPLIST
 
-sudo tar -czf $TMPBACKUPFILE -T $BACKUPLIST >> $LOGFILE 2>&1
+doLog "Create temporary backup archive"
+doLog $(sudo tar -czf $TMPBACKUPFILE -T $BACKUPLIST 2>&1)
 
 [ -f "$ROLLING" ] && ROLLINGOVERWRITTEN=1 && rm -rf $ROLLING
 
-sudo chown -R $USER:$USER $TMPDIR/backup* >> $LOGFILE 2>&1
+doLog $(sudo chown -R $USER:$USER $TMPDIR/backup* 2>&1 )
 
+doLog "Create persistent backup archive"
 if [[ "$BACKUPTYPE" -eq "1" || "$BACKUPTYPE" -eq "3" ]]; then
   cp $TMPBACKUPFILE $BACKUPFILE
 fi
@@ -112,38 +163,41 @@ fi
 
 if [[ "$BACKUPTYPE" -eq "2" || "$BACKUPTYPE" -eq "3" ]]; then
   if [[ "$ROLLINGOVERWRITTEN" -eq 1 ]]; then
-    echo "Rolling Overwritten: True" >> $LOGFILE
+    doLog "Rolling Overwritten: True"
   else
-    echo "Rolling Overwritten: False" >> $LOGFILE
+    doLog "Rolling Overwritten: False"
   fi
 fi
 
-echo "Backup Size (bytes): $(stat --printf="%s" $TMPBACKUPFILE)" >> $LOGFILE
-echo "" >> $LOGFILE
+doLog "Backup Size (bytes): $(stat --printf="%s" $TMPBACKUPFILE)"
+doLog ""
 
-echo "Executing postbackup scripts" >> $LOGFILE
-bash ./scripts/backup_restore/post_backup_complete.sh >> $LOGFILE 2>&1
-echo "" >> $LOGFILE
+doLog "Executing postbackup scripts"
+# Output to log file
+docker-compose logs --no-color >> $LOGFILE
+doLog $(bash ./scripts/backup_restore/post_backup_complete.sh)
+doLog ""
 
-echo "Finished At: $(date +"%Y-%m-%dT%H-%M-%S")" >> $LOGFILE
-echo "" >> $LOGFILE
+doLog "Finished At: $(date +"%Y-%m-%dT%H-%M-%S")"
+doLog ""
 
 if [[ -f "$TMPBACKUPFILE" ]]; then
-  echo "Items backed up:" >> $LOGFILE
-  cat $BACKUPLIST >> $LOGFILE 2>&1
-  echo "" >> $LOGFILE
-  echo "Items Excluded:" >> $LOGFILE
-  echo " - No items" >> $LOGFILE 2>&1
-  rm -rf $BACKUPLIST >> $LOGFILE 2>&1
-  rm -rf $TMPBACKUPFILE >> $LOGFILE 2>&1
+  doLog "Items backed up:"
+  doLog $(cat $BACKUPLIST 2>&1)
+  doLog ""
+  doLog "Items Excluded:"
+  doLog " - No items"
+  doLog $(rm -rf $BACKUPLIST 2>&1)
+  doLog $(rm -rf $TMPBACKUPFILE 2>&1)
 else
-  echo "Something went wrong backing up. The temporary backup file doesn't exist. No temporary files were removed"
-  echo "Files: "
-  echo "  $BACKUPLIST"
+  doLog "Something went wrong backing up. The temporary backup file doesn't exist. No temporary files were removed"
+  doLog "Files: "
+  doLog "  $BACKUPLIST"
 fi
 
-echo "" >> $LOGFILE
-echo "### End of log ###" >> $LOGFILE
-echo "" >> $LOGFILE
+doLog ""
+doLog "### End of log ###"
+doLog ""
 
-cat $LOGFILE
+# we don't need to print LOGFILE if we are in verbose mode
+$VERBOSE && echo "" || cat $LOGFILE
diff --git a/scripts/backup_restore.py b/scripts/backup_restore.py
index e13b8d0a5..73662d994 100755
--- a/scripts/backup_restore.py
+++ b/scripts/backup_restore.py
@@ -14,7 +14,7 @@ def main():
   global mainMenuList
   global currentMenuItemIndex
   global hideHelpText
-  
+
   try: # If not already set, then set it.
     hideHelpText = hideHelpText
   except:
@@ -24,11 +24,13 @@ def main():
 
   def runBackup():
     global needsRender
-    print("Execute Backup:")
-    subprocess.call("./scripts/backup.sh", shell=True)
-    print("")
-    print("Backup completed.")
-    print("Press [Up] or [Down] arrow key to show the menu if it has scrolled too far.")
+    with term.fullscreen(), term.cbreak():
+        print("Execute Backup:")
+        subprocess.call("./scripts/backup.sh", shell=True)
+        print("")
+        print("Backup completed.")
+        print("Press [Up] or [Down] arrow key to show the menu if it has scrolled too far.")
+        term.inkey()
     time.sleep(1)
     needsRender = 1
     return True
@@ -128,12 +130,12 @@ def renderHotZone(term, menu, selection, hotzoneLocation):
       toPrint += "{bv}".format(bv=specialChars[renderMode]["borderVertical"])
 
       toPrint = term.center(toPrint)
-      
+
       print(toPrint)
 
   def mainRender(needsRender, menu, selection):
     term = Terminal()
-    
+
     if needsRender == 1:
       print(term.clear())
       print(term.move_y(6 - hotzoneLocation[0]))
@@ -156,7 +158,7 @@ def mainRender(needsRender, menu, selection):
           print(term.center(commonEmptyLine(renderMode)))
           print(term.center("{bv}      Not enough vertical room to render controls help text                     {bv}".format(bv=specialChars[renderMode]["borderVertical"])))
           print(term.center(commonEmptyLine(renderMode)))
-        else: 
+        else:
           print(term.center(commonEmptyLine(renderMode)))
           print(term.center("{bv}      Controls:                                                                 {bv}".format(bv=specialChars[renderMode]["borderVertical"])))
           print(term.center("{bv}      [Up] and [Down] to move selection cursor                                  {bv}".format(bv=specialChars[renderMode]["borderVertical"])))

From f8a924a3f4d233a725e60a1321d8e3a545146d3a Mon Sep 17 00:00:00 2001
From: DerSkythe <skif@skif.ws>
Date: Sat, 8 Apr 2023 01:21:06 +0400
Subject: [PATCH 2/6] fix typo and add double quotes to prevent some bad things

---
 scripts/backup.sh | 66 ++++++++++++++++++++++++-----------------------
 1 file changed, 34 insertions(+), 32 deletions(-)

diff --git a/scripts/backup.sh b/scripts/backup.sh
index 0e2bd73d7..86a26f3e3 100755
--- a/scripts/backup.sh
+++ b/scripts/backup.sh
@@ -53,7 +53,7 @@ fi
 # 1 = console and file
 VERBOSE=true
 # But we can't do it here thus too many positional args
-# changing will broke compartibility
+# changing will broke compatibility
 #while getopts "v" OPTION
 #do
 #  case $OPTION in
@@ -95,7 +95,7 @@ doLog(){
   then
     MSG=$(echo "$*")
   else
-    MSG=$(echo "`date '+%Y.%m.%dT%H:%M:%S'` - ""$*")
+    MSG=$(echo "$(date '+%Y.%m.%dT%H:%M:%S') - ""$*")
   fi
   # with colors code it's looks very ugly
   # taken from: https://stackoverflow.com/a/51141872/660753
@@ -109,12 +109,12 @@ doLog(){
   fi
 
   # write to LOGFILE
-  echo "$MSG" >> $LOGFILE
+  echo "$MSG" >> "$LOGFILE"
 }
 #eof func doLog
 
-touch $LOGFILE
-doLog ""  > $LOGFILE
+touch "$LOGFILE"
+doLog ""
 doLog "### IOTstack backup generator log ###"
 doLog "Started At: $(date +"%Y-%m-%dT%H-%M-%S")"
 doLog "Current Directory: $(pwd)"
@@ -128,40 +128,42 @@ if [[ "$BACKUPTYPE" -eq "2" || "$BACKUPTYPE" -eq "3" ]]; then
   doLog "Rolling File: $ROLLING"
 fi
 
-echo "" >> $BACKUPLIST
+echo "" >> "$BACKUPLIST"
 
 doLog ""
-doLog "Executing prebackup scripts"
+doLog "Executing pre-backup scripts"
 # Output to log file
-docker-compose logs --no-color >> $LOGFILE
-doLog $(bash ./scripts/backup_restore/pre_backup_complete.sh)
-
-echo "./services/" >> $BACKUPLIST
-echo "./volumes/" >> $BACKUPLIST
-[ -f "./docker-compose.yml" ] && echo "./docker-compose.yml" >> $BACKUPLIST
-[ -f "./compose-override.yml" ] && echo "./compose-override.yml" >> $BACKUPLIST
-[ -f "./extra" ] && echo "./extra" >> $BACKUPLIST
-[ -f "./.tmp/databases_backup" ] && echo "./.tmp/databases_backup" >> $BACKUPLIST
-[ -f "./postbuild.sh" ] && echo "./postbuild.sh" >> $BACKUPLIST
-[ -f "./post_backup.sh" ] && echo "./post_backup.sh" >> $BACKUPLIST
-[ -f "./pre_backup.sh" ] && echo "./pre_backup.sh" >> $BACKUPLIST
+docker-compose logs --no-color >> "$LOGFILE"
+doLog "$(bash ./scripts/backup_restore/pre_backup_complete.sh)"
+
+echo "./services/" >> "$BACKUPLIST"
+echo "./volumes/" >> "$BACKUPLIST"
+[ -f "./docker-compose.yml" ] && echo "./docker-compose.yml" >> "$BACKUPLIST"
+[ -f "./compose-override.yml" ] && echo "./compose-override.yml" >> "$BACKUPLIST"
+[ -f "./extra" ] && echo "./extra" >> "$BACKUPLIST"
+[ -f "./.tmp/databases_backup" ] && echo "./.tmp/databases_backup" >> "$BACKUPLIST"
+[ -f "./postbuild.sh" ] && echo "./postbuild.sh" >> "$BACKUPLIST"
+[ -f "./post_backup.sh" ] && echo "./post_backup.sh" >> "$BACKUPLIST"
+[ -f "./pre_backup.sh" ] && echo "./pre_backup.sh" >> "$BACKUPLIST"
 
 doLog "Create temporary backup archive"
-doLog $(sudo tar -czf $TMPBACKUPFILE -T $BACKUPLIST 2>&1)
+doLog "$(sudo tar -czf "$TMPBACKUPFILE" -T "$BACKUPLIST" 2>&1)"
 
-[ -f "$ROLLING" ] && ROLLINGOVERWRITTEN=1 && rm -rf $ROLLING
+[ -f "$ROLLING" ] && ROLLINGOVERWRITTEN=1 && rm -rf "$ROLLING"
 
-doLog $(sudo chown -R $USER:$USER $TMPDIR/backup* 2>&1 )
+doLog "$(sudo chown -R "$USER":"$USER" $TMPDIR/backup* 2>&1 )"
 
 doLog "Create persistent backup archive"
 if [[ "$BACKUPTYPE" -eq "1" || "$BACKUPTYPE" -eq "3" ]]; then
-  cp $TMPBACKUPFILE $BACKUPFILE
+  # TODO: Double check do we really need a copy instead of move?
+  cp "$TMPBACKUPFILE" "$BACKUPFILE"
 fi
 if [[ "$BACKUPTYPE" -eq "2" || "$BACKUPTYPE" -eq "3" ]]; then
-  cp $TMPBACKUPFILE $ROLLING
+  cp "$TMPBACKUPFILE" "$ROLLING"
 fi
 
 if [[ "$BACKUPTYPE" -eq "2" || "$BACKUPTYPE" -eq "3" ]]; then
+  # TODO: $ROLLINGOVERWRITTEN is not set!
   if [[ "$ROLLINGOVERWRITTEN" -eq 1 ]]; then
     doLog "Rolling Overwritten: True"
   else
@@ -169,13 +171,13 @@ if [[ "$BACKUPTYPE" -eq "2" || "$BACKUPTYPE" -eq "3" ]]; then
   fi
 fi
 
-doLog "Backup Size (bytes): $(stat --printf="%s" $TMPBACKUPFILE)"
+doLog "Backup Size (bytes): $(stat --printf="%s" "$TMPBACKUPFILE")"
 doLog ""
 
-doLog "Executing postbackup scripts"
+doLog "Executing post-backup scripts"
 # Output to log file
-docker-compose logs --no-color >> $LOGFILE
-doLog $(bash ./scripts/backup_restore/post_backup_complete.sh)
+docker-compose logs --no-color >> "$LOGFILE"
+doLog "$(bash ./scripts/backup_restore/post_backup_complete.sh)"
 doLog ""
 
 doLog "Finished At: $(date +"%Y-%m-%dT%H-%M-%S")"
@@ -183,12 +185,12 @@ doLog ""
 
 if [[ -f "$TMPBACKUPFILE" ]]; then
   doLog "Items backed up:"
-  doLog $(cat $BACKUPLIST 2>&1)
+  doLog "$(cat "$BACKUPLIST" 2>&1)"
   doLog ""
   doLog "Items Excluded:"
   doLog " - No items"
-  doLog $(rm -rf $BACKUPLIST 2>&1)
-  doLog $(rm -rf $TMPBACKUPFILE 2>&1)
+  doLog "$(rm -rf "$BACKUPLIST" 2>&1)"
+  doLog "$(rm -rf "$TMPBACKUPFILE" 2>&1)"
 else
   doLog "Something went wrong backing up. The temporary backup file doesn't exist. No temporary files were removed"
   doLog "Files: "
@@ -200,4 +202,4 @@ doLog "### End of log ###"
 doLog ""
 
 # we don't need to print LOGFILE if we are in verbose mode
-$VERBOSE && echo "" || cat $LOGFILE
+$VERBOSE != true && cat "$LOGFILE"

From 42de931d428bc78cc0eaeccc3a401b3bb1b77f51 Mon Sep 17 00:00:00 2001
From: DerSkythe <skif@skif.ws>
Date: Sat, 8 Apr 2023 01:54:15 +0400
Subject: [PATCH 3/6] feat: take only 50 lines of each log file

---
 scripts/backup.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/backup.sh b/scripts/backup.sh
index 86a26f3e3..fa6cfaed3 100755
--- a/scripts/backup.sh
+++ b/scripts/backup.sh
@@ -133,7 +133,7 @@ echo "" >> "$BACKUPLIST"
 doLog ""
 doLog "Executing pre-backup scripts"
 # Output to log file
-docker-compose logs --no-color >> "$LOGFILE"
+docker-compose logs --no-color --tail="50" >> "$LOGFILE"
 doLog "$(bash ./scripts/backup_restore/pre_backup_complete.sh)"
 
 echo "./services/" >> "$BACKUPLIST"
@@ -176,7 +176,7 @@ doLog ""
 
 doLog "Executing post-backup scripts"
 # Output to log file
-docker-compose logs --no-color >> "$LOGFILE"
+docker-compose logs --no-color --tail="50" >> "$LOGFILE"
 doLog "$(bash ./scripts/backup_restore/post_backup_complete.sh)"
 doLog ""
 

From af2f0f416d5902712ec3aad2bfe86235238a012b Mon Sep 17 00:00:00 2001
From: snyk-bot <snyk-bot@snyk.io>
Date: Sun, 2 Jul 2023 21:13:41 +0000
Subject: [PATCH 4/6] fix: requirements-mkdocs.txt to reduce vulnerabilities

The following vulnerabilities are fixed by pinning transitive dependencies:
- https://snyk.io/vuln/SNYK-PYTHON-PYGMENTS-5750273
---
 requirements-mkdocs.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-mkdocs.txt b/requirements-mkdocs.txt
index dcf0f8cd7..23d7a5580 100644
--- a/requirements-mkdocs.txt
+++ b/requirements-mkdocs.txt
@@ -13,7 +13,7 @@ mkdocs-material-extensions==1.0.3
 mkdocs-pagenav-generator @ git+https://github.com/Andre601/mkdocs-pagenav-generator@acb5b1561695e8f69d67fda029779f40e4b0beef
 mkdocs-redirects==1.0.3
 packaging==21.3
-Pygments==2.12.0
+Pygments==2.15.0
 pymdown-extensions==9.4
 pyparsing==3.0.7
 python-dateutil==2.8.2

From 7c6999d9ec5d5878c62245c15825a9214730173c Mon Sep 17 00:00:00 2001
From: snyk-bot <snyk-bot@snyk.io>
Date: Fri, 12 Jan 2024 05:23:34 +0000
Subject: [PATCH 5/6] fix: requirements-mkdocs.txt to reduce vulnerabilities

The following vulnerabilities are fixed by pinning transitive dependencies:
- https://snyk.io/vuln/SNYK-PYTHON-JINJA2-6150717
---
 requirements-mkdocs.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements-mkdocs.txt b/requirements-mkdocs.txt
index 23d7a5580..dcdc85462 100644
--- a/requirements-mkdocs.txt
+++ b/requirements-mkdocs.txt
@@ -2,7 +2,7 @@ bracex==2.2.1
 click==8.0.4
 ghp-import==2.0.2
 importlib-metadata==4.11.2
-Jinja2==3.0.3
+Jinja2==3.1.3
 Markdown==3.3.6
 MarkupSafe==2.1.0
 mergedeep==1.3.4

From b550d29a6edb89fa359c995f4e278166aa8515d5 Mon Sep 17 00:00:00 2001
From: snyk-bot <snyk-bot@snyk.io>
Date: Wed, 22 May 2024 11:58:35 +0000
Subject: [PATCH 6/6] fix: requirements-mkdocs.txt to reduce vulnerabilities

The following vulnerabilities are fixed by pinning transitive dependencies:
- https://snyk.io/vuln/SNYK-PYTHON-REQUESTS-6928867
---
 requirements-mkdocs.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements-mkdocs.txt b/requirements-mkdocs.txt
index 56851693d..e1c945dc2 100644
--- a/requirements-mkdocs.txt
+++ b/requirements-mkdocs.txt
@@ -23,3 +23,4 @@ six
 watchdog
 wcmatch
 zipp
+requests>=2.32.0 # not directly required, pinned by Snyk to avoid a vulnerability