🐛 fix(logging): increase log file size, may still face issue (#16)

* 🧹 chore: savepoint * 🧹 chore: savepoint * 🧹 chore: savepoint * 🧹 chore: savepoint * 🧹 chore: savepoint * 🎨 refactor: auto code check * 🧹 chore: savepoint * Prepare to switch to original multiprocessing * Still face like not using the same queue * Update src/modules/create_pdf.py Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> * Update src/modules/create_pdf.py Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> * 🧹 chore: savepoint * Give it a try * Log in 1 file while multiprocessing * increase log file size --------- Co-authored-by: KevinNitroG <[email protected]> Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com>
KevinNitroG · Mar 12, 2024 · 43aad55 · 43aad55
1 parent b995540
commit 43aad55
Show file tree

Hide file tree

Showing 20 changed files with 207 additions and 67 deletions.
diff --git a/.github/workflows/code-check.yml b/.github/workflows/code-check.yml
@@ -1,11 +1,13 @@
 name: Code Check
 
 concurrency:
-  cancel-in-progress: true
+  cancel-in-progress: false
   group: code-check
 
 on:
   push:
+    branches:
+      - main
   pull_request:
   workflow_dispatch:
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -21,7 +21,7 @@ repos:
         args:
           - "--allow-multiple-documents"
       - id: trailing-whitespace
-        exclude: "^(docs/README.md|src/constants.py)"
+        exclude: "^(docs/README.md|src/constants.py|assets/utils/ascii_banner.txt)"
       - id: end-of-file-fixer
       - id: debug-statements
       # - id: double-quote-string-fixer

diff --git a/assets/utils/ascii_banner.txt b/assets/utils/ascii_banner.txt
@@ -1,11 +1,11 @@
-██╗   ██╗███╗   ██╗██╗   ██╗██╗     ██╗██████╗
+██╗   ██╗███╗   ██╗██╗   ██╗██╗     ██╗██████╗ 
 ██║   ██║████╗  ██║██║   ██║██║     ██║██╔══██╗
 ██║   ██║██╔██╗ ██║██║   ██║██║     ██║██████╔╝
 ╚██╗ ██╔╝██║╚██╗██║██║   ██║██║     ██║██╔══██╗
  ╚████╔╝ ██║ ╚████║╚██████╔╝███████╗██║██████╔╝
-  ╚═══╝  ╚═╝  ╚═══╝ ╚═════╝ ╚══════╝╚═╝╚═════╝
+  ╚═══╝  ╚═╝  ╚═══╝ ╚═════╝ ╚══════╝╚═╝╚═════╝ 
 
-██████╗  ██████╗ ██╗    ██╗███╗   ██╗██╗      ██████╗  █████╗ ██████╗ ███████╗██████╗
+██████╗  ██████╗ ██╗    ██╗███╗   ██╗██╗      ██████╗  █████╗ ██████╗ ███████╗██████╗ 
 ██╔══██╗██╔═══██╗██║    ██║████╗  ██║██║     ██╔═══██╗██╔══██╗██╔══██╗██╔════╝██╔══██╗
 ██║  ██║██║   ██║██║ █╗ ██║██╔██╗ ██║██║     ██║   ██║███████║██║  ██║█████╗  ██████╔╝
 ██║  ██║██║   ██║██║███╗██║██║╚██╗██║██║     ██║   ██║██╔══██║██║  ██║██╔══╝  ██╔══██╗

diff --git a/docs/README.md b/docs/README.md
@@ -267,6 +267,7 @@ python main.py
 
 ## 📓 TODO
 
+- [ ] Logging to file check ⚠️ due to `RotatingFileHandler` on Windows
 - [ ] Support more browsers _(inherited from which class?)_
 - [ ] Options to enable Multithreading, Multiprocessing
 - [ ] Update tool option using command line arguments

diff --git a/main.py b/main.py
@@ -2,8 +2,10 @@
 
 from __future__ import annotations
 
+
 from pprint import pformat
 from multiprocessing import freeze_support
+from logging import getLogger
 from src import (
     Action,
     Browser,
@@ -15,8 +17,8 @@
     Login,
     PrintIntro,
     UserOptions,
+    ToolLogger,
     create_directory,
-    logger,
     pause,
     print_title,
 )
@@ -25,6 +27,9 @@
 
 def main() -> None:
     """Main function to run VNULIB Downloader."""
+    ToolLogger().setup()
+    logger = getLogger("vnulib_downloader")
+
     PrintIntro()
 
     print_title("USER OPTIONS")
@@ -56,9 +61,8 @@ def main() -> None:
             ).action()
     logger.debug(msg=f"LINKS OBJECT:\n{pformat(user_options.links)}")
 
-    create_directory(DOWNLOAD_DIR, force=False)
-
     print_title("DOWNLOAD")
+    create_directory(DOWNLOAD_DIR, force=False)
     DownloadIMG(
         links=user_options.links,
         download_directory=DOWNLOAD_DIR,
@@ -73,9 +77,10 @@ def main() -> None:
         print_title("DELETE IMAGES")
         CleanIMG(user_options.links, DOWNLOAD_DIR).clean_img()
 
+    print_title("END PROGRAM")
+    pause()
+
 
 if __name__ == "__main__":
     freeze_support()  # For pyinstaller to fix multiprocessing in Windows
     main()
-    print_title("END PROGRAM")
-    pause()
diff --git a/src/__init__.py b/src/__init__.py
@@ -13,4 +13,9 @@
     UserOptions,
     setup_argparse,
 )
-from .utils import create_directory, logger, pause, print_title
+from .utils import (
+    ToolLogger,
+    create_directory,
+    pause,
+    print_title,
+)
diff --git a/src/bot/action.py b/src/bot/action.py
@@ -5,12 +5,16 @@
 from __future__ import annotations
 
 from urllib.parse import parse_qs, urlparse
+from logging import getLogger
 from selenium.webdriver.chrome.webdriver import WebDriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.remote.webelement import WebElement
 from .utils import wait_element_visible
 from ..modules import Link, LinkFile
-from ..utils import datetime_name, logger, slugify
+from ..utils import datetime_name, slugify
+
+
+logger = getLogger("vnulib_downloader")
 
 
 class Action:
@@ -150,11 +154,11 @@ def action(self) -> list[Link]:
         for link in self.links:
             match link.original_type:
                 case "book":
-                    converted_links.append(self.book_handler(link=link))
                     logger.info(msg=f'"{link.original_link}": "{link.original_type}"')
+                    converted_links.append(self.book_handler(link=link))
                 case "preview":
-                    converted_links.append(self.preview_handler(link=link))
                     logger.info(msg=f'"{link.original_link}": "{link.original_type}"')
+                    converted_links.append(self.preview_handler(link=link))
                 case "page":
                     converted_links.append(link)
         return converted_links
diff --git a/src/bot/browser.py b/src/bot/browser.py
@@ -4,16 +4,18 @@
 
 import logging
 import os
+from logging import getLogger
 import urllib3
 from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.chrome.webdriver import WebDriver
 from webdriver_manager.chrome import ChromeDriverManager
 from webdriver_manager.core.logger import set_logger
 from src.constants import BROWSER_ARGUMENTS
-from ..utils import logger
 
 
+logger = getLogger("vnulib_downloader")
+
 set_logger(logger)
 os.environ["WDM_LOG"] = str(logging.DEBUG)
 os.environ["WDM_SSL_VERIFY"] = "0"

diff --git a/src/bot/login.py b/src/bot/login.py
@@ -2,12 +2,15 @@
 
 from __future__ import annotations
 
+from logging import getLogger
 from selenium.webdriver.chrome.webdriver import WebDriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.remote.webelement import WebElement
 from .utils import wait_element_clickable
 from ..constants import LOGIN_URL
-from ..utils import logger
+
+
+logger = getLogger("vnulib_downloader")
 
 
 class Login:

diff --git a/src/constants.py b/src/constants.py
@@ -10,7 +10,7 @@
     BUNDLE_DIR = ""
 
 
-VERSION: str = "1.12"
+VERSION: str = "1.13"
 AUTHORS: str = "KevinNitroG & NTGNguyen"
 BANNER_FILE: str = f"{BUNDLE_DIR}assets/utils/ascii_banner.txt"
 with open(BANNER_FILE, encoding="utf-8") as banner_content:

diff --git a/src/logging_configuration.yml b/src/logging_configuration.yml
@@ -8,26 +8,35 @@ formatters:
   detail:
     "format": "%(asctime)s [%(levelname)-8s] - %(message)s (%(pathname)s:%(lineno)d)"
 handlers:
-  stdout:
+  stdout_handler:
     class: logging.StreamHandler
     stream: ext://sys.stdout
     formatter: simple
     level: INFO
-  file:
+  file_handler:
+    class: logging.FileHandler
+    formatter: verbose
+    filename: "log/tool.log"
+    level: DEBUG
+    encoding: utf-8
+  rotating_file_handler: # Problem of RotatingFileHandler: https://python-forum.io/thread-19104.html
     class: logging.handlers.RotatingFileHandler
     formatter: verbose
     filename: "log/tool.log"
-    maxBytes: 20000 # 2KB
+    maxBytes: 50000 # 5KB
     backupCount: 3
     level: DEBUG
     encoding: utf-8
+    delay: false
 loggers:
   vnulib_downloader:
     level: DEBUG
     handlers:
-      - stdout
-      - file
+      - stdout_handler
+      - rotating_file_handler
+  vnulib_downloader_queue:
+    level: DEBUG
+    handlers:
+      - stdout_handler
+      - file_handler
     propogate: false
-root:
-  level: CRITICAL
-  propogate: false
diff --git a/src/modules/clean_img.py b/src/modules/clean_img.py
@@ -3,8 +3,11 @@
 from __future__ import annotations
 
 import os
+from logging import getLogger
 from .link_parse import Link
-from ..utils import logger
+
+
+logger = getLogger("vnulib_downloader")
 
 
 class CleanIMG:

diff --git a/src/modules/config.py b/src/modules/config.py
@@ -4,8 +4,11 @@
 
 from os import path
 from shutil import copyfile
+from logging import getLogger
 from ..constants import CONFIG_FILE, CONFIG_SAMPLE_FILE
-from ..utils import logger
+
+
+logger = getLogger("vnulib_downloader")
 
 
 class Config: