From ece5f6198f54fe925ad5b506a248483d1dbcbdf4 Mon Sep 17 00:00:00 2001 From: KevinNitroG Date: Wed, 13 Mar 2024 13:14:26 +0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=92=A9=20chore(logging):=20write=20loggin?= =?UTF-8?q?g=20in=20the=20right=20way?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .editorconfig | 1 + main.py | 25 +++++++++++------------ src/__init__.py | 1 + src/bot/action.py | 2 +- src/bot/browser.py | 11 +---------- src/bot/login.py | 2 +- src/constants.py | 2 +- src/logging_configuration.yml | 37 ++++++++++------------------------- src/modules/clean_img.py | 2 +- src/modules/config.py | 2 +- src/modules/create_pdf.py | 6 +++--- src/modules/download_img.py | 2 +- src/modules/link_parse.py | 2 +- src/modules/user_options.py | 2 +- src/utils/__init__.py | 2 +- src/utils/logger.py | 25 ++++++++++++++++++----- src/utils/utils.py | 2 +- 17 files changed, 59 insertions(+), 67 deletions(-) diff --git a/.editorconfig b/.editorconfig index ea35227..5379eb1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -19,6 +19,7 @@ tab_width = 4 [*.{js,ts,tsx,json,yml,yaml,toml}] indent_size = 2 +tab_width = 2 [*.sh] end_of_line = lf diff --git a/main.py b/main.py index 512ebe9..ea78d11 100644 --- a/main.py +++ b/main.py @@ -3,16 +3,13 @@ from __future__ import annotations -from src.utils.logger import ToolLogger - -# Have to read the logger config file before all. -ToolLogger().setup() - - -from pprint import pformat # pylint: disable=wrong-import-position, wrong-import-order -from multiprocessing import freeze_support # pylint: disable=wrong-import-position, wrong-import-order -from logging import getLogger, Logger # pylint: disable=wrong-import-position, wrong-import-order -from src import ( # pylint: disable=wrong-import-position, wrong-import-order +from pprint import pformat +from multiprocessing import freeze_support +from logging import getLogger, Logger +from urllib3 import disable_warnings as urllib3_disable_warnings +from urllib3.exceptions import InsecureRequestWarning +from src import ( + ToolLogger, Action, Browser, Config, @@ -27,12 +24,16 @@ pause, print_title, ) -from src.constants import DOWNLOAD_DIR # pylint: disable=wrong-import-position +from src.constants import DOWNLOAD_DIR + + +ToolLogger().setup() +urllib3_disable_warnings(InsecureRequestWarning) def main() -> None: """Main function to run VNULIB Downloader.""" - logger: Logger = getLogger("vnulib_downloader") + logger: Logger = getLogger(__name__) PrintIntro() diff --git a/src/__init__.py b/src/__init__.py index c31e4c7..09af8f1 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -14,6 +14,7 @@ setup_argparse, ) from .utils import ( + ToolLogger, create_directory, pause, print_title, diff --git a/src/bot/action.py b/src/bot/action.py index f6ca91e..0d1a9c5 100644 --- a/src/bot/action.py +++ b/src/bot/action.py @@ -14,7 +14,7 @@ from ..utils import datetime_name, slugify -logger = getLogger("vnulib_downloader") +logger = getLogger(__name__) class Action: diff --git a/src/bot/browser.py b/src/bot/browser.py index 3a7cb05..5b8bd64 100644 --- a/src/bot/browser.py +++ b/src/bot/browser.py @@ -2,24 +2,15 @@ from __future__ import annotations -import logging -import os from logging import getLogger -import urllib3 from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.chrome.webdriver import WebDriver from webdriver_manager.chrome import ChromeDriverManager -from webdriver_manager.core.logger import set_logger from src.constants import BROWSER_ARGUMENTS -logger = getLogger("vnulib_downloader") - -set_logger(logger) -os.environ["WDM_LOG"] = str(logging.DEBUG) -os.environ["WDM_SSL_VERIFY"] = "0" -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +logger = getLogger(__name__) class Browser: diff --git a/src/bot/login.py b/src/bot/login.py index 4b16719..96bb826 100644 --- a/src/bot/login.py +++ b/src/bot/login.py @@ -10,7 +10,7 @@ from ..constants import LOGIN_URL -logger = getLogger("vnulib_downloader") +logger = getLogger(__name__) class Login: diff --git a/src/constants.py b/src/constants.py index 1faf714..8ffa537 100644 --- a/src/constants.py +++ b/src/constants.py @@ -10,7 +10,7 @@ BUNDLE_DIR = "" -VERSION: str = "1.14" +VERSION: str = "1.15" AUTHORS: str = "KevinNitroG & NTGNguyen" BANNER_FILE: str = f"{BUNDLE_DIR}assets/utils/ascii_banner.txt" with open(BANNER_FILE, encoding="utf-8") as banner_content: diff --git a/src/logging_configuration.yml b/src/logging_configuration.yml index 22f5e1f..99d582d 100644 --- a/src/logging_configuration.yml +++ b/src/logging_configuration.yml @@ -1,46 +1,29 @@ version: 1 -disable_existing_loggers: true +disable_existing_loggers: false formatters: simple: "format": "[%(levelname)s]: %(message)s" verbose: "format": "%(asctime)s [%(levelname)-8s] - %(message)s (%(filename)s:%(lineno)d)" detail: - "format": "%(asctime)s [%(levelname)-8s] - %(message)s (%(pathname)s:%(lineno)d)" + "format": "%(asctime)s [%(levelname)-8s] - %(message)s (%(filename)s:%(lineno)d) - %(name)s" handlers: stdout_handler: class: logging.StreamHandler stream: ext://sys.stdout formatter: simple level: INFO - file_handler: - class: logging.FileHandler - formatter: verbose - filename: "log/tool.log" - level: DEBUG - encoding: utf-8 rotating_file_handler: # Problem of RotatingFileHandler: https://python-forum.io/thread-19104.html class: logging.handlers.RotatingFileHandler - formatter: verbose + formatter: detail filename: "log/tool.log" - maxBytes: 50000 # 5KB + maxBytes: 50000 # 50KB backupCount: 3 level: DEBUG encoding: utf-8 - delay: false -loggers: - vnulib_downloader: - level: DEBUG - handlers: - - stdout_handler - - rotating_file_handler - propogate: false - vnulib_downloader_queue_listener: - level: DEBUG - handlers: - - stdout_handler - - file_handler - propogate: false - vnulib_downloader_sub_process: - level: DEBUG - propogate: false +root: + level: DEBUG + propogate: false + handlers: + - stdout_handler + - rotating_file_handler diff --git a/src/modules/clean_img.py b/src/modules/clean_img.py index 25ce6bb..a108457 100644 --- a/src/modules/clean_img.py +++ b/src/modules/clean_img.py @@ -7,7 +7,7 @@ from .link_parse import Link -logger = getLogger("vnulib_downloader") +logger = getLogger(__name__) class CleanIMG: diff --git a/src/modules/config.py b/src/modules/config.py index cee339c..2cc0f10 100644 --- a/src/modules/config.py +++ b/src/modules/config.py @@ -8,7 +8,7 @@ from ..constants import CONFIG_FILE, CONFIG_SAMPLE_FILE -logger = getLogger("vnulib_downloader") +logger = getLogger(__name__) class Config: diff --git a/src/modules/create_pdf.py b/src/modules/create_pdf.py index b1c51fb..426901f 100644 --- a/src/modules/create_pdf.py +++ b/src/modules/create_pdf.py @@ -10,7 +10,7 @@ from logging import Logger import img2pdf from .link_parse import Link -from ..utils import get_queue_logger, logger_listener +from ..utils import get_subprocess_logger, logger_listener class CreatePDF: @@ -49,7 +49,7 @@ def process(directory: str, name: str, queue: Queue) -> None: directory (str): The directory containing the images. name (str): Name of pdf file. """ - logger: Logger = get_queue_logger(queue) + logger: Logger = get_subprocess_logger(queue) pdf_file_name: str = os.path.join(directory, f"{name}.pdf") logger.info('Creating PDF: "%s"', pdf_file_name) files: list[str] = [os.path.join(directory, item) for item in os.listdir(directory)] @@ -109,7 +109,7 @@ def preview_and_page_handler(self, download_directory: str, name: str) -> None: def create_pdf(self) -> None: """Create PDF.""" - listener = Process(target=logger_listener, args=("vnulib_downloader_queue_listener", self.queue)) + listener = Process(target=logger_listener, args=(__name__, self.queue)) listener.start() for link in self.links: match link.original_type: diff --git a/src/modules/download_img.py b/src/modules/download_img.py index f7878d3..442e678 100644 --- a/src/modules/download_img.py +++ b/src/modules/download_img.py @@ -19,7 +19,7 @@ from ..utils import create_directory -logger = getLogger("vnulib_downloader") +logger = getLogger(__name__) def get_error_page_bytes() -> bytes: diff --git a/src/modules/link_parse.py b/src/modules/link_parse.py index d86edb0..d30fc97 100644 --- a/src/modules/link_parse.py +++ b/src/modules/link_parse.py @@ -11,7 +11,7 @@ from ..utils.utils import datetime_name -logger = getLogger("vnulib_downloader") +logger = getLogger(__name__) PATTERN_BOOK = re_compile(r"^https?:\/\/ir\.vnulib\.edu\.vn\/handle\/VNUHCM\/\d+$") diff --git a/src/modules/user_options.py b/src/modules/user_options.py index 9cb511f..30011e4 100644 --- a/src/modules/user_options.py +++ b/src/modules/user_options.py @@ -13,7 +13,7 @@ from .argpase import setup_argparse -logger = getLogger("vnulib_downloader") +logger = getLogger(__name__) @dataclass(slots=True) diff --git a/src/utils/__init__.py b/src/utils/__init__.py index 6f462cf..177a0e4 100644 --- a/src/utils/__init__.py +++ b/src/utils/__init__.py @@ -2,6 +2,6 @@ from __future__ import annotations -from .logger import logger_listener, get_queue_logger +from .logger import ToolLogger, logger_listener, get_subprocess_logger from .prints import print_title from .utils import pause, create_directory, datetime_name, slugify diff --git a/src/utils/logger.py b/src/utils/logger.py index c9fd727..0e5bf59 100644 --- a/src/utils/logger.py +++ b/src/utils/logger.py @@ -2,11 +2,12 @@ from __future__ import annotations -from logging import Logger, getLogger +from logging import Logger, getLogger, DEBUG from logging.config import dictConfig from logging.handlers import QueueHandler from multiprocessing import Queue -from os import makedirs, path +from os import makedirs, path, environ as os_environ +from webdriver_manager.core.logger import set_logger as webdriver_manager_set_logger from yaml import safe_load from src.constants import LOGGING_CONFIG_FILE_PATH, LOGGING_PATH @@ -39,10 +40,22 @@ def read_logging_config(self) -> None: with open(self.config_path, encoding="utf-8") as config_file: dictConfig(safe_load(config_file)) # skipcq: PY-A6006 + @staticmethod + def setup_other_logger() -> None: + """Disable other loggers and change some loggers.""" + getLogger("img2pdf").disabled = True + getLogger("PIL.PngImagePlugin").disabled = True + getLogger("urllib3.connectionpool").disabled = True + getLogger("selenium.webdriver.remote.remote_connection").disabled = True + webdriver_manager_set_logger(getLogger()) + os_environ["WDM_LOG"] = str(DEBUG) + os_environ["WDM_SSL_VERIFY"] = "0" + def setup(self) -> None: """Setup the logger folder and read logging config file.""" self.log_folder() self.read_logging_config() + self.setup_other_logger() def logger_listener(logger_name: str, queue: Queue) -> None: @@ -60,8 +73,8 @@ def logger_listener(logger_name: str, queue: Queue) -> None: logger.handle(record=record) -def get_queue_logger(queue: Queue) -> Logger: - """Get the queue logger which is pre-configured in config file. +def get_subprocess_logger(queue: Queue) -> Logger: + """Get the subprocess logger which is pre-configured in config file. Then add the Queue into the Logger in order to send records to the Queue. Args: @@ -70,6 +83,8 @@ def get_queue_logger(queue: Queue) -> Logger: Returns: Logger: The logger. """ - logger: Logger = getLogger("vnulib_downloader_sub_process") + logger: Logger = getLogger(__name__ + ".subprocess") logger.addHandler(QueueHandler(queue)) + logger.propagate = False + logger.setLevel(DEBUG) return logger diff --git a/src/utils/utils.py b/src/utils/utils.py index 82ca9f2..8d83928 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -11,7 +11,7 @@ from ..constants import USER_INPUT_YES -logger = getLogger("vnulib_downloader") +logger = getLogger(__name__) def pause() -> None: