diff --git a/main.py b/main.py index 6a4fe77..e68897e 100644 --- a/main.py +++ b/main.py @@ -2,9 +2,15 @@ from __future__ import annotations +from src.utils.logger import ToolLogger + +# Have to read the logger config file before all. +ToolLogger().setup() + + from pprint import pformat from multiprocessing import freeze_support -from logging import Logger +from logging import getLogger from src import ( Action, Browser, @@ -17,7 +23,6 @@ PrintIntro, UserOptions, create_directory, - ToolLogger, pause, print_title, ) @@ -26,7 +31,7 @@ def main() -> None: """Main function to run VNULIB Downloader.""" - logger: Logger = ToolLogger().get_logger("vnulib_downloader") + logger = getLogger("vnulib_downloader") PrintIntro() diff --git a/src/__init__.py b/src/__init__.py index e21592c..2200e4b 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -13,4 +13,4 @@ UserOptions, setup_argparse, ) -from .utils import create_directory, ToolLogger, pause, print_title +from .utils import create_directory, pause, print_title diff --git a/src/bot/action.py b/src/bot/action.py index 8bd4ceb..f6ca91e 100644 --- a/src/bot/action.py +++ b/src/bot/action.py @@ -5,15 +5,16 @@ from __future__ import annotations from urllib.parse import parse_qs, urlparse +from logging import getLogger from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement from .utils import wait_element_visible from ..modules import Link, LinkFile -from ..utils import ToolLogger, datetime_name, slugify +from ..utils import datetime_name, slugify -logger = ToolLogger().get_logger("vnulib_downloader") +logger = getLogger("vnulib_downloader") class Action: diff --git a/src/bot/browser.py b/src/bot/browser.py index d636e88..2e4d3da 100644 --- a/src/bot/browser.py +++ b/src/bot/browser.py @@ -5,16 +5,16 @@ import logging import os import urllib3 +from logging import getLogger from selenium import webdriver from selenium.webdriver.chrome.service import Service from selenium.webdriver.chrome.webdriver import WebDriver from webdriver_manager.chrome import ChromeDriverManager from webdriver_manager.core.logger import set_logger from src.constants import BROWSER_ARGUMENTS -from ..utils import ToolLogger -logger = ToolLogger().get_logger("vnulib_downloader") +logger = getLogger("vnulib_downloader") set_logger(logger) os.environ["WDM_LOG"] = str(logging.DEBUG) diff --git a/src/bot/login.py b/src/bot/login.py index 1072c18..4b16719 100644 --- a/src/bot/login.py +++ b/src/bot/login.py @@ -2,15 +2,15 @@ from __future__ import annotations +from logging import getLogger from selenium.webdriver.chrome.webdriver import WebDriver from selenium.webdriver.common.by import By from selenium.webdriver.remote.webelement import WebElement from .utils import wait_element_clickable from ..constants import LOGIN_URL -from ..utils import ToolLogger -logger = ToolLogger().get_logger("vnulib_downloader") +logger = getLogger("vnulib_downloader") class Login: diff --git a/src/constants.py b/src/constants.py index ae4f677..06dc09b 100644 --- a/src/constants.py +++ b/src/constants.py @@ -10,7 +10,7 @@ BUNDLE_DIR = "" -VERSION: str = "1.12" +VERSION: str = "1.13.1-beta" AUTHORS: str = "KevinNitroG & NTGNguyen" BANNER_FILE: str = f"{BUNDLE_DIR}assets/utils/ascii_banner.txt" with open(BANNER_FILE, encoding="utf-8") as banner_content: diff --git a/src/logging_configuration.yml b/src/logging_configuration.yml index f235c53..9447cab 100644 --- a/src/logging_configuration.yml +++ b/src/logging_configuration.yml @@ -17,7 +17,7 @@ handlers: class: logging.handlers.RotatingFileHandler formatter: verbose filename: "log/tool.log" - maxBytes: 5000 # 20000 # 2KB + maxBytes: 10000 # 20000 # 2KB backupCount: 3 level: DEBUG encoding: utf-8 diff --git a/src/modules/clean_img.py b/src/modules/clean_img.py index e3571bd..25ce6bb 100644 --- a/src/modules/clean_img.py +++ b/src/modules/clean_img.py @@ -3,11 +3,11 @@ from __future__ import annotations import os +from logging import getLogger from .link_parse import Link -from ..utils import ToolLogger -logger = ToolLogger().get_logger("vnulib_downloader") +logger = getLogger("vnulib_downloader") class CleanIMG: diff --git a/src/modules/config.py b/src/modules/config.py index fb6ca76..cee339c 100644 --- a/src/modules/config.py +++ b/src/modules/config.py @@ -4,11 +4,11 @@ from os import path from shutil import copyfile +from logging import getLogger from ..constants import CONFIG_FILE, CONFIG_SAMPLE_FILE -from ..utils import ToolLogger -logger = ToolLogger().get_logger("vnulib_downloader") +logger = getLogger("vnulib_downloader") class Config: diff --git a/src/modules/create_pdf.py b/src/modules/create_pdf.py index f899243..42a8731 100644 --- a/src/modules/create_pdf.py +++ b/src/modules/create_pdf.py @@ -8,10 +8,9 @@ import os from multiprocessing import Queue, Process from logging import Logger -from typing import Any import img2pdf from .link_parse import Link -from ..utils import QueueHandlerRun +from ..utils import get_queue_logger, logger_listener class CreatePDF: @@ -24,8 +23,7 @@ class CreatePDF: def __init__(self, links: list[Link], download_directory: str) -> None: self.links: list[Link] = links self.download_directory: str = download_directory - self.queue_handler: QueueHandlerRun = QueueHandlerRun("queue_handler") - self.queue: Queue = self.queue_handler.queue + self.queue: Queue = Queue(-1) self.workers: list[Process] = [] @staticmethod @@ -51,8 +49,7 @@ def process(directory: str, name: str, queue: Queue) -> None: directory (str): The directory containing the images. name (str): Name of pdf file. """ - print("Da vao") - logger: Logger = QueueHandlerRun.get_logger(queue) + logger: Logger = get_queue_logger(queue) pdf_file_name: str = os.path.join(directory, f"{name}.pdf") logger.info('Creating PDF: "%s"', pdf_file_name) files: list[str] = [os.path.join(directory, item) for item in os.listdir(directory)] @@ -112,8 +109,8 @@ def preview_and_page_handler(self, download_directory: str, name: str) -> None: def create_pdf(self) -> None: """Create PDF.""" - logger_listener = Process(target=QueueHandlerRun.logger_listener, args=(self.queue,)) - logger_listener.start() + listener = Process(target=logger_listener, args=("vnulib_downloader", self.queue)) + listener.start() for link in self.links: match link.original_type: case "book": @@ -131,4 +128,4 @@ def create_pdf(self) -> None: for worker in self.workers: worker.join() self.queue.put_nowait(None) - logger_listener.join() + listener.join() diff --git a/src/modules/download_img.py b/src/modules/download_img.py index 98b970d..892b500 100644 --- a/src/modules/download_img.py +++ b/src/modules/download_img.py @@ -12,13 +12,14 @@ from requests.sessions import Session from requests import Response import urllib3 +from logging import getLogger from alive_progress import alive_bar from .link_parse import Link, LinkFile from ..constants import ERROR_PAGE_IMAGE_PATH -from ..utils import create_directory, ToolLogger +from ..utils import create_directory -logger = ToolLogger().get_logger("vnulib_downloader") +logger = getLogger("vnulib_downloader") def get_error_page_bytes() -> bytes: diff --git a/src/modules/link_parse.py b/src/modules/link_parse.py index 5df14d7..d86edb0 100644 --- a/src/modules/link_parse.py +++ b/src/modules/link_parse.py @@ -6,12 +6,12 @@ from re import search as re_search from time import sleep from urllib.parse import parse_qs, urlencode, urlparse, urlunparse +from logging import getLogger from .user_options import Link, LinkFile from ..utils.utils import datetime_name -from ..utils import ToolLogger -logger = ToolLogger().get_logger("vnulib_downloader") +logger = getLogger("vnulib_downloader") PATTERN_BOOK = re_compile(r"^https?:\/\/ir\.vnulib\.edu\.vn\/handle\/VNUHCM\/\d+$") diff --git a/src/modules/user_options.py b/src/modules/user_options.py index 28517d5..9cb511f 100644 --- a/src/modules/user_options.py +++ b/src/modules/user_options.py @@ -7,13 +7,13 @@ from argparse import Namespace from dataclasses import dataclass from pprint import pformat +from logging import getLogger from yaml import safe_load from ..constants import CONFIG_FILE, USER_INPUT_NO, USER_INPUT_YES -from ..utils import ToolLogger from .argpase import setup_argparse -logger = ToolLogger().get_logger("vnulib_downloader") +logger = getLogger("vnulib_downloader") @dataclass(slots=True) diff --git a/src/utils/__init__.py b/src/utils/__init__.py index a9f2e60..6f462cf 100644 --- a/src/utils/__init__.py +++ b/src/utils/__init__.py @@ -2,7 +2,6 @@ from __future__ import annotations -from .logger import ToolLogger -from .queue_logger import QueueHandlerRun +from .logger import logger_listener, get_queue_logger from .prints import print_title from .utils import pause, create_directory, datetime_name, slugify diff --git a/src/utils/logger.py b/src/utils/logger.py index 10501d9..8d24870 100644 --- a/src/utils/logger.py +++ b/src/utils/logger.py @@ -2,8 +2,10 @@ from __future__ import annotations -from logging import Logger, getLogger +from logging import Logger, getLogger, DEBUG from logging.config import dictConfig +from logging.handlers import QueueHandler +from multiprocessing import Queue from os import makedirs, path from yaml import safe_load from src.constants import LOGGING_CONFIG_FILE_PATH, LOGGING_PATH @@ -37,7 +39,37 @@ def read_logging_config(self) -> None: with open(self.config_path, encoding="utf-8") as config_file: dictConfig(safe_load(config_file)) # skipcq: PY-A6006 - def get_logger(self, logger_name: str) -> Logger: + def setup(self) -> None: self.log_folder() self.read_logging_config() - return getLogger(logger_name) + + +def logger_listener(logger_name: str, queue: Queue) -> None: + """Logger listener. Suitable for multiprocessing logger. + + Args: + logger_name (str): The name of logger to get. + queue (Queue): The queue to take record to handle. + """ + logger: Logger = getLogger(logger_name) + while True: + record = queue.get() + if record is None: + break + logger.handle(record=record) + + +def get_queue_logger(queue: Queue) -> Logger: + """Get the queue logger with standard setup. + + Args: + queue (Queue): The queue to add record to. + + Returns: + Logger: The logger. + """ + logger: Logger = getLogger("queue_logger") + logger.propagate = False + logger.addHandler(QueueHandler(queue)) + logger.setLevel(DEBUG) + return logger diff --git a/src/utils/queue_logger.py b/src/utils/queue_logger.py deleted file mode 100644 index 6f78193..0000000 --- a/src/utils/queue_logger.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - -from multiprocessing import Queue -from logging import Logger, Handler, getLogger, getHandlerByName, DEBUG -from logging.handlers import QueueHandler -from .logger import ToolLogger - - -class QueueHandlerRun: - """Start the Queue Handler thread. - Use it by context manager. - """ - - def __init__(self, handler_name: str) -> None: - """Initialise. - - Args: - handler_name (str): Name of the handler to get, not logger :)). - """ - self.handler_name: str = handler_name - ToolLogger().read_logging_config() - self._queue_handler: Handler | None = getHandlerByName(handler_name) - self.queue = Queue(-1) - - @staticmethod - def logger_listener(queue: Queue): - logger: Logger = ToolLogger().get_logger("vnulib_downloader") - while True: - record = queue.get() - if record is None: - break - logger.handle(record=record) - - @staticmethod - def get_logger(queue: Queue) -> Logger: - queue_handler = QueueHandler(queue) - logger: Logger = getLogger("queue_logger") - logger.propagate = False - logger.addHandler(queue_handler) - logger.setLevel(DEBUG) - return logger diff --git a/src/utils/utils.py b/src/utils/utils.py index 6f562cd..82ca9f2 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -7,11 +7,11 @@ from datetime import datetime from os import makedirs, path from shutil import rmtree +from logging import getLogger from ..constants import USER_INPUT_YES -from ..utils import ToolLogger -logger = ToolLogger().get_logger("vnulib_downloader") +logger = getLogger("vnulib_downloader") def pause() -> None: