From 24cc36033feee5a2bce4cc7ec6d97dde21949332 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:17:16 +0800 Subject: [PATCH] feat(db): add support for SQLite WAL mode --- .gitignore | 2 +- app/core/config.py | 2 ++ app/db/__init__.py | 45 ++++++++++++++++++++++++++++++++++++++------- config/app.env | 2 ++ 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 705c952f2..c17d9b7f9 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ app/helper/*.bin app/plugins/** !app/plugins/__init__.py config/cookies/** -config/user.db +config/user.db* config/sites/** config/logs/ config/temp/ diff --git a/app/core/config.py b/app/core/config.py index 8eaee4020..dbdce8add 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -69,6 +69,8 @@ class Config: DB_MAX_OVERFLOW: int = 500 # SQLite 的 busy_timeout 参数，默认为 60 秒 DB_TIMEOUT: int = 60 + # SQLite 是否启用 WAL 模式，默认关闭 + DB_WAL_ENABLE: bool = False # 配置文件目录 CONFIG_DIR: Optional[str] = None # 超级管理员 diff --git a/app/db/__init__.py b/app/db/__init__.py index cf7ab9acb..01bc2d1d2 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -1,22 +1,25 @@ from typing import Any, Generator, List, Optional, Self, Tuple -from sqlalchemy import NullPool, QueuePool, and_, create_engine, inspect +from sqlalchemy import NullPool, QueuePool, and_, create_engine, inspect, text from sqlalchemy.orm import Session, as_declarative, declared_attr, scoped_session, sessionmaker from app.core.config import settings # 根据池类型设置 poolclass 和相关参数 pool_class = NullPool if settings.DB_POOL_TYPE == "NullPool" else QueuePool +connect_args = { + "timeout": settings.DB_TIMEOUT +} +# 启用 WAL 模式时的额外配置 +if settings.DB_WAL_ENABLE: + connect_args["check_same_thread"] = False kwargs = { "url": f"sqlite:///{settings.CONFIG_PATH}/user.db", "pool_pre_ping": settings.DB_POOL_PRE_PING, "echo": settings.DB_ECHO, "poolclass": pool_class, "pool_recycle": settings.DB_POOL_RECYCLE, - "connect_args": { - # "check_same_thread": False, - "timeout": settings.DB_TIMEOUT - } + "connect_args": connect_args } # 当使用 QueuePool 时，添加 QueuePool 特有的参数 if pool_class == QueuePool: @@ -27,6 +30,11 @@ }) # 创建数据库引擎 Engine = create_engine(**kwargs) +# 根据配置设置日志模式 +journal_mode = "WAL" if settings.DB_WAL_ENABLE else "DELETE" +with Engine.connect() as connection: + current_mode = connection.execute(text(f"PRAGMA journal_mode={journal_mode};")).scalar() + print(f"Database journal mode set to: {current_mode}") # 会话工厂 SessionFactory = sessionmaker(bind=Engine) @@ -49,11 +57,34 @@ def get_db() -> Generator: db.close() +def perform_checkpoint(mode: str = "PASSIVE"): + """ + 执行 SQLite 的 checkpoint 操作，将 WAL 文件内容写回主数据库 + :param mode: checkpoint 模式，可选值包括 "PASSIVE"、"FULL"、"RESTART"、"TRUNCATE" + 默认为 "PASSIVE"，即不锁定 WAL 文件的轻量级同步 + """ + if not settings.DB_WAL_ENABLE: + return + valid_modes = {"PASSIVE", "FULL", "RESTART", "TRUNCATE"} + if mode.upper() not in valid_modes: + raise ValueError(f"Invalid checkpoint mode '{mode}'. Must be one of {valid_modes}") + try: + # 使用指定的 checkpoint 模式，确保 WAL 文件数据被正确写回主数据库 + with Engine.connect() as conn: + conn.execute(text(f"PRAGMA wal_checkpoint({mode.upper()});")) + except Exception as e: + print(f"Error during WAL checkpoint: {e}") + + def close_database(): """ - 关闭所有数据库连接 + 关闭所有数据库连接并清理资源 """ - Engine.dispose() + try: + # 释放连接池，SQLite 会自动清空 WAL 文件，这里不单独再调用 checkpoint + Engine.dispose() + except Exception as e: + print(f"Error while disposing database connections: {e}") def get_args_db(args: tuple, kwargs: dict) -> Optional[Session]: diff --git a/config/app.env b/config/app.env index 78ba50bf1..4786acdeb 100644 --- a/config/app.env +++ b/config/app.env @@ -15,6 +15,8 @@ DB_POOL_SIZE=100 DB_MAX_OVERFLOW=500 # SQLite 的 busy_timeout 参数，可适当增加如180以减少锁定错误 DB_TIMEOUT=60 +# SQLite 是否启用 WAL 模式，启用可提升读写并发性能，但可能在异常情况下增加数据丢失的风险 +DB_WAL_ENABLE=false # 【*】超级管理员，设置后一但重启将固化到数据库中，修改将无效（初始化超级管理员密码仅会生成一次，请在日志中查看并自行登录系统修改） SUPERUSER=admin # 辅助认证，允许通过外部服务进行认证、单点登录以及自动创建用户