From c6abb1f9f1e5b361219caa298e48b7e5f69da381 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Thu, 26 Sep 2024 14:00:10 +0800 Subject: [PATCH] =?UTF-8?q?fix=20=E7=AB=99=E7=82=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=88=B7=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/chain/site.py | 4 ++-- app/db/site_oper.py | 5 +++++ app/modules/indexer/__init__.py | 22 +++++++++++++--------- app/modules/indexer/parser/__init__.py | 9 ++++----- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/app/chain/site.py b/app/chain/site.py index a9b38254a..f4be1f129 100644 --- a/app/chain/site.py +++ b/app/chain/site.py @@ -65,10 +65,10 @@ def refresh_userdata(self, site: CommentedMap = None) -> Optional[SiteUserData]: :param site: 站点 :return: 用户数据 """ - userdata = self.run_module("refresh_userdata", site=site) + userdata: SiteUserData = self.run_module("refresh_userdata", site=site) if userdata: self.siteoper.update_userdata(domain=StringUtils.get_url_domain(site.get("domain")), - payload=userdata) + payload=userdata.dict()) return userdata def refresh_userdatas(self) -> None: diff --git a/app/db/site_oper.py b/app/db/site_oper.py index 9120cba4e..beb1984fb 100644 --- a/app/db/site_oper.py +++ b/app/db/site_oper.py @@ -1,9 +1,11 @@ +import json from datetime import datetime from typing import Tuple, List from app.db import DbOper from app.db.models.site import Site from app.db.models.siteuserdata import SiteUserData +from app.utils.object import ObjectUtils class SiteOper(DbOper): @@ -120,6 +122,9 @@ def update_userdata(self, domain: str, payload: dict) -> Tuple[bool, str]: SiteUserData.update(self._db, payload) else: # 不存在则插入 + for key, value in payload.items(): + if ObjectUtils.is_obj(value): + payload[key] = json.dumps(value) SiteUserData(**payload).create(self._db) return True, "更新站点用户数据成功" diff --git a/app/modules/indexer/__init__.py b/app/modules/indexer/__init__.py index e4ccef76d..47c693a18 100644 --- a/app/modules/indexer/__init__.py +++ b/app/modules/indexer/__init__.py @@ -10,6 +10,7 @@ from app.helper.sites import SitesHelper from app.log import logger from app.modules import _ModuleBase +from app.modules.indexer.parser import SiteParserBase from app.modules.indexer.spider import TorrentSpider from app.modules.indexer.spider.haidan import HaiDanSpider from app.modules.indexer.spider.mtorrent import MTorrentSpider @@ -30,8 +31,10 @@ class IndexerModule(_ModuleBase): def init_module(self) -> None: # 加载模块 - self._site_schemas = ModuleHelper.load('app.modules.indexer.parser', - filter_func=lambda _, obj: hasattr(obj, 'schema')) + self._site_schemas = ModuleHelper.load( + 'app.modules.indexer.parser', + filter_func=lambda _, obj: hasattr(obj, 'schema') and getattr(obj, 'schema') is not None) + pass @staticmethod def get_name() -> str: @@ -214,12 +217,12 @@ def refresh_userdata(self, site: CommentedMap) -> Optional[SiteUserData]: :return: 用户数据 """ - def __get_site_obj(): + def __get_site_obj() -> Optional[SiteParserBase]: """ 获取站点解析器 """ for site_schema in self._site_schemas: - if site_schema.schema == site.get("schema"): + if site_schema.schema.value == site.get("schema"): return site_schema( site_name=site.get("name"), url=site.get("url"), @@ -232,13 +235,14 @@ def __get_site_obj(): site_obj = __get_site_obj() if not site_obj: - logger.warn(f"站点 {site.get('name')} 未找到站点解析器: {site.get('schema')}") + if not site.get("public"): + logger.warn(f"站点 {site.get('name')} 未找到站点解析器，schema：{site.get('schema')}") return None # 获取用户数据 - logger.debug(f"站点 {site.get('name')} 开始以 {site.get('schema')} 模型解析") + logger.info(f"站点 {site.get('name')} 开始以 {site.get('schema')} 模型解析数据...") site_obj.parse() - logger.debug(f"站点 {site.get('name')} 解析完成") + logger.debug(f"站点 {site.get('name')} 数据解析完成") return SiteUserData( domain=StringUtils.get_url_domain(site.get("url")), userid=site_obj.userid, @@ -251,11 +255,11 @@ def __get_site_obj(): bonus=site_obj.bonus, seeding=site_obj.seeding, seeding_size=site_obj.seeding_size, - seeding_info=site_obj.seeding_info, + seeding_info=site_obj.seeding_info or [], leeching=site_obj.leeching, leeching_size=site_obj.leeching_size, message_unread=site_obj.message_unread, - message_unread_contents=site_obj.message_unread_contents, + message_unread_contents=site_obj.message_unread_contents or [], updated_at=datetime.now().strftime('%Y-%m-%d'), err_msg=site_obj.err_msg ) diff --git a/app/modules/indexer/parser/__init__.py b/app/modules/indexer/parser/__init__.py index 97ae48d18..e1e82d6fe 100644 --- a/app/modules/indexer/parser/__init__.py +++ b/app/modules/indexer/parser/__init__.py @@ -36,7 +36,7 @@ class SiteSchema(Enum): class SiteParserBase(metaclass=ABCMeta): # 站点模版 - schema = SiteSchema.NexusPhp + schema = None # 请求模式 cookie/apikey request_mode = "cookie" @@ -45,7 +45,6 @@ def __init__(self, site_name: str, site_cookie: str, apikey: str, token: str, - index_html: str, session: Session = None, ua: str = None, emulate: bool = False, @@ -61,12 +60,11 @@ def __init__(self, site_name: str, self._site_domain = __split_url.netloc self._base_url = f"{__split_url.scheme}://{__split_url.netloc}" self._site_cookie = site_cookie - self._index_html = index_html self._session = session if session else None self._ua = ua self._emulate = emulate self._proxy = proxy - + self._index_html = "" # 用户信息 self.username = None self.userid = None @@ -156,6 +154,8 @@ def parse(self): 解析站点信息 :return: """ + # 获取站点首页html + self._index_html = self._get_page_content(url=self._site_url) # 检查是否已经登录 if not self._parse_logged_in(self._index_html): return @@ -194,7 +194,6 @@ def parse(self): ) # 解析用户做种信息 self._parse_seeding_pages() - self.seeding_info = json.dumps(self.seeding_info) def _pase_unread_msgs(self): """