fix 站点数据刷新

app/chain/site.py

@@ -65,10 +65,10 @@ def refresh_userdata(self, site: CommentedMap = None) -> Optional[SiteUserData]:
         :param site:  站点
         :return: 用户数据
         """
-        userdata = self.run_module("refresh_userdata", site=site)
+        userdata: SiteUserData = self.run_module("refresh_userdata", site=site)
         if userdata:
             self.siteoper.update_userdata(domain=StringUtils.get_url_domain(site.get("domain")),
-                                          payload=userdata)
+                                          payload=userdata.dict())
         return userdata
 
     def refresh_userdatas(self) -> None:

app/db/site_oper.py

@@ -1,9 +1,11 @@
+import json
 from datetime import datetime
 from typing import Tuple, List
 
 from app.db import DbOper
 from app.db.models.site import Site
 from app.db.models.siteuserdata import SiteUserData
+from app.utils.object import ObjectUtils
 
 
 class SiteOper(DbOper):
@@ -120,6 +122,9 @@ def update_userdata(self, domain: str, payload: dict) -> Tuple[bool, str]:
             SiteUserData.update(self._db, payload)
         else:
             # 不存在则插入
+            for key, value in payload.items():
+                if ObjectUtils.is_obj(value):
+                    payload[key] = json.dumps(value)
             SiteUserData(**payload).create(self._db)
         return True, "更新站点用户数据成功"
 

app/modules/indexer/__init__.py

@@ -10,6 +10,7 @@
 from app.helper.sites import SitesHelper
 from app.log import logger
 from app.modules import _ModuleBase
+from app.modules.indexer.parser import SiteParserBase
 from app.modules.indexer.spider import TorrentSpider
 from app.modules.indexer.spider.haidan import HaiDanSpider
 from app.modules.indexer.spider.mtorrent import MTorrentSpider
@@ -30,8 +31,10 @@ class IndexerModule(_ModuleBase):
 
     def init_module(self) -> None:
         # 加载模块
-        self._site_schemas = ModuleHelper.load('app.modules.indexer.parser',
-                                               filter_func=lambda _, obj: hasattr(obj, 'schema'))
+        self._site_schemas = ModuleHelper.load(
+            'app.modules.indexer.parser',
+            filter_func=lambda _, obj: hasattr(obj, 'schema') and getattr(obj, 'schema') is not None)
+        pass
 
     @staticmethod
     def get_name() -> str:
@@ -214,12 +217,12 @@ def refresh_userdata(self, site: CommentedMap) -> Optional[SiteUserData]:
         :return: 用户数据
         """
 
-        def __get_site_obj():
+        def __get_site_obj() -> Optional[SiteParserBase]:
             """
             获取站点解析器
             """
             for site_schema in self._site_schemas:
-                if site_schema.schema == site.get("schema"):
+                if site_schema.schema.value == site.get("schema"):
                     return site_schema(
                         site_name=site.get("name"),
                         url=site.get("url"),
@@ -232,13 +235,14 @@ def __get_site_obj():
 
         site_obj = __get_site_obj()
         if not site_obj:
-            logger.warn(f"站点  {site.get('name')} 未找到站点解析器: {site.get('schema')}")
+            if not site.get("public"):
+                logger.warn(f"站点  {site.get('name')} 未找到站点解析器，schema：{site.get('schema')}")
             return None
 
         # 获取用户数据
-        logger.debug(f"站点 {site.get('name')} 开始以 {site.get('schema')} 模型解析")
+        logger.info(f"站点 {site.get('name')} 开始以 {site.get('schema')} 模型解析数据...")
         site_obj.parse()
-        logger.debug(f"站点 {site.get('name')} 解析完成")
+        logger.debug(f"站点 {site.get('name')} 数据解析完成")
         return SiteUserData(
             domain=StringUtils.get_url_domain(site.get("url")),
             userid=site_obj.userid,
@@ -251,11 +255,11 @@ def __get_site_obj():
             bonus=site_obj.bonus,
             seeding=site_obj.seeding,
             seeding_size=site_obj.seeding_size,
-            seeding_info=site_obj.seeding_info,
+            seeding_info=site_obj.seeding_info or [],
             leeching=site_obj.leeching,
             leeching_size=site_obj.leeching_size,
             message_unread=site_obj.message_unread,
-            message_unread_contents=site_obj.message_unread_contents,
+            message_unread_contents=site_obj.message_unread_contents or [],
             updated_at=datetime.now().strftime('%Y-%m-%d'),
             err_msg=site_obj.err_msg
         )

app/modules/indexer/parser/__init__.py

@@ -36,7 +36,7 @@ class SiteSchema(Enum):
 
 class SiteParserBase(metaclass=ABCMeta):
     # 站点模版
-    schema = SiteSchema.NexusPhp
+    schema = None
     # 请求模式 cookie/apikey
     request_mode = "cookie"
 
@@ -45,7 +45,6 @@ def __init__(self, site_name: str,
                  site_cookie: str,
                  apikey: str,
                  token: str,
-                 index_html: str,
                  session: Session = None,
                  ua: str = None,
                  emulate: bool = False,
@@ -61,12 +60,11 @@ def __init__(self, site_name: str,
         self._site_domain = __split_url.netloc
         self._base_url = f"{__split_url.scheme}://{__split_url.netloc}"
         self._site_cookie = site_cookie
-        self._index_html = index_html
         self._session = session if session else None
         self._ua = ua
         self._emulate = emulate
         self._proxy = proxy
-
+        self._index_html = ""
         # 用户信息
         self.username = None
         self.userid = None
@@ -156,6 +154,8 @@ def parse(self):
         解析站点信息
         :return:
         """
+        # 获取站点首页html
+        self._index_html = self._get_page_content(url=self._site_url)
         # 检查是否已经登录
         if not self._parse_logged_in(self._index_html):
             return
@@ -194,7 +194,6 @@ def parse(self):
             )
         # 解析用户做种信息
         self._parse_seeding_pages()
-        self.seeding_info = json.dumps(self.seeding_info)
 
     def _pase_unread_msgs(self):
         """