Skip to content

Commit

Permalink
Fix linting
Browse files Browse the repository at this point in the history
  • Loading branch information
@dipu-bd
dipu-bd authored Dec 13, 2023
1 parent bbd07dc commit 97b9a2b
Showing 1 changed file with 26 additions and 21 deletions.
47 changes: 26 additions & 21 deletions sources/en/w/wuxiacom.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from lncrawl.models import Chapter, Volume
from lncrawl.templates.browser.basic import BasicBrowserTemplate
from lncrawl.webdriver.elements import By
import selenium

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -55,7 +54,7 @@ def login(self, email: str, password: str) -> None:
if storage.has(self.localstorageuser):
self.bearer_token = '{token_type} {access_token}'.format(**json.loads(storage[self.localstorageuser]))
except Exception as e:
logger.debug("login Email: Failed")
logger.debug("login Email: Failed", e)

def read_novel_info_in_scraper(self) -> None:
slug = re.findall(r"/novel/([^/]+)", self.novel_url)[0]
Expand Down Expand Up @@ -207,7 +206,7 @@ def read_novel_info_in_browser(self) -> None:
self.novel_author = author_tag.text.strip()

# Open chapters menu (note: the order of tabs in novel info change whether if you are logged in or not)
if len(self.browser.find_all('//*[starts-with(@id, "full-width-tab-")]',By.XPATH)) == 3:
if len(self.browser.find_all('//*[starts-with(@id, "full-width-tab-")]', By.XPATH)) == 3:
self.browser.click("#novel-tabs #full-width-tab-0")
self.browser.wait("#full-width-tabpanel-0 .MuiAccordion-root")
else:
Expand Down Expand Up @@ -263,14 +262,18 @@ def download_chapter_body_in_browser(self, chapter: Chapter) -> str:
storage = LocalStorage(self.browser._driver)
logger.debug("LocalStorage: %s", storage)
if not storage.has(self.localstorageuser):
storage[self.localstorageuser] = '{"access_token":"%s","token_type":"%s"}' % tuple(self.bearer_token.split(" ",1)[::-1])
token_type, token = self.bearer_token.split(" ", 1)
storage[self.localstorageuser] = json.dumps({
"access_token": token,
"token_type": token_type,
})
logger.debug("LocalStorage: %s", storage)
self.visit('https://www.wuxiaworld.com/manage/profile/')
try:
self.browser.wait("//h2[normalize-space()='Your Profile']", By.XPATH, 10)
self.browser.find("//h2[normalize-space()='Your Profile']", By.XPATH)
except Exception as e:
logger.debug("login Email: Failed")
logger.debug("login Email: Failed", e)
self.start_download_chapter_body_in_browser = True
self.visit(chapter.url)
try:
Expand All @@ -280,33 +283,35 @@ def download_chapter_body_in_browser(self, chapter: Chapter) -> str:
self.browser.wait("//button[normalize-space()='Favorite']", By.XPATH, 10)
self.browser.find("//button[normalize-space()='Favorite']", By.XPATH)
except Exception as e:
logger.debug("error loading chapter (%s) or chapter is locked", str(chapter.url))
logger.debug("error loading (%s)", str(chapter.url), e)
content = self.browser.find("chapter-content", By.CLASS_NAME).as_tag()
self.cleaner.clean_contents(content)
return content.decode_contents()

# this class to read browser LocalStorage
class LocalStorage:

# Class for reading localStorage from Browser
class LocalStorage:
def __init__(self, driver) :
self.driver = driver

def __len__(self):
return self.driver.execute_script("return window.localStorage.length;")

def items(self) :
return self.driver.execute_script( \
"var ls = window.localStorage, items = {}; " \
"for (var i = 0, k; i < ls.length; ++i) " \
" items[k = ls.key(i)] = ls.getItem(k); " \
"return items; ")
return self.driver.execute_script(
"var ls = window.localStorage, items = {}; "
"for (var i = 0, k; i < ls.length; ++i) "
" items[k = ls.key(i)] = ls.getItem(k); "
"return items; "
)

def keys(self) :
return self.driver.execute_script( \
"var ls = window.localStorage, keys = []; " \
"for (var i = 0; i < ls.length; ++i) " \
" keys[i] = ls.key(i); " \
"return keys; ")
return self.driver.execute_script(
"var ls = window.localStorage, keys = []; "
"for (var i = 0; i < ls.length; ++i) "
" keys[i] = ls.key(i); "
"return keys; "
)

def get(self, key):
return self.driver.execute_script("return window.localStorage.getItem(arguments[0]);", key)
Expand All @@ -325,8 +330,8 @@ def clear(self):

def __getitem__(self, key) :
value = self.get(key)
if value is None :
raise KeyError(key)
if value is None:
raise KeyError(key)
return value

def __setitem__(self, key, value):
Expand All @@ -342,7 +347,7 @@ def __repr__(self):
return self.items().__str__()

WUXIWORLD_PROTO = json.loads(
"""
"""
{"file": [{"name": "google/protobuf/wrappers.proto", "package": "google.protobuf", "messageType": [{"name": "DoubleValue", "field": [{"name": "value", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_DOUBLE", "jsonName": "value"}]}, {"name": "FloatValue", "field": [{"name": "value", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_FLOAT", "jsonName": "value"}]}, {"name": "Int64Value", "field": [{"name": "value", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT64", "jsonName": "value"}]}, {"name": "UInt64Value", "field": [{"name": "value", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_UINT64", "jsonName": "value"}]}, {"name": "Int32Value", "field": [{"name": "value", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "value"}]}, {"name": "UInt32Value", "field": [{"name": "value", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_UINT32", "jsonName": "value"}]}, {"name": "BoolValue", "field": [{"name": "value", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "value"}]}, {"name": "StringValue", "field": [{"name": "value", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "jsonName": "value"}]}, {"name": "BytesValue", "field": [{"name": "value", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_BYTES", "jsonName": "value"}]}], "options": {"javaPackage": "com.google.protobuf", "javaOuterClassname": "WrappersProto", "javaMultipleFiles": true, "goPackage": "google.golang.org/protobuf/types/known/wrapperspb", "ccEnableArenas": true, "objcClassPrefix": "GPB", "csharpNamespace": "Google.Protobuf.WellKnownTypes"}, "syntax": "proto3"}, {"name": "google/protobuf/timestamp.proto", "package": "google.protobuf", "messageType": [{"name": "Timestamp", "field": [{"name": "seconds", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT64", "jsonName": "seconds"}, {"name": "nanos", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "nanos"}]}], "options": {"javaPackage": "com.google.protobuf", "javaOuterClassname": "TimestampProto", "javaMultipleFiles": true, "goPackage": "google.golang.org/protobuf/types/known/timestamppb", "ccEnableArenas": true, "objcClassPrefix": "GPB", "csharpNamespace": "Google.Protobuf.WellKnownTypes"}, "syntax": "proto3"}, {"name": "wuxia.proto", "package": "wuxiaworld.api.v2", "dependency": ["google/protobuf/wrappers.proto", "google/protobuf/timestamp.proto"], "messageType": [{"name": "RelatedChapterUserInfo", "field": [{"name": "isChapterUnlocked", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.BoolValue", "jsonName": "isChapterUnlocked"}, {"name": "isNovelUnlocked", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.BoolValue", "jsonName": "isNovelUnlocked"}, {"name": "isChapterFavorite", "number": 3, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.BoolValue", "jsonName": "isChapterFavorite"}, {"name": "isNovelOwned", "number": 4, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.BoolValue", "jsonName": "isNovelOwned"}, {"name": "isChapterOwned", "number": 5, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.BoolValue", "jsonName": "isChapterOwned"}]}, {"name": "ChapterSponsor", "field": [{"name": "advanceChapter", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "advanceChapter"}, {"name": "advanceChapterNumber", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.Int32Value", "jsonName": "advanceChapterNumber"}, {"name": "plans", "number": 3, "label": "LABEL_REPEATED", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.ChapterSponsor.AdvanceChapterPlan", "jsonName": "plans"}], "nestedType": [{"name": "AdvanceChapterPlan", "field": [{"name": "name", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "jsonName": "name"}, {"name": "advanceChapterCount", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "advanceChapterCount"}]}]}, {"name": "ChapterPricing", "field": [{"name": "isFree", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "isFree"}, {"name": "isLastHoldback", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "isLastHoldback"}]}, {"name": "ChapterItem", "field": [{"name": "entityId", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "entityId"}, {"name": "name", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "jsonName": "name"}, {"name": "slug", "number": 3, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "jsonName": "slug"}, {"name": "content", "number": 5, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.StringValue", "jsonName": "content"}, {"name": "novelId", "number": 6, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "novelId"}, {"name": "visible", "number": 7, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "visible"}, {"name": "isTeaser", "number": 8, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "isTeaser"}, {"name": "spoilerTitle", "number": 10, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "spoilerTitle"}, {"name": "sponsorInfo", "number": 15, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.ChapterSponsor", "jsonName": "sponsorInfo"}, {"name": "relatedUserInfo", "number": 16, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.RelatedChapterUserInfo", "jsonName": "relatedUserInfo"}, {"name": "publishedAt", "number": 18, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.Timestamp", "jsonName": "publishedAt"}, {"name": "translatorThoughts", "number": 19, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.StringValue", "jsonName": "translatorThoughts"}, {"name": "pricingInfo", "number": 20, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.ChapterPricing", "jsonName": "pricingInfo"}]}, {"name": "ChapterGroupItem", "field": [{"name": "id", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "id"}, {"name": "title", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "jsonName": "title"}, {"name": "order", "number": 3, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "order"}, {"name": "chapterList", "number": 6, "label": "LABEL_REPEATED", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.ChapterItem", "jsonName": "chapterList"}]}, {"name": "SponsorPlanItem", "field": [{"name": "id", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "id"}, {"name": "name", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "jsonName": "name"}, {"name": "enabled", "number": 4, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "enabled"}, {"name": "visible", "number": 5, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "visible"}, {"name": "advanceChapterCount", "number": 6, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "advanceChapterCount"}, {"name": "paused", "number": 10, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "paused"}]}, {"name": "NovelItem", "field": [{"name": "id", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "id"}, {"name": "name", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "jsonName": "name"}, {"name": "coverUrl", "number": 10, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.StringValue", "jsonName": "coverUrl"}, {"name": "translatorName", "number": 11, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.StringValue", "jsonName": "translatorName"}, {"name": "authorName", "number": 13, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.StringValue", "jsonName": "authorName"}, {"name": "isSneakPeek", "number": 18, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "isSneakPeek"}]}, {"name": "UnlockedItem", "field": [{"name": "novelId", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "oneofIndex": 0, "jsonName": "novelId"}, {"name": "chapterId", "number": 3, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "oneofIndex": 0, "jsonName": "chapterId"}], "oneofDecl": [{"name": "id"}]}, {"name": "VipItem", "field": [{"name": "id", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "id"}, {"name": "name", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "jsonName": "name"}, {"name": "enabled", "number": 7, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "enabled"}, {"name": "visible", "number": 8, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "visible"}]}, {"name": "SubscriptionItem", "field": [{"name": "id", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "id"}, {"name": "active", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_BOOL", "jsonName": "active"}, {"name": "plan", "number": 3, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.SubscriptionItem.Plan", "jsonName": "plan"}], "nestedType": [{"name": "Plan", "field": [{"name": "vip", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.VipItem", "oneofIndex": 0, "jsonName": "vip"}, {"name": "sponsor", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.SponsorPlanItem", "oneofIndex": 0, "jsonName": "sponsor"}], "oneofDecl": [{"name": "plan"}]}]}, {"name": "GetChapterByProperty", "field": [{"name": "chapterId", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "oneofIndex": 0, "jsonName": "chapterId"}, {"name": "slugs", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.GetChapterByProperty.ByNovelAndChapterSlug", "oneofIndex": 0, "jsonName": "slugs"}], "nestedType": [{"name": "ByNovelAndChapterSlug", "field": [{"name": "novelSlug", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "jsonName": "novelSlug"}, {"name": "chapterSlug", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "jsonName": "chapterSlug"}]}], "oneofDecl": [{"name": "byProperty"}]}, {"name": "GetNovelRequest", "field": [{"name": "id", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "oneofIndex": 0, "jsonName": "id"}, {"name": "slug", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_STRING", "oneofIndex": 0, "jsonName": "slug"}], "oneofDecl": [{"name": "selector"}]}, {"name": "GetNovelResponse", "field": [{"name": "item", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.NovelItem", "jsonName": "item"}]}, {"name": "GetChapterListRequest", "field": [{"name": "novelId", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "novelId"}, {"name": "filter", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.GetChapterListRequest.FilterChapters", "jsonName": "filter"}], "nestedType": [{"name": "FilterChapters", "field": [{"name": "chapterGroupId", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.Int32Value", "jsonName": "chapterGroupId"}, {"name": "isAdvanceChapter", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".google.protobuf.BoolValue", "jsonName": "isAdvanceChapter"}]}]}, {"name": "GetChapterListResponse", "field": [{"name": "items", "number": 1, "label": "LABEL_REPEATED", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.ChapterGroupItem", "jsonName": "items"}, {"name": "novelInfo", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.NovelItem", "jsonName": "novelInfo"}]}, {"name": "GetChapterRequest", "field": [{"name": "chapterProperty", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.GetChapterByProperty", "jsonName": "chapterProperty"}]}, {"name": "GetChapterResponse", "field": [{"name": "item", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.ChapterItem", "jsonName": "item"}]}, {"name": "UnlockItemRequest", "field": [{"name": "unlockMethod", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_ENUM", "typeName": ".wuxiaworld.api.v2.UnlockItemMethod", "jsonName": "unlockMethod"}, {"name": "item", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.UnlockedItem", "jsonName": "item"}]}, {"name": "UnlockItemResponse", "field": [{"name": "unlockedItem", "number": 1, "label": "LABEL_OPTIONAL", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.UnlockedItem", "jsonName": "unlockedItem"}]}, {"name": "GetSubscriptionsRequest", "field": [{"name": "novelId", "number": 2, "label": "LABEL_OPTIONAL", "type": "TYPE_INT32", "jsonName": "novelId"}]}, {"name": "GetSubscriptionsResponse", "field": [{"name": "items", "number": 1, "label": "LABEL_REPEATED", "type": "TYPE_MESSAGE", "typeName": ".wuxiaworld.api.v2.SubscriptionItem", "jsonName": "items"}]}], "enumType": [{"name": "UnlockItemMethod", "value": [{"name": "UnlockMethodNone", "number": 0}, {"name": "UnlockMethodKarma", "number": 1}, {"name": "UnlockMethodVip", "number": 2}, {"name": "UnlockMethodSponsor", "number": 3}]}], "service": [{"name": "Novels", "method": [{"name": "GetNovel", "inputType": ".wuxiaworld.api.v2.GetNovelRequest", "outputType": ".wuxiaworld.api.v2.GetNovelResponse"}]}, {"name": "Chapters", "method": [{"name": "GetChapterList", "inputType": ".wuxiaworld.api.v2.GetChapterListRequest", "outputType": ".wuxiaworld.api.v2.GetChapterListResponse"}, {"name": "GetChapter", "inputType": ".wuxiaworld.api.v2.GetChapterRequest", "outputType": ".wuxiaworld.api.v2.GetChapterResponse"}]}, {"name": "Unlocks", "method": [{"name": "UnlockItem", "inputType": ".wuxiaworld.api.v2.UnlockItemRequest", "outputType": ".wuxiaworld.api.v2.UnlockItemResponse"}]}, {"name": "Subscriptions", "method": [{"name": "GetSubscriptions", "inputType": ".wuxiaworld.api.v2.GetSubscriptionsRequest", "outputType": ".wuxiaworld.api.v2.GetSubscriptionsResponse"}]}], "publicDependency": [0, 1], "syntax": "proto3"}]}
""" # noqa: E501
)

0 comments on commit 97b9a2b

Please sign in to comment.