Merge pull request #176 from YaoZeyuan/yaozeyuan/dev/etf-save-world-v3

Yaozeyuan/dev/etf save world v3

changelog.md

@@ -1,3 +1,8 @@
+# 3.5.3
+
+1.  bug fix
+-   修复生成的pdf中, 文章内容错位/丢失问题
+
 # 3.5.2
 
 1.  bug fix

client/src/pages/customer_task/utils.ts

@@ -137,7 +137,7 @@ export function startBackupTask() {
 }
 
 export async function asyncCheckNeedUpdate() {
-  let checkUpgradeUri = 'https://cdn.jsdelivr.net/gh/YaoZeyuan/stablog@master/upgrade_config/version.json';
+  let checkUpgradeUri = 'https://gitee.com/yaozeyuan/stablog/raw/master/upgrade_config/version.json';
   let remoteVersionConfig = await http
     .asyncGet(checkUpgradeUri, {
       params: {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "stablog",
-  "version": "3.5.2",
+  "version": "3.5.3",
   "description": "稳部落. 专业备份微blog, 老铁, 稳!",
   "repository": {
     "type": "git",
@@ -70,13 +70,14 @@
     "@adonisjs/ace": "^5.0.8",
     "archiver": "^3.0.0",
     "axios": "^1.7.7",
+    "better-sqlite3": "^11.5.0",
     "cross-env": "^5.2.0",
     "dayjs": "^1.11.13",
     "decode-html": "^2.0.0",
     "dotenv": "^4.0.0",
     "fflate": "^0.8",
     "fs-extra": "^10.0.0",
-    "image-size": "^0.8.3",
+    "image-size": "^1.1.1",
     "json5": "^2.1.0",
     "jspdf": "^2.3.0",
     "knex": "^2.2.0",
@@ -89,8 +90,7 @@
     "request": "^2.88.2",
     "request-promise": "^4.2.6",
     "sharp": "^0.33.5",
-    "shelljs": "^0.8.5",
-    "better-sqlite3": "^11.5.0"
+    "shelljs": "^0.8.5"
   },
   "devDependencies": {
     "@babel/cli": "^7.18.6",

src/command/debug.ts

@@ -8,8 +8,6 @@ import path from 'path'
 // import { BrowserWindow } from 'electron'
 // import CommonUtil from '~/src/library/util/common'
 // import { TypeTransConfigPackageList } from './generate/trans_config'
-import imageSize from 'image-size'
-import * as mozjpeg from "mozjpeg-js"
 import sharp from "sharp"
 
 // const outputUri = path.resolve('F:/www/share/github/stablog/缓存文件/pdf_debug_1.pdf')

src/command/generate/base.ts

@@ -109,11 +109,16 @@ class GenerateBase extends Base {
   processContent(content: string) {
     let that = this
     // 删除noscript标签内的元素
-    function removeNoScript(rawHtml: string) {
+    function removeUselessEle(rawHtml: string) {
+      // 规范br标签
       rawHtml = _.replace(rawHtml, /<\/br>/g, '')
       rawHtml = _.replace(rawHtml, /<br>/g, '<br/>')
+      // 修复跳转链接
       rawHtml = _.replace(rawHtml, /href="\/\/link.zhihu.com'/g, 'href="https://link.zhihu.com') // 修复跳转链接
+      // 移除noscript标签
       rawHtml = _.replace(rawHtml, /\<noscript\>.*?\<\/noscript\>/g, '')
+      // 移除script标签
+      rawHtml = _.replace(rawHtml, /\<script .*?\<\/script\>/g, '')
       return rawHtml
     }
 
@@ -176,7 +181,7 @@ class GenerateBase extends Base {
       let processedHtml = strMergeList.join('')
       return processedHtml
     }
-    content = removeNoScript(content)
+    content = removeUselessEle(content)
     let tinyContentList = content.split(`<div data-key='single-page'`).map(value => {
       return replaceImgSrc(value)
     })

src/command/generate/customer.ts

@@ -26,9 +26,9 @@ import { BrowserWindow } from 'electron'
 import CommonUtil from '~/src/library/util/common'
 
 /**
- * 单张页面渲染时间不能超过60秒
+ * 单张页面渲染时间不能超过120秒(降低屏幕高度后, 渲染时间增加, 所以对应的最大等待时间也要增加)
  */
-const Const_Render_Html_Timeout_Second = 60
+const Const_Render_Html_Timeout_Second = 120
 /**
  * 宽为760px的图片, 在电脑端打开正常, 但是pdf中会被拉伸到全屏大小, 成为原先的200%, 导致模糊.
  * 为了保证pdf中图片清晰, 因此需要在截图时, 主动x2. 代价是pdf文件更大, 但可接受
@@ -37,7 +37,11 @@ const Pixel_Zoom_Rate = 2
 /**
  * 渲染webview最大高度(经实验, 当Electron窗口高度超过16380时, 会直接黑屏卡死, 所以需要专门限制下)
  */
-const Const_Max_Webview_Render_Height_Px = 5000
+const Const_Max_Webview_Render_Height_Px = 2000 * Pixel_Zoom_Rate
+/**
+ * webview中, js滚动返回和实际完成滚动时间不一致, 因此需要额外休眠等待. 等待时间过短会截取到错误图片
+ */
+const Const_Webview_Js_Scroll_Sleep_Second = 0.5
 /**
  * 单卷中最多只能有5000条微博
  */
@@ -638,14 +642,15 @@ class GenerateCustomer extends Base {
 
           while (remainHeight >= Const_Max_Webview_Render_Height_Px) {
             let imgIndex = imgContentList.length;
-            let currentOffsetHeight = Const_Max_Webview_Render_Height_Px * imgIndex
+            // 在页面内滚动时, 需要将实际像素重新转为逻辑像素
+            let currentOffsetHeight = Const_Max_Webview_Render_Height_Px / Pixel_Zoom_Rate * imgIndex
             // 先移动到offset高度
             let command = `document.children[0].children[1].scrollTop = ${currentOffsetHeight}`
             await webview.executeJavaScript(command);
 
             // 然后对界面截屏
             // js指令执行后, 滚动到指定位置还需要时间, 所以截屏前需要sleep一下
-            await CommonUtil.asyncSleep(1000 * 0.2)
+            await CommonUtil.asyncSleep(1000 * Const_Webview_Js_Scroll_Sleep_Second)
             let nativeImg = await webview.capturePage();
             let content = await nativeImg.toJPEG(100)
 
@@ -673,15 +678,16 @@ class GenerateCustomer extends Base {
             // 首先调整页面高度
             await subWindow.setContentSize(Const_Default_Webview_Width, remainHeight);
             // 然后走流程, 捕捉界面
-            let currentOffsetHeight = Const_Max_Webview_Render_Height_Px * imgContentList.length
+            // 在页面内滚动时, 需要将实际像素重新转为逻辑像素
+            let currentOffsetHeight = Const_Max_Webview_Render_Height_Px / Pixel_Zoom_Rate * imgContentList.length
             let imgIndex = imgContentList.length;
 
             // 先移动到offset高度
             let command = `document.children[0].children[1].scrollTop = ${currentOffsetHeight}`
             await webview.executeJavaScript(command);
             // 然后对界面截屏
             // js指令执行后, 滚动到指定位置还需要时间, 所以截屏前需要sleep一下
-            await CommonUtil.asyncSleep(1000 * 0.2)
+            await CommonUtil.asyncSleep(1000 * Const_Webview_Js_Scroll_Sleep_Second)
             let nativeImg = await webview.capturePage();
 
             let content = await nativeImg.toJPEG(100)

src/config/common.ts

@@ -5,6 +5,6 @@ let version = parseFloat(packageJson.version)
 class Common {
   static readonly version = version
   // 感谢jsdeliver, 免费cdn
-  static readonly checkUpgradeUri = 'https://cdn.jsdelivr.net/gh/YaoZeyuan/stablog@master/upgrade_config/version.json'
+  static readonly checkUpgradeUri = 'https://gitee.com/yaozeyuan/stablog/raw/master/upgrade_config/version.json'
 }
 export default Common

src/index.ts

@@ -125,13 +125,13 @@ function createWindow() {
 
   async function debugCaputure() {
     let targetSource =
-      'file:///D:/win_www/stablog/%E7%A8%B3%E9%83%A8%E8%90%BD%E8%BE%93%E5%87%BA%E7%9A%84%E7%94%B5%E5%AD%90%E4%B9%A6/%E4%BA%8C%E7%BA%A7%E5%B8%82%E5%9C%BA%E6%8D%A1%E8%BE%A3%E9%B8%A1%E5%86%A0%E5%86%9B-%E5%BE%AE%E5%8D%9A%E6%95%B4%E7%90%86-(2020-11-06~2023-01-19)/html_to_pdf/2023-01-13%2013%EF%BC%9A54%EF%BC%9A49_4857447900517349.html'
+      'file:///C:/Users/yao/AppData/Local/Programs/stablog/resources/app/%E7%A8%B3%E9%83%A8%E8%90%BD%E8%BE%93%E5%87%BA%E7%9A%84%E7%94%B5%E5%AD%90%E4%B9%A6/ETF%E6%8B%AF%E6%95%91%E4%B8%96%E7%95%8C-%E5%BE%AE%E5%8D%9A%E6%95%B4%E7%90%86-%E7%AC%AC1%EF%BC%8F3%E5%8D%B7-(2015-09-28~2018-11-15)/html_to_pdf/2016-07-01%2011%EF%BC%9A31%EF%BC%9A16_3992391939353203.html'
     let demoUri = path.resolve(__dirname, '../demo.jpg')
 
     // 图片放大系数, 系数越大, pdf越清晰, 文件体积越大
     const Pixel_Zoom_Radio = 2
 
-    const Const_Max_Webview_Render_Height_Px = 5000 * Pixel_Zoom_Radio
+    const Const_Max_Webview_Render_Height_Px = 2000 * Pixel_Zoom_Radio
     const Const_Default_Webview_Width = 760 * Pixel_Zoom_Radio
     const Const_Default_Webview_Height = 10
 
@@ -172,11 +172,14 @@ function createWindow() {
 
       while (remainHeight >= Const_Max_Webview_Render_Height_Px) {
         let imgIndex = imgContentList.length
-        let currentOffsetHeight = Const_Max_Webview_Render_Height_Px * imgIndex
+        // 将实际像素转回逻辑像素
+        let currentOffsetHeight = Const_Max_Webview_Render_Height_Px / Pixel_Zoom_Radio * imgIndex
+
+        Logger.log(`[${remainHeight}]开始执行页面滚动`)
         // 先移动到offset高度
         let command = `document.children[0].children[1].scrollTop = ${currentOffsetHeight}`
         await webview.executeJavaScript(command)
-
+        Logger.log(`[${remainHeight}]页面滚动执行完毕`)
         // 然后对界面截屏
         // js指令执行后, 滚动到指定位置还需要时间, 所以截屏前需要sleep一下
         await CommonUtil.asyncSleep(1000 * 0.5)
@@ -196,7 +199,8 @@ function createWindow() {
         // 首先调整页面高度
         await subWindow.setContentSize(Const_Default_Webview_Width, remainHeight)
         // 然后走流程, 捕捉界面
-        let currentOffsetHeight = Const_Max_Webview_Render_Height_Px * imgContentList.length
+        // 将实际像素转回逻辑像素
+        let currentOffsetHeight = Const_Max_Webview_Render_Height_Px / Pixel_Zoom_Radio * imgContentList.length
         let imgIndex = imgContentList.length
 
         // 先移动到offset高度

upgrade_config/readme.md

@@ -1,10 +1,9 @@
-利用jsdelivr作为静态资源接口
+利用gitee.com作为静态资源接口
 
 接口api地址:
-- https://cdn.jsdelivr.net/gh/YaoZeyuan/stablog@master/upgrade_config/thank_you/list.json
-- https://cdn.jsdelivr.net/gh/YaoZeyuan/stablog@master/upgrade_config/version.json
+- https://gitee.com/yaozeyuan/stablog/raw/master/upgrade_config/version.json
+- https://gitee.com/yaozeyuan/stablog/raw/master/upgrade_config/thank_you/list.json
 
 刷新接口内容:
 
-- https://purge.jsdelivr.net/gh/YaoZeyuan/stablog@master/upgrade_config/thank_you/list.json
-- https://purge.jsdelivr.net/gh/YaoZeyuan/stablog@master/upgrade_config/version.json
+- https://gitee.com/yaozeyuan/stablog  点击强制同步