From f61233af82c55bb351d18bd145752896535b6773 Mon Sep 17 00:00:00 2001 From: doufen-org Date: Fri, 23 Aug 2019 17:20:35 +0800 Subject: [PATCH] release v0.5.0 --- extension/backup.html | 10 +- extension/explorer.js | 2 +- extension/manifest.json | 4 +- extension/options.html | 15 ++- extension/options.js | 37 ++++--- extension/service.js | 24 +++-- extension/settings.js | 12 ++- extension/storage.js | 3 + extension/tasks/doumail.js | 2 +- extension/tasks/files.js | 190 ++++++++++++++++++++++++++++++++++++ extension/tasks/interest.js | 1 - extension/tasks/photo.js | 32 +++++- 12 files changed, 299 insertions(+), 33 deletions(-) create mode 100644 extension/tasks/files.js diff --git a/extension/backup.html b/extension/backup.html index 0fc1bf2..d3ea258 100644 --- a/extension/backup.html +++ b/extension/backup.html @@ -75,9 +75,6 @@ -
-

注意：仅备份当前登录用户的数据。如需备份其他账号数据，请先退出当前账号，再登录需要备份的账号。

-
@@ -202,6 +199,13 @@

+
+
+ +
+

diff --git a/extension/explorer.js b/extension/explorer.js index 5acb86e..c37c0c0 100644 --- a/extension/explorer.js +++ b/extension/explorer.js @@ -795,7 +795,7 @@ class Photo extends Panel { for (let { photo, version } of collection) { let $photo = $(TEMPLATE_PHOTO); $photo.find('.image img').attr('src', photo.cover).click(() => { - PictureModal.show(photo.cover.replace('/m/','/l/')); + PictureModal.show(photo.raw); }); $photo.find('.description').text(photo.description); version < currentVersion && $photo.addClass('is-obsolete'); diff --git a/extension/manifest.json b/extension/manifest.json index c54ea55..decc722 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -2,8 +2,8 @@ "name": "豆伴：豆瓣账号备份工具", "short_name": "豆伴", "author": "豆坟", - "version": "0.4.2", - "version_name": "0.4.2 beta", + "version": "0.5.0", + "version_name": "0.5.0 beta", "description": "能将指定的豆瓣用户数据备份到本地，并支持导出 Excel 文档。", "manifest_version": 2, "icons": { diff --git a/extension/options.html b/extension/options.html index 8054566..ad0a4e7 100644 --- a/extension/options.html +++ b/extension/options.html @@ -98,7 +98,7 @@

- +

请求间隔过短可能导致被豆瓣服务器屏蔽。

@@ -112,6 +112,19 @@
+
+
+ +

允许将广播、日记、评论以及相册中的图片上传到 Cloudinary 云存储中。

+
+
+
+

+ +

+
+
+

保存 diff --git a/extension/options.js b/extension/options.js index d550839..d3a5170 100644 --- a/extension/options.js +++ b/extension/options.js @@ -1,5 +1,6 @@ import Settings from './settings.js'; -import {SERVICE_SETTINGS, Task} from './service.js'; +import {SERVICE_SETTINGS} from './service.js'; +import {TASK_FILES_SETTINGS} from './tasks/files.js'; import Notification from './ui/notification.js'; import TabPanel from './ui/tab.js'; @@ -123,10 +124,21 @@ class GeneralPanel { } }; + let TextInput = class extends Control { + set value(value) { + value && (this.element.value = value); + } + + get value() { + return this.element.value || ''; + } + }; + const CONTROL_METAS = [ //{name: 'assistant.enable', type: BoolSwitch}, {name: 'service.debug', type: BoolSwitch}, {name: 'service.requestInterval', type: TimeInput}, + {name: '同步图片.cloudName', type: TextInput}, ]; this.controls = new Object(); @@ -138,34 +150,37 @@ class GeneralPanel { this.saveButton = document.querySelector('.button[name="save"]'); this.resetButton = document.querySelector('.button[name="reset"]'); - this.saveButton.addEventListener('click', event => { + this.saveButton.addEventListener('click', async () => { for (let name in this.controls) { this.settings[name] = this.controls[name].value; } - this.save(this.settings); + await this.save(this.settings); }); - this.resetButton.addEventListener('click', event => { + this.resetButton.addEventListener('click', async () => { for (let name in this.controls) { this.controls[name].value = this.defaults[name]; } - this.save(this.defaults); + await this.save(this.defaults); }); } - save(settings) { + async save(settings) { try { - chrome.storage.sync.set(settings, result => { - Notification.show('保存成功'); - }); + await Settings.save(settings); + let service = (await new Promise(resolve => { + chrome.runtime.getBackgroundPage(resolve); + })).service; + service.loadSettings(); + Notification.show('保存成功'); } catch (e) { Notification.show('保存失败', {type: 'danger'}); } } static async render() { - let settings = await Settings.load(SERVICE_SETTINGS); - let defaults = SERVICE_SETTINGS; + let settings = await Settings.load(SERVICE_SETTINGS, TASK_FILES_SETTINGS); + let defaults = Object.assign({}, SERVICE_SETTINGS, TASK_FILES_SETTINGS); return new GeneralPanel('.page-tab-content[name="general"]', settings, defaults); } } diff --git a/extension/service.js b/extension/service.js index 15b5782..395ad30 100644 --- a/extension/service.js +++ b/extension/service.js @@ -9,6 +9,7 @@ import Storage from './storage.js'; export const SERVICE_SETTINGS = { 'service.debug': false, 'service.requestInterval': 1000, + 'service.cloudinary': '', }; @@ -488,6 +489,9 @@ class StateChangeEvent extends Event { * Class Service */ export default class Service extends EventTarget { + /** + * Constructor + */ constructor() { super(); Object.assign(this, { @@ -504,6 +508,16 @@ export default class Service extends EventTarget { chrome.runtime.onConnect.addListener(port => this.onConnect(port)); } + /** + * Load settings + */ + async loadSettings() { + let settings = await Settings.load(SERVICE_SETTINGS); + Settings.apply(this, settings); + this.logger.debug('Service settings loaded.'); + return this; + } + /** * Get name */ @@ -775,13 +789,9 @@ export default class Service extends EventTarget { static async startup() { const RUN_FOREVER = true; - let service = Service.instance; + let service = await Service.instance.loadSettings(); let logger = service.logger; - let settings = await Settings.load(SERVICE_SETTINGS); - Settings.apply(service, settings); - logger.debug('Service settings loaded.'); - let browserMainVersion = (/Chrome\/([0-9]+)/.exec(navigator.userAgent)||[,0])[1]; let extraOptions = (browserMainVersion >= 72) ? ['blocking', 'requestHeaders', 'extraHeaders'] : ['blocking', 'requestHeaders']; @@ -795,10 +805,10 @@ export default class Service extends EventTarget { return {requestHeaders: details.requestHeaders}; }, {urls: ['http://*.douban.com/*', 'https://*.douban.com/*']}, extraOptions); let lastRequest = 0; - let fetchURL = (resource, init) => { + let fetchURL = (resource, init, continuous = false) => { let promise = service.continue(); let requestInterval = lastRequest + service.requestInterval - Date.now(); - if (requestInterval > 0) { + if (!continuous && requestInterval > 0) { promise = promise.then(() => { return new Promise(resolve => { setTimeout(resolve, requestInterval); diff --git a/extension/settings.js b/extension/settings.js index 1e212a0..6b2270e 100644 --- a/extension/settings.js +++ b/extension/settings.js @@ -2,10 +2,6 @@ * Class Settings */ export default class Settings { - constructor (settings, defaults) { - return Object.assign(defaults, settings); - } - static apply(target, settings) { for (let key in settings) { try { @@ -28,6 +24,12 @@ export default class Settings { let settings = await new Promise(resolve => { chrome.storage.sync.get(Object.keys(defaults), resolve); }); - return new Settings(settings, defaults); + return Object.assign({}, defaults, settings); + } + + static async save(settings) { + return await new Promise(resolve => { + chrome.storage.sync.set(settings, resolve); + }); } } diff --git a/extension/storage.js b/extension/storage.js index 401885b..3f8464f 100644 --- a/extension/storage.js +++ b/extension/storage.js @@ -30,6 +30,9 @@ const SCHEMA_LOCAL = [ doumailContact: 'id, rank', version: 'table, version', }, + { + files: '++id, &url', + }, ]; diff --git a/extension/tasks/doumail.js b/extension/tasks/doumail.js index a8cb237..513613a 100644 --- a/extension/tasks/doumail.js +++ b/extension/tasks/doumail.js @@ -49,7 +49,7 @@ export default class Photo extends Task { }; let readMore = true; for (let start = 0; readMore; start += PAGE_SIZE) { - let postData = new FormData(); + let postData = new URLSearchParams(); postData.append('start', start); postData.append('target_id', userId); postData.append('ck', this.session.cookies.ck); diff --git a/extension/tasks/files.js b/extension/tasks/files.js new file mode 100644 index 0000000..70915a3 --- /dev/null +++ b/extension/tasks/files.js @@ -0,0 +1,190 @@ +'use strict'; +import {TaskError, Task} from '../service.js'; +import Settings from '../settings.js'; + + +export const TASK_FILES_SETTINGS = { + '同步图片.cloudName': '', +}; + +const UPLOAD_URL = 'https://api.cloudinary.com/v1_1/{cloud}/image/upload'; +const PAGE_SIZE = 100; + + +function encodeContext(context) { + let contextArray = []; + for (let key in context) { + let value = context[key]; + key = key.replace('|', '\|').replace('=', '\='); + value = value.replace('|', '\|').replace('=', '\='); + contextArray.push(`${key}=${value}`); + } + return contextArray.join('|'); +} + + +export default class Files extends Task { + async addFile(url, tags, meta, path) { + try { + await this.storage.files.add({ + url: url, + tags: tags, + meta: meta, + path: path, + }); + } catch (e) { + if (e.name != 'ConstraintError') { + this.logger.warning(e.message); + } + } + } + + async extractImages() { + await this.storage.transaction('rw', this.storage.album, this.storage.files, async () => { + await this.storage.album.each(async item => { + await this.addFile( + item.album.cover_url, + ['相册'], + { + caption: item.album.title, + alt: item.album.description, + from: item.album.url, + }, + 'thumbnail' + ); + }); + }); + await this.storage.transaction('rw', this.storage.album, this.storage.photo, this.storage.files, async () => { + await this.storage.photo.each(async item => { + let {album} = await this.storage.album.get(item.album); + let meta = { + caption: album.title, + alt: item.photo.description, + from: item.photo.url, + }; + await this.addFile( + item.photo.cover, + ['照片'], + meta, + 'thumbnail' + ); + await this.addFile( + item.photo.raw, + ['照片'], + meta, + '相册/' + album.title + ); + }); + }); + await this.storage.transaction('rw', this.storage.note, this.storage.files, async () => { + await this.storage.note.each(async item => { + let images = this.parseHTML(item.note.fulltext).querySelectorAll('img'); + for (let image of images) { + await this.addFile( + image.src, + ['日记'], + { + caption: item.note.title, + alt: item.note.abstract, + from: item.note.url, + }, + '日记/' + item.note.title + ); + } + }); + }); + await this.storage.transaction('rw', this.storage.review, this.storage.files, async () => { + await this.storage.review.each(async item => { + let images = this.parseHTML(item.review.fulltext).querySelectorAll('img'); + for (let image of images) { + await this.addFile( + image.src, + ['评论'], + { + caption: item.review.title, + alt: item.review.abstract, + from: item.review.url, + }, + '评论/' + item.review.title + ); + } + }); + }); + await this.storage.transaction('rw', this.storage.status, this.storage.files, async () => { + await this.storage.status.each(async item => { + if (item.status.images) { + let statusUrl = item.status.sharing_url; + for (let image of item.status.images) { + await this.addFile(image.large.url, ['广播'], { from: statusUrl }, '广播'); + await this.addFile(image.normal.url, ['广播'], { from: statusUrl }, 'thumbnail'); + } + } + if (item.status.reshared_status && item.status.reshared_status.images) { + let statusUrl = item.status.reshared_status.sharing_url; + for (let image of item.status.reshared_status.images) { + await this.addFile(image.large.url, ['广播'], { from: statusUrl }, '广播'); + await this.addFile(image.normal.url, ['广播'], { from: statusUrl }, 'thumbnail'); + } + } + }); + }); + } + + async run() { + let settings = await Settings.load(TASK_FILES_SETTINGS); + Settings.apply(this, settings); + if (!this.cloudName) { + this.logger.warning('Missing setting of cloudinary cloud name.'); + return; + } + + await this.extractImages(); + + this.total = await this.storage.files.filter(row => { + return !(row.save); + }).count(); + if (this.total == 0) { + return; + } + + let uploadURL = UPLOAD_URL.replace('{cloud}', this.cloudName); + + let pageCount = Math.ceil(this.total / PAGE_SIZE); + for (let i = 0; i < pageCount; i ++) { + let rows = await this.storage.files.filter(row => { + return !(row.save); + }).limit(PAGE_SIZE).toArray(); + + for (let row of rows) { + let postData = new URLSearchParams(); + postData.append('file', row.url); + postData.append('upload_preset', 'douban'); + postData.append('tags', row.tags); + postData.append('context', encodeContext(row.meta)); + postData.append('folder', `${this.targetUser.uid}/${row.path}`); + + let response = await this.fetch(uploadURL, { + method: 'POST', + body: postData, + }, true); + if (response.status >= 500) { + throw new TaskError('Cloudinary 接口异常'); + } + let savedData = await response.json(); + if (response.status == 400 && !savedData['error']['message'].startsWith('Error in loading http')) { + throw new TaskError('Cloudinary 接口返回错误'); + } + await this.storage.files.update(row.id, { + save: savedData, + }) + this.step(); + } + } + + this.complete(); + } + + get name() { + return '同步图片'; + } +} diff --git a/extension/tasks/interest.js b/extension/tasks/interest.js index 07df6fe..1db2a0f 100644 --- a/extension/tasks/interest.js +++ b/extension/tasks/interest.js @@ -98,7 +98,6 @@ export default class Interest extends Task { } } } - delete this._name; this.complete(); } diff --git a/extension/tasks/photo.js b/extension/tasks/photo.js index 5ed9e02..c3069f8 100644 --- a/extension/tasks/photo.js +++ b/extension/tasks/photo.js @@ -12,6 +12,19 @@ export default class Photo extends Task { return true; } + async fetchPhotoDetail(url) { + let response = await this.fetch(url); + if (response.status != 200) { + if (response.status < 500) { + return false; + } + throw new TaskError('豆瓣服务器返回错误'); + } + let html = this.parseHTML(await response.text()); + let photo = html.querySelector('.mainphoto>img'); + return photo.src; + } + async run() { let version = this.jobId; this.total = this.targetUser.photo_albums_count; @@ -34,6 +47,7 @@ export default class Photo extends Task { pageCount = Math.ceil(json.total / PAGE_SIZE); for (let album of json.photo_albums) { let albumId = parseInt(album.id); + let albumPrivacy = album.privacy; if (isNaN(albumId)) continue; let row = await this.storage.album.get(albumId); if (row) { @@ -73,10 +87,20 @@ export default class Photo extends Task { if (row) { let lastVersion = row.version; row.version = version; - if (row.photo.description != photoDescription) { + if (row.photo.description != photoDescription || + row.photo.cover != photoImg.src) { !row.history && (row.history = {}); row.history[lastVersion] = row.photo; row.photo.description = photoDescription; + if (row.photo.cover != photoImg.src) { + if (albumPrivacy != 'public') { + let rawUrl = await this.fetchPhotoDetail(photoAnchor.href); + row.photo.raw = rawUrl || photoImg.src; + } else { + row.photo.raw = photoImg.src.replace('/m/', '/l/'); + } + row.photo.cover = photoImg.src; + } } } else { row = { @@ -89,6 +113,12 @@ export default class Photo extends Task { description: photoDescription, } } + if (albumPrivacy != 'public') { + let rawUrl = await this.fetchPhotoDetail(photoAnchor.href); + row.photo.raw = rawUrl || photoImg.src; + } else { + row.photo.raw = photoImg.src.replace('/m/', '/l/'); + } } await this.storage.photo.put(row); }