From 29ee5eca3a0113047df631b269796adc9323e7b0 Mon Sep 17 00:00:00 2001 From: Daniel Yrovas Date: Sun, 13 Aug 2023 12:09:35 +1000 Subject: [PATCH] keep rss & web content | rebase --- internal/api/entry.go | 8 +- internal/database/migrations.go | 7 + internal/locale/translations/de_DE.json | 2 + internal/locale/translations/el_EL.json | 2 + internal/locale/translations/en_US.json | 2 + internal/locale/translations/es_ES.json | 2 + internal/locale/translations/fi_FI.json | 2 + internal/locale/translations/fr_FR.json | 2 + internal/locale/translations/hi_IN.json | 2 + internal/locale/translations/id_ID.json | 2 + internal/locale/translations/it_IT.json | 2 + internal/locale/translations/ja_JP.json | 2 + internal/locale/translations/nl_NL.json | 2 + internal/locale/translations/pl_PL.json | 2 + internal/locale/translations/pt_BR.json | 2 + internal/locale/translations/ru_RU.json | 2 + internal/locale/translations/tr_TR.json | 2 + internal/locale/translations/uk_UA.json | 2 + internal/locale/translations/zh_CN.json | 2 + internal/locale/translations/zh_TW.json | 2 + internal/model/entry.go | 1 + internal/reader/processor/processor.go | 10 +- internal/reader/rewrite/rewriter.go | 42 +-- internal/reader/rewrite/rewriter_test.go | 288 +++++++++---------- internal/storage/entry.go | 28 +- internal/storage/entry_query_builder.go | 3 +- internal/template/templates/views/entry.html | 22 +- internal/ui/entry_scraper.go | 44 +++ internal/ui/static/js/app.js | 28 +- internal/ui/ui.go | 1 + 30 files changed, 325 insertions(+), 193 deletions(-) diff --git a/internal/api/entry.go b/internal/api/entry.go index 6a299a01e58..d2f9c7426df 100644 --- a/internal/api/entry.go +++ b/internal/api/entry.go @@ -335,7 +335,13 @@ func (h *handler) fetchContent(w http.ResponseWriter, r *http.Request) { return } - json.OK(w, r, map[string]string{"content": entry.Content}) + if request.QueryBoolParam(r, "save", false) { + if err := h.store.UpdateEntryTitleAndContent(entry); err != nil { + json.ServerError(w, r, err) + } + } + + json.OK(w, r, map[string]string{"content": entry.Content, "web_content": entry.WebContent}) } func (h *handler) flushHistory(w http.ResponseWriter, r *http.Request) { diff --git a/internal/database/migrations.go b/internal/database/migrations.go index d40e5d2f813..b8ae43a8b12 100644 --- a/internal/database/migrations.go +++ b/internal/database/migrations.go @@ -882,4 +882,11 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, + func(tx *sql.Tx) (err error) { + sql := ` + ALTER TABLE entries ADD COLUMN web_content text default ''; + ` + _, err = tx.Exec(sql) + return err + }, } diff --git a/internal/locale/translations/de_DE.json b/internal/locale/translations/de_DE.json index cc60035a0eb..e49d7506a47 100644 --- a/internal/locale/translations/de_DE.json +++ b/internal/locale/translations/de_DE.json @@ -72,6 +72,8 @@ "entry.save.title": "Diesen Artikel speichern", "entry.save.completed": "Erledigt!", "entry.save.toast.completed": "Artikel gespeichert", + "entry.scraper.label.rss": "RSS-Inhalte anzeigen", + "entry.scraper.title.rss": "RSS-Inhalte abrufen", "entry.scraper.label": "Herunterladen", "entry.scraper.title": "Inhalt herunterladen", "entry.scraper.completed": "Erledigt!", diff --git a/internal/locale/translations/el_EL.json b/internal/locale/translations/el_EL.json index 822821633db..5ff20e22667 100644 --- a/internal/locale/translations/el_EL.json +++ b/internal/locale/translations/el_EL.json @@ -72,6 +72,8 @@ "entry.save.title": "Αποθηκεύστε αυτό το άρθρο", "entry.save.completed": "Έγινε!", "entry.save.toast.completed": "Το άρθρο αποθηκεύτηκε", + "entry.scraper.label.rss": "Εμφάνιση περιεχομένου RSS", + "entry.scraper.title.rss": "Λήψη περιεχομένου RSS", "entry.scraper.label": "Λήψη", "entry.scraper.title": "Λήψη αρχικού περιεχομένου", "entry.scraper.completed": "Έγινε!", diff --git a/internal/locale/translations/en_US.json b/internal/locale/translations/en_US.json index 97e58fe5a2b..b463d43388b 100644 --- a/internal/locale/translations/en_US.json +++ b/internal/locale/translations/en_US.json @@ -72,6 +72,8 @@ "entry.save.title": "Save this entry", "entry.save.completed": "Done!", "entry.save.toast.completed": "Entry saved", + "entry.scraper.label.rss": "Show RSS Content", + "entry.scraper.title.rss": "Fetch RSS content", "entry.scraper.label": "Download", "entry.scraper.title": "Fetch original content", "entry.scraper.completed": "Done!", diff --git a/internal/locale/translations/es_ES.json b/internal/locale/translations/es_ES.json index 914d8994220..6e6c003d112 100644 --- a/internal/locale/translations/es_ES.json +++ b/internal/locale/translations/es_ES.json @@ -72,6 +72,8 @@ "entry.save.title": "Guardar este artículo", "entry.save.completed": "¡Hecho!", "entry.save.toast.completed": "Artículos guardados", + "entry.scraper.label.rss": "Mostrar contenido RSS", + "entry.scraper.title.rss": "Obtener contenido RSS", "entry.scraper.label": "Descargar", "entry.scraper.title": "Obtener contenido original", "entry.scraper.completed": "¡Hecho!", diff --git a/internal/locale/translations/fi_FI.json b/internal/locale/translations/fi_FI.json index 41bd2e50d04..95337d8e674 100644 --- a/internal/locale/translations/fi_FI.json +++ b/internal/locale/translations/fi_FI.json @@ -72,6 +72,8 @@ "entry.save.title": "Tallenna tämä artikkeli", "entry.save.completed": "Valmis!", "entry.save.toast.completed": "Artikkeli tallennettu", + "entry.scraper.label.rss": "Näytä RSS-sisältö", + "entry.scraper.title.rss": "Hae RSS-sisältöä", "entry.scraper.label": "Lataa", "entry.scraper.title": "Nouda alkuperäinen sisältö", "entry.scraper.completed": "Valmis!", diff --git a/internal/locale/translations/fr_FR.json b/internal/locale/translations/fr_FR.json index d616eb3eb3a..a8d7ceccda2 100644 --- a/internal/locale/translations/fr_FR.json +++ b/internal/locale/translations/fr_FR.json @@ -72,6 +72,8 @@ "entry.save.title": "Sauvegarder cet article", "entry.save.completed": "Terminé !", "entry.save.toast.completed": "Article sauvegardé", + "entry.scraper.label.rss": "Afficher le contenu RSS", + "entry.scraper.title.rss": "Obtenir le contenu RSS", "entry.scraper.label": "Télécharger", "entry.scraper.title": "Récupérer le contenu original", "entry.scraper.completed": "Terminé !", diff --git a/internal/locale/translations/hi_IN.json b/internal/locale/translations/hi_IN.json index 199c66af1a6..de076e1c0ab 100644 --- a/internal/locale/translations/hi_IN.json +++ b/internal/locale/translations/hi_IN.json @@ -72,6 +72,8 @@ "entry.save.title": "एस लेख को सहेजे", "entry.save.completed": "कार्य समाप्त हुआ!", "entry.save.toast.completed": "लेख को सहेज लिया", + "entry.scraper.label.rss": "RSS सामग्री दिखाएँ", + "entry.scraper.title.rss": "RSS सामग्री प्राप्त करें", "entry.scraper.label": "डाउनलोड", "entry.scraper.title": "मूल विषयवस्तु लाए", "entry.scraper.completed": "कार्य समाप्त हुआ!", diff --git a/internal/locale/translations/id_ID.json b/internal/locale/translations/id_ID.json index ee06ac61f34..b7cb54b31f1 100644 --- a/internal/locale/translations/id_ID.json +++ b/internal/locale/translations/id_ID.json @@ -72,6 +72,8 @@ "entry.save.title": "Simpan artikel ini", "entry.save.completed": "Selesai!", "entry.save.toast.completed": "Artikel tersimpan", + "entry.scraper.label.rss": "Tampilkan Konten RSS", + "entry.scraper.title.rss": "Dapatkan Konten RSS", "entry.scraper.label": "Unduh", "entry.scraper.title": "Ambil konten asli", "entry.scraper.completed": "Selesai!", diff --git a/internal/locale/translations/it_IT.json b/internal/locale/translations/it_IT.json index 6e808a02053..4f0eee49b3c 100644 --- a/internal/locale/translations/it_IT.json +++ b/internal/locale/translations/it_IT.json @@ -72,6 +72,8 @@ "entry.save.title": "Salva questo articolo", "entry.save.completed": "Fatto!", "entry.save.toast.completed": "Articolo salvato", + "entry.scraper.label.rss": "Mostra contenuto RSS", + "entry.scraper.title.rss": "Ottieni contenuto RSS", "entry.scraper.label": "Scarica", "entry.scraper.title": "Scarica il contenuto integrale", "entry.scraper.completed": "Fatto!", diff --git a/internal/locale/translations/ja_JP.json b/internal/locale/translations/ja_JP.json index 8c767b55162..bd6697101bf 100644 --- a/internal/locale/translations/ja_JP.json +++ b/internal/locale/translations/ja_JP.json @@ -72,6 +72,8 @@ "entry.save.title": "この記事を保存", "entry.save.completed": "完了!", "entry.save.toast.completed": "記事は保存されました", + "entry.scraper.label.rss": "RSS コンテンツを表示", + "entry.scraper.title.rss": "RSS コンテンツを取得", "entry.scraper.label": "ダウンロード", "entry.scraper.title": "オリジナルの内容を取得", "entry.scraper.completed": "完了!", diff --git a/internal/locale/translations/nl_NL.json b/internal/locale/translations/nl_NL.json index d47510e9518..3a1e5fd61f2 100644 --- a/internal/locale/translations/nl_NL.json +++ b/internal/locale/translations/nl_NL.json @@ -72,6 +72,8 @@ "entry.save.title": "Artikel opslaan", "entry.save.completed": "Done!", "entry.save.toast.completed": "Artikel opgeslagen", + "entry.scraper.label.rss": "Toon RSS-content", + "entry.scraper.title.rss": "Haal RSS-content op", "entry.scraper.label": "Downloaden", "entry.scraper.title": "Fetch original content", "entry.scraper.completed": "Klaar!", diff --git a/internal/locale/translations/pl_PL.json b/internal/locale/translations/pl_PL.json index 32f3e4f8ceb..de9c267b2fe 100644 --- a/internal/locale/translations/pl_PL.json +++ b/internal/locale/translations/pl_PL.json @@ -72,6 +72,8 @@ "entry.save.title": "Zapisz ten artykuł", "entry.save.completed": "Gotowe!", "entry.save.toast.completed": "Artykuł zapisany", + "entry.scraper.label.rss": "Pokaż treść RSS", + "entry.scraper.title.rss": "Pobierz treść RSS", "entry.scraper.label": "Ściągnij", "entry.scraper.title": "Pobierz oryginalną treść", "entry.scraper.completed": "Gotowe!", diff --git a/internal/locale/translations/pt_BR.json b/internal/locale/translations/pt_BR.json index 56861a3a080..5c213d21146 100644 --- a/internal/locale/translations/pt_BR.json +++ b/internal/locale/translations/pt_BR.json @@ -72,6 +72,8 @@ "entry.save.title": "Salvar esse item", "entry.save.completed": "Feito!", "entry.save.toast.completed": "Item guardado", + "entry.scraper.label.rss": "Mostrar conteúdo RSS", + "entry.scraper.title.rss": "Obter conteúdo RSS", "entry.scraper.label": "Baixar", "entry.scraper.title": "Obter conteúdo completo", "entry.scraper.completed": "Feito!", diff --git a/internal/locale/translations/ru_RU.json b/internal/locale/translations/ru_RU.json index 69c139f0183..a0d51f7a83a 100644 --- a/internal/locale/translations/ru_RU.json +++ b/internal/locale/translations/ru_RU.json @@ -72,6 +72,8 @@ "entry.save.title": "Сохранить эту статью", "entry.save.completed": "Готово!", "entry.save.toast.completed": "Статья сохранена", + "entry.scraper.label.rss": "Показать содержимое RSS", + "entry.scraper.title.rss": "Получить содержимое RSS", "entry.scraper.label": "Скачать", "entry.scraper.title": "Извлечь оригинальное содержимое", "entry.scraper.completed": "Готово!", diff --git a/internal/locale/translations/tr_TR.json b/internal/locale/translations/tr_TR.json index 85238400541..502ecc6c292 100644 --- a/internal/locale/translations/tr_TR.json +++ b/internal/locale/translations/tr_TR.json @@ -72,6 +72,8 @@ "entry.save.title": "Bu makaleyi kaydet", "entry.save.completed": "Bitti!", "entry.save.toast.completed": "Makale kaydedildi", + "entry.scraper.label.rss": "RSS İçeriğini Göster", + "entry.scraper.title.rss": "RSS İçeriğini Al", "entry.scraper.label": "İndir", "entry.scraper.title": "Orijinal içeriği çek", "entry.scraper.completed": "Bitti!", diff --git a/internal/locale/translations/uk_UA.json b/internal/locale/translations/uk_UA.json index eeb305d73b4..8de17076867 100644 --- a/internal/locale/translations/uk_UA.json +++ b/internal/locale/translations/uk_UA.json @@ -69,7 +69,9 @@ "entry.state.saving": "Зберігаю...", "entry.state.loading": "Завантаження...", "entry.save.label": "Зберегти", + "entry.scraper.label.rss": "Показати вміст RSS", "entry.save.title": "Зберегти цю статтю", + "entry.scraper.title.rss": "Отримати вміст RSS", "entry.save.completed": "Готово!", "entry.save.toast.completed": "Стаття збережена", "entry.scraper.label": "Завантажити", diff --git a/internal/locale/translations/zh_CN.json b/internal/locale/translations/zh_CN.json index dc1079c420f..ea356a9c958 100644 --- a/internal/locale/translations/zh_CN.json +++ b/internal/locale/translations/zh_CN.json @@ -72,6 +72,8 @@ "entry.save.title": "保存这篇文章", "entry.save.completed": "完成", "entry.save.toast.completed": "已保存文章", + "entry.scraper.label.rss": "显示 RSS 内容", + "entry.scraper.title.rss": "获取 RSS 内容", "entry.scraper.label": "抓取全文", "entry.scraper.title": "抓取全文内容", "entry.scraper.completed": "抓取完成", diff --git a/internal/locale/translations/zh_TW.json b/internal/locale/translations/zh_TW.json index 4b4316f3b0e..92f840936e6 100644 --- a/internal/locale/translations/zh_TW.json +++ b/internal/locale/translations/zh_TW.json @@ -72,6 +72,8 @@ "entry.save.title": "儲存這篇文章", "entry.save.completed": "完成", "entry.save.toast.completed": "已儲存文章", + "entry.scraper.label.rss": "顯示 RSS 內容", + "entry.scraper.title.rss": "獲取 RSS 內容", "entry.scraper.label": "下載原文", "entry.scraper.title": "下載原文內容", "entry.scraper.completed": "下載完成", diff --git a/internal/model/entry.go b/internal/model/entry.go index 7631119ab6b..93de92a49bd 100644 --- a/internal/model/entry.go +++ b/internal/model/entry.go @@ -30,6 +30,7 @@ type Entry struct { CreatedAt time.Time `json:"created_at"` ChangedAt time.Time `json:"changed_at"` Content string `json:"content"` + WebContent string `json:"web_content,omitempty"` Author string `json:"author"` ShareCode string `json:"share_code"` Starred bool `json:"starred"` diff --git a/internal/reader/processor/processor.go b/internal/reader/processor/processor.go index 913ae0b3bfc..0bb20527b1e 100644 --- a/internal/reader/processor/processor.go +++ b/internal/reader/processor/processor.go @@ -98,14 +98,14 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us ) } else if content != "" { // We replace the entry content only if the scraper doesn't return any error. - entry.Content = content + entry.WebContent = content } } rewrite.Rewriter(websiteURL, entry, feed.RewriteRules) // The sanitizer should always run at the end of the process to make sure unsafe HTML is filtered. - entry.Content = sanitizer.Sanitize(websiteURL, entry.Content) + entry.WebContent = sanitizer.Sanitize(websiteURL, entry.Content) updateEntryReadingTime(store, feed, entry, entryIsNew, user) filteredEntries = append(filteredEntries, entry) @@ -209,14 +209,14 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) } if content != "" { - entry.Content = content + entry.WebContent = content if user.ShowReadingTime { - entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed) + entry.ReadingTime = readingtime.EstimateReadingTime(entry.WebContent, user.DefaultReadingSpeed, user.CJKReadingSpeed) } } rewrite.Rewriter(websiteURL, entry, entry.Feed.RewriteRules) - entry.Content = sanitizer.Sanitize(websiteURL, entry.Content) + entry.WebContent = sanitizer.Sanitize(websiteURL, entry.WebContent) return nil } diff --git a/internal/reader/rewrite/rewriter.go b/internal/reader/rewrite/rewriter.go index b577f687517..37d9243787c 100644 --- a/internal/reader/rewrite/rewriter.go +++ b/internal/reader/rewrite/rewriter.go @@ -24,35 +24,35 @@ type rule struct { func (rule rule) applyRule(entryURL string, entry *model.Entry) { switch rule.name { case "add_image_title": - entry.Content = addImageTitle(entryURL, entry.Content) + entry.WebContent = addImageTitle(entryURL, entry.WebContent) case "add_mailto_subject": - entry.Content = addMailtoSubject(entryURL, entry.Content) + entry.WebContent = addMailtoSubject(entryURL, entry.WebContent) case "add_dynamic_image": - entry.Content = addDynamicImage(entryURL, entry.Content) + entry.WebContent = addDynamicImage(entryURL, entry.WebContent) case "add_dynamic_iframe": - entry.Content = addDynamicIframe(entryURL, entry.Content) + entry.WebContent = addDynamicIframe(entryURL, entry.WebContent) case "add_youtube_video": - entry.Content = addYoutubeVideo(entryURL, entry.Content) + entry.WebContent = addYoutubeVideo(entryURL, entry.WebContent) case "add_invidious_video": - entry.Content = addInvidiousVideo(entryURL, entry.Content) + entry.WebContent = addInvidiousVideo(entryURL, entry.WebContent) case "add_youtube_video_using_invidious_player": - entry.Content = addYoutubeVideoUsingInvidiousPlayer(entryURL, entry.Content) + entry.WebContent = addYoutubeVideoUsingInvidiousPlayer(entryURL, entry.WebContent) case "add_youtube_video_from_id": - entry.Content = addYoutubeVideoFromId(entry.Content) + entry.WebContent = addYoutubeVideoFromId(entry.WebContent) case "add_pdf_download_link": - entry.Content = addPDFLink(entryURL, entry.Content) + entry.WebContent = addPDFLink(entryURL, entry.WebContent) case "nl2br": - entry.Content = strings.ReplaceAll(entry.Content, "\n", "
") + entry.WebContent = strings.ReplaceAll(entry.WebContent, "\n", "
") case "convert_text_link", "convert_text_links": - entry.Content = replaceTextLinks(entry.Content) + entry.WebContent = replaceTextLinks(entry.WebContent) case "fix_medium_images": - entry.Content = fixMediumImages(entryURL, entry.Content) + entry.WebContent = fixMediumImages(entryURL, entry.WebContent) case "use_noscript_figure_images": - entry.Content = useNoScriptImages(entryURL, entry.Content) + entry.WebContent = useNoScriptImages(entryURL, entry.WebContent) case "replace": // Format: replace("search-term"|"replace-term") if len(rule.args) >= 2 { - entry.Content = replaceCustom(entry.Content, rule.args[0], rule.args[1]) + entry.WebContent = replaceCustom(entry.WebContent, rule.args[0], rule.args[1]) } else { slog.Warn("Cannot find search and replace terms for replace rule", slog.Any("rule", rule), @@ -72,7 +72,7 @@ func (rule rule) applyRule(entryURL string, entry *model.Entry) { case "remove": // Format: remove("#selector > .element, .another") if len(rule.args) >= 1 { - entry.Content = removeCustom(entry.Content, rule.args[0]) + entry.WebContent = removeCustom(entry.WebContent, rule.args[0]) } else { slog.Warn("Cannot find selector for remove rule", slog.Any("rule", rule), @@ -80,21 +80,21 @@ func (rule rule) applyRule(entryURL string, entry *model.Entry) { ) } case "add_castopod_episode": - entry.Content = addCastopodEpisode(entryURL, entry.Content) + entry.WebContent = addCastopodEpisode(entryURL, entry.WebContent) case "base64_decode": selector := "body" if len(rule.args) >= 1 { selector = rule.args[0] } - entry.Content = applyFuncOnTextContent(entry.Content, selector, decodeBase64Content) + entry.WebContent = applyFuncOnTextContent(entry.WebContent, selector, decodeBase64Content) case "add_hn_links_using_hack": - entry.Content = addHackerNewsLinksUsing(entry.Content, "hack") + entry.WebContent = addHackerNewsLinksUsing(entry.WebContent, "hack") case "add_hn_links_using_opener": - entry.Content = addHackerNewsLinksUsing(entry.Content, "opener") + entry.WebContent = addHackerNewsLinksUsing(entry.WebContent, "opener") case "parse_markdown": - entry.Content = parseMarkdown(entry.Content) + entry.WebContent = parseMarkdown(entry.WebContent) case "remove_tables": - entry.Content = removeTables(entry.Content) + entry.WebContent = removeTables(entry.WebContent) case "remove_clickbait": entry.Title = cases.Title(language.English).String(strings.ToLower(entry.Title)) } diff --git a/internal/reader/rewrite/rewriter_test.go b/internal/reader/rewrite/rewriter_test.go index fa2b765bd2e..2dcceb6b46a 100644 --- a/internal/reader/rewrite/rewriter_test.go +++ b/internal/reader/rewrite/rewriter_test.go @@ -50,12 +50,12 @@ func TestReplaceTextLinks(t *testing.T) { func TestRewriteWithNoMatchingRule(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `Some text.`, + Title: `A title`, + WebContent: `Some text.`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Some text.`, + Title: `A title`, + WebContent: `Some text.`, } Rewriter("https://example.org/article", testEntry, ``) @@ -68,12 +68,12 @@ func TestRewriteWithYoutubeLink(t *testing.T) { config.Opts = config.NewOptions() controlEntry := &model.Entry{ - Title: `A title`, - Content: `
Video Description`, + Title: `A title`, + WebContent: `
Video Description`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Video Description`, + Title: `A title`, + WebContent: `Video Description`, } Rewriter("https://www.youtube.com/watch?v=1234", testEntry, ``) @@ -95,12 +95,12 @@ func TestRewriteWithYoutubeLinkAndCustomEmbedURL(t *testing.T) { } controlEntry := &model.Entry{ - Title: `A title`, - Content: `
Video Description`, + Title: `A title`, + WebContent: `
Video Description`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Video Description`, + Title: `A title`, + WebContent: `Video Description`, } Rewriter("https://www.youtube.com/watch?v=1234", testEntry, ``) @@ -111,12 +111,12 @@ func TestRewriteWithYoutubeLinkAndCustomEmbedURL(t *testing.T) { func TestRewriteWithInexistingCustomRule(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `Video Description`, + Title: `A title`, + WebContent: `Video Description`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Video Description`, + Title: `A title`, + WebContent: `Video Description`, } Rewriter("https://www.youtube.com/watch?v=1234", testEntry, `some rule`) @@ -127,12 +127,12 @@ func TestRewriteWithInexistingCustomRule(t *testing.T) { func TestRewriteWithXkcdLink(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `
Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.

Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.

`, + Title: `A title`, + WebContent: `
Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.

Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.

`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.`, + Title: `A title`, + WebContent: `Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.`, } Rewriter("https://xkcd.com/1912/", testEntry, ``) @@ -143,12 +143,12 @@ func TestRewriteWithXkcdLink(t *testing.T) { func TestRewriteWithXkcdLinkHtmlInjection(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `
<foo>

<foo>

`, + Title: `A title`, + WebContent: `
<foo>

<foo>

`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `<foo>`, + Title: `A title`, + WebContent: `<foo>`, } Rewriter("https://xkcd.com/1912/", testEntry, ``) @@ -159,12 +159,12 @@ func TestRewriteWithXkcdLinkHtmlInjection(t *testing.T) { func TestRewriteWithXkcdLinkAndImageNoTitle(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.`, + Title: `A title`, + WebContent: `Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.`, + Title: `A title`, + WebContent: `Your problem is so terrible, I worry that, if I help you, I risk drawing the attention of whatever god of technology inflicted it on you.`, } Rewriter("https://xkcd.com/1912/", testEntry, ``) @@ -175,12 +175,12 @@ func TestRewriteWithXkcdLinkAndImageNoTitle(t *testing.T) { func TestRewriteWithXkcdLinkAndNoImage(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `test`, + Title: `A title`, + WebContent: `test`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `test`, + Title: `A title`, + WebContent: `test`, } Rewriter("https://xkcd.com/1912/", testEntry, ``) @@ -191,12 +191,12 @@ func TestRewriteWithXkcdLinkAndNoImage(t *testing.T) { func TestRewriteWithXkcdAndNoImage(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `test`, + Title: `A title`, + WebContent: `test`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `test`, + Title: `A title`, + WebContent: `test`, } Rewriter("https://xkcd.com/1912/", testEntry, ``) @@ -207,12 +207,12 @@ func TestRewriteWithXkcdAndNoImage(t *testing.T) { func TestRewriteMailtoLink(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `contact [blah blah]`, + Title: `A title`, + WebContent: `contact [blah blah]`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `contact`, + Title: `A title`, + WebContent: `contact`, } Rewriter("https://www.qwantz.com/", testEntry, ``) @@ -223,12 +223,12 @@ func TestRewriteMailtoLink(t *testing.T) { func TestRewriteWithPDFLink(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `PDF
test`, + Title: `A title`, + WebContent: `PDF
test`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `test`, + Title: `A title`, + WebContent: `test`, } Rewriter("https://example.org/document.pdf", testEntry, ``) @@ -239,12 +239,12 @@ func TestRewriteWithPDFLink(t *testing.T) { func TestRewriteWithNoLazyImage(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `Image`, + Title: `A title`, + WebContent: `Image`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Image`, + Title: `A title`, + WebContent: `Image`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") @@ -255,12 +255,12 @@ func TestRewriteWithNoLazyImage(t *testing.T) { func TestRewriteWithLazyImage(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `Image`, + Title: `A title`, + WebContent: `Image`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Image`, + Title: `A title`, + WebContent: `Image`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") @@ -271,12 +271,12 @@ func TestRewriteWithLazyImage(t *testing.T) { func TestRewriteWithLazyDivImage(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `Image`, + Title: `A title`, + WebContent: `Image`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `
`, + Title: `A title`, + WebContent: `
`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") @@ -287,12 +287,12 @@ func TestRewriteWithLazyDivImage(t *testing.T) { func TestRewriteWithUnknownLazyNoScriptImage(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `ImageFallback`, + Title: `A title`, + WebContent: `ImageFallback`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Image`, + Title: `A title`, + WebContent: `Image`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") @@ -303,12 +303,12 @@ func TestRewriteWithUnknownLazyNoScriptImage(t *testing.T) { func TestRewriteWithLazySrcset(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `Image`, + Title: `A title`, + WebContent: `Image`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Image`, + Title: `A title`, + WebContent: `Image`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") @@ -319,12 +319,12 @@ func TestRewriteWithLazySrcset(t *testing.T) { func TestRewriteWithImageAndLazySrcset(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `Image`, + Title: `A title`, + WebContent: `Image`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Image`, + Title: `A title`, + WebContent: `Image`, } Rewriter("https://example.org/article", testEntry, "add_dynamic_image") @@ -335,12 +335,12 @@ func TestRewriteWithImageAndLazySrcset(t *testing.T) { func TestRewriteWithNoLazyIframe(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: ``, + Title: `A title`, + WebContent: ``, } testEntry := &model.Entry{ - Title: `A title`, - Content: ``, + Title: `A title`, + WebContent: ``, } Rewriter("https://example.org/article", testEntry, "add_dynamic_iframe") @@ -351,12 +351,12 @@ func TestRewriteWithNoLazyIframe(t *testing.T) { func TestRewriteWithLazyIframe(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: ``, + Title: `A title`, + WebContent: ``, } testEntry := &model.Entry{ - Title: `A title`, - Content: ``, + Title: `A title`, + WebContent: ``, } Rewriter("https://example.org/article", testEntry, "add_dynamic_iframe") @@ -367,12 +367,12 @@ func TestRewriteWithLazyIframe(t *testing.T) { func TestRewriteWithLazyIframeAndSrc(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: ``, + Title: `A title`, + WebContent: ``, } testEntry := &model.Entry{ - Title: `A title`, - Content: ``, + Title: `A title`, + WebContent: ``, } Rewriter("https://example.org/article", testEntry, "add_dynamic_iframe") @@ -383,12 +383,12 @@ func TestRewriteWithLazyIframeAndSrc(t *testing.T) { func TestNewLineRewriteRule(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `A
B
C`, + Title: `A title`, + WebContent: `A
B
C`, } testEntry := &model.Entry{ - Title: `A title`, - Content: "A\nB\nC", + Title: `A title`, + WebContent: "A\nB\nC", } Rewriter("https://example.org/article", testEntry, "nl2br") @@ -399,12 +399,12 @@ func TestNewLineRewriteRule(t *testing.T) { func TestConvertTextLinkRewriteRule(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `Test: http://example.org/a/b`, + Title: `A title`, + WebContent: `Test: http://example.org/a/b`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Test: http://example.org/a/b`, + Title: `A title`, + WebContent: `Test: http://example.org/a/b`, } Rewriter("https://example.org/article", testEntry, "convert_text_link") @@ -415,12 +415,12 @@ func TestConvertTextLinkRewriteRule(t *testing.T) { func TestMediumImage(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `Image for post`, + Title: `A title`, + WebContent: `Image for post`, } testEntry := &model.Entry{ Title: `A title`, - Content: ` + WebContent: `
@@ -441,7 +441,7 @@ func TestMediumImage(t *testing.T) { `, } Rewriter("https://example.org/article", testEntry, "fix_medium_images") - testEntry.Content = strings.TrimSpace(testEntry.Content) + testEntry.WebContent = strings.TrimSpace(testEntry.WebContent) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) @@ -450,15 +450,15 @@ func TestMediumImage(t *testing.T) { func TestRewriteNoScriptImageWithoutNoScriptTag(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `
The beautiful MDN logo.
MDN Logo
`, + Title: `A title`, + WebContent: `
The beautiful MDN logo.
MDN Logo
`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `
The beautiful MDN logo.
MDN Logo
`, + Title: `A title`, + WebContent: `
The beautiful MDN logo.
MDN Logo
`, } Rewriter("https://example.org/article", testEntry, "use_noscript_figure_images") - testEntry.Content = strings.TrimSpace(testEntry.Content) + testEntry.WebContent = strings.TrimSpace(testEntry.WebContent) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) @@ -467,15 +467,15 @@ func TestRewriteNoScriptImageWithoutNoScriptTag(t *testing.T) { func TestRewriteNoScriptImageWithNoScriptTag(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `
MDN Logo
`, + Title: `A title`, + WebContent: `
MDN Logo
`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `
The beautiful MDN logo.
MDN Logo
`, + Title: `A title`, + WebContent: `
The beautiful MDN logo.
MDN Logo
`, } Rewriter("https://example.org/article", testEntry, "use_noscript_figure_images") - testEntry.Content = strings.TrimSpace(testEntry.Content) + testEntry.WebContent = strings.TrimSpace(testEntry.WebContent) if !reflect.DeepEqual(testEntry, controlEntry) { t.Errorf(`Not expected output: got "%+v" instead of "%+v"`, testEntry, controlEntry) @@ -484,12 +484,12 @@ func TestRewriteNoScriptImageWithNoScriptTag(t *testing.T) { func TestRewriteReplaceCustom(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: ``, + Title: `A title`, + WebContent: ``, } testEntry := &model.Entry{ - Title: `A title`, - Content: ``, + Title: `A title`, + WebContent: ``, } Rewriter("https://example.org/article", testEntry, `replace("article/(.*).svg"|"article/$1.png")`) @@ -500,12 +500,12 @@ func TestRewriteReplaceCustom(t *testing.T) { func TestRewriteReplaceTitleCustom(t *testing.T) { controlEntry := &model.Entry{ - Title: `Ouch, a thistle`, - Content: `The replace_title rewrite rule should not modify the content.`, + Title: `Ouch, a thistle`, + WebContent: `The replace_title rewrite rule should not modify the content.`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `The replace_title rewrite rule should not modify the content.`, + Title: `A title`, + WebContent: `The replace_title rewrite rule should not modify the content.`, } Rewriter("https://example.org/article", testEntry, `replace_title("(?i)^a\\s*ti"|"Ouch, a this")`) @@ -516,12 +516,12 @@ func TestRewriteReplaceTitleCustom(t *testing.T) { func TestRewriteRemoveCustom(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `
Lorem Ipsum Super important info
`, + Title: `A title`, + WebContent: `
Lorem Ipsum Super important info
`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `
Lorem Ipsum I dont want to see thisSuper important info
`, + Title: `A title`, + WebContent: `
Lorem Ipsum I dont want to see thisSuper important info
`, } Rewriter("https://example.org/article", testEntry, `remove(".spam, .ads:not(.keep)")`) @@ -532,12 +532,12 @@ func TestRewriteRemoveCustom(t *testing.T) { func TestRewriteAddCastopodEpisode(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `
Episode Description`, + Title: `A title`, + WebContent: `
Episode Description`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `Episode Description`, + Title: `A title`, + WebContent: `Episode Description`, } Rewriter("https://podcast.demo/@demo/episodes/test", testEntry, `add_castopod_episode`) @@ -548,12 +548,12 @@ func TestRewriteAddCastopodEpisode(t *testing.T) { func TestRewriteBase64Decode(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `This is some base64 encoded content`, + Title: `A title`, + WebContent: `This is some base64 encoded content`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=`, + Title: `A title`, + WebContent: `VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=`, } Rewriter("https://example.org/article", testEntry, `base64_decode`) @@ -564,12 +564,12 @@ func TestRewriteBase64Decode(t *testing.T) { func TestRewriteBase64DecodeInHTML(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `
Lorem Ipsum not valid base64This is some base64 encoded content
`, + Title: `A title`, + WebContent: `
Lorem Ipsum not valid base64This is some base64 encoded content
`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `
Lorem Ipsum not valid base64VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`, + Title: `A title`, + WebContent: `
Lorem Ipsum not valid base64VGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`, } Rewriter("https://example.org/article", testEntry, `base64_decode`) @@ -580,12 +580,12 @@ func TestRewriteBase64DecodeInHTML(t *testing.T) { func TestRewriteBase64DecodeArgs(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `
Lorem IpsumThis is some base64 encoded content
`, + Title: `A title`, + WebContent: `
Lorem IpsumThis is some base64 encoded content
`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `
Lorem IpsumVGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`, + Title: `A title`, + WebContent: `
Lorem IpsumVGhpcyBpcyBzb21lIGJhc2U2NCBlbmNvZGVkIGNvbnRlbnQ=
`, } Rewriter("https://example.org/article", testEntry, `base64_decode(".base64")`) @@ -596,12 +596,12 @@ func TestRewriteBase64DecodeArgs(t *testing.T) { func TestRewriteRemoveTables(t *testing.T) { controlEntry := &model.Entry{ - Title: `A title`, - Content: `

Test

Hello World!

Test

`, + Title: `A title`, + WebContent: `

Test

Hello World!

Test

`, } testEntry := &model.Entry{ - Title: `A title`, - Content: `

Test

Hello World!

Test

`, + Title: `A title`, + WebContent: `

Test

Hello World!

Test

`, } Rewriter("https://example.org/article", testEntry, `remove_tables`) @@ -612,12 +612,12 @@ func TestRewriteRemoveTables(t *testing.T) { func TestRemoveClickbait(t *testing.T) { controlEntry := &model.Entry{ - Title: `This Is Amazing`, - Content: `Some description`, + Title: `This Is Amazing`, + WebContent: `Some description`, } testEntry := &model.Entry{ - Title: `THIS IS AMAZING`, - Content: `Some description`, + Title: `THIS IS AMAZING`, + WebContent: `Some description`, } Rewriter("https://example.org/article", testEntry, `remove_clickbait`) @@ -629,7 +629,7 @@ func TestRemoveClickbait(t *testing.T) { func TestAddHackerNewsLinksUsingHack(t *testing.T) { testEntry := &model.Entry{ Title: `A title`, - Content: `

Article URL: https://example.org/article

+ WebContent: `

Article URL: https://example.org/article

Comments URL: https://news.ycombinator.com/item?id=37620043

Points: 23

# Comments: 38

`, @@ -637,7 +637,7 @@ func TestAddHackerNewsLinksUsingHack(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, - Content: `

Article URL: https://example.org/article

+ WebContent: `

Article URL: https://example.org/article

Comments URL: https://news.ycombinator.com/item?id=37620043 Open with HACK

Points: 23

# Comments: 38

`, @@ -652,7 +652,7 @@ func TestAddHackerNewsLinksUsingHack(t *testing.T) { func TestAddHackerNewsLinksUsingOpener(t *testing.T) { testEntry := &model.Entry{ Title: `A title`, - Content: `

Article URL: https://example.org/article

+ WebContent: `

Article URL: https://example.org/article

Comments URL: https://news.ycombinator.com/item?id=37620043

Points: 23

# Comments: 38

`, @@ -660,7 +660,7 @@ func TestAddHackerNewsLinksUsingOpener(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, - Content: `

Article URL: https://example.org/article

+ WebContent: `

Article URL: https://example.org/article

Comments URL: https://news.ycombinator.com/item?id=37620043 Open with Opener

Points: 23

# Comments: 38

`, @@ -675,7 +675,7 @@ func TestAddHackerNewsLinksUsingOpener(t *testing.T) { func TestAddImageTitle(t *testing.T) { testEntry := &model.Entry{ Title: `A title`, - Content: ` + WebContent: ` "onerror=alert(1) a=" "onerror=alert(1) a=" @@ -688,7 +688,7 @@ func TestAddImageTitle(t *testing.T) { controlEntry := &model.Entry{ Title: `A title`, - Content: `

pouf

+ WebContent: `

pouf

pouf

pouf

;"onerror=alert(1) a=;"

pouf

diff --git a/internal/storage/entry.go b/internal/storage/entry.go index 1a7cc6d73dc..bd4e342eee7 100644 --- a/internal/storage/entry.go +++ b/internal/storage/entry.go @@ -77,13 +77,14 @@ func (s *Storage) UpdateEntryTitleAndContent(entry *model.Entry) error { SET title=$1, content=$2, - reading_time=$3, - document_vectors = setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($2, ''), 500000)), 'B') + web_content=$3, + reading_time=$4, + document_vectors = setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($2, ''), 500000)), 'B') || setweight(to_tsvector(left(coalesce($3, ''), 500000)), 'C') WHERE - id=$4 AND user_id=$5 + id=$5 AND user_id=$6 ` - if _, err := s.db.Exec(query, entry.Title, entry.Content, entry.ReadingTime, entry.ID, entry.UserID); err != nil { + if _, err := s.db.Exec(query, entry.Title, entry.Content, entry.WebContent, entry.ReadingTime, entry.ID, entry.UserID); err != nil { return fmt.Errorf(`store: unable to update entry #%d: %v`, entry.ID, err) } @@ -101,6 +102,7 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error { comments_url, published_at, content, + web_content, author, user_id, feed_id, @@ -121,9 +123,10 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error { $8, $9, $10, + $11, now(), - setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($6, ''), 500000)), 'B'), - $11 + setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($6, ''), 500000)), 'B') || setweight(to_tsvector(left(coalesce($7, ''), 500000)), 'C'), + $12 ) RETURNING id, status, created_at, changed_at @@ -136,6 +139,7 @@ func (s *Storage) createEntry(tx *sql.Tx, entry *model.Entry) error { entry.CommentsURL, entry.Date, entry.Content, + entry.WebContent, entry.Author, entry.UserID, entry.FeedID, @@ -176,12 +180,13 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error { url=$2, comments_url=$3, content=$4, - author=$5, - reading_time=$6, - document_vectors = setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($4, ''), 500000)), 'B'), - tags=$10 + web_content=$5, + author=$6, + reading_time=$7, + document_vectors = setweight(to_tsvector(left(coalesce($1, ''), 500000)), 'A') || setweight(to_tsvector(left(coalesce($4, ''), 500000)), 'B') || setweight(to_tsvector(left(coalesce($5, ''), 500000)), 'C'), + tags=$11 WHERE - user_id=$7 AND feed_id=$8 AND hash=$9 + user_id=$8 AND feed_id=$9 AND hash=$10 RETURNING id ` @@ -191,6 +196,7 @@ func (s *Storage) updateEntry(tx *sql.Tx, entry *model.Entry) error { entry.URL, entry.CommentsURL, entry.Content, + entry.WebContent, entry.Author, entry.ReadingTime, entry.UserID, diff --git a/internal/storage/entry_query_builder.go b/internal/storage/entry_query_builder.go index 680fbedb25e..c70500c5a46 100644 --- a/internal/storage/entry_query_builder.go +++ b/internal/storage/entry_query_builder.go @@ -271,6 +271,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { e.author, e.share_code, e.content, + e.web_content, e.status, e.starred, e.reading_time, @@ -337,6 +338,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { &entry.Author, &entry.ShareCode, &entry.Content, + &entry.WebContent, &entry.Status, &entry.Starred, &entry.ReadingTime, @@ -361,7 +363,6 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { &iconID, &tz, ) - if err != nil { return nil, fmt.Errorf("store: unable to fetch entry row: %v", err) } diff --git a/internal/template/templates/views/entry.html b/internal/template/templates/views/entry.html index 4284f097013..a6ec78e1b9f 100644 --- a/internal/template/templates/views/entry.html +++ b/internal/template/templates/views/entry.html @@ -85,11 +85,17 @@

  • + >{{ icon "scraper" }}{{ if .entry.WebContent }}{{ t "entry.scraper.label" }}{{ else }}{{ t "entry.scraper.label.rss" }}{{ end }}
    • {{ if .entry.CommentsURL }}
  • @@ -201,9 +207,17 @@

    {{ end }} {{end}} {{ if .user }} - {{ noescape (proxyFilter .entry.Content) }} + {{ if .entry.WebContent }} + {{ noescape (proxyFilter .entry.WebContent) }} + {{ else }} + {{ noescape (proxyFilter .entry.Content) }} + {{ end }} {{ else }} - {{ noescape .entry.Content }} + {{ if .entry.WebContent }} + {{ noescape .entry.WebContent }} + {{ else }} + {{ noescape .entry.Content }} + {{ end }} {{ end }} {{ if .entry.Enclosures }} diff --git a/internal/ui/entry_scraper.go b/internal/ui/entry_scraper.go index 8eaaffc077d..0a4526ab65f 100644 --- a/internal/ui/entry_scraper.go +++ b/internal/ui/entry_scraper.go @@ -65,5 +65,49 @@ func (h *handler) fetchContent(w http.ResponseWriter, r *http.Request) { readingTime := locale.NewPrinter(user.Language).Plural("entry.estimated_reading_time", entry.ReadingTime, entry.ReadingTime) + json.OK(w, r, map[string]string{"content": mediaproxy.RewriteDocumentWithRelativeProxyURL(h.router, entry.WebContent), "reading_time": readingTime}) +} + +func (h *handler) fetchOriginal(w http.ResponseWriter, r *http.Request) { + loggedUserID := request.UserID(r) + entryID := request.RouteInt64Param(r, "entryID") + + entryBuilder := h.store.NewEntryQueryBuilder(loggedUserID) + entryBuilder.WithEntryID(entryID) + entryBuilder.WithoutStatus(model.EntryStatusRemoved) + + entry, err := entryBuilder.GetEntry() + if err != nil { + json.ServerError(w, r, err) + return + } + + if entry == nil { + json.NotFound(w, r) + return + } + + user, err := h.store.UserByID(loggedUserID) + if err != nil { + json.ServerError(w, r, err) + return + } + + feedBuilder := storage.NewFeedQueryBuilder(h.store, loggedUserID) + feedBuilder.WithFeedID(entry.FeedID) + feed, err := feedBuilder.GetFeed() + if err != nil { + json.ServerError(w, r, err) + return + } + + if feed == nil { + json.NotFound(w, r) + return + } + + readingTime := locale.NewPrinter(user.Language).Plural("entry.estimated_reading_time", entry.ReadingTime, entry.ReadingTime) + json.OK(w, r, map[string]string{"content": mediaproxy.RewriteDocumentWithRelativeProxyURL(h.router, entry.Content), "reading_time": readingTime}) + } diff --git a/internal/ui/static/js/app.js b/internal/ui/static/js/app.js index 79ffb4b58d1..27bd0146141 100644 --- a/internal/ui/static/js/app.js +++ b/internal/ui/static/js/app.js @@ -339,16 +339,32 @@ function handleFetchOriginalContent() { if (!buttonElement) { return; } + let labelElement = buttonElement.querySelector("span.icon-label"); + buttonElement.removeChild(labelElement); + labelElement = labelElement.cloneNode(true); + appendIconLabel(buttonElement, buttonElement.dataset.labelLoading); - const previousElement = buttonElement.cloneNode(true); + let reqUrl = buttonElement.dataset.fetchContentUrl + let switchTo = "rss"; + let label = buttonElement.dataset.labelRss; + let title = buttonElement.dataset.titleRss; - buttonElement.textContent = ""; - appendIconLabel(buttonElement, buttonElement.dataset.labelLoading); + if (buttonElement.dataset.switchTo === "rss") { + reqUrl = buttonElement.dataset.fetchOriginalContentUrl + switchTo = "web"; + label = buttonElement.dataset.label; + title = buttonElement.dataset.title; + } + + const request = new RequestBuilder(reqUrl); - const request = new RequestBuilder(buttonElement.dataset.fetchContentUrl); request.withCallback((response) => { - buttonElement.textContent = ''; - buttonElement.appendChild(previousElement); + + buttonElement.removeChild(buttonElement.querySelector("span.icon-label")) + labelElement.textContent = label + buttonElement.title = title + buttonElement.dataset.switchTo = switchTo + buttonElement.appendChild(labelElement) response.json().then((data) => { if (data.hasOwnProperty("content") && data.hasOwnProperty("reading_time")) { diff --git a/internal/ui/ui.go b/internal/ui/ui.go index 6d8e729cf61..5eeaafa435d 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -98,6 +98,7 @@ func Serve(router *mux.Router, store *storage.Storage, pool *worker.Pool) { uiRouter.HandleFunc("/entry/save/{entryID}", handler.saveEntry).Name("saveEntry").Methods(http.MethodPost) uiRouter.HandleFunc("/entry/enclosure/{enclosureID}/save-progression", handler.saveEnclosureProgression).Name("saveEnclosureProgression").Methods(http.MethodPost) uiRouter.HandleFunc("/entry/download/{entryID}", handler.fetchContent).Name("fetchContent").Methods(http.MethodPost) + uiRouter.HandleFunc("/entry/original/{entryID}", handler.fetchOriginal).Name("fetchOriginal").Methods(http.MethodPost) uiRouter.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", handler.mediaProxy).Name("proxy").Methods(http.MethodGet) uiRouter.HandleFunc("/entry/bookmark/{entryID}", handler.toggleBookmark).Name("toggleBookmark").Methods(http.MethodPost)