From 036ace553060370555eece0fda5e09d94cfb7fc8 Mon Sep 17 00:00:00 2001 From: Ben Adrian Sarmiento Date: Wed, 30 Aug 2023 00:39:26 +0200 Subject: [PATCH] 2.7.2 (#53) improve worker --- package.json | 2 +- src/pages/api/dbsize.ts | 2 +- src/pages/api/moviesearch.ts | 7 + src/pages/api/requested.ts | 192 +++++++++++++++++++++++++ src/pages/api/scrape.ts | 13 +- src/pages/api/tvsearch.ts | 11 +- src/pages/index.tsx | 10 ++ src/services/btdigg-v2.ts | 270 ++++++++++++++++++----------------- src/services/planetscale.ts | 23 +++ tor.Dockerfile | 2 +- 10 files changed, 391 insertions(+), 141 deletions(-) create mode 100644 src/pages/api/requested.ts diff --git a/package.json b/package.json index 784e8de..2b420c6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "debrid-media-manager", - "version": "2.7.1", + "version": "2.7.2", "private": false, "scripts": { "dev": "next dev", diff --git a/src/pages/api/dbsize.ts b/src/pages/api/dbsize.ts index ca01652..6b420fe 100644 --- a/src/pages/api/dbsize.ts +++ b/src/pages/api/dbsize.ts @@ -5,7 +5,7 @@ const cache = new PlanetScaleCache(); const handler: NextApiHandler = async (req, res) => { try { - const size = await cache.getDbSize(); + const size = await cache.getScrapedDbSize(); res.status(200).json({ size }); } catch (err: any) { console.error(err); diff --git a/src/pages/api/moviesearch.ts b/src/pages/api/moviesearch.ts index 2c96e28..17d0ba9 100644 --- a/src/pages/api/moviesearch.ts +++ b/src/pages/api/moviesearch.ts @@ -18,6 +18,13 @@ const handler: NextApiHandler = async (req, res) => { return; } + const isProcessing = await db.keyExists(`processing:${imdbId}`); + if (isProcessing) { + res.status(204).json(null); + return; + } + + await db.saveScrapedResults(`requested:${imdbId.toString().trim()}`, []); res.status(204).json(null); } catch (error: any) { console.error('encountered a db issue', error); diff --git a/src/pages/api/requested.ts b/src/pages/api/requested.ts new file mode 100644 index 0000000..b8efbbd --- /dev/null +++ b/src/pages/api/requested.ts @@ -0,0 +1,192 @@ +import { SearchResult } from '@/services/btdigg-v2'; +import { + createAxiosInstance, + scrapeResults, + flattenAndRemoveDuplicates, + groupByParsedTitle, +} from '@/services/btdigg-v2'; +import { PlanetScaleCache } from '@/services/planetscale'; +import axios from 'axios'; +import { NextApiRequest, NextApiResponse } from 'next'; +import { SocksProxyAgent } from 'socks-proxy-agent'; + +const tmdbKey = process.env.TMDB_KEY; +const mdblistKey = process.env.MDBLIST_KEY; +const getTmdbInfo = (imdbId: string) => + `https://api.themoviedb.org/3/find/${imdbId}?api_key=${tmdbKey}&external_source=imdb_id`; +const getMdbInfo = (imdbId: string) => `https://mdblist.com/api/?apikey=${mdblistKey}&i=${imdbId}`; +function padWithZero(num: number) { + if (num < 10) { + return '0' + num; + } else { + return num.toString(); + } +} +const cleanSearchQuery = (search: string): string => { + return search + .split(/[\s\=:\?\.\-\(\)\/]/) // split the search query into an array of elements + .filter((e) => e !== '') // filter out any empty elements + .map((e) => e.toLowerCase()) // convert each element to lowercase + .join(' ') // join the remaining elements with a single space + .replace(/[áàäâ]/g, 'a') // replace certain characters with their equivalent + .replace(/[éèëê]/g, 'e') + .replace(/[íìïî]/g, 'i') + .replace(/[óòöô]/g, 'o') + .replace(/[úùüû]/g, 'u') + .replace(/\s+/g, ' ') // replace multiple spaces with a single space + .trim(); +}; + +const db = new PlanetScaleCache(); + +export type ScrapeResponse = { + status: string; + errorMessage?: string; +}; +1; +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + const { scrapePassword } = req.query; + if (process.env.SEARCH_SPEED_PASSWORD && scrapePassword !== process.env.SEARCH_SPEED_PASSWORD) { + res.status(403).json({ status: 'error', errorMessage: 'You are not authorized to use this feature' }); + return; + } + + const imdbId = await db.getLatestRequest(); + if (!imdbId) { + res.status(200).json({ status: 'done' }); + return; + } + + const isProcessing = await db.keyExists(`processing:${imdbId}`); + if (isProcessing) { + res.status(200).json({ status: 'processing' }); + return; + } + + // imdbId to search for + const tmdbResponse = await axios.get(getTmdbInfo(imdbId)); + + const movieTitles: string[] = []; + const tvTitles: string[] = []; + + let tmdbItem: any = {}; + let itemType: 'movie' | 'tv' = 'movie'; + + if (tmdbResponse.data.movie_results.length > 0) { + itemType = 'movie'; + tmdbItem = tmdbResponse.data.movie_results[0]; + movieTitles.push(`"${cleanSearchQuery(tmdbItem.title)}"`); + movieTitles.push(`"${cleanSearchQuery(tmdbItem.title)}" ${tmdbItem.release_date.substring(0, 4)}`); + + if (tmdbItem.original_title && tmdbItem.original_title !== tmdbItem.title) { + movieTitles.push(`"${tmdbItem.original_title}"`); + movieTitles.push(`"${tmdbItem.original_title}" ${tmdbItem.release_date.substring(0, 4)}`); + const mdbItem = await axios.get(getMdbInfo(imdbId)); + for (let rating of mdbItem.data.ratings) { + if (rating.source === 'tomatoes') { + const cleanedTitle = ( + itemType === 'movie' ? rating.url.split('/m/') : rating.url.split('/tv/') + )[1].replaceAll('_', ' '); + movieTitles.push(`"${cleanedTitle}"`); + movieTitles.push(`"${cleanedTitle}" ${tmdbItem.release_date.substring(0, 4)}`); + } + } + } + + await db.saveScrapedResults(`processing:${imdbId}`, []); + + try { + const results = []; + for (const movieTitle of movieTitles) { + for (const lType of ['720p', '1080p', '2160p', '']) { + results.push( + await scrapeResults( + createAxiosInstance( + new SocksProxyAgent(process.env.PROXY!, { timeout: 10000 }) + ), + `${movieTitle} ${lType}`.trim(), + lType || '1080p', + ) + ); + } + } + let processedResults = flattenAndRemoveDuplicates(results); + if (processedResults.length) processedResults = groupByParsedTitle(processedResults); + + await db.saveScrapedResults(`movie:${imdbId}`, processedResults); + + res.status(200).json({ status: `scraped: ${processedResults.length} items` }); + } catch (error: any) { + res.status(500).json({ + status: 'error', + errorMessage: `An error occurred while scraping Btdigg (${error.message})`, + }); + } + } + + if (tmdbResponse.data.tv_results.length > 0) { + itemType = 'tv'; + tmdbItem = tmdbResponse.data.tv_results[0]; + tvTitles.push(`"${cleanSearchQuery(tmdbItem.name)}"`); + tvTitles.push(`"${cleanSearchQuery(tmdbItem.name)}" ${tmdbItem.first_air_date.substring(0, 4)}`); + + if (tmdbItem.original_name && tmdbItem.original_name !== tmdbItem.name) { + tvTitles.push(`"${tmdbItem.original_name}"`); + tvTitles.push(`"${tmdbItem.original_name}" ${tmdbItem.first_air_date.substring(0, 4)}`); + } + + await db.saveScrapedResults(`processing:${imdbId}`, []); + + let totalResultsCount = 0; + const showResponse = await axios.get(getMdbInfo(imdbId)); + for (const season of showResponse.data.seasons + ? showResponse.data.seasons + : [{ season_number: 1, episode_count: 0 }]) { + if (season.season_number === 0) continue; + let seasonQueries = tvTitles.map((q) => `${q} "s${padWithZero(season.season_number)}"`); + try { + const results = []; + for (const finalQuery of seasonQueries) { + for (const lType of ['720p', '1080p', '2160p', '']) { + results.push( + await scrapeResults( + createAxiosInstance( + new SocksProxyAgent(process.env.PROXY!, { timeout: 10000 }) + ), + `${finalQuery} ${lType}`.trim(), + lType || '1080p', + ) + ); + } + } + let processedResults = flattenAndRemoveDuplicates(results); + if (processedResults.length) processedResults = groupByParsedTitle(processedResults); + + await db.saveScrapedResults(`tv:${imdbId}:${season.season_number}`, processedResults); + + totalResultsCount += processedResults.length; + } catch (error: any) { + res.status(500).json({ + status: 'error', + errorMessage: `An error occurred while scraping Btdigg (${error.message})`, + }); + } + // if (season.episode_count === 0) continue; + // for (let i = 1; i <= season.episode_count; i++) { + // seasonQueries = seasonQueries.concat( + // tvTitles.map( + // (q) => + // `${q} "s${padWithZero(season.season_number)}e${padWithZero(i)}"` + // ) + // ); + // } + } + + res.status(200).json({ status: `scraped: ${totalResultsCount} items` }); + } + + await db.markAsDone(imdbId); +} diff --git a/src/pages/api/scrape.ts b/src/pages/api/scrape.ts index c046632..38361c8 100644 --- a/src/pages/api/scrape.ts +++ b/src/pages/api/scrape.ts @@ -55,11 +55,16 @@ export default async function handler( } if (!imdbId || !(typeof imdbId === 'string')) { - console.log(imdbId); res.status(400).json({ status: 'error', errorMessage: 'Missing "imdbId" query parameter' }); return; } + const isProcessing = await db.keyExists(`processing:${imdbId}`); + if (isProcessing) { + res.status(200).json({ status: 'processing' }); + return; + } + // imdbId to search for const tmdbResponse = await axios.get(getTmdbInfo(imdbId.toString().trim())); @@ -98,6 +103,8 @@ export default async function handler( } } + await db.saveScrapedResults(`processing:${imdbId}`, []); + try { const results = []; for (const movieTitle of movieTitles) { @@ -146,6 +153,8 @@ export default async function handler( tvTitles.push(`"${tmdbItem.original_name}" ${tmdbItem.first_air_date.substring(0, 4)}`); } + await db.saveScrapedResults(`processing:${imdbId}`, []); + let totalResultsCount = 0; const showResponse = await axios.get(getMdbInfo(imdbId.toString().trim())); for (const season of showResponse.data.seasons @@ -193,4 +202,6 @@ export default async function handler( res.status(200).json({ status: `scraped: ${totalResultsCount} items` }); } + + await db.markAsDone(imdbId.toString().trim()); } diff --git a/src/pages/api/tvsearch.ts b/src/pages/api/tvsearch.ts index bf002d9..a8a8d2b 100644 --- a/src/pages/api/tvsearch.ts +++ b/src/pages/api/tvsearch.ts @@ -18,14 +18,19 @@ const handler: NextApiHandler = async (req, res) => { } try { - const searchResults = await db.getScrapedResults( - `tv:${imdbId.toString().trim()}:${parseInt(seasonNum.toString().trim(), 10)}` - ); + const searchResults = await db.getScrapedResults(`tv:${imdbId.toString().trim()}:${parseInt(seasonNum.toString().trim(), 10)}`); if (searchResults) { res.status(200).json({ results: searchResults }); return; } + const isProcessing = await db.keyExists(`processing:${imdbId.toString().trim()}`); + if (isProcessing) { + res.status(204).json(null); + return; + } + + await db.saveScrapedResults(`requested:${imdbId.toString().trim()}`, []); res.status(204).json(null); } catch (error: any) { console.error('encountered a db issue', error); diff --git a/src/pages/index.tsx b/src/pages/index.tsx index 2f5c7cf..05d6cad 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -16,6 +16,10 @@ function IndexPage() { router.push('/search'); }; + const handleSearchV2Click = () => { + router.push('/search-v2'); + }; + const handleLogout = () => { if (typeof window === 'undefined') { // Running on the server, return null @@ -80,6 +84,12 @@ function IndexPage() { > My Library +