Import movies and shows from Plex (#1122)

* docs: order imports by their order of impotance

* docs: add info about plex importing

* chore(docs): remove useless line

* feat(backend): start impl for plex importing

* feat(frontend): add support for plex import

* chore(frontend): remove usage of password input

* chore(backend): change name of import source

* chore(frontend): adapt to new gql api

* feat(backend): change names of import sources to be more uniform

* chore(frontend): adapt to new gql api

* refactor(frontend): remove lot of un-needed duplicated code

* feat(services/importer): implement basic plex import

* feat(services/importer): add error handling to import process

* feat(services/integration): remove useless struct

* feat(services/importer): add error handling to import process

* fix(services/importer): use correct attribute name

* chore(services/importer): add more logging for each directory processed

* build(ts): upgrade deps

* feat(services/importer): get the view count for metadata items

* feat(services/importer): start importing movies

* feat(services/importer): start importing shows

* fix(services/importer): remove useless attribute

* fix(services/importer): de-duplicate show seen elements

* ci: Run CI

* fix(frontend): make select input searchable

* refactor(enums): change order of attributes and macros

* ci: Run CI

* chore(services/importer): change order of attributes

* chore(services/importer): remove useless annotations

* chore(services/importer): bring back annotations

* refactor(backend): change names of structs

* refactor(backend): change schema for importing data

* build(backend): upgrade tracing deps

* build(backend): hoist all deps to project root

* build(backend): port over all deps to workspace root

* build(backend): update deps

* build(backend): update deps

* chore(backend): apply new lints

* fix(utils/dependent): display correct number of item imported

* chore(backend): display the type of media being imported

* feat(migrations): add columns for import report progress

* chore(backend): adapt to new gql schema

* feat(backend): emit progress for importing correctly

* feat(services/importer): update progress of import report

* feat(gql): request import progress from backend

* refactor(frontend): extract into function

* feat(frontend): display progress of imports

* ci: Run CI

* build(backend): upgrade schematic deps

* chore(gql): collapse queries into one

* chore(backend): collapse two queries into

* chore(backend): remove useless part of query

* fix(backend): deploy job to update user workouts when name of exercise edited

* chore(services/fitness): use new api for sea orm paginator

* chore(services/fitness): use new api for sea orm paginator

* feat(backend): add basic mutation to merge exercises

* fix(services/fitness): do not iterate when not required

* refactor(services/fitness): deploy job using function

* refactor(services/fitness): extract rename implementation to a function

* feat(services/fitness): complete implementation to merge exercises

* fix(services/fitness): make sure exercise lots are same before merging

* ci: Run CI

* feat(backend): bring back old query that i removed in this PR

* refactor(backend): move exercise parameters to core details query

* chore(frontend): adapt to new gql schema

* refactor(backend): move lot to source mappings to backend

* chore(frontend): adapt to new gql schema

* feat(frontend): allow merging exercises

* refactor(frontend): common function to construct url to exercise details

* ci: Run CI

* fix(services/fitness): return only exercises which are from github or are created by the current user

* chore(services/fitness): more descriptive error

* chore(services/fitness): add another check

* feat(frontend): display alert when merging/replacing exercises

* fix(services/importer): get strong android import working again

* chore(services/importer): use debug display

* chore(services/importer): determine the exercise lot

* chore(backend): allow creating default exercises

* chore(backend): allow hashing exercises

* chore(services/importer): determine unique exercises

* feat(backend): allow importing custom exercises

* feat(services/importer): start importing exercises for story app

* feat(backend): start creating exercises for story app

* fix(backend): create unique custom exercises

* feat(backend): import strong data via generic csv

* fix(backend): no separate input for strong app

* chore(frontend): adapt to new gql schema

* feat(backend): allow importing strong measurements

* feat(frontend): adapt to new gql schema

* docs: correct instructions for importing strong app data

* feat(migrations): add source result column to import report

* feat(backend): save source result into import report

* chore(services/importer): create custom exercises with nanoid

* chore(services/importer): remove useless dep

* refactor(services/importer): extract importing exercises into a function

* refactor(services/integration): do not use fully qualified path

* build(backend): add `indexmap` deps

* feat(services/importer): new strategy to import strong app exercises

* fix(services/importer): use better candidate to determine exercise name

* docs: update strong app docs

* chore(services/importer): change order of attributes

Cargo.toml

@@ -47,7 +47,8 @@ members = [
 resolver = "2"
 
 [workspace.dependencies]
-anyhow = "=1.0.91"
+askama = "0.12.1"
+anyhow = "=1.0.93"
 apalis = { version = "=0.5.5", features = ["cron", "limit"] }
 argon2 = "=0.6.0-pre.1"
 async-graphql = { version = "=7.0.11", features = [
@@ -56,35 +57,56 @@ async-graphql = { version = "=7.0.11", features = [
     "log",
     "tracing",
 ] }
+async-graphql-axum = "=7.0.11"
 async-trait = "=0.1.83"
-aws-sdk-s3 = { version = "=1.58.0", features = ["behavior-version-latest"] }
-axum = { version = "=0.7.7", features = ["macros", "multipart"] }
+aws-sdk-s3 = { version = "=1.63.0", features = ["behavior-version-latest"] }
+axum = { version = "=0.7.9", features = ["macros", "multipart"] }
 boilermates = "=0.3.0"
 chrono = "=0.4.38"
 chrono-tz = "=0.10.0"
+compile-time = "=0.2.0"
 const-str = "=0.5.7"
 convert_case = "=0.6.0"
+csv = "=1.3.1"
+data-encoding = "=2.6.0"
 derive_more = { version = "=1.0.0", features = [
     "add",
     "add_assign",
     "sum",
 ], default-features = false }
+dotenvy = "=0.15.7"
+dotenv-build = "0.1.1"
 educe = { version = "=0.6.0", features = [
     "Debug",
     "Default",
     "full",
 ], default-features = false }
 enum_meta = "=0.7.0"
-itertools = "=0.13.0"
+eventsource-stream = "=0.2.3"
+flate2 = "=1.0.35"
 futures = "=0.3.31"
+graphql_client = "=0.14.0"
 hashbag = "=0.1.12"
+http = "=1.1.0"
+indexmap = "=2.7.0"
+indoc = "=2.0.5"
+isolang = { version = "=2.4.0", features = ["list_languages"] }
+itertools = "=0.13.0"
+jsonwebtoken = { version = "=9.3.0", default-features = false }
+lettre = "=0.11.10"
+logs-wheel = "=0.3.1"
 markdown = "=1.0.0-alpha.21"
+mime_guess = "=2.0.5"
 nanoid = "=0.4.0"
 openidconnect = "=3.5.0"
+paginate = "=1.1.11"
+radarr-api-rs = "=3.0.1"
+rand = "=0.9.0-beta.0"
 regex = "=1.11.1"
 rust_decimal = "=1.36.0"
 rust_decimal_macros = "=1.36.0"
-schematic = { version = "=0.17.5", features = [
+rust_iso3166 = "=0.1.13"
+schematic = { version = "=0.17.7", features = [
     "config",
     "env",
     "json",
@@ -98,7 +120,8 @@ schematic = { version = "=0.17.5", features = [
     "validate",
     "yaml",
 ], default-features = false }
-sea-orm = { version = "=1.1.0", features = [
+scraper = "=0.21.0"
+sea-orm = { version = "=1.1.1", features = [
     "debug-print",
     "postgres-array",
     "macros",
@@ -109,16 +132,24 @@ sea-orm = { version = "=1.1.0", features = [
     "with-rust_decimal",
     "with-uuid",
 ], default-features = false }
-sea-orm-migration = "=1.1.0"
+sea-orm-migration = "=1.1.1"
 sea-query = "=0.32.0"
-serde = { version = "=1.0.214", features = ["derive"] }
-serde_json = "=1.0.132"
+serde = { version = "=1.0.215", features = ["derive"] }
+serde_json = "=1.0.133"
 serde_with = { version = "=3.11.0", features = ["chrono_0_4"] }
+serde-xml-rs = "=0.6.0"
 slug = "=0.1.6"
+sonarr-api-rs = "=3.0.0"
 strum = { version = "=0.26.3", features = ["derive"] }
+struson = { version = "=0.6.0", features = ["serde"] }
 reqwest = { version = "=0.12.9", features = ["json", "stream"] }
-tokio = { version = "=1.41.0", features = ["full"] }
-tracing = { version = "=0.1.40", features = ["attributes"] }
+tokio = { version = "=1.41.1", features = ["full"] }
+tokio-util = { version = "=0.7.12", features = ["codec"] }
+tower = { version = "=0.5.1", features = ["buffer"] }
+tower-http = { version = "=0.6.2", features = ["catch-panic", "cors", "trace"] }
+tracing = { version = "=0.1.41", features = ["attributes"] }
+tracing-subscriber = "=0.3.19"
+unkey = "=0.5.0"
 uuid = { version = "=1.11.0", features = ["v4"], default-features = false }
 
 [profile.release]

apps/backend/Cargo.toml

@@ -10,7 +10,7 @@ anyhow = { workspace = true }
 apalis = { workspace = true }
 application-utils = { path = "../../crates/utils/application" }
 async-graphql = { workspace = true }
-async-graphql-axum = "=7.0.11"
+async-graphql-axum = { workspace = true }
 axum = { workspace = true }
 aws-sdk-s3 = { workspace = true }
 background = { path = "../../crates/background" }
@@ -23,20 +23,20 @@ common-utils = { path = "../../crates/utils/common" }
 config = { path = "../../crates/config" }
 database-models = { path = "../../crates/models/database" }
 dependent-models = { path = "../../crates/models/dependent" }
-dotenvy = "=0.15.7"
+dotenvy = { workspace = true }
 env-utils = { path = "../../crates/utils/env" }
 exporter-resolver = { path = "../../crates/resolvers/exporter" }
 exporter-service = { path = "../../crates/services/exporter" }
 file-storage-resolver = { path = "../../crates/resolvers/file-storage" }
 file-storage-service = { path = "../../crates/services/file-storage" }
 fitness-resolver = { path = "../../crates/resolvers/fitness" }
 fitness-service = { path = "../../crates/services/fitness" }
-http = "=1.1.0"
+http = { workspace = true }
 importer-resolver = { path = "../../crates/resolvers/importer" }
 importer-service = { path = "../../crates/services/importer" }
 integration-service = { path = "../../crates/services/integration" }
 itertools = { workspace = true }
-logs-wheel = "=0.3.1"
+logs-wheel = { workspace = true }
 media-models = { path = "../../crates/models/media" }
 migrations = { path = "../../crates/migrations" }
 miscellaneous-resolver = { path = "../../crates/resolvers/miscellaneous" }
@@ -54,10 +54,10 @@ statistics-resolver = { path = "../../crates/resolvers/statistics" }
 statistics-service = { path = "../../crates/services/statistics" }
 supporting-service = { path = "../../crates/services/supporting" }
 tokio = { workspace = true }
-tower = { version = "=0.4.13", features = ["buffer"] }
-tower-http = { version = "=0.5.2", features = ["catch-panic", "cors", "trace"] }
+tower = { workspace = true }
+tower-http = { workspace = true }
 tracing = { workspace = true }
-tracing-subscriber = "=0.3.18"
-unkey = "=0.5.0"
+tracing-subscriber = { workspace = true }
+unkey = { workspace = true }
 user-resolver = { path = "../../crates/resolvers/user" }
 user-service = { path = "../../crates/services/user" }

apps/frontend/app/components/fitness.tsx

@@ -45,7 +45,12 @@ import { $path } from "remix-routes";
 import { match } from "ts-pattern";
 import { withFragment } from "ufo";
 import { BaseMediaDisplayItem } from "~/components/common";
-import { FitnessEntity, dayjsLib, getSetColor } from "~/lib/generals";
+import {
+	FitnessEntity,
+	dayjsLib,
+	getExerciseDetailsPath,
+	getSetColor,
+} from "~/lib/generals";
 import { useGetRandomMantineColor, useUserUnitSystem } from "~/lib/hooks";
 import {
 	getExerciseDetailsQuery,
@@ -271,9 +276,7 @@ export const ExerciseHistory = (props: {
 													}),
 													props.exerciseIdx.toString(),
 												)
-											: $path("/fitness/exercises/item/:id", {
-													id: encodeURIComponent(exercise.name),
-												})
+											: getExerciseDetailsPath(exercise.name)
 									}
 									fw="bold"
 									lineClamp={1}
@@ -424,10 +427,8 @@ export const ExerciseDisplayItem = (props: {
 			innerRef={ref}
 			name={exerciseDetails?.id}
 			isLoading={isExerciseDetailsLoading}
-			onImageClickBehavior={$path("/fitness/exercises/item/:id", {
-				id: encodeURIComponent(props.exerciseId),
-			})}
 			imageUrl={exerciseDetails?.attributes.images.at(0)}
+			onImageClickBehavior={getExerciseDetailsPath(props.exerciseId)}
 			labels={{
 				left: isNumber(times)
 					? `${times} time${times > 1 ? "s" : ""}`

apps/frontend/app/lib/generals.ts

@@ -32,6 +32,7 @@ import timezone from "dayjs/plugin/timezone";
 import utc from "dayjs/plugin/utc";
 import { GraphQLClient } from "graphql-request";
 import Cookies from "js-cookie";
+import { $path } from "remix-routes";
 import { match } from "ts-pattern";
 import { z } from "zod";
 
@@ -382,9 +383,6 @@ const fitnessQueryKeys = createQueryKeys("fitness", {
 	workoutDetails: (workoutId: string) => ({
 		queryKey: ["workoutDetails", workoutId],
 	}),
-	exerciseParameters: () => ({
-		queryKey: ["exerciseParameters"],
-	}),
 	workoutTemplateDetails: (workoutTemplateId: string) => ({
 		queryKey: ["workoutTemplateDetails", workoutTemplateId],
 	}),
@@ -452,3 +450,8 @@ export const refreshUserMetadataDetails = (metadataId: string) =>
 			queryKey: queryFactory.media.userMetadataDetails(metadataId).queryKey,
 		});
 	}, 1500);
+
+export const getExerciseDetailsPath = (exerciseId: string) =>
+	$path("/fitness/exercises/item/:id", {
+		id: encodeURIComponent(exerciseId),
+	});

apps/frontend/app/lib/state/fitness.ts

@@ -261,6 +261,10 @@ const measurementsDrawerOpenAtom = atom(false);
 export const useMeasurementsDrawerOpen = () =>
 	useAtom(measurementsDrawerOpenAtom);
 
+export const mergingExerciseAtom = atom<string | null>(null);
+
+export const useMergingExercise = () => useAtom(mergingExerciseAtom);
+
 export const duplicateOldWorkout = async (
 	name: string,
 	fitnessEntity: FitnessAction,

apps/frontend/app/lib/utilities.server.ts

@@ -9,7 +9,6 @@ import {
 import {
 	BackendError,
 	CoreDetailsDocument,
-	ExerciseParametersDocument,
 	GetPresignedS3UrlDocument,
 	PresignedPutS3UrlDocument,
 	UserCollectionsListDocument,
@@ -194,16 +193,6 @@ const getCachedUserDetails = async (request: Request) => {
 	});
 };
 
-export const getCachedExerciseParameters = async () => {
-	return queryClient.ensureQueryData({
-		queryKey: queryFactory.fitness.exerciseParameters().queryKey,
-		queryFn: () =>
-			serverGqlService
-				.request(ExerciseParametersDocument)
-				.then((data) => data.exerciseParameters),
-	});
-};
-
 export const getCachedUserPreferences = async (request: Request) => {
 	const userDetails = await redirectIfNotAuthenticatedOrUpdated(request);
 	return userDetails.preferences;

apps/frontend/app/routes/_dashboard.fitness.$action.tsx

@@ -110,6 +110,7 @@ import {
 	PRO_REQUIRED_MESSAGE,
 	clientGqlService,
 	dayjsLib,
+	getExerciseDetailsPath,
 	getSetColor,
 	getSurroundingElements,
 	postMessageToServiceWorker,
@@ -1115,12 +1116,10 @@ const ExerciseDisplay = (props: {
 					<Menu shadow="md" width={200} position="left-end">
 						<Group justify="space-between" pos="relative" wrap="nowrap">
 							<Anchor
-								component={Link}
-								to={$path("/fitness/exercises/item/:id", {
-									id: encodeURIComponent(exercise.exerciseId),
-								})}
 								fw="bold"
 								lineClamp={1}
+								component={Link}
+								to={getExerciseDetailsPath(exercise.exerciseId)}
 							>
 								{exercise.exerciseId}
 							</Anchor>

apps/frontend/app/routes/_dashboard.fitness.exercises.$action.tsx

@@ -44,6 +44,7 @@ import { match } from "ts-pattern";
 import { withQuery } from "ufo";
 import { z } from "zod";
 import { zx } from "zodix";
+import { getExerciseDetailsPath } from "~/lib/generals";
 import { useCoreDetails } from "~/lib/hooks";
 import {
 	createToastHeaders,
@@ -115,11 +116,7 @@ export const action = async ({ request }: ActionFunctionArgs) => {
 						CreateCustomExerciseDocument,
 						{ input },
 					);
-				return redirect(
-					$path("/fitness/exercises/item/:id", {
-						id: encodeURIComponent(createCustomExercise),
-					}),
-				);
+				return redirect(getExerciseDetailsPath(createCustomExercise));
 			})
 			.with(Action.Update, async () => {
 				invariant(submission.oldName);