From 08e7155e5254f8f31f92d017a966d8f16e230c04 Mon Sep 17 00:00:00 2001 From: SimplyBoo6 Date: Thu, 11 Jan 2024 12:01:48 +0000 Subject: [PATCH] Expanded import/export to cover playlists and deleted media --- common/src/media.d.ts | 5 +++++ server/src/database/mongodb/index.ts | 30 +++++++++++++++++++++++++--- server/src/types/database.ts | 3 +++ server/src/types/dump.d.ts | 8 +++++++- server/src/types/index.ts | 2 +- server/src/utils/export-json.ts | 14 ++++++++++++- server/src/utils/import-json.ts | 21 +++++++++++++++---- 7 files changed, 73 insertions(+), 10 deletions(-) diff --git a/common/src/media.d.ts b/common/src/media.d.ts index 483814f..ac237b8 100755 --- a/common/src/media.d.ts +++ b/common/src/media.d.ts @@ -110,3 +110,8 @@ export interface MediaResolution { export interface Media extends BaseMedia { absolutePath: string; } + +export interface DeletedMedia { + hash: string; + path: string; +} diff --git a/server/src/database/mongodb/index.ts b/server/src/database/mongodb/index.ts index 01bac2b..8872d62 100755 --- a/server/src/database/mongodb/index.ts +++ b/server/src/database/mongodb/index.ts @@ -3,6 +3,7 @@ import Path from 'path'; import { BaseMedia, Configuration, + DeletedMedia, Media, MediaPlaylist, Playlist, @@ -54,11 +55,15 @@ export class MongoConnector extends Database { if (!config) { throw new Error('database config missing'); } + if (config.provider !== 'mongodb') { + throw new Error('Missing mongo config'); + } + const mongoConfig = config; // eslint-disable-next-line no-constant-condition while (true) { try { - return await MongoClient.connect(config.uri, { + return await MongoClient.connect(mongoConfig.uri, { useNewUrlParser: true, useUnifiedTopology: true, }); @@ -74,7 +79,16 @@ export class MongoConnector extends Database { private constructor(server: MongoClient) { super(); this.server = server; - const dbName = Config.get().database?.db; + + const config = Config.get().database; + if (!config) { + throw new Error('database config missing'); + } + if (config.provider !== 'mongodb') { + throw new Error('Missing mongo config'); + } + const mongoConfig = config; + const dbName = mongoConfig.db; if (!dbName) { throw new Error('missing database config'); } @@ -603,17 +617,27 @@ export class MongoConnector extends Database { } if (media && ignoreInImport) { - await this.db.collection('media.deleted').update({ hash: media.hash }, media, { upsert: true }); + await this.addDeleted(media); } await mediaCollection.deleteOne({ hash }); } + public async addDeleted(deleted: DeletedMedia): Promise { + await this.db.collection('media.deleted').update({ hash: deleted.hash }, deleted, { upsert: true }); + } + public async isDeletedPath(path: string): Promise { const media = this.db.collection('media.deleted'); const result = await media.findOne({ path }); return Boolean(result); } + public async getDeletedMedia(): Promise { + const media = this.db.collection('media.deleted'); + const result = await media.find({}).toArray(); + return result.map((res) => ({ hash: res.hash, path: res.path })); + } + public async addMediaTag(hash: string, tag: string): Promise { await this.db.collection('media').updateOne({ hash }, { $addToSet: { tags: tag } }); } diff --git a/server/src/types/database.ts b/server/src/types/database.ts index 9655752..474b3ec 100755 --- a/server/src/types/database.ts +++ b/server/src/types/database.ts @@ -1,6 +1,7 @@ import type { BaseMedia, Configuration, + DeletedMedia, Media, MediaPlaylist, Playlist, @@ -19,6 +20,8 @@ export abstract class Database { public abstract saveBulkMedia(constraints: SubsetConstraints, media: UpdateMedia): Promise; public abstract removeMedia(hash: string, ignoreInImport: boolean): Promise; public abstract isDeletedPath(path: string): Promise; + public abstract getDeletedMedia(): Promise; + public abstract addDeleted(deleted: DeletedMedia): Promise; // Media - tags public abstract addMediaTag(hash: string, tag: string): Promise; public abstract removeMediaTag(hash: string, tag: string): Promise; diff --git a/server/src/types/dump.d.ts b/server/src/types/dump.d.ts index f187bcb..f2e0fcb 100644 --- a/server/src/types/dump.d.ts +++ b/server/src/types/dump.d.ts @@ -1,9 +1,15 @@ -import { BaseMedia, Configuration } from '@vimtur/common'; +import { BaseMedia, Configuration, DeletedMedia, Playlist } from '@vimtur/common'; + +export interface DumpPlaylist extends Playlist { + hashes: string[]; +} export interface DumpFile { tags: string[]; actors: string[]; media: BaseMedia[]; config?: Configuration.Partial; + deleted?: DeletedMedia[]; version?: number; + playlists?: DumpPlaylist[]; } diff --git a/server/src/types/index.ts b/server/src/types/index.ts index 9e5f3e5..6ed9f87 100755 --- a/server/src/types/index.ts +++ b/server/src/types/index.ts @@ -1,4 +1,4 @@ export { Database } from './database'; -export type { DumpFile } from './dump'; +export type { DumpFile, DumpPlaylist } from './dump'; export type { TaskRunnerCallback, TaskRunner, Task, RouterTask } from './task'; export type { DownloaderCallback, DownloaderRunner, Downloader } from './downloader'; diff --git a/server/src/utils/export-json.ts b/server/src/utils/export-json.ts index 83f7b58..997d69e 100755 --- a/server/src/utils/export-json.ts +++ b/server/src/utils/export-json.ts @@ -3,7 +3,7 @@ import Util from 'util'; import Config from '../config'; import { setup as setupDb } from '../database'; -import type { DumpFile } from '../types'; +import type { DumpFile, DumpPlaylist } from '../types'; async function main(): Promise { const file = process.argv[2]; @@ -20,13 +20,25 @@ async function main(): Promise { const userConfigOverlay = await db.getUserConfig(); Config.setUserOverlay(userConfigOverlay); + const outputPlaylists: DumpPlaylist[] = []; const output: DumpFile = { tags: await db.getTags(), media: [], actors: await db.getActors(), config: userConfigOverlay, + playlists: outputPlaylists, + deleted: await db.getDeletedMedia(), version: 4, }; + + const playlists = await db.getPlaylists(); + for (const playlist of playlists) { + outputPlaylists.push({ + ...playlist, + hashes: await db.subset({ playlist: playlist.id, sortBy: 'order' }), + }); + } + // Save tags const map = await db.subset({}); for (const hash of map) { diff --git a/server/src/utils/import-json.ts b/server/src/utils/import-json.ts index 50fe81b..22357bd 100755 --- a/server/src/utils/import-json.ts +++ b/server/src/utils/import-json.ts @@ -10,6 +10,7 @@ import Config from '../config'; import { setup as setupDb } from '../database'; import type { Database, DumpFile } from '../types'; +// TODO This must also cover playlists and deleted media. async function importMedia(db: Database, media: BaseMedia, version?: number): Promise { if (!media.hash) { throw new Error('Missing hash'); @@ -142,6 +143,10 @@ async function main(): Promise { console.log('Adding all to database. This can take some time.'); const start = new Date(); + console.log('Adding config...'); + if (imported.config && Object.keys(imported.config).length) { + await db.saveUserConfig(imported.config); + } console.log('Adding tags...'); for (const tag of imported.tags) { await db.addTag(tag); @@ -150,10 +155,6 @@ async function main(): Promise { for (const actor of imported.actors) { await db.addActor(actor); } - console.log('Adding config...'); - if (imported.config && Object.keys(imported.config).length) { - await db.saveUserConfig(imported.config); - } console.log('Adding media...'); let progress = 0; for (let i = 0; i < imported.media.length; i++) { @@ -175,6 +176,18 @@ async function main(): Promise { ); } } + + console.log('Adding deleted media...'); + for (const deleted of imported.deleted || []) { + await db.addDeleted(deleted); + } + + console.log('Adding playlists...'); + for (const playlist of imported.playlists || []) { + // Note this discards the ID. + await db.addPlaylist(playlist); + } + console.log('Import complete'); await db.close();