Skip to content
This repository has been archived by the owner on Jan 31, 2024. It is now read-only.

Commit

Permalink
Merge branch 'develop' into import-more-key-notes
Browse files Browse the repository at this point in the history
  • Loading branch information
Marie authored Nov 29, 2023
2 parents 93f18d7 + 62bcd42 commit 94a3579
Show file tree
Hide file tree
Showing 7 changed files with 96 additions and 22 deletions.
84 changes: 72 additions & 12 deletions packages/backend/src/queue/processors/ImportNotesProcessorService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import * as vm from 'node:vm';
import { Inject, Injectable } from '@nestjs/common';
import { ZipReader } from 'slacc';
import { DI } from '@/di-symbols.js';
import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository, MiUser } from '@/models/_.js';
import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository, MiUser, DriveFoldersRepository, MiDriveFolder } from '@/models/_.js';
import type Logger from '@/logger.js';
import { DownloadService } from '@/core/DownloadService.js';
import { bindThis } from '@/decorators.js';
Expand All @@ -14,6 +14,7 @@ import { DriveService } from '@/core/DriveService.js';
import { MfmService } from '@/core/MfmService.js';
import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js';
import { extractApHashtagObjects } from '@/core/activitypub/models/tag.js';
import { IdService } from '@/core/IdService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbKeyNoteImportToDbJobData } from '../types.js';
Expand All @@ -29,6 +30,9 @@ export class ImportNotesProcessorService {
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,

@Inject(DI.driveFoldersRepository)
private driveFoldersRepository: DriveFoldersRepository,

@Inject(DI.notesRepository)
private notesRepository: NotesRepository,

Expand All @@ -38,20 +42,21 @@ export class ImportNotesProcessorService {
private apNoteService: ApNoteService,
private driveService: DriveService,
private downloadService: DownloadService,
private idService: IdService,
private queueLoggerService: QueueLoggerService,
) {
this.logger = this.queueLoggerService.logger.createSubLogger('import-notes');
}

@bindThis
private async uploadFiles(dir: string, user: MiUser) {
private async uploadFiles(dir: string, user: MiUser, folder?: MiDriveFolder['id']) {
const fileList = fs.readdirSync(dir);
for await (const file of fileList) {
const name = `${dir}/${file}`;
if (fs.statSync(name).isDirectory()) {
await this.uploadFiles(name, user);
await this.uploadFiles(name, user, folder);
} else {
const exists = await this.driveFilesRepository.findOneBy({ name: file, userId: user.id });
const exists = await this.driveFilesRepository.findOneBy({ name: file, userId: user.id, folderId: folder });

if (file.endsWith('.srt')) return;

Expand All @@ -60,6 +65,7 @@ export class ImportNotesProcessorService {
user: user,
path: name,
name: file,
folderId: folder,
});
}
}
Expand Down Expand Up @@ -134,6 +140,12 @@ export class ImportNotesProcessorService {
return;
}

let folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id });
if (folder == null) {
await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Imports', userId: job.data.user.id });
folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id });
}

const type = job.data.type;

if (type === 'Twitter' || file.name.startsWith('twitter') && file.name.endsWith('.zip')) {
Expand Down Expand Up @@ -172,7 +184,7 @@ export class ImportNotesProcessorService {
const tweets = Object.keys(fakeWindow.window.YTD.tweets.part0).reduce((m, key, i, obj) => {
return m.concat(fakeWindow.window.YTD.tweets.part0[key].tweet);
}, []);
const processedTweets = await this.recreateChain("id_str", "in_reply_to_status_id_str", tweets, false);
const processedTweets = await this.recreateChain('id_str', 'in_reply_to_status_id_str', tweets, false);
this.queueService.createImportTweetsToDbJob(job.data.user, processedTweets, null);
} finally {
cleanup();
Expand Down Expand Up @@ -200,7 +212,12 @@ export class ImportNotesProcessorService {
ZipReader.withDestinationPath(outputPath).viaBuffer(await fs.promises.readFile(destPath));
const postsJson = fs.readFileSync(outputPath + '/your_activity_across_facebook/posts/your_posts__check_ins__photos_and_videos_1.json', 'utf-8');
const posts = JSON.parse(postsJson);
await this.uploadFiles(outputPath + '/your_activity_across_facebook/posts/media', user);
const facebookFolder = await this.driveFoldersRepository.findOneBy({ name: 'Facebook', userId: job.data.user.id, parentId: folder?.id });
if (facebookFolder == null && folder) {
await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Facebook', userId: job.data.user.id, parentId: folder.id });
const createdFolder = await this.driveFoldersRepository.findOneBy({ name: 'Facebook', userId: job.data.user.id, parentId: folder.id });
if (createdFolder) await this.uploadFiles(outputPath + '/your_activity_across_facebook/posts/media', user, createdFolder.id);
}
this.queueService.createImportFBToDbJob(job.data.user, posts);
} finally {
cleanup();
Expand Down Expand Up @@ -231,7 +248,12 @@ export class ImportNotesProcessorService {
if (isInstagram) {
const postsJson = fs.readFileSync(outputPath + '/content/posts_1.json', 'utf-8');
const posts = JSON.parse(postsJson);
await this.uploadFiles(outputPath + '/media/posts', user);
const igFolder = await this.driveFoldersRepository.findOneBy({ name: 'Instagram', userId: job.data.user.id, parentId: folder?.id });
if (igFolder == null && folder) {
await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Instagram', userId: job.data.user.id, parentId: folder.id });
const createdFolder = await this.driveFoldersRepository.findOneBy({ name: 'Instagram', userId: job.data.user.id, parentId: folder.id });
if (createdFolder) await this.uploadFiles(outputPath + '/media/posts', user, createdFolder.id);
}
this.queueService.createImportIGToDbJob(job.data.user, posts);
} else if (isOutbox) {
const actorJson = fs.readFileSync(outputPath + '/actor.json', 'utf-8');
Expand All @@ -244,7 +266,14 @@ export class ImportNotesProcessorService {
} else {
const outboxJson = fs.readFileSync(outputPath + '/outbox.json', 'utf-8');
const outbox = JSON.parse(outboxJson);
if (fs.existsSync(outputPath + '/media_attachments/files')) await this.uploadFiles(outputPath + '/media_attachments/files', user);
let mastoFolder = await this.driveFoldersRepository.findOneBy({ name: 'Mastodon', userId: job.data.user.id, parentId: folder?.id });
if (mastoFolder == null && folder) {
await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Mastodon', userId: job.data.user.id, parentId: folder.id });
mastoFolder = await this.driveFoldersRepository.findOneBy({ name: 'Mastodon', userId: job.data.user.id, parentId: folder.id });
}
if (fs.existsSync(outputPath + '/media_attachments/files') && mastoFolder) {
await this.uploadFiles(outputPath + '/media_attachments/files', user, mastoFolder.id);
}
this.queueService.createImportMastoToDbJob(job.data.user, outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note'));
}
}
Expand All @@ -268,7 +297,7 @@ export class ImportNotesProcessorService {

const notesJson = fs.readFileSync(path, 'utf-8');
const notes = JSON.parse(notesJson);
const processedNotes = await this.recreateChain("id", "replyId", notes, true);
const processedNotes = await this.recreateChain('id', 'replyId', notes, true);
this.queueService.createImportKeyNotesToDbJob(job.data.user, processedNotes, null);
cleanup();
}
Expand All @@ -288,16 +317,25 @@ export class ImportNotesProcessorService {

const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null;

const folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id });
if (folder == null) return;

const files: MiDriveFile[] = [];
const date = new Date(note.createdAt);

if (note.files && this.isIterable(note.files)) {
let keyFolder = await this.driveFoldersRepository.findOneBy({ name: 'Misskey', userId: job.data.user.id, parentId: folder.id });
if (keyFolder == null) {
await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Misskey', userId: job.data.user.id, parentId: folder.id });
keyFolder = await this.driveFoldersRepository.findOneBy({ name: 'Misskey', userId: job.data.user.id, parentId: folder.id });
}

for await (const file of note.files) {
const [filePath, cleanup] = await createTemp();
const slashdex = file.url.lastIndexOf('/');
const name = file.url.substring(slashdex + 1);

const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id });
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: name, userId: user.id, folderId: keyFolder?.id });

if (!exists) {
try {
Expand All @@ -309,6 +347,7 @@ export class ImportNotesProcessorService {
user: user,
path: filePath,
name: name,
folderId: keyFolder?.id,
});
files.push(driveFile);
} else {
Expand Down Expand Up @@ -381,6 +420,9 @@ export class ImportNotesProcessorService {
const files: MiDriveFile[] = [];
let reply: MiNote | null = null;

const folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id });
if (folder == null) return;

if (post.object.inReplyTo != null) {
try {
reply = await this.apNoteService.resolveNote(post.object.inReplyTo);
Expand All @@ -400,12 +442,18 @@ export class ImportNotesProcessorService {
}

if (post.object.attachment && this.isIterable(post.object.attachment)) {
let pleroFolder = await this.driveFoldersRepository.findOneBy({ name: 'Pleroma', userId: job.data.user.id, parentId: folder.id });
if (pleroFolder == null) {
await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Pleroma', userId: job.data.user.id, parentId: folder.id });
pleroFolder = await this.driveFoldersRepository.findOneBy({ name: 'Pleroma', userId: job.data.user.id, parentId: folder.id });
}

for await (const file of post.object.attachment) {
const slashdex = file.url.lastIndexOf('/');
const name = file.url.substring(slashdex + 1);
const [filePath, cleanup] = await createTemp();

const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id });
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: name, userId: user.id, folderId: pleroFolder?.id });

if (!exists) {
try {
Expand All @@ -417,6 +465,7 @@ export class ImportNotesProcessorService {
user: user,
path: filePath,
name: name,
folderId: pleroFolder?.id,
});
files.push(driveFile);
} else {
Expand Down Expand Up @@ -483,6 +532,9 @@ export class ImportNotesProcessorService {
return;
}

const folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id });
if (folder == null) return;

const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null;

async function replaceTwitterUrls(full_text: string, urls: any) {
Expand All @@ -508,13 +560,19 @@ export class ImportNotesProcessorService {
const files: MiDriveFile[] = [];

if (tweet.extended_entities && this.isIterable(tweet.extended_entities.media)) {
let twitFolder = await this.driveFoldersRepository.findOneBy({ name: 'Twitter', userId: job.data.user.id, parentId: folder.id });
if (twitFolder == null) {
await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Twitter', userId: job.data.user.id, parentId: folder.id });
twitFolder = await this.driveFoldersRepository.findOneBy({ name: 'Twitter', userId: job.data.user.id, parentId: folder.id });
}

for await (const file of tweet.extended_entities.media) {
if (file.video_info) {
const [filePath, cleanup] = await createTemp();
const slashdex = file.video_info.variants[0].url.lastIndexOf('/');
const name = file.video_info.variants[0].url.substring(slashdex + 1);

const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id });
const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: name, userId: user.id, folderId: twitFolder?.id });

const videos = file.video_info.variants.filter((x: any) => x.content_type === 'video/mp4');

Expand All @@ -528,6 +586,7 @@ export class ImportNotesProcessorService {
user: user,
path: filePath,
name: name,
folderId: twitFolder?.id,
});
files.push(driveFile);
} else {
Expand All @@ -553,6 +612,7 @@ export class ImportNotesProcessorService {
user: user,
path: filePath,
name: name,
folderId: twitFolder?.id,
});
files.push(driveFile);
} else {
Expand Down
2 changes: 1 addition & 1 deletion packages/frontend/src/components/MkNote.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ const translating = ref(false);
const showTicker = (defaultStore.state.instanceTicker === 'always') || (defaultStore.state.instanceTicker === 'remote' && appearNote.user.instance);
const canRenote = computed(() => ['public', 'home'].includes(appearNote.visibility) || (appearNote.visibility === 'followers' && appearNote.userId === $i.id));
let renoteCollapsed = $ref(defaultStore.state.collapseRenotes && isRenote && (($i && ($i.id === note.userId || $i.id === appearNote.userId)) || (appearNote.myReaction != null)));
const defaultLike = computed(() => defaultStore.state.like !== '❤️' ? defaultStore.state.like : null);
const defaultLike = computed(() => defaultStore.state.like ? defaultStore.state.like : null);

const keymap = {
'r': () => reply(true),
Expand Down
2 changes: 1 addition & 1 deletion packages/frontend/src/components/MkNoteDetailed.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@ const conversation = ref<Misskey.entities.Note[]>([]);
const replies = ref<Misskey.entities.Note[]>([]);
const quotes = ref<Misskey.entities.Note[]>([]);
const canRenote = computed(() => ['public', 'home'].includes(appearNote.visibility) || appearNote.userId === $i.id);
const defaultLike = computed(() => defaultStore.state.like !== '❤️' ? defaultStore.state.like : null);
const defaultLike = computed(() => defaultStore.state.like ? defaultStore.state.like : null);

watch(() => props.expandAllCws, (expandAllCws) => {
if (expandAllCws !== showContent.value) showContent.value = expandAllCws;
Expand Down
2 changes: 1 addition & 1 deletion packages/frontend/src/components/MkNoteSimple.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ SPDX-License-Identifier: AGPL-3.0-only
<div>
<p v-if="note.cw != null" :class="$style.cw">
<Mfm v-if="note.cw != ''" style="margin-right: 8px;" :text="note.cw" :author="note.user" :nyaize="'respect'" :emojiUrls="note.emojis"/>
<MkCwButton v-model="showContent" :note="note"/>
<MkCwButton v-model="showContent" :note="note" v-on:click.stop/>
</p>
<div v-show="note.cw == null || showContent">
<MkSubNoteContent :hideFiles="hideFiles" :class="$style.text" :note="note"/>
Expand Down
2 changes: 1 addition & 1 deletion packages/frontend/src/components/MkNoteSub.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ const menuButton = shallowRef<HTMLElement>();
const likeButton = shallowRef<HTMLElement>();

let appearNote = $computed(() => isRenote ? props.note.renote as Misskey.entities.Note : props.note);
const defaultLike = computed(() => defaultStore.state.like !== '❤️' ? defaultStore.state.like : null);
const defaultLike = computed(() => defaultStore.state.like ? defaultStore.state.like : null);

const isRenote = (
props.note.renote != null &&
Expand Down
24 changes: 19 additions & 5 deletions packages/frontend/src/pages/settings/reaction.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ SPDX-License-Identifier: AGPL-3.0-only

<FromSlot>
<template #label>Default like emoji</template>
<MkCustomEmoji v-if="like.startsWith(':')" style="max-height: 3em; font-size: 1.1em;" :useOriginalSize="false" :class="$style.reaction" :name="like" :normal="true" :noStyle="true"/>
<MkEmoji v-else :emoji="like" style="max-height: 3em; font-size: 1.1em;" :normal="true" :noStyle="true"/>
<MkCustomEmoji v-if="like && like.startsWith(':')" style="max-height: 3em; font-size: 1.1em;" :useOriginalSize="false" :class="$style.reaction" :name="like" :normal="true" :noStyle="true"/>
<MkEmoji v-else-if="like && !like.startsWith(':')" :emoji="like" style="max-height: 3em; font-size: 1.1em;" :normal="true" :noStyle="true"/>
<span v-else-if="!like">{{ i18n.ts.notSet }}</span>
<div class="_buttons" style="padding-top: 8px;">
<MkButton rounded :small="true" inline @click="chooseNewLike"><i class="ph-smiley ph-bold ph-lg"></i> Change</MkButton>
<MkButton rounded :small="true" inline @click="resetLike"><i class="ph-arrow-clockwise ph-bold ph-lg"></i> Reset</MkButton>
Expand Down Expand Up @@ -82,6 +83,7 @@ import { defaultStore } from '@/store.js';
import { i18n } from '@/i18n.js';
import { definePageMetadata } from '@/scripts/page-metadata.js';
import { deepClone } from '@/scripts/clone.js';
import { unisonReload } from '@/scripts/unison-reload.js';

let reactions = $ref(deepClone(defaultStore.state.reactions));
const like = $computed(defaultStore.makeGetterSetter('like'));
Expand All @@ -91,6 +93,16 @@ const reactionPickerWidth = $computed(defaultStore.makeGetterSetter('reactionPic
const reactionPickerHeight = $computed(defaultStore.makeGetterSetter('reactionPickerHeight'));
const reactionPickerUseDrawerForMobile = $computed(defaultStore.makeGetterSetter('reactionPickerUseDrawerForMobile'));

async function reloadAsk() {
const { canceled } = await os.confirm({
type: 'info',
text: i18n.ts.reloadToApplySetting,
});
if (canceled) return;

unisonReload();
}

function save() {
defaultStore.set('reactions', reactions);
}
Expand Down Expand Up @@ -134,13 +146,15 @@ function chooseEmoji(ev: MouseEvent) {
function chooseNewLike(ev: MouseEvent) {
os.pickEmoji(ev.currentTarget ?? ev.target, {
showPinned: false,
}).then(emoji => {
}).then(async emoji => {
defaultStore.set('like', emoji as string);
await reloadAsk();
});
}

function resetLike() {
defaultStore.set('like', '❤️');
async function resetLike() {
defaultStore.set('like', null);
await reloadAsk();
}

watch($$(reactions), () => {
Expand Down
2 changes: 1 addition & 1 deletion packages/frontend/src/store.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ export const defaultStore = markRaw(new Storage('base', {
},
like: {
where: 'account',
default: '❤️',
default: null as string | null,
},
mutedAds: {
where: 'account',
Expand Down

0 comments on commit 94a3579

Please sign in to comment.