Merge Beta3

.github/workflows/docker.yml

@@ -1,6 +1,12 @@
 name: Publish Docker image
 
 on:
+  push:
+    branches:
+      - stable
+    paths:
+      - packages/**
+      - locales/**
   release:
     types: [published]
   workflow_dispatch:

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "sharkey",
-	"version": "2023.9.1.beta2",
+	"version": "2023.9.1.beta3",
 	"codename": "shonk",
 	"repository": {
 		"type": "git",

packages/backend/src/core/entities/NoteEntityService.ts

@@ -343,6 +343,9 @@ export class NoteEntityService implements OnModuleInit {
 			uri: note.uri ?? undefined,
 			url: note.url ?? undefined,
 			updatedAt: note.updatedAt != null ? note.updatedAt.toISOString() : undefined,
+			...(meId ? {
+				myReaction: this.populateMyReaction(note, meId, options?._hint_),
+			} : {}),
 
 			...(opts.detail ? {
 				clippedCount: note.clippedCount,
@@ -358,10 +361,6 @@ export class NoteEntityService implements OnModuleInit {
 				}) : undefined,
 
 				poll: note.hasPoll ? this.populatePoll(note, meId) : undefined,
-
-				...(meId ? {
-					myReaction: this.populateMyReaction(note, meId, options?._hint_),
-				} : {}),
 			} : {}),
 		});
 

packages/backend/src/server/api/endpoints/ap/show.ts

@@ -27,7 +27,7 @@ export const meta = {
 	requireCredential: true,
 
 	limit: {
-		duration: ms('1hour'),
+		duration: ms('1minute'),
 		max: 30,
 	},
 

packages/backend/src/server/api/mastodon/MastodonApiServerService.ts

@@ -3,7 +3,7 @@ import megalodon, { Entity, MegalodonInterface } from 'megalodon';
 import querystring from 'querystring';
 import { IsNull } from 'typeorm';
 import multer from 'fastify-multer';
-import type { UsersRepository } from '@/models/_.js';
+import type { NotesRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js';
 import { DI } from '@/di-symbols.js';
 import { bindThis } from '@/decorators.js';
 import type { Config } from '@/config.js';
@@ -12,6 +12,7 @@ import { convertId, IdConvertType as IdType, convertAccount, convertAnnouncement
 import { getInstance } from './endpoints/meta.js';
 import { ApiAuthMastodon, ApiAccountMastodon, ApiFilterMastodon, ApiNotifyMastodon, ApiSearchMastodon, ApiTimelineMastodon, ApiStatusMastodon } from './endpoints.js';
 import type { FastifyInstance, FastifyPluginOptions } from 'fastify';
+import { UserEntityService } from '@/core/entities/UserEntityService.js';
 
 export function getClient(BASE_URL: string, authorization: string | undefined): MegalodonInterface {
 	const accessTokenArr = authorization?.split(' ') ?? [null];
@@ -26,9 +27,14 @@ export class MastodonApiServerService {
 	constructor(
         @Inject(DI.usersRepository)
         private usersRepository: UsersRepository,
+		@Inject(DI.notesRepository)
+        private notesRepository: NotesRepository,
+		@Inject(DI.userProfilesRepository)
+		private userProfilesRepository: UserProfilesRepository,
         @Inject(DI.config)
         private config: Config,
         private metaService: MetaService,
+		private userEntityService: UserEntityService,
 	) { }
 
 	@bindThis
@@ -256,8 +262,10 @@ export class MastodonApiServerService {
 			const client = getClient(BASE_URL, accessTokens); // we are using this here, because in private mode some info isnt
 			// displayed without being logged in
 			try {
-				const account = new ApiAccountMastodon(_request, client, BASE_URL);
-				reply.send(await account.lookup());
+				const data = await client.search((_request.query as any).acct, { type: 'accounts' });
+				const profile = await this.userProfilesRepository.findOneBy({ userId: data.data.accounts[0].id });
+				data.data.accounts[0].fields = profile?.fields.map(f => ({ ...f, verified_at: null })) || [];
+				reply.send(convertAccount(data.data.accounts[0]));
 			} catch (e: any) {
 				/* console.error(e); */
 				reply.code(401).send(e.response.data);
@@ -294,6 +302,8 @@ export class MastodonApiServerService {
 			try {
 				const sharkId = convertId(_request.params.id, IdType.SharkeyId);
 				const data = await client.getAccount(sharkId);
+				const profile = await this.userProfilesRepository.findOneBy({ userId: sharkId });
+				data.data.fields = profile?.fields.map(f => ({ ...f, verified_at: null })) || [];
 				reply.send(convertAccount(data.data));
 			} catch (e: any) {
 				/* console.error(e);
@@ -744,7 +754,7 @@ export class MastodonApiServerService {
 		//#endregion
 
 		//#region Timelines
-		const TLEndpoint = new ApiTimelineMastodon(fastify);
+		const TLEndpoint = new ApiTimelineMastodon(fastify, this.config, this.usersRepository, this.notesRepository, this.userEntityService);
 
 		// GET Endpoints
 		TLEndpoint.getTL();
@@ -769,7 +779,7 @@ export class MastodonApiServerService {
 		//#endregion
 
 		//#region Status
-		const NoteEndpoint = new ApiStatusMastodon(fastify);
+		const NoteEndpoint = new ApiStatusMastodon(fastify, this.config, this.usersRepository, this.notesRepository, this.userEntityService);
 
 		// GET Endpoints
 		NoteEndpoint.getStatus();

packages/backend/src/server/api/mastodon/converters.ts

@@ -1,4 +1,14 @@
+import type { Config } from '@/config.js';
+import { MfmService } from '@/core/MfmService.js';
+import { DI } from '@/di-symbols.js';
+import { Inject } from '@nestjs/common';
 import { Entity } from 'megalodon';
+import { parse } from 'mfm-js';
+import { GetterService } from '../GetterService.js';
+import type { IMentionedRemoteUsers } from '@/models/Note.js';
+import type { MiUser } from '@/models/User.js';
+import type { NotesRepository, UsersRepository } from '@/models/_.js';
+import { UserEntityService } from '@/core/entities/UserEntityService.js';
 
 const CHAR_COLLECTION = '0123456789abcdefghijklmnopqrstuvwxyz';
 
@@ -7,6 +17,91 @@ export enum IdConvertType {
     SharkeyId,
 }
 
+export const escapeMFM = (text: string): string => text
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&#39;")
+    .replace(/`/g, "&#x60;")
+    .replace(/\r?\n/g, "<br>");
+
+export class MastoConverters {
+	private MfmService: MfmService;
+	private GetterService: GetterService;
+
+	constructor(
+		@Inject(DI.config)
+		private config: Config,
+
+		@Inject(DI.usersRepository)
+		private usersRepository: UsersRepository,
+
+		@Inject(DI.notesRepository)
+		private notesRepository: NotesRepository,
+
+		private userEntityService: UserEntityService
+	) {
+		this.MfmService = new MfmService(this.config);
+		this.GetterService = new GetterService(this.usersRepository, this.notesRepository, this.userEntityService);
+	}
+
+	private encode(u: MiUser, m: IMentionedRemoteUsers): MastodonEntity.Mention {
+		let acct = u.username;
+		let acctUrl = `https://${u.host || this.config.host}/@${u.username}`;
+		let url: string | null = null;
+		if (u.host) {
+			const info = m.find(r => r.username === u.username && r.host === u.host);
+			acct = `${u.username}@${u.host}`;
+			acctUrl = `https://${u.host}/@${u.username}`;
+			if (info) url = info.url ?? info.uri;
+		}
+		return {
+			id: u.id,
+			username: u.username,
+			acct: acct,
+			url: url ?? acctUrl,
+		};
+	}
+
+	public async getUser(id: string): Promise<MiUser> {
+		return this.GetterService.getUser(id).then(p => {
+			return p;
+		});
+	}
+
+	public async convertStatus(status: Entity.Status) {
+		status.account = convertAccount(status.account);
+		const note = await this.GetterService.getNote(status.id);
+		status.id = convertId(status.id, IdConvertType.MastodonId);
+		if (status.in_reply_to_account_id) status.in_reply_to_account_id = convertId(
+			status.in_reply_to_account_id,
+			IdConvertType.MastodonId,
+		);
+		if (status.in_reply_to_id) status.in_reply_to_id = convertId(status.in_reply_to_id, IdConvertType.MastodonId);
+		status.media_attachments = status.media_attachments.map((attachment) =>
+			convertAttachment(attachment),
+		);
+		// This will eventually be improved with a rewrite of this file
+		const mentions = Promise.all(note.mentions.map(p =>
+			this.getUser(p)
+				.then(u => this.encode(u, JSON.parse(note.mentionedRemoteUsers)))
+				.catch(() => null)))
+			.then(p => p.filter(m => m)) as Promise<MastodonEntity.Mention[]>;
+		status.mentions = await mentions;
+		status.mentions = status.mentions.map((mention) => ({
+			...mention,
+			id: convertId(mention.id, IdConvertType.MastodonId),
+		}));
+		const convertedMFM = this.MfmService.toHtml(parse(status.content), JSON.parse(note.mentionedRemoteUsers));
+		status.content = status.content ? convertedMFM?.replace(/&amp;/g, "&").replaceAll(`<span>&</span><a href="${this.config.url}/tags/39;" rel="tag">#39;</a>`, "<span>\'</span>") as string : status.content;
+		if (status.poll) status.poll = convertPoll(status.poll);
+		if (status.reblog) status.reblog = convertStatus(status.reblog);
+
+		return status;
+	}
+}
+
 export function convertId(in_id: string, id_convert_type: IdConvertType): string {
 	switch (id_convert_type) {
 		case IdConvertType.MastodonId: {

packages/backend/src/server/api/mastodon/endpoints/account.ts

@@ -63,7 +63,7 @@ export class ApiAccountMastodon {
 			const data = await this.client.search((this.request.query as any).acct, { type: 'accounts' });
 			return convertAccount(data.data.accounts[0]);
 		} catch (e: any) {
-			/* console.error(e);
+			/* console.error(e)
 			console.error(e.response.data); */
 			return e.response;
 		}

packages/backend/src/server/api/mastodon/endpoints/status.ts

@@ -1,10 +1,13 @@
 import querystring from 'querystring';
 import { emojiRegexAtStartToEnd } from '@/misc/emoji-regex.js';
-import { convertId, IdConvertType as IdType, convertAccount, convertAttachment, convertPoll, convertStatus, convertStatusSource } from '../converters.js';
+import { convertId, IdConvertType as IdType, convertAccount, convertAttachment, convertPoll, convertStatusSource, MastoConverters } from '../converters.js';
 import { getClient } from '../MastodonApiServerService.js';
 import { convertTimelinesArgsId, limitToInt } from './timeline.js';
 import type { Entity } from 'megalodon';
 import type { FastifyInstance } from 'fastify';
+import type { Config } from '@/config.js';
+import { NotesRepository, UsersRepository } from '@/models/_.js';
+import { UserEntityService } from '@/core/entities/UserEntityService.js';
 
 function normalizeQuery(data: any) {
 	const str = querystring.stringify(data);
@@ -13,9 +16,11 @@ function normalizeQuery(data: any) {
 
 export class ApiStatusMastodon {
 	private fastify: FastifyInstance;
+	private mastoconverter: MastoConverters;
 
-	constructor(fastify: FastifyInstance) {
+	constructor(fastify: FastifyInstance, config: Config, usersrepo: UsersRepository, notesrepo: NotesRepository, userentity: UserEntityService) {
 		this.fastify = fastify;
+		this.mastoconverter = new MastoConverters(config, usersrepo, notesrepo, userentity);
 	}
 
 	public async getStatus() {
@@ -25,7 +30,7 @@ export class ApiStatusMastodon {
 			const client = getClient(BASE_URL, accessTokens);
 			try {
 				const data = await client.getStatus(convertId(_request.params.id, IdType.SharkeyId));
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(_request.is404 ? 404 : 401).send(e.response.data);
@@ -59,8 +64,8 @@ export class ApiStatusMastodon {
 					convertId(_request.params.id, IdType.SharkeyId),
 					convertTimelinesArgsId(limitToInt(query)),
 				);
-				data.data.ancestors = data.data.ancestors.map((status: Entity.Status) => convertStatus(status));
-				data.data.descendants = data.data.descendants.map((status: Entity.Status) => convertStatus(status));
+				data.data.ancestors = await Promise.all(data.data.ancestors.map(async (status: Entity.Status) => await this.mastoconverter.convertStatus(status)));
+				data.data.descendants = await Promise.all(data.data.descendants.map(async (status: Entity.Status) => await this.mastoconverter.convertStatus(status)));
 				reply.send(data.data);
 			} catch (e: any) {
 				console.error(e);
@@ -219,7 +224,7 @@ export class ApiStatusMastodon {
 				}
 
 				const data = await client.postStatus(text, body);
-				reply.send(convertStatus(data.data as Entity.Status));
+				reply.send(await this.mastoconverter.convertStatus(data.data as Entity.Status));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);
@@ -240,7 +245,7 @@ export class ApiStatusMastodon {
 					body.media_ids = (body.media_ids as string[]).map((p) => convertId(p, IdType.SharkeyId));
 				}
 				const data = await client.editStatus(convertId(_request.params.id, IdType.SharkeyId), body);
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(_request.is404 ? 404 : 401).send(e.response.data);
@@ -258,7 +263,7 @@ export class ApiStatusMastodon {
 					convertId(_request.params.id, IdType.SharkeyId),
 					'❤',
 				)) as any;
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);
@@ -276,7 +281,7 @@ export class ApiStatusMastodon {
 					convertId(_request.params.id, IdType.SharkeyId),
 					'❤',
 				);
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);
@@ -291,7 +296,7 @@ export class ApiStatusMastodon {
 			const client = getClient(BASE_URL, accessTokens);
 			try {
 				const data = await client.reblogStatus(convertId(_request.params.id, IdType.SharkeyId));
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);
@@ -306,7 +311,7 @@ export class ApiStatusMastodon {
 			const client = getClient(BASE_URL, accessTokens);
 			try {
 				const data = await client.unreblogStatus(convertId(_request.params.id, IdType.SharkeyId));
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);
@@ -321,7 +326,7 @@ export class ApiStatusMastodon {
 			const client = getClient(BASE_URL, accessTokens);
 			try {
 				const data = await client.bookmarkStatus(convertId(_request.params.id, IdType.SharkeyId));
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);
@@ -336,7 +341,7 @@ export class ApiStatusMastodon {
 			const client = getClient(BASE_URL, accessTokens);
 			try {
 				const data = await client.unbookmarkStatus(convertId(_request.params.id, IdType.SharkeyId));
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);
@@ -351,7 +356,7 @@ export class ApiStatusMastodon {
 			const client = getClient(BASE_URL, accessTokens);
 			try {
 				const data = await client.pinStatus(convertId(_request.params.id, IdType.SharkeyId));
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);
@@ -366,7 +371,7 @@ export class ApiStatusMastodon {
 			const client = getClient(BASE_URL, accessTokens);
 			try {
 				const data = await client.unpinStatus(convertId(_request.params.id, IdType.SharkeyId));
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);
@@ -381,7 +386,7 @@ export class ApiStatusMastodon {
 			const client = getClient(BASE_URL, accessTokens);
 			try {
 				const data = await client.createEmojiReaction(convertId(_request.params.id, IdType.SharkeyId), _request.params.name);
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);
@@ -396,7 +401,7 @@ export class ApiStatusMastodon {
 			const client = getClient(BASE_URL, accessTokens);
 			try {
 				const data = await client.deleteEmojiReaction(convertId(_request.params.id, IdType.SharkeyId), _request.params.name);
-				reply.send(convertStatus(data.data));
+				reply.send(await this.mastoconverter.convertStatus(data.data));
 			} catch (e: any) {
 				console.error(e);
 				reply.code(401).send(e.response.data);