Misskey® Reactions Buffering Technology™ (misskey-dev#14579)

* wip * wip * Update ReactionsBufferingService.ts * Update ReactionsBufferingService.ts * wip * wip * wip * Update ReactionsBufferingService.ts * wip * wip * wip * Update NoteEntityService.ts * wip * wip * wip * wip * Update CHANGELOG.md
kanarikanaru · Sep 20, 2024 · 0b062f1 · 0b062f1
1 parent f585f70
commit 0b062f1
Show file tree

Hide file tree

Showing 29 changed files with 498 additions and 92 deletions.
diff --git a/.config/cypress-devcontainer.yml b/.config/cypress-devcontainer.yml
@@ -103,6 +103,14 @@ redis:
 #  #prefix: example-prefix
 #  #db: 1
 
+#redisForReactions:
+#  host: redis
+#  port: 6379
+#  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+#  #pass: example-pass
+#  #prefix: example-prefix
+#  #db: 1
+
 #   ┌───────────────────────────┐
 #───┘ MeiliSearch configuration └─────────────────────────────
 

diff --git a/.config/docker_example.yml b/.config/docker_example.yml
@@ -106,6 +106,14 @@ redis:
 #  #prefix: example-prefix
 #  #db: 1
 
+#redisForReactions:
+#  host: redis
+#  port: 6379
+#  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+#  #pass: example-pass
+#  #prefix: example-prefix
+#  #db: 1
+
 #   ┌───────────────────────────┐
 #───┘ MeiliSearch configuration └─────────────────────────────
 

diff --git a/.config/example.yml b/.config/example.yml
@@ -172,6 +172,16 @@ redis:
 #  # You can specify more ioredis options...
 #  #username: example-username
 
+#redisForReactions:
+#  host: localhost
+#  port: 6379
+#  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+#  #pass: example-pass
+#  #prefix: example-prefix
+#  #db: 1
+#  # You can specify more ioredis options...
+#  #username: example-username
+
 #   ┌───────────────────────────┐
 #───┘ MeiliSearch configuration └─────────────────────────────
 

diff --git a/.devcontainer/devcontainer.yml b/.devcontainer/devcontainer.yml
@@ -103,6 +103,14 @@ redis:
 #  #prefix: example-prefix
 #  #db: 1
 
+#redisForReactions:
+#  host: redis
+#  port: 6379
+#  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+#  #pass: example-pass
+#  #prefix: example-prefix
+#  #db: 1
+
 #   ┌───────────────────────────┐
 #───┘ MeiliSearch configuration └─────────────────────────────
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,7 @@
 - Fix: 設定変更時のリロード確認ダイアログが複数個表示されることがある問題を修正
 
 ### Server
+- Feat: Misskey® Reactions Buffering Technology™ (RBT)により、リアクションの作成負荷を低減することが可能に
 - Fix: アンテナの書き込み時にキーワードが与えられなかった場合のエラーをApiErrorとして投げるように
   - この変更により、公式フロントエンドでは入力の不備が内部エラーとして報告される代わりに一般的なエラーダイアログで報告されます
 - Fix: ファイルがサイズの制限を超えてアップロードされた際にエラーを返さなかった問題を修正

diff --git a/chart/files/default.yml b/chart/files/default.yml
@@ -124,6 +124,14 @@ redis:
 #  #prefix: example-prefix
 #  #db: 1
 
+#redisForReactions:
+#  host: redis
+#  port: 6379
+#  #family: 0  # 0=Both, 4=IPv4, 6=IPv6
+#  #pass: example-pass
+#  #prefix: example-prefix
+#  #db: 1
+
 #   ┌───────────────────────────┐
 #───┘ MeiliSearch configuration └─────────────────────────────
 

diff --git a/locales/index.d.ts b/locales/index.d.ts
@@ -5583,6 +5583,10 @@ export interface Locale extends ILocale {
          * 有効にすると、タイムラインがキャッシュされていない場合にDBへ追加で問い合わせを行うフォールバック処理を行います。無効にすると、フォールバック処理を行わないことでさらにサーバーの負荷を軽減することができますが、タイムラインが取得できる範囲に制限が生じます。
          */
         "fanoutTimelineDbFallbackDescription": string;
+        /**
+         * 有効にすると、リアクション作成時のパフォーマンスが大幅に向上し、データベースへの負荷を軽減することが可能です。ただし、Redisのメモリ使用量は増加します。
+         */
+        "reactionsBufferingDescription": string;
         /**
          * 問い合わせ先URL
          */

diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml
@@ -1411,6 +1411,7 @@ _serverSettings:
   fanoutTimelineDescription: "有効にすると、各種タイムラインを取得する際のパフォーマンスが大幅に向上し、データベースへの負荷を軽減することが可能です。ただし、Redisのメモリ使用量は増加します。サーバーのメモリ容量が少ない場合、または動作が不安定な場合は無効にすることができます。"
   fanoutTimelineDbFallback: "データベースへのフォールバック"
   fanoutTimelineDbFallbackDescription: "有効にすると、タイムラインがキャッシュされていない場合にDBへ追加で問い合わせを行うフォールバック処理を行います。無効にすると、フォールバック処理を行わないことでさらにサーバーの負荷を軽減することができますが、タイムラインが取得できる範囲に制限が生じます。"
+  reactionsBufferingDescription: "有効にすると、リアクション作成時のパフォーマンスが大幅に向上し、データベースへの負荷を軽減することが可能です。ただし、Redisのメモリ使用量は増加します。"
   inquiryUrl: "問い合わせ先URL"
   inquiryUrlDescription: "サーバー運営者へのお問い合わせフォームのURLや、運営者の連絡先等が記載されたWebページのURLを指定します。"
 

diff --git a/packages/backend/migration/1726804538569-reactions-buffering.js b/packages/backend/migration/1726804538569-reactions-buffering.js
@@ -0,0 +1,16 @@
+/*
+ * SPDX-FileCopyrightText: syuilo and misskey-project
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+export class ReactionsBuffering1726804538569 {
+    name = 'ReactionsBuffering1726804538569'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" ADD "enableReactionsBuffering" boolean NOT NULL DEFAULT false`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableReactionsBuffering"`);
+    }
+}
diff --git a/packages/backend/src/GlobalModule.ts b/packages/backend/src/GlobalModule.ts
@@ -78,11 +78,19 @@ const $redisForTimelines: Provider = {
 	inject: [DI.config],
 };
 
+const $redisForReactions: Provider = {
+	provide: DI.redisForReactions,
+	useFactory: (config: Config) => {
+		return new Redis.Redis(config.redisForReactions);
+	},
+	inject: [DI.config],
+};
+
 @Global()
 @Module({
 	imports: [RepositoryModule],
-	providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines],
-	exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, RepositoryModule],
+	providers: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions],
+	exports: [$config, $db, $meilisearch, $redis, $redisForPub, $redisForSub, $redisForTimelines, $redisForReactions, RepositoryModule],
 })
 export class GlobalModule implements OnApplicationShutdown {
 	constructor(
@@ -91,6 +99,7 @@ export class GlobalModule implements OnApplicationShutdown {
 		@Inject(DI.redisForPub) private redisForPub: Redis.Redis,
 		@Inject(DI.redisForSub) private redisForSub: Redis.Redis,
 		@Inject(DI.redisForTimelines) private redisForTimelines: Redis.Redis,
+		@Inject(DI.redisForReactions) private redisForReactions: Redis.Redis,
 	) { }
 
 	public async dispose(): Promise<void> {
@@ -103,6 +112,7 @@ export class GlobalModule implements OnApplicationShutdown {
 			this.redisForPub.disconnect(),
 			this.redisForSub.disconnect(),
 			this.redisForTimelines.disconnect(),
+			this.redisForReactions.disconnect(),
 		]);
 	}
 

diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts
@@ -49,6 +49,7 @@ type Source = {
 	redisForPubsub?: RedisOptionsSource;
 	redisForJobQueue?: RedisOptionsSource;
 	redisForTimelines?: RedisOptionsSource;
+	redisForReactions?: RedisOptionsSource;
 	meilisearch?: {
 		host: string;
 		port: string;
@@ -171,6 +172,7 @@ export type Config = {
 	redisForPubsub: RedisOptions & RedisOptionsSource;
 	redisForJobQueue: RedisOptions & RedisOptionsSource;
 	redisForTimelines: RedisOptions & RedisOptionsSource;
+	redisForReactions: RedisOptions & RedisOptionsSource;
 	sentryForBackend: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; } | undefined;
 	sentryForFrontend: { options: Partial<Sentry.NodeOptions> } | undefined;
 	perChannelMaxNoteCacheCount: number;
@@ -251,6 +253,7 @@ export function loadConfig(): Config {
 		redisForPubsub: config.redisForPubsub ? convertRedisOptions(config.redisForPubsub, host) : redis,
 		redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,
 		redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis,
+		redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis,
 		sentryForBackend: config.sentryForBackend,
 		sentryForFrontend: config.sentryForFrontend,
 		id: config.id,

diff --git a/packages/backend/src/const.ts b/packages/backend/src/const.ts
@@ -8,6 +8,8 @@ export const MAX_NOTE_TEXT_LENGTH = 3000;
 export const USER_ONLINE_THRESHOLD = 1000 * 60 * 10; // 10min
 export const USER_ACTIVE_THRESHOLD = 1000 * 60 * 60 * 24 * 3; // 3days
 
+export const PER_NOTE_REACTION_USER_PAIR_CACHE_MAX = 16;
+
 //#region hard limits
 // If you change DB_* values, you must also change the DB schema.
 

diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts
@@ -50,6 +50,7 @@ import { PollService } from './PollService.js';
 import { PushNotificationService } from './PushNotificationService.js';
 import { QueryService } from './QueryService.js';
 import { ReactionService } from './ReactionService.js';
+import { ReactionsBufferingService } from './ReactionsBufferingService.js';
 import { RelayService } from './RelayService.js';
 import { RoleService } from './RoleService.js';
 import { S3Service } from './S3Service.js';
@@ -193,6 +194,7 @@ const $ProxyAccountService: Provider = { provide: 'ProxyAccountService', useExis
 const $PushNotificationService: Provider = { provide: 'PushNotificationService', useExisting: PushNotificationService };
 const $QueryService: Provider = { provide: 'QueryService', useExisting: QueryService };
 const $ReactionService: Provider = { provide: 'ReactionService', useExisting: ReactionService };
+const $ReactionsBufferingService: Provider = { provide: 'ReactionsBufferingService', useExisting: ReactionsBufferingService };
 const $RelayService: Provider = { provide: 'RelayService', useExisting: RelayService };
 const $RoleService: Provider = { provide: 'RoleService', useExisting: RoleService };
 const $S3Service: Provider = { provide: 'S3Service', useExisting: S3Service };
@@ -342,6 +344,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		PushNotificationService,
 		QueryService,
 		ReactionService,
+		ReactionsBufferingService,
 		RelayService,
 		RoleService,
 		S3Service,
@@ -487,6 +490,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		$PushNotificationService,
 		$QueryService,
 		$ReactionService,
+		$ReactionsBufferingService,
 		$RelayService,
 		$RoleService,
 		$S3Service,
@@ -633,6 +637,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		PushNotificationService,
 		QueryService,
 		ReactionService,
+		ReactionsBufferingService,
 		RelayService,
 		RoleService,
 		S3Service,
@@ -777,6 +782,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
 		$PushNotificationService,
 		$QueryService,
 		$ReactionService,
+		$ReactionsBufferingService,
 		$RelayService,
 		$RoleService,
 		$S3Service,

diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts
@@ -87,6 +87,12 @@ export class QueueService {
 			repeat: { pattern: '*/5 * * * *' },
 			removeOnComplete: true,
 		});
+
+		this.systemQueue.add('bakeBufferedReactions', {
+		}, {
+			repeat: { pattern: '0 0 * * *' },
+			removeOnComplete: true,
+		});
 	}
 
 	@bindThis

diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts
@@ -4,7 +4,6 @@
  */
 
 import { Inject, Injectable } from '@nestjs/common';
-import * as Redis from 'ioredis';
 import { DI } from '@/di-symbols.js';
 import type { EmojisRepository, NoteReactionsRepository, UsersRepository, NotesRepository } from '@/models/_.js';
 import { IdentifiableError } from '@/misc/identifiable-error.js';
@@ -30,9 +29,10 @@ import { RoleService } from '@/core/RoleService.js';
 import { FeaturedService } from '@/core/FeaturedService.js';
 import { trackPromise } from '@/misc/promise-tracker.js';
 import { isQuote, isRenote } from '@/misc/is-renote.js';
+import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
+import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js';
 
 const FALLBACK = '\u2764';
-const PER_NOTE_REACTION_USER_PAIR_CACHE_MAX = 16;
 
 const legacies: Record<string, string> = {
 	'like': '👍',
@@ -71,9 +71,6 @@ const decodeCustomEmojiRegexp = /^:([\w+-]+)(?:@([\w.-]+))?:$/;
 @Injectable()
 export class ReactionService {
 	constructor(
-		@Inject(DI.redis)
-		private redisClient: Redis.Redis,
-
 		@Inject(DI.usersRepository)
 		private usersRepository: UsersRepository,
 
@@ -93,6 +90,7 @@ export class ReactionService {
 		private userEntityService: UserEntityService,
 		private noteEntityService: NoteEntityService,
 		private userBlockingService: UserBlockingService,
+		private reactionsBufferingService: ReactionsBufferingService,
 		private idService: IdService,
 		private featuredService: FeaturedService,
 		private globalEventService: GlobalEventService,
@@ -174,7 +172,6 @@ export class ReactionService {
 			reaction,
 		};
 
-		// Create reaction
 		try {
 			await this.noteReactionsRepository.insert(record);
 		} catch (e) {
@@ -198,16 +195,25 @@ export class ReactionService {
 		}
 
 		// Increment reactions count
-		const sql = `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + 1)::text::jsonb)`;
-		await this.notesRepository.createQueryBuilder().update()
-			.set({
-				reactions: () => sql,
-				...(note.reactionAndUserPairCache.length < PER_NOTE_REACTION_USER_PAIR_CACHE_MAX ? {
-					reactionAndUserPairCache: () => `array_append("reactionAndUserPairCache", '${user.id}/${reaction}')`,
-				} : {}),
-			})
-			.where('id = :id', { id: note.id })
-			.execute();
+		if (meta.enableReactionsBuffering) {
+			await this.reactionsBufferingService.create(note.id, user.id, reaction, note.reactionAndUserPairCache);
+
+			// for debugging
+			if (reaction === ':angry_ai:') {
+				this.reactionsBufferingService.bake();
+			}
+		} else {
+			const sql = `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + 1)::text::jsonb)`;
+			await this.notesRepository.createQueryBuilder().update()
+				.set({
+					reactions: () => sql,
+					...(note.reactionAndUserPairCache.length < PER_NOTE_REACTION_USER_PAIR_CACHE_MAX ? {
+						reactionAndUserPairCache: () => `array_append("reactionAndUserPairCache", '${user.id}/${reaction}')`,
+					} : {}),
+				})
+				.where('id = :id', { id: note.id })
+				.execute();
+		}
 
 		// 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新
 		if (
@@ -304,15 +310,21 @@ export class ReactionService {
 			throw new IdentifiableError('60527ec9-b4cb-4a88-a6bd-32d3ad26817d', 'not reacted');
 		}
 
+		const meta = await this.metaService.fetch();
+
 		// Decrement reactions count
-		const sql = `jsonb_set("reactions", '{${exist.reaction}}', (COALESCE("reactions"->>'${exist.reaction}', '0')::int - 1)::text::jsonb)`;
-		await this.notesRepository.createQueryBuilder().update()
-			.set({
-				reactions: () => sql,
-				reactionAndUserPairCache: () => `array_remove("reactionAndUserPairCache", '${user.id}/${exist.reaction}')`,
-			})
-			.where('id = :id', { id: note.id })
-			.execute();
+		if (meta.enableReactionsBuffering) {
+			await this.reactionsBufferingService.delete(note.id, user.id, exist.reaction);
+		} else {
+			const sql = `jsonb_set("reactions", '{${exist.reaction}}', (COALESCE("reactions"->>'${exist.reaction}', '0')::int - 1)::text::jsonb)`;
+			await this.notesRepository.createQueryBuilder().update()
+				.set({
+					reactions: () => sql,
+					reactionAndUserPairCache: () => `array_remove("reactionAndUserPairCache", '${user.id}/${exist.reaction}')`,
+				})
+				.where('id = :id', { id: note.id })
+				.execute();
+		}
 
 		this.globalEventService.publishNoteStream(note.id, 'unreacted', {
 			reaction: this.decodeReaction(exist.reaction).reaction,