From 3f4ee9840560d8c423111f1ab93b2e36218a1314 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 15 Oct 2023 10:36:22 +0900 Subject: [PATCH 1/3] perf(backend): improve streaming api performance (#12033) * wip * Update NoteEntityService.ts * wip * wip * wip * wip --- .../backend/src/core/NoteCreateService.ts | 2 +- .../src/core/entities/NoteEntityService.ts | 13 +++++--- .../src/server/api/stream/channels/channel.ts | 18 +++-------- .../api/stream/channels/global-timeline.ts | 18 +++-------- .../src/server/api/stream/channels/hashtag.ts | 12 +++---- .../api/stream/channels/home-timeline.ts | 30 ++++++----------- .../api/stream/channels/hybrid-timeline.ts | 30 ++++++----------- .../api/stream/channels/local-timeline.ts | 18 +++-------- .../server/api/stream/channels/user-list.ts | 32 +++++++------------ 9 files changed, 58 insertions(+), 115 deletions(-) diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 4496be3e7d..e12945172f 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -577,7 +577,7 @@ export class NoteCreateService implements OnApplicationShutdown { } // Pack the note - const noteObj = await this.noteEntityService.pack(note); + const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true }); this.globalEventService.publishNotesStream(noteObj); diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index 316367f23a..f871ba50a8 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -16,6 +16,7 @@ import type { UsersRepository, NotesRepository, FollowingsRepository, PollsRepos import { bindThis } from '@/decorators.js'; import { isNotNull } from '@/misc/is-not-null.js'; import { DebounceLoader } from '@/misc/loader.js'; +import { IdService } from '@/core/IdService.js'; import type { OnModuleInit } from '@nestjs/common'; import type { CustomEmojiService } from '../CustomEmojiService.js'; import type { ReactionService } from '../ReactionService.js'; @@ -28,6 +29,7 @@ export class NoteEntityService implements OnModuleInit { private driveFileEntityService: DriveFileEntityService; private customEmojiService: CustomEmojiService; private reactionService: ReactionService; + private idService: IdService; private noteLoader = new DebounceLoader(this.findNoteOrFail); constructor( @@ -66,6 +68,7 @@ export class NoteEntityService implements OnModuleInit { this.driveFileEntityService = this.moduleRef.get('DriveFileEntityService'); this.customEmojiService = this.moduleRef.get('CustomEmojiService'); this.reactionService = this.moduleRef.get('ReactionService'); + this.idService = this.moduleRef.get('IdService'); } @bindThis @@ -167,11 +170,11 @@ export class NoteEntityService implements OnModuleInit { } @bindThis - private async populateMyReaction(note: MiNote, meId: MiUser['id'], _hint_?: { + public async populateMyReaction(noteId: MiNote['id'], meId: MiUser['id'], _hint_?: { myReactions: Map; }) { if (_hint_?.myReactions) { - const reaction = _hint_.myReactions.get(note.id); + const reaction = _hint_.myReactions.get(noteId); if (reaction) { return this.reactionService.convertLegacyReaction(reaction.reaction); } else if (reaction === null) { @@ -181,13 +184,13 @@ export class NoteEntityService implements OnModuleInit { } // パフォーマンスのためノートが作成されてから2秒以上経っていない場合はリアクションを取得しない - if (note.createdAt.getTime() + 2000 > Date.now()) { + if (this.idService.parse(noteId).date.getTime() + 2000 > Date.now()) { return undefined; } const reaction = await this.noteReactionsRepository.findOneBy({ userId: meId, - noteId: note.id, + noteId: noteId, }); if (reaction) { @@ -355,7 +358,7 @@ export class NoteEntityService implements OnModuleInit { poll: note.hasPoll ? this.populatePoll(note, meId) : undefined, ...(meId ? { - myReaction: this.populateMyReaction(note, meId, options?._hint_), + myReaction: this.populateMyReaction(note.id, meId, options?._hint_), } : {}), } : {}), }); diff --git a/packages/backend/src/server/api/stream/channels/channel.ts b/packages/backend/src/server/api/stream/channels/channel.ts index a01714e76d..e4c34e00ce 100644 --- a/packages/backend/src/server/api/stream/channels/channel.ts +++ b/packages/backend/src/server/api/stream/channels/channel.ts @@ -38,19 +38,6 @@ class ChannelChannel extends Channel { private async onNote(note: Packed<'Note'>) { if (note.channelId !== this.channelId) return; - // リプライなら再pack - if (note.replyId != null) { - note.reply = await this.noteEntityService.pack(note.replyId, this.user, { - detail: true, - }); - } - // Renoteなら再pack - if (note.renoteId != null) { - note.renote = await this.noteEntityService.pack(note.renoteId, this.user, { - detail: true, - }); - } - // 流れてきたNoteがミュートしているユーザーが関わるものだったら無視する if (isUserRelated(note, this.userIdsWhoMeMuting)) return; // 流れてきたNoteがブロックされているユーザーが関わるものだったら無視する @@ -58,6 +45,11 @@ class ChannelChannel extends Channel { if (note.renote && !note.text && isUserRelated(note, this.userIdsWhoMeMutingRenotes)) return; + if (this.user && note.renoteId && !note.text) { + const myRenoteReaction = await this.noteEntityService.populateMyReaction(note.renoteId, this.user.id); + note.renote!.myReaction = myRenoteReaction; + } + this.connection.cacheNote(note); this.send('note', note); diff --git a/packages/backend/src/server/api/stream/channels/global-timeline.ts b/packages/backend/src/server/api/stream/channels/global-timeline.ts index 03f2dff62b..c499d1787e 100644 --- a/packages/backend/src/server/api/stream/channels/global-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/global-timeline.ts @@ -52,19 +52,6 @@ class GlobalTimelineChannel extends Channel { if (note.visibility !== 'public') return; if (note.channelId != null) return; - // リプライなら再pack - if (note.replyId != null) { - note.reply = await this.noteEntityService.pack(note.replyId, this.user, { - detail: true, - }); - } - // Renoteなら再pack - if (note.renoteId != null) { - note.renote = await this.noteEntityService.pack(note.renoteId, this.user, { - detail: true, - }); - } - // 関係ない返信は除外 if (note.reply && !this.following[note.userId]?.withReplies) { const reply = note.reply; @@ -84,6 +71,11 @@ class GlobalTimelineChannel extends Channel { if (note.renote && !note.text && isUserRelated(note, this.userIdsWhoMeMutingRenotes)) return; + if (this.user && note.renoteId && !note.text) { + const myRenoteReaction = await this.noteEntityService.populateMyReaction(note.renoteId, this.user.id); + note.renote!.myReaction = myRenoteReaction; + } + this.connection.cacheNote(note); this.send('note', note); diff --git a/packages/backend/src/server/api/stream/channels/hashtag.ts b/packages/backend/src/server/api/stream/channels/hashtag.ts index 3945b1a1eb..2cfe9572d3 100644 --- a/packages/backend/src/server/api/stream/channels/hashtag.ts +++ b/packages/backend/src/server/api/stream/channels/hashtag.ts @@ -43,13 +43,6 @@ class HashtagChannel extends Channel { const matched = this.q.some(tags => tags.every(tag => noteTags.includes(normalizeForSearch(tag)))); if (!matched) return; - // Renoteなら再pack - if (note.renoteId != null) { - note.renote = await this.noteEntityService.pack(note.renoteId, this.user, { - detail: true, - }); - } - // 流れてきたNoteがミュートしているユーザーが関わるものだったら無視する if (isUserRelated(note, this.userIdsWhoMeMuting)) return; // 流れてきたNoteがブロックされているユーザーが関わるものだったら無視する @@ -57,6 +50,11 @@ class HashtagChannel extends Channel { if (note.renote && !note.text && isUserRelated(note, this.userIdsWhoMeMutingRenotes)) return; + if (this.user && note.renoteId && !note.text) { + const myRenoteReaction = await this.noteEntityService.populateMyReaction(note.renoteId, this.user.id); + note.renote!.myReaction = myRenoteReaction; + } + this.connection.cacheNote(note); this.send('note', note); diff --git a/packages/backend/src/server/api/stream/channels/home-timeline.ts b/packages/backend/src/server/api/stream/channels/home-timeline.ts index 24be590504..de755cccb9 100644 --- a/packages/backend/src/server/api/stream/channels/home-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/home-timeline.ts @@ -51,27 +51,10 @@ class HomeTimelineChannel extends Channel { // Ignore notes from instances the user has muted if (isInstanceMuted(note, new Set(this.userProfile!.mutedInstances ?? []))) return; - if (['followers', 'specified'].includes(note.visibility)) { - note = await this.noteEntityService.pack(note.id, this.user!, { - detail: true, - }); - - if (note.isHidden) { - return; - } - } else { - // リプライなら再pack - if (note.replyId != null) { - note.reply = await this.noteEntityService.pack(note.replyId, this.user!, { - detail: true, - }); - } - // Renoteなら再pack - if (note.renoteId != null) { - note.renote = await this.noteEntityService.pack(note.renoteId, this.user!, { - detail: true, - }); - } + if (note.visibility === 'followers') { + if (!Object.hasOwn(this.following, note.userId)) return; + } else if (note.visibility === 'specified') { + if (!note.visibleUserIds!.includes(this.user!.id)) return; } // 関係ない返信は除外 @@ -90,6 +73,11 @@ class HomeTimelineChannel extends Channel { if (note.renote && !note.text && isUserRelated(note, this.userIdsWhoMeMutingRenotes)) return; + if (this.user && note.renoteId && !note.text) { + const myRenoteReaction = await this.noteEntityService.populateMyReaction(note.renoteId, this.user.id); + note.renote!.myReaction = myRenoteReaction; + } + this.connection.cacheNote(note); this.send('note', note); diff --git a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts index adedca5152..83f0bccd90 100644 --- a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts @@ -62,27 +62,10 @@ class HybridTimelineChannel extends Channel { (note.channelId != null && this.followingChannels.has(note.channelId)) )) return; - if (['followers', 'specified'].includes(note.visibility)) { - note = await this.noteEntityService.pack(note.id, this.user!, { - detail: true, - }); - - if (note.isHidden) { - return; - } - } else { - // リプライなら再pack - if (note.replyId != null) { - note.reply = await this.noteEntityService.pack(note.replyId, this.user!, { - detail: true, - }); - } - // Renoteなら再pack - if (note.renoteId != null) { - note.renote = await this.noteEntityService.pack(note.renoteId, this.user!, { - detail: true, - }); - } + if (note.visibility === 'followers') { + if (!Object.hasOwn(this.following, note.userId)) return; + } else if (note.visibility === 'specified') { + if (!note.visibleUserIds!.includes(this.user!.id)) return; } // Ignore notes from instances the user has muted @@ -104,6 +87,11 @@ class HybridTimelineChannel extends Channel { if (note.renote && !note.text && isUserRelated(note, this.userIdsWhoMeMutingRenotes)) return; + if (this.user && note.renoteId && !note.text) { + const myRenoteReaction = await this.noteEntityService.populateMyReaction(note.renoteId, this.user.id); + note.renote!.myReaction = myRenoteReaction; + } + this.connection.cacheNote(note); this.send('note', note); diff --git a/packages/backend/src/server/api/stream/channels/local-timeline.ts b/packages/backend/src/server/api/stream/channels/local-timeline.ts index 69aa366f00..a211041134 100644 --- a/packages/backend/src/server/api/stream/channels/local-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/local-timeline.ts @@ -54,19 +54,6 @@ class LocalTimelineChannel extends Channel { if (note.visibility !== 'public') return; if (note.channelId != null && !this.followingChannels.has(note.channelId)) return; - // リプライなら再pack - if (note.replyId != null) { - note.reply = await this.noteEntityService.pack(note.replyId, this.user, { - detail: true, - }); - } - // Renoteなら再pack - if (note.renoteId != null) { - note.renote = await this.noteEntityService.pack(note.renoteId, this.user, { - detail: true, - }); - } - // 関係ない返信は除外 if (note.reply && this.user && !this.following[note.userId]?.withReplies && !this.withReplies) { const reply = note.reply; @@ -83,6 +70,11 @@ class LocalTimelineChannel extends Channel { if (note.renote && !note.text && isUserRelated(note, this.userIdsWhoMeMutingRenotes)) return; + if (this.user && note.renoteId && !note.text) { + const myRenoteReaction = await this.noteEntityService.populateMyReaction(note.renoteId, this.user.id); + note.renote!.myReaction = myRenoteReaction; + } + this.connection.cacheNote(note); this.send('note', note); diff --git a/packages/backend/src/server/api/stream/channels/user-list.ts b/packages/backend/src/server/api/stream/channels/user-list.ts index 240822d9ab..b73cedaa8b 100644 --- a/packages/backend/src/server/api/stream/channels/user-list.ts +++ b/packages/backend/src/server/api/stream/channels/user-list.ts @@ -82,27 +82,10 @@ class UserListChannel extends Channel { if (!Object.hasOwn(this.membershipsMap, note.userId)) return; - if (['followers', 'specified'].includes(note.visibility)) { - note = await this.noteEntityService.pack(note.id, this.user, { - detail: true, - }); - - if (note.isHidden) { - return; - } - } else { - // リプライなら再pack - if (note.replyId != null) { - note.reply = await this.noteEntityService.pack(note.replyId, this.user, { - detail: true, - }); - } - // Renoteなら再pack - if (note.renoteId != null) { - note.renote = await this.noteEntityService.pack(note.renoteId, this.user, { - detail: true, - }); - } + if (note.visibility === 'followers') { + if (!Object.hasOwn(this.following, note.userId)) return; + } else if (note.visibility === 'specified') { + if (!note.visibleUserIds!.includes(this.user!.id)) return; } // 関係ない返信は除外 @@ -119,6 +102,13 @@ class UserListChannel extends Channel { if (note.renote && !note.text && isUserRelated(note, this.userIdsWhoMeMutingRenotes)) return; + if (this.user && note.renoteId && !note.text) { + const myRenoteReaction = await this.noteEntityService.populateMyReaction(note.renoteId, this.user.id); + note.renote!.myReaction = myRenoteReaction; + } + + this.connection.cacheNote(note); + this.send('note', note); } From 5117818f6fcaa5ff6d033daaab980b956f2b20ad Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 15 Oct 2023 10:37:03 +0900 Subject: [PATCH 2/3] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50334560c9..9b51d2c036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ - Enhance: TLの返信表示オプションを記憶するように ### Server -- +- Enhance: ストリーミングAPIのパフォーマンスを向上 ## 2023.10.1 ### General From 15706c8c2bdb0a6edf2b34a7c1f25d80e4610a60 Mon Sep 17 00:00:00 2001 From: syuilo Date: Sun, 15 Oct 2023 16:25:42 +0900 Subject: [PATCH 3/3] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b51d2c036..85a998de10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ ### NOTE - 2023.9.2で導入されたノート編集機能はクオリティの高い実装が困難であることが判明したため撤回されました - アップデートを行うと、タイムラインが一時的にリセットされます + - アンテナ内のノートも含む - ソフトミュート設定はクライアントではなくサーバー側に保存されるようになったため、アップデートを行うとソフトミュートの設定がリセットされます ### Changes