From 978a9bbb3beafb7322aaf81d097430de6dbf527e Mon Sep 17 00:00:00 2001 From: tamaina Date: Thu, 12 Jan 2023 21:03:02 +0900 Subject: [PATCH] perf(backend): Use undici instead of node-fetch and got (#9459) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Implement? HttpFetchService * :v: * remove node-fetch * fix * refactor * fix * gateway timeout * UndiciFetcherクラスを追加 (仮コミット, ビルドもstartもさせていない) * fix * add logger and fix url preview * fix ip check * enhance logger and error handling * fix * fix * clean up * Use custom fetcher for ApRequest / ApResolver * bypassProxyはproxyBypassHostsに判断を委譲するように * set maxRedirections (default 3, ApRequest/ApResolver: 0) * fix comment * handle error s3 upload * add debug message * no return await * Revert "no return await" This reverts commit b5b0dc58a342393d260492e3a6f58304372f53b2. * reduce maxSockets * apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger * Revert "apResolverのUndiciFetcherを廃止しapRequestのものを使う、 add ap logger" This reverts commit 997243915c8e1f8472da64f607f88c36cb1d5cb4. * add logger * fix * change logger name * safe * デフォルトでUser-Agentを設定 --- .config/example.yml | 10 +- packages/backend/package.json | 3 +- packages/backend/src/core/CaptchaService.ts | 26 +- packages/backend/src/core/DownloadService.ts | 90 ++--- packages/backend/src/core/DriveService.ts | 17 +- .../src/core/FetchInstanceMetadataService.ts | 7 +- .../backend/src/core/HttpRequestService.ts | 349 +++++++++++++----- packages/backend/src/core/S3Service.ts | 2 +- packages/backend/src/core/WebfingerService.ts | 2 +- .../src/core/activitypub/ApRequestService.ts | 36 +- .../src/core/activitypub/ApResolverService.ts | 15 +- .../core/activitypub/LdSignatureService.ts | 20 +- .../WebhookDeliverProcessorService.ts | 38 +- .../src/server/api/endpoints/fetch-rss.ts | 19 +- .../server/api/endpoints/notes/translate.ts | 35 +- .../api/integration/DiscordServerService.ts | 4 +- .../api/integration/GithubServerService.ts | 4 +- .../src/server/web/UrlPreviewService.ts | 3 +- yarn.lock | 19 +- 19 files changed, 444 insertions(+), 255 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 93dea1f254..cabf167fba 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -122,10 +122,12 @@ id: 'aid' # Proxy for HTTP/HTTPS #proxy: http://127.0.0.1:3128 -#proxyBypassHosts: [ -# 'example.com', -# '192.0.2.8' -#] +proxyBypassHosts: + - api.deepl.com + - api-free.deepl.com + - www.recaptcha.net + - hcaptcha.com + - challenges.cloudflare.com # Proxy for SMTP/SMTPS #proxySmtp: http://127.0.0.1:3128 # use HTTP/1.1 CONNECT diff --git a/packages/backend/package.json b/packages/backend/package.json index 293b630ad6..fbce696a3e 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -77,7 +77,6 @@ "misskey-js": "0.0.14", "ms": "3.0.0-canary.1", "nested-property": "4.0.0", - "node-fetch": "3.3.0", "nodemailer": "6.8.0", "nsfwjs": "2.4.2", "oauth": "^0.10.0", @@ -118,6 +117,7 @@ "twemoji-parser": "14.0.0", "typeorm": "0.3.11", "ulid": "2.3.0", + "undici": "^5.14.0", "unzipper": "0.10.11", "uuid": "9.0.0", "vary": "1.1.2", @@ -180,6 +180,7 @@ "execa": "6.1.0", "jest": "29.3.1", "jest-mock": "^29.3.1", + "node-fetch": "3.3.0", "typescript": "4.9.4" } } diff --git a/packages/backend/src/core/CaptchaService.ts b/packages/backend/src/core/CaptchaService.ts index 0207cf58a0..1e98914052 100644 --- a/packages/backend/src/core/CaptchaService.ts +++ b/packages/backend/src/core/CaptchaService.ts @@ -1,7 +1,4 @@ -import { Inject, Injectable } from '@nestjs/common'; -import { DI } from '@/di-symbols.js'; -import type { UsersRepository } from '@/models/index.js'; -import type { Config } from '@/config.js'; +import { Injectable } from '@nestjs/common'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; @@ -13,9 +10,6 @@ type CaptchaResponse = { @Injectable() export class CaptchaService { constructor( - @Inject(DI.config) - private config: Config, - private httpRequestService: HttpRequestService, ) { } @@ -27,16 +21,16 @@ export class CaptchaService { response, }); - const res = await fetch(url, { - method: 'POST', - body: params, - headers: { - 'User-Agent': this.config.userAgent, + const res = await this.httpRequestService.fetch( + url, + { + method: 'POST', + body: params, }, - // TODO - //timeout: 10 * 1000, - agent: (url, bypassProxy) => this.httpRequestService.getAgentByUrl(url, bypassProxy), - }).catch(err => { + { + noOkError: true, + } + ).catch(err => { throw `${err.message ?? err}`; }); diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 62123246a7..a3078bff45 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -8,11 +8,12 @@ import got, * as Got from 'got'; import chalk from 'chalk'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; -import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; import { createTemp } from '@/misc/create-temp.js'; import { StatusError } from '@/misc/status-error.js'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; +import { buildConnector } from 'undici'; const pipeline = util.promisify(stream.pipeline); import { bindThis } from '@/decorators.js'; @@ -20,6 +21,7 @@ import { bindThis } from '@/decorators.js'; @Injectable() export class DownloadService { private logger: Logger; + private undiciFetcher: UndiciFetcher; constructor( @Inject(DI.config) @@ -29,70 +31,42 @@ export class DownloadService { private loggerService: LoggerService, ) { this.logger = this.loggerService.getLogger('download'); + + this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption( + { + connect: process.env.NODE_ENV === 'development' ? + this.httpRequestService.clientDefaults.connect + : + this.httpRequestService.getConnectorWithIpCheck( + buildConnector({ + ...this.httpRequestService.clientDefaults.connect, + }), + (ip) => !this.isPrivateIp(ip) + ), + bodyTimeout: 30 * 1000, + }, + { + connect: this.httpRequestService.clientDefaults.connect, + } + ), this.logger); } @bindThis public async downloadUrl(url: string, path: string): Promise { this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); - + const timeout = 30 * 1000; const operationTimeout = 60 * 1000; const maxSize = this.config.maxFileSize ?? 262144000; - - const req = got.stream(url, { - headers: { - 'User-Agent': this.config.userAgent, - }, - timeout: { - lookup: timeout, - connect: timeout, - secureConnect: timeout, - socket: timeout, // read timeout - response: timeout, - send: timeout, - request: operationTimeout, // whole operation timeout - }, - agent: { - http: this.httpRequestService.httpAgent, - https: this.httpRequestService.httpsAgent, - }, - http2: false, // default - retry: { - limit: 0, - }, - }).on('response', (res: Got.Response) => { - if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) { - if (this.isPrivateIp(res.ip)) { - this.logger.warn(`Blocked address: ${res.ip}`); - req.destroy(); - } - } - - const contentLength = res.headers['content-length']; - if (contentLength != null) { - const size = Number(contentLength); - if (size > maxSize) { - this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`); - req.destroy(); - } - } - }).on('downloadProgress', (progress: Got.Progress) => { - if (progress.transferred > maxSize) { - this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); - req.destroy(); - } - }); - - try { - await pipeline(req, fs.createWriteStream(path)); - } catch (e) { - if (e instanceof Got.HTTPError) { - throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); - } else { - throw e; - } + + const response = await this.undiciFetcher.fetch(url); + + if (response.body === null) { + throw new StatusError('No body', 400, 'No body'); } - + + await pipeline(stream.Readable.fromWeb(response.body), fs.createWriteStream(path)); + this.logger.succ(`Download finished: ${chalk.cyan(url)}`); } @@ -114,7 +88,7 @@ export class DownloadService { cleanup(); } } - + @bindThis private isPrivateIp(ip: string): boolean { for (const net of this.config.allowedPrivateNetworks ?? []) { @@ -124,6 +98,6 @@ export class DownloadService { } } - return PrivateIp(ip); + return PrivateIp(ip) ?? false; } } diff --git a/packages/backend/src/core/DriveService.ts b/packages/backend/src/core/DriveService.ts index 5954abba91..747dfd6130 100644 --- a/packages/backend/src/core/DriveService.ts +++ b/packages/backend/src/core/DriveService.ts @@ -375,8 +375,19 @@ export class DriveService { partSize: s3.endpoint.hostname === 'storage.googleapis.com' ? 500 * 1024 * 1024 : 8 * 1024 * 1024, }); - const result = await upload.promise(); - if (result) this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); + await upload.promise() + .then( + result => { + if (result) { + this.registerLogger.debug(`Uploaded: ${result.Bucket}/${result.Key} => ${result.Location}`); + } else { + this.registerLogger.error(`Upload Result Empty: key = ${key}, filename = ${filename}`); + } + }, + err => { + this.registerLogger.error(`Upload Failed: key = ${key}, filename = ${filename}`, err); + } + ); } @bindThis @@ -462,6 +473,8 @@ export class DriveService { } } + this.registerLogger.debug(`ADD DRIVE FILE: user ${user?.id ?? 'not set'}, name ${detectedName}, tmp ${path}`); + //#region Check drive usage if (user && !isLink) { const usage = await this.driveFileEntityService.calcDriveUsageOf(user); diff --git a/packages/backend/src/core/FetchInstanceMetadataService.ts b/packages/backend/src/core/FetchInstanceMetadataService.ts index 7eea45200e..cb9d099a22 100644 --- a/packages/backend/src/core/FetchInstanceMetadataService.ts +++ b/packages/backend/src/core/FetchInstanceMetadataService.ts @@ -1,7 +1,6 @@ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import { JSDOM } from 'jsdom'; -import fetch from 'node-fetch'; import tinycolor from 'tinycolor2'; import type { Instance } from '@/models/entities/Instance.js'; import type { InstancesRepository } from '@/models/index.js'; @@ -191,11 +190,7 @@ export class FetchInstanceMetadataService { const faviconUrl = url + '/favicon.ico'; - const favicon = await fetch(faviconUrl, { - // TODO - //timeout: 10000, - agent: url => this.httpRequestService.getAgentByUrl(url), - }); + const favicon = await this.httpRequestService.fetch(faviconUrl, {}, { noOkError: true }); if (favicon.ok) { return faviconUrl; diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 49b28ae523..8639b5713d 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -1,67 +1,257 @@ import * as http from 'node:http'; import * as https from 'node:https'; import CacheableLookup from 'cacheable-lookup'; -import fetch from 'node-fetch'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; -import type { Response } from 'node-fetch'; -import type { URL } from 'node:url'; +import * as undici from 'undici'; +import { LookupFunction } from 'node:net'; +import { LoggerService } from '@/core/LoggerService.js'; +import type Logger from '@/logger.js'; + +// true to allow, false to deny +export type IpChecker = (ip: string) => boolean; + +/* + * Child class to create and save Agent for fetch. + * You should construct this when you want + * to change timeout, size limit, socket connect function, etc. + */ +export class UndiciFetcher { + /** + * Get http non-proxy agent (undici) + */ + public nonProxiedAgent: undici.Agent; + + /** + * Get http proxy or non-proxy agent (undici) + */ + public agent: undici.ProxyAgent | undici.Agent; + + private proxyBypassHosts: string[]; + private userAgent: string | undefined; + + private logger: Logger | undefined; + + constructor( + args: { + agentOptions: undici.Agent.Options; + proxy?: { + uri: string; + options?: undici.Agent.Options; // Override of agentOptions + }, + proxyBypassHosts?: string[]; + userAgent?: string; + }, + logger?: Logger, + ) { + this.logger = logger; + this.logger?.debug('UndiciFetcher constructor', args); + + this.proxyBypassHosts = args.proxyBypassHosts ?? []; + this.userAgent = args.userAgent; + + this.nonProxiedAgent = new undici.Agent({ + ...args.agentOptions, + connect: (process.env.NODE_ENV !== 'production' && typeof args.agentOptions.connect !== 'function') + ? (options, cb) => { + // Custom connector for debug + undici.buildConnector(args.agentOptions.connect as undici.buildConnector.BuildOptions)(options, (err, socket) => { + this.logger?.debug('Socket connector called', socket); + if (err) { + this.logger?.debug(`Socket error`, err); + cb(new Error(`Error while socket connecting\n${err}`), null); + return; + } + this.logger?.debug(`Socket connected: port ${socket.localPort} => remote ${socket.remoteAddress}`); + cb(null, socket); + }); + } : args.agentOptions.connect, + }); + + this.agent = args.proxy + ? new undici.ProxyAgent({ + ...args.agentOptions, + ...args.proxy.options, + + uri: args.proxy.uri, + + connect: (process.env.NODE_ENV !== 'production' && typeof (args.proxy?.options?.connect ?? args.agentOptions.connect) !== 'function') + ? (options, cb) => { + // Custom connector for debug + undici.buildConnector((args.proxy?.options?.connect ?? args.agentOptions.connect) as undici.buildConnector.BuildOptions)(options, (err, socket) => { + this.logger?.debug('Socket connector called (secure)', socket); + if (err) { + this.logger?.debug(`Socket error`, err); + cb(new Error(`Error while socket connecting\n${err}`), null); + return; + } + this.logger?.debug(`Socket connected (secure): port ${socket.localPort} => remote ${socket.remoteAddress}`); + cb(null, socket); + }); + } : (args.proxy?.options?.connect ?? args.agentOptions.connect), + }) + : this.nonProxiedAgent; + } + + /** + * Get agent by URL + * @param url URL + * @param bypassProxy Allways bypass proxy + */ + @bindThis + public getAgentByUrl(url: URL, bypassProxy = false): undici.Agent | undici.ProxyAgent { + if (bypassProxy || this.proxyBypassHosts.includes(url.hostname)) { + return this.nonProxiedAgent; + } else { + return this.agent; + } + } + + @bindThis + public async fetch( + url: string | URL, + options: undici.RequestInit = {}, + privateOptions: { noOkError?: boolean; bypassProxy?: boolean; } = { noOkError: false, bypassProxy: false } + ): Promise { + const res = await undici.fetch(url, { + dispatcher: this.getAgentByUrl(new URL(url), privateOptions.bypassProxy), + ...options, + headers: { + 'User-Agent': this.userAgent ?? '', + ...(options.headers ?? {}), + }, + }).catch((err) => { + this.logger?.error('fetch error', err); + throw new StatusError('Resource Unreachable', 500, 'Resource Unreachable'); + }); + if (!res.ok && !privateOptions.noOkError) { + throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); + } + return res; + } + + @bindThis + public async getJson(url: string, accept = 'application/json, */*', headers?: Record): Promise { + const res = await this.fetch( + url, + { + headers: Object.assign({ + Accept: accept, + }, headers ?? {}), + } + ); + + return await res.json() as T; + } + + @bindThis + public async getHtml(url: string, accept = 'text/html, */*', headers?: Record): Promise { + const res = await this.fetch( + url, + { + headers: Object.assign({ + Accept: accept, + }, headers ?? {}), + } + ); + + return await res.text(); + } +} @Injectable() export class HttpRequestService { - /** - * Get http non-proxy agent - */ + public defaultFetcher: UndiciFetcher; + public fetch: UndiciFetcher['fetch']; + public getHtml: UndiciFetcher['getHtml']; + public defaultJsonFetcher: UndiciFetcher; + public getJson: UndiciFetcher['getJson']; + + //#region for old http/https, only used in S3Service + // http non-proxy agent private http: http.Agent; - /** - * Get https non-proxy agent - */ + // https non-proxy agent private https: https.Agent; - /** - * Get http proxy or non-proxy agent - */ + // http proxy or non-proxy agent public httpAgent: http.Agent; - /** - * Get https proxy or non-proxy agent - */ + // https proxy or non-proxy agent public httpsAgent: https.Agent; + //#endregion + + public readonly dnsCache: CacheableLookup; + public readonly clientDefaults: undici.Agent.Options; + private maxSockets: number; + + private logger: Logger; constructor( @Inject(DI.config) private config: Config, + private loggerService: LoggerService, ) { - const cache = new CacheableLookup({ + this.logger = this.loggerService.getLogger('http-request'); + + this.dnsCache = new CacheableLookup({ maxTtl: 3600, // 1hours errorTtl: 30, // 30secs lookup: false, // nativeのdns.lookupにfallbackしない }); - + + this.clientDefaults = { + keepAliveTimeout: 30 * 1000, + keepAliveMaxTimeout: 10 * 60 * 1000, + keepAliveTimeoutThreshold: 1 * 1000, + strictContentLength: true, + headersTimeout: 10 * 1000, + bodyTimeout: 10 * 1000, + maxHeaderSize: 16364, // default + maxResponseSize: 10 * 1024 * 1024, + maxRedirections: 3, + connect: { + timeout: 10 * 1000, // コネクションが確立するまでのタイムアウト + maxCachedSessions: 300, // TLSセッションのキャッシュ数 https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L80 + lookup: this.dnsCache.lookup as LookupFunction, // https://github.com/nodejs/undici/blob/v5.14.0/lib/core/connect.js#L98 + }, + } + + this.maxSockets = Math.max(64, this.config.deliverJobConcurrency ?? 128); + + this.defaultFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption(), this.logger); + + this.fetch = this.defaultFetcher.fetch; + this.getHtml = this.defaultFetcher.getHtml; + + this.defaultJsonFetcher = new UndiciFetcher(this.getStandardUndiciFetcherOption({ + maxResponseSize: 1024 * 256, + }), this.logger); + + this.getJson = this.defaultJsonFetcher.getJson; + + //#region for old http/https, only used in S3Service this.http = new http.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, + lookup: this.dnsCache.lookup, } as http.AgentOptions); this.https = new https.Agent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - lookup: cache.lookup, + lookup: this.dnsCache.lookup, } as https.AgentOptions); - - const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128); - + this.httpAgent = config.proxy ? new HttpProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - maxSockets, + maxSockets: this.maxSockets, maxFreeSockets: 256, scheduling: 'lifo', proxy: config.proxy, @@ -72,21 +262,42 @@ export class HttpRequestService { ? new HttpsProxyAgent({ keepAlive: true, keepAliveMsecs: 30 * 1000, - maxSockets, + maxSockets: this.maxSockets, maxFreeSockets: 256, scheduling: 'lifo', proxy: config.proxy, }) : this.https; + //#endregion + } + + @bindThis + public getStandardUndiciFetcherOption(opts: undici.Agent.Options = {}, proxyOpts: undici.Agent.Options = {}) { + return { + agentOptions: { + ...this.clientDefaults, + ...opts, + }, + ...(this.config.proxy ? { + proxy: { + uri: this.config.proxy, + options: { + connections: this.maxSockets, + ...proxyOpts, + } + } + } : {}), + userAgent: this.config.userAgent, + } } /** - * Get agent by URL + * Get http agent by URL * @param url URL * @param bypassProxy Allways bypass proxy */ @bindThis - public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { + public getHttpAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { if (bypassProxy || (this.config.proxyBypassHosts || []).includes(url.hostname)) { return url.protocol === 'http:' ? this.http : this.https; } else { @@ -94,67 +305,37 @@ export class HttpRequestService { } } + /** + * check ip + */ @bindThis - public async getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record): Promise { - const res = await this.getResponse({ - url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': this.config.userAgent, - Accept: accept, - }, headers ?? {}), - timeout, - size: 1024 * 256, - }); + public getConnectorWithIpCheck(connector: undici.buildConnector.connector, checkIp: IpChecker): undici.buildConnector.connectorAsync { + return (options, cb) => { + connector(options, (err, socket) => { + this.logger.debug('Socket connector (with ip checker) called', socket); + if (err) { + this.logger.error(`Socket error`, err) + cb(new Error(`Error while socket connecting\n${err}`), null); + return; + } - return await res.json(); - } + if (socket.remoteAddress == undefined) { + this.logger.error(`Socket error: remoteAddress is undefined`); + cb(new Error('remoteAddress is undefined (maybe socket destroyed)'), null); + return; + } - @bindThis - public async getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record): Promise { - const res = await this.getResponse({ - url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': this.config.userAgent, - Accept: accept, - }, headers ?? {}), - timeout, - }); + // allow + if (checkIp(socket.remoteAddress)) { + this.logger.debug(`Socket connected (ip ok): ${socket.localPort} => ${socket.remoteAddress}`); + cb(null, socket); + return; + } - return await res.text(); - } - - @bindThis - public async getResponse(args: { - url: string, - method: string, - body?: string, - headers: Record, - timeout?: number, - size?: number, - }): Promise { - const timeout = args.timeout ?? 10 * 1000; - - const controller = new AbortController(); - setTimeout(() => { - controller.abort(); - }, timeout * 6); - - const res = await fetch(args.url, { - method: args.method, - headers: args.headers, - body: args.body, - timeout, - size: args.size ?? 10 * 1024 * 1024, - agent: (url) => this.getAgentByUrl(url), - signal: controller.signal, - }); - - if (!res.ok) { - throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText); - } - - return res; + this.logger.error('IP is not allowed', socket); + cb(new StatusError('IP is not allowed', 403, 'IP is not allowed'), null); + socket.destroy(); + }); + }; } } diff --git a/packages/backend/src/core/S3Service.ts b/packages/backend/src/core/S3Service.ts index 0ce69aaa74..930188ce6e 100644 --- a/packages/backend/src/core/S3Service.ts +++ b/packages/backend/src/core/S3Service.ts @@ -33,7 +33,7 @@ export class S3Service { ? false : meta.objectStorageS3ForcePathStyle, httpOptions: { - agent: this.httpRequestService.getAgentByUrl(new URL(u), !meta.objectStorageUseProxy), + agent: this.httpRequestService.getHttpAgentByUrl(new URL(u), !meta.objectStorageUseProxy), }, }); } diff --git a/packages/backend/src/core/WebfingerService.ts b/packages/backend/src/core/WebfingerService.ts index 4c91ab8438..69df2d0c1b 100644 --- a/packages/backend/src/core/WebfingerService.ts +++ b/packages/backend/src/core/WebfingerService.ts @@ -30,7 +30,7 @@ export class WebfingerService { public async webfinger(query: string): Promise { const url = this.genUrl(query); - return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json') as IWebFinger; + return await this.httpRequestService.getJson(url, 'application/jrd+json, application/json'); } @bindThis diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index d1edd579fa..d44d06a442 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -5,8 +5,10 @@ import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { User } from '@/models/entities/User.js'; import { UserKeypairStoreService } from '@/core/UserKeypairStoreService.js'; -import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; +import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; +import type Logger from '@/logger.js'; type Request = { url: string; @@ -28,13 +30,21 @@ type PrivateKey = { @Injectable() export class ApRequestService { + private undiciFetcher: UndiciFetcher; + private logger: Logger; + constructor( @Inject(DI.config) private config: Config, private userKeypairStoreService: UserKeypairStoreService, private httpRequestService: HttpRequestService, + private loggerService: LoggerService, ) { + this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる + this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({ + maxRedirections: 0, + }), this.logger ); } @bindThis @@ -148,16 +158,17 @@ export class ApRequestService { url, body, additionalHeaders: { - 'User-Agent': this.config.userAgent, }, }); - await this.httpRequestService.getResponse({ + await this.undiciFetcher.fetch( url, - method: req.request.method, - headers: req.request.headers, - body, - }); + { + method: req.request.method, + headers: req.request.headers, + body, + } + ); } /** @@ -176,15 +187,16 @@ export class ApRequestService { }, url, additionalHeaders: { - 'User-Agent': this.config.userAgent, }, }); - const res = await this.httpRequestService.getResponse({ + const res = await this.httpRequestService.fetch( url, - method: req.request.method, - headers: req.request.headers, - }); + { + method: req.request.method, + headers: req.request.headers, + } + ); return await res.json(); } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index e96c84f148..dc84f06a6a 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -4,7 +4,7 @@ import { InstanceActorService } from '@/core/InstanceActorService.js'; import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository } from '@/models/index.js'; import type { Config } from '@/config.js'; import { MetaService } from '@/core/MetaService.js'; -import { HttpRequestService } from '@/core/HttpRequestService.js'; +import { HttpRequestService, UndiciFetcher } from '@/core/HttpRequestService.js'; import { DI } from '@/di-symbols.js'; import { UtilityService } from '@/core/UtilityService.js'; import { bindThis } from '@/decorators.js'; @@ -12,11 +12,15 @@ import { isCollectionOrOrderedCollection } from './type.js'; import { ApDbResolverService } from './ApDbResolverService.js'; import { ApRendererService } from './ApRendererService.js'; import { ApRequestService } from './ApRequestService.js'; +import { LoggerService } from '@/core/LoggerService.js'; import type { IObject, ICollection, IOrderedCollection } from './type.js'; +import type Logger from '@/logger.js'; export class Resolver { private history: Set; private user?: ILocalUser; + private undiciFetcher: UndiciFetcher; + private logger: Logger; constructor( private config: Config, @@ -31,9 +35,14 @@ export class Resolver { private httpRequestService: HttpRequestService, private apRendererService: ApRendererService, private apDbResolverService: ApDbResolverService, + private loggerService: LoggerService, private recursionLimit = 100, ) { this.history = new Set(); + this.logger = this.loggerService?.getLogger('ap-resolve'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる + this.undiciFetcher = new UndiciFetcher(this.httpRequestService.getStandardUndiciFetcherOption({ + maxRedirections: 0, + }), this.logger); } @bindThis @@ -96,8 +105,8 @@ export class Resolver { } const object = (this.user - ? await this.apRequestService.signedGet(value, this.user) - : await this.httpRequestService.getJson(value, 'application/activity+json, application/ld+json')) as IObject; + ? await this.apRequestService.signedGet(value, this.user) as IObject + : await this.undiciFetcher.getJson(value, 'application/activity+json, application/ld+json')); if (object == null || ( Array.isArray(object['@context']) ? diff --git a/packages/backend/src/core/activitypub/LdSignatureService.ts b/packages/backend/src/core/activitypub/LdSignatureService.ts index b71320ed0b..4e4b7dce2d 100644 --- a/packages/backend/src/core/activitypub/LdSignatureService.ts +++ b/packages/backend/src/core/activitypub/LdSignatureService.ts @@ -1,6 +1,5 @@ import * as crypto from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; -import fetch from 'node-fetch'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { CONTEXTS } from './misc/contexts.js'; @@ -116,14 +115,19 @@ class LdSignature { @bindThis private async fetchDocument(url: string) { - const json = await fetch(url, { - headers: { - Accept: 'application/ld+json, application/json', + const json = await this.httpRequestService.fetch( + url, + { + headers: { + Accept: 'application/ld+json, application/json', + }, + // TODO + //timeout: this.loderTimeout, }, - // TODO - //timeout: this.loderTimeout, - agent: u => u.protocol === 'http:' ? this.httpRequestService.httpAgent : this.httpRequestService.httpsAgent, - }).then(res => { + { + noOkError: true, + } + ).then(res => { if (!res.ok) { throw `${res.status} ${res.statusText}`; } else { diff --git a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts index 183ef07477..f0543a5ed1 100644 --- a/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/WebhookDeliverProcessorService.ts @@ -33,24 +33,26 @@ export class WebhookDeliverProcessorService { try { this.logger.debug(`delivering ${job.data.webhookId}`); - const res = await this.httpRequestService.getResponse({ - url: job.data.to, - method: 'POST', - headers: { - 'User-Agent': 'Misskey-Hooks', - 'X-Misskey-Host': this.config.host, - 'X-Misskey-Hook-Id': job.data.webhookId, - 'X-Misskey-Hook-Secret': job.data.secret, - }, - body: JSON.stringify({ - hookId: job.data.webhookId, - userId: job.data.userId, - eventId: job.data.eventId, - createdAt: job.data.createdAt, - type: job.data.type, - body: job.data.content, - }), - }); + const res = await this.httpRequestService.fetch( + job.data.to, + { + method: 'POST', + headers: { + 'User-Agent': 'Misskey-Hooks', + 'X-Misskey-Host': this.config.host, + 'X-Misskey-Hook-Id': job.data.webhookId, + 'X-Misskey-Hook-Secret': job.data.secret, + }, + body: JSON.stringify({ + hookId: job.data.webhookId, + userId: job.data.userId, + eventId: job.data.eventId, + createdAt: job.data.createdAt, + type: job.data.type, + body: job.data.content, + }), + } + ); this.webhooksRepository.update({ id: job.data.webhookId }, { latestSentAt: new Date(), diff --git a/packages/backend/src/server/api/endpoints/fetch-rss.ts b/packages/backend/src/server/api/endpoints/fetch-rss.ts index 58fa01ac48..ae6a87513d 100644 --- a/packages/backend/src/server/api/endpoints/fetch-rss.ts +++ b/packages/backend/src/server/api/endpoints/fetch-rss.ts @@ -33,15 +33,16 @@ export default class extends Endpoint { private httpRequestService: HttpRequestService, ) { super(meta, paramDef, async (ps, me) => { - const res = await this.httpRequestService.getResponse({ - url: ps.url, - method: 'GET', - headers: Object.assign({ - 'User-Agent': config.userAgent, - Accept: 'application/rss+xml, */*', - }), - timeout: 5000, - }); + const res = await this.httpRequestService.fetch( + ps.url, + { + method: 'GET', + headers: { + Accept: 'application/rss+xml, */*', + }, + // timeout: 5000, + } + ); const text = await res.text(); diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index ec16965998..ab19771676 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -1,5 +1,4 @@ import { URLSearchParams } from 'node:url'; -import fetch from 'node-fetch'; import { Inject, Injectable } from '@nestjs/common'; import type { NotesRepository } from '@/models/index.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; @@ -84,25 +83,27 @@ export default class extends Endpoint { const endpoint = instance.deeplIsPro ? 'https://api.deepl.com/v2/translate' : 'https://api-free.deepl.com/v2/translate'; - const res = await fetch(endpoint, { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - 'User-Agent': config.userAgent, - Accept: 'application/json, */*', + const res = await this.httpRequestService.fetch( + endpoint, + { + method: 'POST', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + Accept: 'application/json, */*', + }, + body: params.toString(), }, - body: params, - // TODO - //timeout: 10000, - agent: (url) => this.httpRequestService.getAgentByUrl(url), - }); + { + noOkError: false, + } + ); const json = (await res.json()) as { - translations: { - detected_source_language: string; - text: string; - }[]; - }; + translations: { + detected_source_language: string; + text: string; + }[]; + }; return { sourceLang: json.translations[0].detected_source_language, diff --git a/packages/backend/src/server/api/integration/DiscordServerService.ts b/packages/backend/src/server/api/integration/DiscordServerService.ts index 805056da8b..0ac2733817 100644 --- a/packages/backend/src/server/api/integration/DiscordServerService.ts +++ b/packages/backend/src/server/api/integration/DiscordServerService.ts @@ -181,7 +181,7 @@ export class DiscordServerService { } })); - const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, { + const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', { 'Authorization': `Bearer ${accessToken}`, })) as Record; @@ -249,7 +249,7 @@ export class DiscordServerService { } })); - const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', 10 * 1000, { + const { id, username, discriminator } = (await this.httpRequestService.getJson('https://discord.com/api/users/@me', '*/*', { 'Authorization': `Bearer ${accessToken}`, })) as Record; if (typeof id !== 'string' || typeof username !== 'string' || typeof discriminator !== 'string') { diff --git a/packages/backend/src/server/api/integration/GithubServerService.ts b/packages/backend/src/server/api/integration/GithubServerService.ts index 6f38c262a1..a8c745d2dc 100644 --- a/packages/backend/src/server/api/integration/GithubServerService.ts +++ b/packages/backend/src/server/api/integration/GithubServerService.ts @@ -174,7 +174,7 @@ export class GithubServerService { } })); - const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, { + const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', { 'Authorization': `bearer ${accessToken}`, })) as Record; if (typeof login !== 'string' || typeof id !== 'string') { @@ -223,7 +223,7 @@ export class GithubServerService { } })); - const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', 10 * 1000, { + const { login, id } = (await this.httpRequestService.getJson('https://api.github.com/user', 'application/vnd.github.v3+json', { 'Authorization': `bearer ${accessToken}`, })) as Record; diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index baef8fa993..802b404ce6 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -63,9 +63,8 @@ export class UrlPreviewService { this.logger.info(meta.summalyProxy ? `(Proxy) Getting preview of ${url}@${lang} ...` : `Getting preview of ${url}@${lang} ...`); - try { - const summary = meta.summalyProxy ? await this.httpRequestService.getJson(`${meta.summalyProxy}?${query({ + const summary = meta.summalyProxy ? await this.httpRequestService.getJson>(`${meta.summalyProxy}?${query({ url: url, lang: lang ?? 'ja-JP', })}`) : await summaly.default(url, { diff --git a/yarn.lock b/yarn.lock index 2e45bc1440..cdf2a19e1d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4272,6 +4272,7 @@ __metadata: typeorm: 0.3.11 typescript: 4.9.4 ulid: 2.3.0 + undici: ^5.14.0 unzipper: 0.10.11 uuid: 9.0.0 vary: 1.1.2 @@ -16665,6 +16666,15 @@ __metadata: languageName: node linkType: hard +"undici@npm:^5.14.0, undici@npm:^5.5.1": + version: 5.14.0 + resolution: "undici@npm:5.14.0" + dependencies: + busboy: ^1.6.0 + checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26 + languageName: node + linkType: hard + "undici@npm:^5.2.0": version: 5.13.0 resolution: "undici@npm:5.13.0" @@ -16674,15 +16684,6 @@ __metadata: languageName: node linkType: hard -"undici@npm:^5.5.1": - version: 5.14.0 - resolution: "undici@npm:5.14.0" - dependencies: - busboy: ^1.6.0 - checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26 - languageName: node - linkType: hard - "union-value@npm:^1.0.0": version: 1.0.1 resolution: "union-value@npm:1.0.1"