From ca66acac2b9ac29804a92c496b4f08056e0233b3 Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Sat, 11 Apr 2020 18:28:40 +0900 Subject: [PATCH] =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=AE?= =?UTF-8?q?=E3=83=80=E3=82=A6=E3=83=B3=E3=83=AD=E3=83=BC=E3=83=89=E3=81=8C?= =?UTF-8?q?=E3=82=BF=E3=82=A4=E3=83=A0=E3=82=A2=E3=82=A6=E3=83=88=E3=81=97?= =?UTF-8?q?=E3=81=AA=E3=81=8F=E3=81=AA=E3=81=A3=E3=81=A6=E3=81=84=E3=82=8B?= =?UTF-8?q?=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3=E3=81=AA=E3=81=A9=20(#6233?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refactor download / file-info * body read timeout on download Fix syuilo#6232 --- package.json | 1 + src/misc/donwload-url.ts | 48 +++++++++++++-------------------------- src/misc/get-file-info.ts | 29 ++++++++--------------- 3 files changed, 26 insertions(+), 52 deletions(-) diff --git a/package.json b/package.json index 67fd51565d..0214f8f3a9 100644 --- a/package.json +++ b/package.json @@ -102,6 +102,7 @@ "@types/websocket": "1.0.0", "@types/ws": "7.2.3", "@typescript-eslint/parser": "2.26.0", + "abort-controller": "3.0.0", "animejs": "3.1.0", "apexcharts": "3.17.1", "autobind-decorator": "2.4.0", diff --git a/src/misc/donwload-url.ts b/src/misc/donwload-url.ts index cd15bbd731..3f42fb3bef 100644 --- a/src/misc/donwload-url.ts +++ b/src/misc/donwload-url.ts @@ -1,55 +1,39 @@ import * as fs from 'fs'; +import * as stream from 'stream'; +import * as util from 'util'; import fetch from 'node-fetch'; import { httpAgent, httpsAgent } from './fetch'; +import { AbortController } from 'abort-controller'; import config from '../config'; import * as chalk from 'chalk'; import Logger from '../services/logger'; +const pipeline = util.promisify(stream.pipeline); + export async function downloadUrl(url: string, path: string) { const logger = new Logger('download'); logger.info(`Downloading ${chalk.cyan(url)} ...`); + const controller = new AbortController(); + setTimeout(() => { + controller.abort(); + }, 11 * 1000); const response = await fetch(new URL(url).href, { headers: { 'User-Agent': config.userAgent }, timeout: 10 * 1000, + signal: controller.signal, agent: u => u.protocol == 'http:' ? httpAgent : httpsAgent, - }).then(response => { - if (!response.ok) { - logger.error(`Got ${response.status} (${url})`); - throw response.status; - } else { - return response; - } }); - await new Promise((res, rej) => { - const writable = fs.createWriteStream(path); + if (!response.ok) { + logger.error(`Got ${response.status} (${url})`); + throw response.status; + } - response.body.on('error', (error: any) => { - logger.error(`Failed to start download: ${chalk.cyan(url)}: ${error}`, { - url: url, - e: error - }); - writable.close(); - rej(error); - }); + await pipeline(response.body, fs.createWriteStream(path)); - writable.on('finish', () => { - logger.succ(`Download finished: ${chalk.cyan(url)}`); - res(); - }); - - writable.on('error', error => { - logger.error(`Download failed: ${chalk.cyan(url)}: ${error}`, { - url: url, - e: error - }); - rej(error); - }); - - response.body.pipe(writable); - }); + logger.succ(`Download finished: ${chalk.cyan(url)}`); } diff --git a/src/misc/get-file-info.ts b/src/misc/get-file-info.ts index 5ccb280260..b838900f61 100644 --- a/src/misc/get-file-info.ts +++ b/src/misc/get-file-info.ts @@ -1,10 +1,14 @@ import * as fs from 'fs'; import * as crypto from 'crypto'; +import * as stream from 'stream'; +import * as util from 'util'; import * as fileType from 'file-type'; import isSvg from 'is-svg'; import * as probeImageSize from 'probe-image-size'; import * as sharp from 'sharp'; +const pipeline = util.promisify(stream.pipeline); + export type FileInfo = { size: number; md5: string; @@ -138,32 +142,17 @@ export async function checkSvg(path: string) { * Get file size */ export async function getFileSize(path: string): Promise { - return new Promise((res, rej) => { - fs.stat(path, (err, stats) => { - if (err) return rej(err); - res(stats.size); - }); - }); + const getStat = util.promisify(fs.stat); + return (await getStat(path)).size; } /** * Calculate MD5 hash */ async function calcHash(path: string): Promise { - return new Promise((res, rej) => { - const readable = fs.createReadStream(path); - const hash = crypto.createHash('md5'); - const chunks: Buffer[] = []; - readable - .on('error', rej) - .pipe(hash) - .on('error', rej) - .on('data', chunk => chunks.push(chunk)) - .on('end', () => { - const buffer = Buffer.concat(chunks); - res(buffer.toString('hex')); - }); - }); + const hash = crypto.createHash('md5').setEncoding('hex'); + await pipeline(fs.createReadStream(path), hash); + return hash.read(); } /**