From c9d5c64f98c1f1fe7950de60c58edeaf3ace070d Mon Sep 17 00:00:00 2001 From: Rigel Kent Date: Mon, 27 Aug 2018 13:28:49 +0200 Subject: [PATCH] replace fs by fs-extra to prevent EMFILE error --- package.json | 2 +- scripts/generate-code-contributors.ts | 2 +- scripts/i18n/create-custom-files.ts | 2 +- scripts/i18n/xliff2json.ts | 2 +- scripts/parse-log.ts | 2 +- server/controllers/static.ts | 2 +- server/helpers/captions-utils.ts | 2 +- server/helpers/core-utils.ts | 3 +-- server/helpers/logger.ts | 4 ++-- server/helpers/requests.ts | 2 +- server/helpers/webtorrent.ts | 2 +- server/initializers/migrations/0065-video-file-size.ts | 2 +- server/lib/cache/abstract-video-static-file-cache.ts | 2 +- server/lib/emailer.ts | 2 +- server/lib/schedulers/youtube-dl-update-scheduler.ts | 2 +- server/tests/utils/users/accounts.ts | 2 +- server/tests/utils/videos/videos.ts | 2 +- server/tools/upload.ts | 2 +- 18 files changed, 19 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index 3f4e272ec..635ceaf76 100644 --- a/package.json +++ b/package.json @@ -100,6 +100,7 @@ "express-validator": "^5.0.0", "flat": "^4.1.0", "fluent-ffmpeg": "^2.1.0", + "fs-extra": "^7.0.0", "helmet": "^3.12.1", "ip-anonymize": "^0.0.6", "ipaddr.js": "https://github.com/whitequark/ipaddr.js.git#8e69afeb4053ee32447a101845f860848280eca5", @@ -110,7 +111,6 @@ "jsonld-signatures": "https://github.com/Chocobozzz/jsonld-signatures#rsa2017", "lodash": "^4.17.10", "magnet-uri": "^5.1.4", - "mkdirp": "^0.5.1", "morgan": "^1.5.3", "multer": "^1.1.0", "nodemailer": "^4.4.2", diff --git a/scripts/generate-code-contributors.ts b/scripts/generate-code-contributors.ts index 0cce62180..3d37372d1 100755 --- a/scripts/generate-code-contributors.ts +++ b/scripts/generate-code-contributors.ts @@ -1,5 +1,5 @@ import { doRequest } from '../server/helpers/requests' -import { readFileSync } from 'fs' +import { readFileSync } from 'fs-extra' run() .then(() => process.exit(0)) diff --git a/scripts/i18n/create-custom-files.ts b/scripts/i18n/create-custom-files.ts index c4f8411e1..eed3182a6 100755 --- a/scripts/i18n/create-custom-files.ts +++ b/scripts/i18n/create-custom-files.ts @@ -1,5 +1,5 @@ import * as jsToXliff12 from 'xliff/jsToXliff12' -import { writeFile } from 'fs' +import { writeFile } from 'fs-extra' import { join } from 'path' import { buildLanguages, diff --git a/scripts/i18n/xliff2json.ts b/scripts/i18n/xliff2json.ts index ed6854514..7e6de2893 100755 --- a/scripts/i18n/xliff2json.ts +++ b/scripts/i18n/xliff2json.ts @@ -1,5 +1,5 @@ import * as xliff12ToJs from 'xliff/xliff12ToJs' -import { readFileSync, unlink, writeFile } from 'fs' +import { readFileSync, unlink, writeFile } from 'fs-extra' import { join } from 'path' import { buildFileLocale, I18N_LOCALES, isDefaultLocale } from '../../shared/models/i18n/i18n' import { eachSeries } from 'async' diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts index d22e90266..86aaa7994 100755 --- a/scripts/parse-log.ts +++ b/scripts/parse-log.ts @@ -1,5 +1,5 @@ import * as program from 'commander' -import { createReadStream, readdirSync, statSync } from 'fs' +import { createReadStream, readdirSync, statSync } from 'fs-extra' import { join } from 'path' import { createInterface } from 'readline' import * as winston from 'winston' diff --git a/server/controllers/static.ts b/server/controllers/static.ts index 2a92810f3..6e8f1a07f 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -1,5 +1,5 @@ import * as cors from 'cors' -import { createReadStream } from 'fs' +import { createReadStream } from 'fs-extra' import * as express from 'express' import { CONFIG, STATIC_DOWNLOAD_PATHS, STATIC_MAX_AGE, STATIC_PATHS, ROUTE_CACHE_LIFETIME } from '../initializers' import { VideosPreviewCache } from '../lib/cache' diff --git a/server/helpers/captions-utils.ts b/server/helpers/captions-utils.ts index 8b04f878d..20c9fe5aa 100644 --- a/server/helpers/captions-utils.ts +++ b/server/helpers/captions-utils.ts @@ -3,7 +3,7 @@ import { join } from 'path' import { CONFIG } from '../initializers' import { VideoCaptionModel } from '../models/video/video-caption' import * as srt2vtt from 'srt-to-vtt' -import { createReadStream, createWriteStream } from 'fs' +import { createReadStream, createWriteStream } from 'fs-extra' async function moveAndProcessCaptionFile (physicalFile: { filename: string, path: string }, videoCaption: VideoCaptionModel) { const videoCaptionsDir = CONFIG.STORAGE.CAPTIONS_DIR diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 90d2cd9b3..9830d41a8 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -6,8 +6,7 @@ import * as bcrypt from 'bcrypt' import * as createTorrent from 'create-torrent' import { createHash, pseudoRandomBytes } from 'crypto' -import { copyFile, readdir, readFile, rename, stat, Stats, unlink, writeFile } from 'fs' -import * as mkdirp from 'mkdirp' +import { copyFile, readdir, readFile, rename, stat, Stats, unlink, writeFile, mkdirp } from 'fs-extra' import { isAbsolute, join } from 'path' import * as pem from 'pem' import * as rimraf from 'rimraf' diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index 480c5b49e..ce6e38f15 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts @@ -1,5 +1,5 @@ // Thanks http://tostring.it/2014/06/23/advanced-logging-with-nodejs/ -import * as mkdirp from 'mkdirp' +import { mkdirpSync } from 'fs-extra' import * as path from 'path' import * as winston from 'winston' import { CONFIG } from '../initializers' @@ -7,7 +7,7 @@ import { CONFIG } from '../initializers' const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT // Create the directory if it does not exist -mkdirp.sync(CONFIG.STORAGE.LOG_DIR) +mkdirpSync(CONFIG.STORAGE.LOG_DIR) function loggerReplacer (key: string, value: any) { if (value instanceof Error) { diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 64e3ce663..ee9e80404 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -1,5 +1,5 @@ import * as Bluebird from 'bluebird' -import { createWriteStream } from 'fs' +import { createWriteStream } from 'fs-extra' import * as request from 'request' import { ACTIVITY_PUB } from '../initializers' diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 6f2adb3cb..1c0d00d70 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -1,7 +1,7 @@ import { logger } from './logger' import { generateVideoTmpPath } from './utils' import * as WebTorrent from 'webtorrent' -import { createWriteStream } from 'fs' +import { createWriteStream } from 'fs-extra' import { CONFIG } from '../initializers' import { join } from 'path' import { unlinkPromise } from './core-utils' diff --git a/server/initializers/migrations/0065-video-file-size.ts b/server/initializers/migrations/0065-video-file-size.ts index 4e2075f8b..66f25016a 100644 --- a/server/initializers/migrations/0065-video-file-size.ts +++ b/server/initializers/migrations/0065-video-file-size.ts @@ -1,6 +1,6 @@ import * as Sequelize from 'sequelize' import * as Promise from 'bluebird' -import { stat } from 'fs' +import { stat } from 'fs-extra' import { VideoModel } from '../../models/video/video' function up (utils: { diff --git a/server/lib/cache/abstract-video-static-file-cache.ts b/server/lib/cache/abstract-video-static-file-cache.ts index 8e895cc82..3e20c5d2a 100644 --- a/server/lib/cache/abstract-video-static-file-cache.ts +++ b/server/lib/cache/abstract-video-static-file-cache.ts @@ -1,5 +1,5 @@ import * as AsyncLRU from 'async-lru' -import { createWriteStream } from 'fs' +import { createWriteStream } from 'fs-extra' import { unlinkPromise } from '../../helpers/core-utils' import { logger } from '../../helpers/logger' import { VideoModel } from '../../models/video/video' diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts index c8398c9e7..bf8e5b6c3 100644 --- a/server/lib/emailer.ts +++ b/server/lib/emailer.ts @@ -7,7 +7,7 @@ import { UserModel } from '../models/account/user' import { VideoModel } from '../models/video/video' import { JobQueue } from './job-queue' import { EmailPayload } from './job-queue/handlers/email' -import { readFileSync } from 'fs' +import { readFileSync } from 'fs-extra' class Emailer { diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts index 24cd3f87b..da47378e8 100644 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts @@ -5,7 +5,7 @@ import { AbstractScheduler } from './abstract-scheduler' import { SCHEDULER_INTERVALS_MS } from '../../initializers' import { logger } from '../../helpers/logger' import * as request from 'request' -import { createWriteStream, writeFile } from 'fs' +import { createWriteStream, writeFile } from 'fs-extra' import { join } from 'path' import { mkdirpPromise, root } from '../../helpers/core-utils' diff --git a/server/tests/utils/users/accounts.ts b/server/tests/utils/users/accounts.ts index 30b3c54f8..024a315c7 100644 --- a/server/tests/utils/users/accounts.ts +++ b/server/tests/utils/users/accounts.ts @@ -1,7 +1,7 @@ /* tslint:disable:no-unused-expression */ import { expect } from 'chai' -import { existsSync } from 'fs' +import { existsSync } from 'fs-extra' import { join } from 'path' import { Account } from '../../../../shared/models/actors' import { readdirPromise } from '../../../helpers/core-utils' diff --git a/server/tests/utils/videos/videos.ts b/server/tests/utils/videos/videos.ts index 674a92df9..973bbbe87 100644 --- a/server/tests/utils/videos/videos.ts +++ b/server/tests/utils/videos/videos.ts @@ -1,7 +1,7 @@ /* tslint:disable:no-unused-expression */ import { expect } from 'chai' -import { existsSync, readFile } from 'fs' +import { existsSync, readFile } from 'fs-extra' import * as parseTorrent from 'parse-torrent' import { extname, join } from 'path' import * as request from 'supertest' diff --git a/server/tools/upload.ts b/server/tools/upload.ts index 4d40c8c1a..b5630bb9c 100644 --- a/server/tools/upload.ts +++ b/server/tools/upload.ts @@ -1,5 +1,5 @@ import * as program from 'commander' -import { access, constants } from 'fs' +import { access, constants } from 'fs-extra' import { isAbsolute } from 'path' import { promisify } from 'util' import { getClient, login } from '../tests/utils' -- 2.25.1