From 23e27dd53599be65b2dc2968448ce155a00a96c9 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 19 Jan 2018 13:58:13 +0100 Subject: [PATCH] Add ability to configure log level --- config/default.yaml | 3 + config/production.yaml.example | 3 + package.json | 3 +- scripts/parse-log.ts | 39 ++++--- server.ts | 2 +- server/helpers/core-utils.ts | 2 + server/helpers/logger.ts | 60 ++++++++--- server/initializers/checker.ts | 2 +- server/initializers/constants.ts | 3 + server/models/activitypub/actor-follow.ts | 2 +- server/tests/api/server/config.ts | 4 + support/doc/production.md | 6 ++ yarn.lock | 121 ++++++++++++++++------ 13 files changed, 183 insertions(+), 67 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 07bb26d5f..3d0732069 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -24,6 +24,9 @@ storage: torrents: 'storage/torrents/' cache: 'storage/cache/' +log: + level: 'debug' # debug/info/warning/error + cache: previews: size: 1 # Max number of previews you want to cache diff --git a/config/production.yaml.example b/config/production.yaml.example index 985b25a52..69052fb26 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -25,6 +25,9 @@ storage: torrents: '/home/peertube/storage/torrents/' cache: '/home/peertube/storage/cache/' +log: + level: 'debug' # debug/info/warning/error + cache: previews: size: 100 # Max number of previews you want to cache diff --git a/package.json b/package.json index 23ea8928a..fcac1c7c2 100644 --- a/package.json +++ b/package.json @@ -89,7 +89,7 @@ "uuid": "^3.1.0", "validator": "^9.0.0", "webfinger.js": "^2.6.6", - "winston": "^2.1.1", + "winston": "3.0.0-rc1", "ws": "^3.3.2" }, "devDependencies": { @@ -114,7 +114,6 @@ "@types/supertest": "^2.0.3", "@types/validator": "^6.2.0", "@types/webtorrent": "^0.98.4", - "@types/winston": "^2.3.2", "@types/ws": "^3.0.2", "chai": "^4.1.1", "commander": "^2.9.0", diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts index e2c42bf4c..7e804b3f9 100755 --- a/scripts/parse-log.ts +++ b/scripts/parse-log.ts @@ -2,34 +2,34 @@ import { createReadStream } from 'fs' import { join } from 'path' import { createInterface } from 'readline' import * as winston from 'winston' +import { labelFormatter, loggerFormat, timestampFormatter } from '../server/helpers/logger' import { CONFIG } from '../server/initializers/constants' -const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT - -const logger = new winston.Logger({ +const logger = new winston.createLogger({ transports: [ new winston.transports.Console({ level: 'debug', - label: label, - handleExceptions: true, - humanReadableUnhandledException: true, - json: false, - colorize: true, - prettyPrint: true, - stderrLevels: [] + stderrLevels: [], + format: winston.format.combine( + timestampFormatter, + winston.format.splat(), + labelFormatter, + winston.format.colorize(), + loggerFormat + ) }) ], exitOnError: true }) const logLevels = { - error: logger.error, - warn: logger.warn, - info: logger.info, - debug: logger.debug + error: logger.error.bind(logger), + warn: logger.warn.bind(logger), + info: logger.info.bind(logger), + debug: logger.debug.bind(logger) } -const path = join(CONFIG.STORAGE.LOG_DIR, 'all-logs.log') +const path = join(CONFIG.STORAGE.LOG_DIR, 'peertube.log') console.log('Opening %s.', path) const rl = createInterface({ @@ -38,11 +38,8 @@ const rl = createInterface({ rl.on('line', line => { const log = JSON.parse(line) - const additionalInfo: any = {} - - Object.keys(log).forEach(logKey => { - if (logKey !== 'message' && logKey !== 'level') additionalInfo[logKey] = log[logKey] - }) + // Don't know why but loggerFormat does not remove splat key + Object.assign(log, { splat: undefined }) - logLevels[log.level](log.message, additionalInfo) + logLevels[log.level](log) }) diff --git a/server.ts b/server.ts index 99077a173..a138b6359 100644 --- a/server.ts +++ b/server.ts @@ -82,7 +82,7 @@ if (isTestInstance()) { // For the logger app.use(morgan('combined', { - stream: { write: logger.info } + stream: { write: logger.info.bind(logger) } })) // For body requests app.use(bodyParser.json({ diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 77547c528..65f18d644 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -50,6 +50,8 @@ function root () { // Thanks: https://stackoverflow.com/a/12034334 function escapeHTML (stringParam) { + if (!stringParam) return '' + const entityMap = { '&': '&', '<': '<', diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts index 2676133db..6a02f680a 100644 --- a/server/helpers/logger.ts +++ b/server/helpers/logger.ts @@ -9,26 +9,57 @@ const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT // Create the directory if it does not exist mkdirp.sync(CONFIG.STORAGE.LOG_DIR) -const logger = new winston.Logger({ +// Use object for better performances (~ O(1)) +const excludedKeys = { + level: true, + message: true, + splat: true, + timestamp: true, + label: true +} +function keysExcluder (key, value) { + return excludedKeys[key] === true ? undefined : value +} + +const loggerFormat = winston.format.printf((info) => { + let additionalInfos = JSON.stringify(info, keysExcluder, 2) + if (additionalInfos === '{}') additionalInfos = '' + + return `[${info.label}] ${info.timestamp} ${info.level}: ${info.message} ${additionalInfos}` +}) + +const timestampFormatter = winston.format.timestamp({ + format: 'YYYY-MM-dd HH:mm:ss.SSS' +}) +const labelFormatter = winston.format.label({ + label +}) + +const logger = new winston.createLogger({ + level: CONFIG.LOG.LEVEL, transports: [ new winston.transports.File({ - level: 'debug', - filename: path.join(CONFIG.STORAGE.LOG_DIR, 'all-logs.log'), + filename: path.join(CONFIG.STORAGE.LOG_DIR, 'peertube.log'), handleExceptions: true, - json: true, maxsize: 5242880, maxFiles: 5, - colorize: false, - prettyPrint: true + format: winston.format.combine( + timestampFormatter, + labelFormatter, + winston.format.splat(), + winston.format.json() + ) }), new winston.transports.Console({ - level: 'debug', - label: label, handleExceptions: true, humanReadableUnhandledException: true, - json: false, - colorize: true, - prettyPrint: true + format: winston.format.combine( + timestampFormatter, + winston.format.splat(), + labelFormatter, + winston.format.colorize(), + loggerFormat + ) }) ], exitOnError: true @@ -36,4 +67,9 @@ const logger = new winston.Logger({ // --------------------------------------------------------------------------- -export { logger } +export { + timestampFormatter, + labelFormatter, + loggerFormat, + logger +} diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 7cfbc123d..35fab244c 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -21,7 +21,7 @@ function checkMissedConfig () { const required = [ 'listen.port', 'webserver.https', 'webserver.hostname', 'webserver.port', 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', - 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews', 'storage.torrents', 'storage.cache', + 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews', 'storage.torrents', 'storage.cache', 'log.level', 'cache.previews.size', 'admin.email', 'signup.enabled', 'signup.limit', 'transcoding.enabled', 'transcoding.threads', 'user.video_quota' ] const miss: string[] = [] diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index c10213890..cb043251a 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -113,6 +113,9 @@ const CONFIG = { URL: '', HOST: '' }, + LOG: { + LEVEL: config.get('log.level') + }, ADMIN: { get EMAIL () { return config.get('admin.email') } }, diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index 416496607..a32f5f498 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts @@ -376,7 +376,7 @@ export class ActorFollowModel extends Model { [Sequelize.Op.lte]: 0 } }, - logger: false + logging: false } return ActorFollowModel.findAll(query) diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts index 0aa0e2ec1..a1f8212bb 100644 --- a/server/tests/api/server/config.ts +++ b/server/tests/api/server/config.ts @@ -112,6 +112,8 @@ describe('Test config', function () { }) it('Should have the configuration updated after a restart', async function () { + this.timeout(10000) + killallServers([ server ]) await reRunServer(server) @@ -134,6 +136,8 @@ describe('Test config', function () { }) it('Should remove the custom configuration', async function () { + this.timeout(10000) + await deleteCustomConfig(server.url, server.accessToken) const res = await getCustomConfig(server.url, server.accessToken) diff --git a/support/doc/production.md b/support/doc/production.md index 0bdc36da1..d1ebbd291 100644 --- a/support/doc/production.md +++ b/support/doc/production.md @@ -247,6 +247,12 @@ $ SQL_BACKUP_PATH="backup/sql-peertube_prod-$(date -Im).bak" && \ sudo pg_dump -U peertube -W -h localhost -F c peertube_prod -f "$SQL_BACKUP_PATH" ``` +Update your configuration file. **If some keys are missing, your upgraded PeerTube won't start!** + +``` +$ diff <(curl -s https://raw.githubusercontent.com/Chocobozzz/PeerTube/develop/config/production.yaml.example) /home/peertube/config/production.yaml +``` + Upgrade PeerTube: ``` diff --git a/yarn.lock b/yarn.lock index 5fb157e08..7b8950435 100644 --- a/yarn.lock +++ b/yarn.lock @@ -203,12 +203,6 @@ "@types/parse-torrent" "*" "@types/simple-peer" "*" -"@types/winston@^2.3.2": - version "2.3.7" - resolved "https://registry.yarnpkg.com/@types/winston/-/winston-2.3.7.tgz#2ea18b2dc772d459b6af0f587447704df31afec2" - dependencies: - "@types/node" "*" - "@types/ws@^3.0.2": version "3.2.1" resolved "https://registry.yarnpkg.com/@types/ws/-/ws-3.2.1.tgz#b0c1579e58e686f83ce0a97bb9463d29705827fb" @@ -403,14 +397,10 @@ async@>=0.2.9, async@^2.0.0: dependencies: lodash "^4.14.0" -async@^1.5.2: +async@^1.0.0, async@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" -async@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/async/-/async-1.0.0.tgz#f8fc04ca3a13784ade9e1641af98578cfbd647a9" - asynckit@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -889,6 +879,10 @@ code-point-at@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" +color-convert@^0.5.0: + version "0.5.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-0.5.3.tgz#bdb6c69ce660fadffe0b0007cc447e1b9f7282bd" + color-convert@^1.9.0, color-convert@^1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.1.tgz#c1261107aeb2f294ebffec9ed9ecad529a6097ed" @@ -899,6 +893,12 @@ color-name@^1.0.0, color-name@^1.1.1: version "1.1.3" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" +color-string@^0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/color-string/-/color-string-0.3.0.tgz#27d46fb67025c5c2fa25993bfbf579e47841b991" + dependencies: + color-name "^1.0.0" + color-string@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.5.2.tgz#26e45814bc3c9a7cbd6751648a41434514a773a9" @@ -906,6 +906,13 @@ color-string@^1.5.2: color-name "^1.0.0" simple-swizzle "^0.2.2" +color@0.8.x: + version "0.8.0" + resolved "https://registry.yarnpkg.com/color/-/color-0.8.0.tgz#890c07c3fd4e649537638911cf691e5458b6fca5" + dependencies: + color-convert "^0.5.0" + color-string "^0.3.0" + color@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/color/-/color-2.0.1.tgz#e4ed78a3c4603d0891eba5430b04b86314f4c839" @@ -913,9 +920,20 @@ color@^2.0.0: color-convert "^1.9.1" color-string "^1.5.2" -colors@1.0.x: - version "1.0.3" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.0.3.tgz#0433f44d809680fdeb60ed260f1b0c262e82a40b" +colornames@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/colornames/-/colornames-0.0.2.tgz#d811fd6c84f59029499a8ac4436202935b92be31" + +colors@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.1.2.tgz#168a4701756b6a7f51a12ce0c97bfa28c084ed63" + +colorspace@1.0.x: + version "1.0.1" + resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.0.1.tgz#c99c796ed31128b9876a52e1ee5ee03a4a719749" + dependencies: + color "0.8.x" + text-hex "0.0.x" combined-stream@^1.0.5, combined-stream@~1.0.5: version "1.0.5" @@ -1087,10 +1105,6 @@ crypto-random-string@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-1.0.0.tgz#a230f64f568310e1498009940790ec99545bca7e" -cycle@1.0.x: - version "1.0.3" - resolved "https://registry.yarnpkg.com/cycle/-/cycle-1.0.3.tgz#21e80b2be8580f98b468f379430662b046c34ad2" - d@1: version "1.0.0" resolved "https://registry.yarnpkg.com/d/-/d-1.0.0.tgz#754bb5bfe55451da69a58b94d45f4c5b0462d58f" @@ -1201,6 +1215,14 @@ detect-libc@^1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" +diagnostics@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/diagnostics/-/diagnostics-1.1.0.tgz#e1090900b49523e8527be20f081275205f2ae36a" + dependencies: + colorspace "1.0.x" + enabled "1.0.x" + kuler "0.0.x" + dicer@0.2.5: version "0.2.5" resolved "https://registry.yarnpkg.com/dicer/-/dicer-0.2.5.tgz#5996c086bb33218c812c090bddc09cd12facb70f" @@ -1280,6 +1302,12 @@ elliptic@=3.0.3: hash.js "^1.0.0" inherits "^2.0.1" +enabled@1.0.x: + version "1.0.2" + resolved "https://registry.yarnpkg.com/enabled/-/enabled-1.0.2.tgz#965f6513d2c2d1c5f4652b64a2e3396467fc2f93" + dependencies: + env-variable "0.0.x" + encodeurl@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.1.tgz#79e3d58655346909fe6f0f45a5de68103b294d20" @@ -1290,6 +1318,10 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0: dependencies: once "^1.4.0" +env-variable@0.0.x: + version "0.0.3" + resolved "https://registry.yarnpkg.com/env-variable/-/env-variable-0.0.3.tgz#b86c1641be5610267d506f18071ea76d707097cb" + error-ex@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.1.tgz#f855a86ce61adc4e8621c3cda21e7a7612c3a8dc" @@ -1657,10 +1689,6 @@ extsprintf@^1.2.0: version "1.4.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" -eyes@0.1.x: - version "0.1.8" - resolved "https://registry.yarnpkg.com/eyes/-/eyes-0.1.8.tgz#62cf120234c683785d902348a800ef3e0cc20bc0" - fast-deep-equal@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" @@ -1673,6 +1701,10 @@ fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" +fecha@^2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fecha/-/fecha-2.3.2.tgz#360f035dd6edd954bc9581f95f2a4a7f2a3505c1" + figures@^1.3.5: version "1.7.0" resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e" @@ -2544,6 +2576,12 @@ kind-of@^4.0.0: dependencies: is-buffer "^1.1.5" +kuler@0.0.x: + version "0.0.0" + resolved "https://registry.yarnpkg.com/kuler/-/kuler-0.0.0.tgz#b66bb46b934e550f59d818848e0abba4f7f5553c" + dependencies: + colornames "0.0.2" + latest-version@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/latest-version/-/latest-version-3.1.0.tgz#a205383fea322b33b5ae3b18abee0dc2f356ee15" @@ -2661,6 +2699,13 @@ lodash@=3.10.1: version "3.10.1" resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6" +logform@^1.2.1: + version "1.2.2" + resolved "https://registry.yarnpkg.com/logform/-/logform-1.2.2.tgz#7be8847684de07185029ed09c65b11d06191ef93" + dependencies: + colors "^1.1.2" + fecha "^2.3.2" + lowercase-keys@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.0.tgz#4e3366b39e7f5457e35f1324bdf6f88d0bfc7306" @@ -3081,6 +3126,10 @@ once@^1.3.0, once@^1.3.1, once@^1.3.3, once@^1.4.0: dependencies: wrappy "1" +one-time@0.0.4: + version "0.0.4" + resolved "https://registry.yarnpkg.com/one-time/-/one-time-0.0.4.tgz#f8cdf77884826fe4dff93e3a9cc37b1e4480742e" + onetime@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/onetime/-/onetime-1.1.0.tgz#a1f7838f8314c516f05ecefcbc4ccfe04b4ed789" @@ -4263,6 +4312,10 @@ terraformer@~1.0.5: dependencies: "@types/geojson" "^1.0.0" +text-hex@0.0.x: + version "0.0.0" + resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-0.0.0.tgz#578fbc85a6a92636e42dd17b41d0218cce9eb2b3" + text-table@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" @@ -4322,6 +4375,10 @@ tree-kill@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.0.tgz#5846786237b4239014f05db156b643212d4c6f36" +triple-beam@^1.0.1: + version "1.1.0" + resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.1.0.tgz#2ac387c8c4bd04bd26c61df891a6079f8592fe10" + ts-node@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-3.3.0.tgz#c13c6a3024e30be1180dd53038fc209289d4bf69" @@ -4647,16 +4704,22 @@ widest-line@^2.0.0: dependencies: string-width "^2.1.1" -winston@^2.1.1: - version "2.4.0" - resolved "https://registry.yarnpkg.com/winston/-/winston-2.4.0.tgz#808050b93d52661ed9fb6c26b3f0c826708b0aee" +winston-transport@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-3.0.1.tgz#8008b15eef5660c4fb3fa094d58ccbd08528c58d" + +winston@3.0.0-rc1: + version "3.0.0-rc1" + resolved "https://registry.yarnpkg.com/winston/-/winston-3.0.0-rc1.tgz#982bc0ad4ef5c53000ca68036d78a3deaa28cac5" dependencies: - async "~1.0.0" - colors "1.0.x" - cycle "1.0.x" - eyes "0.1.x" + async "^1.0.0" + diagnostics "^1.0.1" isstream "0.1.x" + logform "^1.2.1" + one-time "0.0.4" stack-trace "0.0.x" + triple-beam "^1.0.1" + winston-transport "^3.0.1" wkx@^0.4.1: version "0.4.2" -- 2.25.1