Add tmp and redundancy directories
authorChocobozzz <me@florianbigard.com>
Tue, 4 Dec 2018 15:02:49 +0000 (16:02 +0100)
committerChocobozzz <me@florianbigard.com>
Tue, 4 Dec 2018 15:04:15 +0000 (16:04 +0100)
26 files changed:
config/default.yaml
config/production.yaml.example
config/test-1.yaml
config/test-2.yaml
config/test-3.yaml
config/test-4.yaml
config/test-5.yaml
config/test-6.yaml
config/test.yaml
server/controllers/api/users/me.ts
server/controllers/api/video-channel.ts
server/controllers/api/videos/import.ts
server/controllers/api/videos/index.ts
server/controllers/static.ts
server/helpers/requests.ts
server/helpers/utils.ts
server/helpers/webtorrent.ts
server/helpers/youtube-dl.ts
server/initializers/checker-before-init.ts
server/initializers/constants.ts
server/lib/activitypub/actor.ts
server/lib/activitypub/videos.ts
server/lib/job-queue/handlers/video-import.ts
server/lib/job-queue/handlers/video-views.ts
server/lib/redis.ts
support/docker/production/config/production.yaml

index 257ec7ed166b9eeb59978fbe8fea726a7aac4d49..d95fdc57bc7e07caec31830125673901164b4bd1 100644 (file)
@@ -45,8 +45,10 @@ smtp:
 
 # From the project root directory
 storage:
+  tmp: 'storage/tmp/' # Used to download data (imports etc), store uploaded files before processing...
   avatars: 'storage/avatars/'
   videos: 'storage/videos/'
+  redundancy: 'storage/redundancy/'
   logs: 'storage/logs/'
   previews: 'storage/previews/'
   thumbnails: 'storage/thumbnails/'
@@ -75,7 +77,7 @@ trending:
 redundancy:
   videos:
     check_interval: '1 hour' # How often you want to check new videos to cache
-    strategies:
+    strategies: # Just uncomment strategies you want
 #      -
 #        size: '10GB'
 #        # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
index ac15fc736f5062818b8815f382452a93ba8b8e3f..4c50a550b575f204bcb6c6e830ae04258da90407 100644 (file)
@@ -46,8 +46,10 @@ smtp:
 
 # From the project root directory
 storage:
+  tmp: '/var/www/peertube/storage/tmp/' # Used to download data (imports etc), store uploaded files before processing...
   avatars: '/var/www/peertube/storage/avatars/'
   videos: '/var/www/peertube/storage/videos/'
+  redundancy: '/var/www/peertube/storage/videos/'
   logs: '/var/www/peertube/storage/logs/'
   previews: '/var/www/peertube/storage/previews/'
   thumbnails: '/var/www/peertube/storage/thumbnails/'
@@ -76,7 +78,7 @@ trending:
 redundancy:
   videos:
     check_interval: '1 hour' # How often you want to check new videos to cache
-    strategies:
+    strategies: # Just uncomment strategies you want
 #      -
 #        size: '10GB'
 #        # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
index 503bbc6610632fb813d7591c3f764ef8f0b08ad3..8f4f66d2a9c24e9ea79f41c6dd48baab4c6ff8a3 100644 (file)
@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test1/tmp/'
   avatars: 'test1/avatars/'
   videos: 'test1/videos/'
+  redundancy: 'test1/redundancy/'
   logs: 'test1/logs/'
   previews: 'test1/previews/'
   thumbnails: 'test1/thumbnails/'
index 8c77bf58107bb36dd7c1588341612a3cb0ee7e10..a80ec6e540c3f466a4302d5f83e3d42f6def1aa9 100644 (file)
@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test2/tmp/'
   avatars: 'test2/avatars/'
   videos: 'test2/videos/'
+  redundancy: 'test2/redundancy/'
   logs: 'test2/logs/'
   previews: 'test2/previews/'
   thumbnails: 'test2/thumbnails/'
index 82d89567a7e09551fa30677029180ec7f937e8a6..934401eb07ed5e01ac7af6f9868e92bbba6a6af6 100644 (file)
@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test3/tmp/'
   avatars: 'test3/avatars/'
   videos: 'test3/videos/'
+  redundancy: 'test3/redundancy/'
   logs: 'test3/logs/'
   previews: 'test3/previews/'
   thumbnails: 'test3/thumbnails/'
index 1aa56d041c665962e1e43488ee6d8378389709d9..ee99b250be4013f3061d0d1529017beca48e0a9d 100644 (file)
@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test4/tmp/'
   avatars: 'test4/avatars/'
   videos: 'test4/videos/'
+  redundancy: 'test4/redundancy/'
   logs: 'test4/logs/'
   previews: 'test4/previews/'
   thumbnails: 'test4/thumbnails/'
index 5f1c2f583cce4635da722209352414b09dbfc40c..e2662bdd930a44ee5232ad122f6d59a658860f9f 100644 (file)
@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test5/tmp/'
   avatars: 'test5/avatars/'
   videos: 'test5/videos/'
+  redundancy: 'test5/redundancy/'
   logs: 'test5/logs/'
   previews: 'test5/previews/'
   thumbnails: 'test5/thumbnails/'
index 719629844b162ad795ddb5df82e7e1c133ddefdf..ad39c6a9fd1584ea4e728bef635d29b0021e0196 100644 (file)
@@ -10,8 +10,10 @@ database:
 
 # From the project root directory
 storage:
+  tmp: 'test6/tmp/'
   avatars: 'test6/avatars/'
   videos: 'test6/videos/'
+  redundancy: 'test6/redundancy/'
   logs: 'test6/logs/'
   previews: 'test6/previews/'
   thumbnails: 'test6/thumbnails/'
index 9c051fabc45f3bfa09ea818ccd94832a85f46448..51a77e2fdaa4921af1cbb471ea22258a6bece379 100644 (file)
@@ -67,4 +67,4 @@ import:
       enabled: true
 
 instance:
-  default_nsfw_policy: 'display'
\ No newline at end of file
+  default_nsfw_policy: 'display'
index 82299747dab159ca51b839eefdba2090d3249d22..47f2c9ec7f0dbd050595b7d297e0f5da42577d41 100644 (file)
@@ -42,7 +42,7 @@ import { AccountModel } from '../../../models/account/account'
 
 const auditLogger = auditLoggerFactory('users-me')
 
-const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
+const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR })
 
 const meRouter = express.Router()
 
@@ -348,7 +348,7 @@ async function updateMe (req: express.Request, res: express.Response, next: expr
   return res.sendStatus(204)
 }
 
-async function updateMyAvatar (req: express.Request, res: express.Response, next: express.NextFunction) {
+async function updateMyAvatar (req: express.Request, res: express.Response) {
   const avatarPhysicalFile = req.files[ 'avatarfile' ][ 0 ]
   const user: UserModel = res.locals.oauth.token.user
   const oldUserAuditView = new UserAuditView(user.toFormattedJSON())
index 9bf3c5fd808e1e2c9fa865442f9435a830b739a2..63240dfa1b98a131a1b5860c8f7763b5c2a66203 100644 (file)
@@ -32,7 +32,7 @@ import { resetSequelizeInstance } from '../../helpers/database-utils'
 import { UserModel } from '../../models/account/user'
 
 const auditLogger = auditLoggerFactory('channels')
-const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.AVATARS_DIR })
+const reqAvatarFile = createReqFiles([ 'avatarfile' ], IMAGE_MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR })
 
 const videoChannelRouter = express.Router()
 
index 398fd5a7f68e9379a0cfb1f6091ed38d2e3dabda..f27d648c7da23287d81a3a4d0950f539c093d49b 100644 (file)
@@ -37,9 +37,9 @@ const reqVideoFileImport = createReqFiles(
   [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
   Object.assign({}, TORRENT_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT),
   {
-    thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
-    previewfile: CONFIG.STORAGE.PREVIEWS_DIR,
-    torrentfile: CONFIG.STORAGE.TORRENTS_DIR
+    thumbnailfile: CONFIG.STORAGE.TMP_DIR,
+    previewfile: CONFIG.STORAGE.TMP_DIR,
+    torrentfile: CONFIG.STORAGE.TMP_DIR
   }
 )
 
index 3d1b2e1a225477931a55e4d8a63a27bd75fc5005..4e4697ef4cfc1b24c0edbe8abb3c81592a110a38 100644 (file)
@@ -67,17 +67,17 @@ const reqVideoFileAdd = createReqFiles(
   [ 'videofile', 'thumbnailfile', 'previewfile' ],
   Object.assign({}, VIDEO_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT),
   {
-    videofile: CONFIG.STORAGE.VIDEOS_DIR,
-    thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
-    previewfile: CONFIG.STORAGE.PREVIEWS_DIR
+    videofile: CONFIG.STORAGE.TMP_DIR,
+    thumbnailfile: CONFIG.STORAGE.TMP_DIR,
+    previewfile: CONFIG.STORAGE.TMP_DIR
   }
 )
 const reqVideoFileUpdate = createReqFiles(
   [ 'thumbnailfile', 'previewfile' ],
   IMAGE_MIMETYPE_EXT,
   {
-    thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
-    previewfile: CONFIG.STORAGE.PREVIEWS_DIR
+    thumbnailfile: CONFIG.STORAGE.TMP_DIR,
+    previewfile: CONFIG.STORAGE.TMP_DIR
   }
 )
 
index 75e30353c37a868df3398b482ae4abcf9ddb54d1..f16a7d72bbd802aa0e1c1a7af8efa41876dc30d2 100644 (file)
@@ -34,12 +34,17 @@ staticRouter.use(
 )
 
 // Videos path for webseeding
-const videosPhysicalPath = CONFIG.STORAGE.VIDEOS_DIR
 staticRouter.use(
   STATIC_PATHS.WEBSEED,
   cors(),
-  express.static(videosPhysicalPath)
+  express.static(CONFIG.STORAGE.VIDEOS_DIR)
 )
+staticRouter.use(
+  STATIC_PATHS.WEBSEED,
+  cors(),
+  express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }) // 404, because we don't have this video
+)
+
 staticRouter.use(
   STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension',
   asyncMiddleware(videosGetValidator),
index 5760ad1c1854dc617f164b00aa68fbd8dd5a9fe4..3fc776f1a53299ceb2630313938d8c4757b24b33 100644 (file)
@@ -1,9 +1,9 @@
 import * as Bluebird from 'bluebird'
 import { createWriteStream } from 'fs-extra'
 import * as request from 'request'
-import { ACTIVITY_PUB } from '../initializers'
+import { ACTIVITY_PUB, CONFIG } from '../initializers'
 import { processImage } from './image-utils'
-import { extname } from 'path'
+import { join } from 'path'
 
 function doRequest <T> (
   requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }
@@ -29,10 +29,11 @@ function doRequestAndSaveToFile (requestOptions: request.CoreOptions & request.U
   })
 }
 
-async function downloadImage (url: string, destPath: string, size: { width: number, height: number }) {
-  const tmpPath = destPath + '.tmp' + extname(destPath)
+async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
+  const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
   await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath)
 
+  const destPath = join(destDir, destName)
   await processImage({ path: tmpPath }, destPath, size)
 }
 
index 5c9d6fe2f2b4b5e96eee3a3184c0e913cc1a8b22..9b89e3e61066907e32612fb1a0b6dd714e29e427 100644 (file)
@@ -46,11 +46,11 @@ const getServerActor = memoizee(async function () {
   return actor
 })
 
-function generateVideoTmpPath (target: string | ParseTorrent) {
+function generateVideoImportTmpPath (target: string | ParseTorrent) {
   const id = typeof target === 'string' ? target : target.infoHash
 
   const hash = sha256(id)
-  return join(CONFIG.STORAGE.VIDEOS_DIR, hash + '-import.mp4')
+  return join(CONFIG.STORAGE.TMP_DIR, hash + '-import.mp4')
 }
 
 function getSecureTorrentName (originalName: string) {
@@ -103,6 +103,6 @@ export {
   getSecureTorrentName,
   getServerActor,
   getServerCommit,
-  generateVideoTmpPath,
+  generateVideoImportTmpPath,
   getUUIDFromFilename
 }
index ce35b87dac395135d731a8875012d55f4f0c73b0..3c9a0b96ae87b21d942da729634c1e4250b4766a 100644 (file)
@@ -1,5 +1,5 @@
 import { logger } from './logger'
-import { generateVideoTmpPath } from './utils'
+import { generateVideoImportTmpPath } from './utils'
 import * as WebTorrent from 'webtorrent'
 import { createWriteStream, ensureDir, remove } from 'fs-extra'
 import { CONFIG } from '../initializers'
@@ -9,10 +9,10 @@ async function downloadWebTorrentVideo (target: { magnetUri: string, torrentName
   const id = target.magnetUri || target.torrentName
   let timer
 
-  const path = generateVideoTmpPath(id)
+  const path = generateVideoImportTmpPath(id)
   logger.info('Importing torrent video %s', id)
 
-  const directoryPath = join(CONFIG.STORAGE.VIDEOS_DIR, 'import')
+  const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent')
   await ensureDir(directoryPath)
 
   return new Promise<string>((res, rej) => {
index 2a56630427855fee83ccb79e370cc6aa11c1009d..b74351b4219a72963c04cfe092c8081a5a7ad63f 100644 (file)
@@ -1,7 +1,7 @@
 import { truncate } from 'lodash'
 import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES } from '../initializers'
 import { logger } from './logger'
-import { generateVideoTmpPath } from './utils'
+import { generateVideoImportTmpPath } from './utils'
 import { join } from 'path'
 import { root } from './core-utils'
 import { ensureDir, writeFile, remove } from 'fs-extra'
@@ -40,7 +40,7 @@ function getYoutubeDLInfo (url: string, opts?: string[]): Promise<YoutubeDLInfo>
 }
 
 function downloadYoutubeDLVideo (url: string, timeout: number) {
-  const path = generateVideoTmpPath(url)
+  const path = generateVideoImportTmpPath(url)
   let timer
 
   logger.info('Importing youtubeDL video %s', url)
index 9dfb5d68c375947a009d122e04bcf8ba7637b390..b51c7cfba82c25ca53de5cc370b2d8204fa76b47 100644 (file)
@@ -12,6 +12,7 @@ function checkMissedConfig () {
     'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'database.pool.max',
     'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address',
     'storage.avatars', 'storage.videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache',
+    'storage.redundancy', 'storage.tmp',
     'log.level',
     'user.video_quota', 'user.video_quota_daily',
     'cache.previews.size', 'admin.email',
index 8a8bcd126f08e6f7dd0acaa9c7fe23b1077e6801..876aa1cf534e19f12ef8ef7c71bb85705ea8b0c0 100644 (file)
@@ -185,9 +185,11 @@ const CONFIG = {
     FROM_ADDRESS: config.get<string>('smtp.from_address')
   },
   STORAGE: {
+    TMP_DIR: buildPath(config.get<string>('storage.tmp')),
     AVATARS_DIR: buildPath(config.get<string>('storage.avatars')),
     LOG_DIR: buildPath(config.get<string>('storage.logs')),
     VIDEOS_DIR: buildPath(config.get<string>('storage.videos')),
+    REDUNDANCY_DIR: buildPath(config.get<string>('storage.redundancy')),
     THUMBNAILS_DIR: buildPath(config.get<string>('storage.thumbnails')),
     PREVIEWS_DIR: buildPath(config.get<string>('storage.previews')),
     CAPTIONS_DIR: buildPath(config.get<string>('storage.captions')),
index 504263c9998c8b4e3cd9314ce97cd562bfc6b3cb..bbe48833d7e4e87d43d9cb07b19e571796cad6ee 100644 (file)
@@ -178,9 +178,7 @@ async function fetchAvatarIfExists (actorJSON: ActivityPubActor) {
     const extension = IMAGE_MIMETYPE_EXT[actorJSON.icon.mediaType]
 
     const avatarName = uuidv4() + extension
-    const destPath = join(CONFIG.STORAGE.AVATARS_DIR, avatarName)
-
-    await downloadImage(actorJSON.icon.url, destPath, AVATARS_SIZE)
+    await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE)
 
     return avatarName
   }
index a5d649391f6cd684e942bb735894bfc1989051f6..3d17e6846dba4421d38b9477744da193d9232149 100644 (file)
@@ -95,9 +95,8 @@ function fetchRemoteVideoStaticFile (video: VideoModel, path: string, reject: Fu
 
 function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject) {
   const thumbnailName = video.getThumbnailName()
-  const thumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName)
 
-  return downloadImage(icon.url, thumbnailPath, THUMBNAILS_SIZE)
+  return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
 }
 
 function getOrCreateVideoChannelFromVideoObject (videoObject: VideoTorrentObject) {
index 4de901c0c1f55236e7ff1018955e3aeb5e532cdc..51a0b5faf5a2bc347a515addbec764491839963a 100644 (file)
@@ -7,7 +7,7 @@ import { getDurationFromVideoFile, getVideoFileFPS, getVideoFileResolution } fro
 import { extname, join } from 'path'
 import { VideoFileModel } from '../../../models/video/video-file'
 import { CONFIG, PREVIEWS_SIZE, sequelizeTypescript, THUMBNAILS_SIZE, VIDEO_IMPORT_TIMEOUT } from '../../../initializers'
-import { doRequestAndSaveToFile, downloadImage } from '../../../helpers/requests'
+import { downloadImage } from '../../../helpers/requests'
 import { VideoState } from '../../../../shared'
 import { JobQueue } from '../index'
 import { federateVideoIfNeeded } from '../../activitypub'
@@ -109,6 +109,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
   let tempVideoPath: string
   let videoDestFile: string
   let videoFile: VideoFileModel
+
   try {
     // Download video from youtubeDL
     tempVideoPath = await downloader()
@@ -144,8 +145,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
     // Process thumbnail
     if (options.downloadThumbnail) {
       if (options.thumbnailUrl) {
-        const destThumbnailPath = join(CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName())
-        await downloadImage(options.thumbnailUrl, destThumbnailPath, THUMBNAILS_SIZE)
+        await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.THUMBNAILS_DIR, videoImport.Video.getThumbnailName(), THUMBNAILS_SIZE)
       } else {
         await videoImport.Video.createThumbnail(videoFile)
       }
@@ -156,8 +156,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
     // Process preview
     if (options.downloadPreview) {
       if (options.thumbnailUrl) {
-        const destPreviewPath = join(CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName())
-        await downloadImage(options.thumbnailUrl, destPreviewPath, PREVIEWS_SIZE)
+        await downloadImage(options.thumbnailUrl, CONFIG.STORAGE.PREVIEWS_DIR, videoImport.Video.getPreviewName(), PREVIEWS_SIZE)
       } else {
         await videoImport.Video.createPreview(videoFile)
       }
index 038ef43e2c7ea624aea5841c93e44567c44f39c1..fa1fd13b38da28b6080487b35f416120497fb976 100644 (file)
@@ -23,9 +23,7 @@ async function processVideosViews () {
   for (const videoId of videoIds) {
     try {
       const views = await Redis.Instance.getVideoViews(videoId, hour)
-      if (isNaN(views)) {
-        logger.error('Cannot process videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, views)
-      } else {
+      if (views) {
         logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour)
 
         try {
index abd75d5122d7162276fb245f7c2191ab952ad19b..3e25e6a2c278987d9629cd3cee36bf5c36621d38 100644 (file)
@@ -121,7 +121,14 @@ class Redis {
     const key = this.generateVideoViewKey(videoId, hour)
 
     const valueString = await this.getValue(key)
-    return parseInt(valueString, 10)
+    const valueInt = parseInt(valueString, 10)
+
+    if (isNaN(valueInt)) {
+      logger.error('Cannot get videos views of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString)
+      return undefined
+    }
+
+    return valueInt
   }
 
   async getVideosIdViewed (hour: number) {
index 4970bbccab70497b769c73563d8383e12afc1b12..846c838e85c6d964147e45065cfaf6f52d036aba 100644 (file)
@@ -32,8 +32,10 @@ redis:
 
 # From the project root directory
 storage:
+  tmp: '../data/tmp/'
   avatars: '../data/avatars/'
   videos: '../data/videos/'
+  redundancy: '../data/redundancy/'
   logs: '../data/logs/'
   previews: '../data/previews/'
   thumbnails: '../data/thumbnails/'