Add import.video.torrent configuration
authorChocobozzz <me@florianbigard.com>
Tue, 7 Aug 2018 08:07:53 +0000 (10:07 +0200)
committerChocobozzz <me@florianbigard.com>
Wed, 8 Aug 2018 07:30:31 +0000 (09:30 +0200)
23 files changed:
client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.html
client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts
client/src/app/core/server/server.service.ts
client/src/app/videos/+video-edit/video-add.component.scss
client/src/app/videos/+video-edit/video-add.component.ts
config/default.yaml
config/production.yaml.example
config/test.yaml
server/controllers/api/config.ts
server/controllers/api/users.ts
server/controllers/api/videos/import.ts
server/initializers/constants.ts
server/initializers/migrations/0245-import-magnet.ts [deleted file]
server/lib/job-queue/handlers/video-import.ts
server/middlewares/validators/config.ts
server/middlewares/validators/video-imports.ts
server/models/account/user.ts
server/models/video/video-import.ts
server/tests/api/check-params/config.ts
server/tests/api/server/config.ts
server/tests/utils/server/config.ts
shared/models/server/custom-config.model.ts
shared/models/server/server-config.model.ts

index 13b43306bc44296d14396ff1aa643705db4f1a83..0a032df1269b9c0c6ea3026ad495c7839b5012a9 100644 (file)
         i18n-labelText labelText="Video import with HTTP enabled"
       ></my-peertube-checkbox>
 
+      <my-peertube-checkbox
+        inputName="importVideosTorrentEnabled" formControlName="importVideosTorrentEnabled"
+        i18n-labelText labelText="Video import with a torrent file or a magnet URI enabled"
+      ></my-peertube-checkbox>
+
       <div i18n class="inner-form-title">Administrator</div>
 
       <div class="form-group">
index bc5ce6e5d0ae287bf0919ea214941590fe791b51..fd6784415d45026e8336a47aa173fe353c5ae87e 100644 (file)
@@ -72,6 +72,7 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
       signupEnabled: null,
       signupLimit: this.customConfigValidatorsService.SIGNUP_LIMIT,
       importVideosHttpEnabled: null,
+      importVideosTorrentEnabled: null,
       adminEmail: this.customConfigValidatorsService.ADMIN_EMAIL,
       userVideoQuota: this.userValidatorsService.USER_VIDEO_QUOTA,
       transcodingThreads: this.customConfigValidatorsService.TRANSCODING_THREADS,
@@ -189,6 +190,9 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
         videos: {
           http: {
             enabled: this.form.value['importVideosHttpEnabled']
+          },
+          torrent: {
+            enabled: this.form.value['importVideosTorrentEnabled']
           }
         }
       }
@@ -231,7 +235,8 @@ export class EditCustomConfigComponent extends FormReactive implements OnInit {
       transcodingEnabled: this.customConfig.transcoding.enabled,
       customizationJavascript: this.customConfig.instance.customizations.javascript,
       customizationCSS: this.customConfig.instance.customizations.css,
-      importVideosHttpEnabled: this.customConfig.import.videos.http.enabled
+      importVideosHttpEnabled: this.customConfig.import.videos.http.enabled,
+      importVideosTorrentEnabled: this.customConfig.import.videos.torrent.enabled
     }
 
     for (const resolution of this.resolutions) {
index ab317f0aa6c5c490fa991b5218b4abd8b5d38471..52b50cbe8a851346bff4d4407a8486b1ebbf1094 100644 (file)
@@ -73,6 +73,9 @@ export class ServerService {
       videos: {
         http: {
           enabled: false
+        },
+        torrent: {
+          enabled: false
         }
       }
     }
index 443361f502d6febced7c90ce3184487953bb62e6..c1f96cc374faf92b84d9b2762d60b94567a05e98 100644 (file)
@@ -50,6 +50,7 @@ $background-color:  #F7F7F7;
     border-radius: 3px;
     width: 100%;
     min-height: 440px;
+    padding-bottom: 20px;
     display: flex;
     justify-content: center;
     align-items: center;
index 7d360598dfdae54bcf31b034e8bb2683ca709783..1a9247dbea4a34b4497186d5588fedb6a1237141 100644 (file)
@@ -40,6 +40,6 @@ export class VideoAddComponent implements CanComponentDeactivate {
   }
 
   isVideoImportTorrentEnabled () {
-    return this.serverService.getConfig().import.videos.http.enabled
+    return this.serverService.getConfig().import.videos.torrent.enabled
   }
 }
index 5fa7e59455cae463daf8464c475b9c1c25809e10..60da192b4f8f64d0fcbc77f6e12ee50ff6d6b320 100644 (file)
@@ -97,6 +97,8 @@ import:
   videos:
     http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
       enabled: false
+    torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
+      enabled: false
 
 instance:
   name: 'PeerTube'
index 635a41e9e556af62cb7e46cc61ab5985fc4e299b..9e8b578292351986e90f045a325d5f64bc236637 100644 (file)
@@ -111,6 +111,8 @@ import:
   videos:
     http: # Classic HTTP or all sites supported by youtube-dl https://rg3.github.io/youtube-dl/supportedsites.html
       enabled: false
+    torrent: # Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
+      enabled: false
 
 # Instance settings
 instance:
index 3c51785f2c3d3d92e21bd449f84e3206db197f08..879b6bdd43a35869cab8625449f9f4546a31e981 100644 (file)
@@ -44,6 +44,8 @@ import:
   videos:
     http:
       enabled: true
+    torrent:
+      enabled: true
 
 instance:
   default_nsfw_policy: 'display'
\ No newline at end of file
index 950a1498ea6fac8e404154cc0dcff0078258f4b3..6f05c33dbbb27207d47d287284ee07cd4f0d5192 100644 (file)
@@ -9,7 +9,7 @@ import { CONFIG, CONSTRAINTS_FIELDS, reloadConfig } from '../../initializers'
 import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares'
 import { customConfigUpdateValidator } from '../../middlewares/validators/config'
 import { ClientHtml } from '../../lib/client-html'
-import { CustomConfigAuditView, auditLoggerFactory } from '../../helpers/audit-logger'
+import { auditLoggerFactory, CustomConfigAuditView } from '../../helpers/audit-logger'
 
 const packageJSON = require('../../../../package.json')
 const configRouter = express.Router()
@@ -69,6 +69,9 @@ async function getConfig (req: express.Request, res: express.Response, next: exp
       videos: {
         http: {
           enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED
+        },
+        torrent: {
+          enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
         }
       }
     },
@@ -237,6 +240,9 @@ function customConfig (): CustomConfig {
       videos: {
         http: {
           enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED
+        },
+        torrent: {
+          enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED
         }
       }
     }
index 879ba3f917b5501a4d58f836afe74a96f70f7dd6..36bf0e0fe49713249f95ee5a399f6f80e0d7734f 100644 (file)
@@ -196,7 +196,7 @@ async function getUserVideos (req: express.Request, res: express.Response, next:
 async function getUserVideoImports (req: express.Request, res: express.Response, next: express.NextFunction) {
   const user = res.locals.oauth.token.User as UserModel
   const resultList = await VideoImportModel.listUserVideoImportsForApi(
-    user.Account.id,
+    user.id,
     req.query.start as number,
     req.query.count as number,
     req.query.sort
index df151e79d7026abda921b1d57ceb97baa9490b2d..94dafcdbdc28707699f0a6a714f6b3a89962ca78 100644 (file)
@@ -61,12 +61,13 @@ export {
 function addVideoImport (req: express.Request, res: express.Response) {
   if (req.body.targetUrl) return addYoutubeDLImport(req, res)
 
-  const file = req.files['torrentfile'][0]
+  const file = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined
   if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
 }
 
 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
   const body: VideoImportCreate = req.body
+  const user = res.locals.oauth.token.User
 
   let videoName: string
   let torrentName: string
@@ -100,7 +101,8 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
   const videoImportAttributes = {
     magnetUri,
     torrentName,
-    state: VideoImportState.PENDING
+    state: VideoImportState.PENDING,
+    userId: user.id
   }
   const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
 
@@ -120,6 +122,7 @@ async function addTorrentImport (req: express.Request, res: express.Response, to
 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
   const body: VideoImportCreate = req.body
   const targetUrl = body.targetUrl
+  const user = res.locals.oauth.token.User
 
   let youtubeDLInfo: YoutubeDLInfo
   try {
@@ -140,7 +143,8 @@ async function addYoutubeDLImport (req: express.Request, res: express.Response)
   const tags = body.tags || youtubeDLInfo.tags
   const videoImportAttributes = {
     targetUrl,
-    state: VideoImportState.PENDING
+    state: VideoImportState.PENDING,
+    userId: user.id
   }
   const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
 
index cf7cd3d74ee661c99269a6e92558e13c64922362..80eb3f1e7c887399e231e27982320ba3a2274109 100644 (file)
@@ -15,7 +15,7 @@ let config: IConfig = require('config')
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 245
+const LAST_MIGRATION_VERSION = 240
 
 // ---------------------------------------------------------------------------
 
@@ -211,6 +211,9 @@ const CONFIG = {
     VIDEOS: {
       HTTP: {
         get ENABLED () { return config.get<boolean>('import.videos.http.enabled') }
+      },
+      TORRENT: {
+        get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }
       }
     }
   },
diff --git a/server/initializers/migrations/0245-import-magnet.ts b/server/initializers/migrations/0245-import-magnet.ts
deleted file mode 100644 (file)
index 87603b0..0000000
+++ /dev/null
@@ -1,42 +0,0 @@
-import * as Sequelize from 'sequelize'
-import { Migration } from '../../models/migrations'
-import { CONSTRAINTS_FIELDS } from '../index'
-
-async function up (utils: {
-  transaction: Sequelize.Transaction
-  queryInterface: Sequelize.QueryInterface
-  sequelize: Sequelize.Sequelize
-}): Promise<any> {
-  {
-    const data = {
-      type: Sequelize.STRING,
-      allowNull: true,
-      defaultValue: null
-    } as Migration.String
-    await utils.queryInterface.changeColumn('videoImport', 'targetUrl', data)
-  }
-
-  {
-    const data = {
-      type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max),
-      allowNull: true,
-      defaultValue: null
-    }
-    await utils.queryInterface.addColumn('videoImport', 'magnetUri', data)
-  }
-
-  {
-    const data = {
-      type: Sequelize.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max),
-      allowNull: true,
-      defaultValue: null
-    }
-    await utils.queryInterface.addColumn('videoImport', 'torrentName', data)
-  }
-}
-
-function down (options) {
-  throw new Error('Not implemented.')
-}
-
-export { up, down }
index fd61aecad6bc1e2242fb7aa8c48f754acd211cee..28a03d19ecdfb2dceae4b35ee6cb766e004bd415 100644 (file)
@@ -114,16 +114,21 @@ async function processFile (downloader: () => Promise<string>, videoImport: Vide
     tempVideoPath = await downloader()
 
     // Get information about this video
+    const { size } = await statPromise(tempVideoPath)
+    const isAble = await videoImport.User.isAbleToUploadVideo({ size })
+    if (isAble === false) {
+      throw new Error('The user video quota is exceeded with this video to import.')
+    }
+
     const { videoFileResolution } = await getVideoFileResolution(tempVideoPath)
     const fps = await getVideoFileFPS(tempVideoPath)
-    const stats = await statPromise(tempVideoPath)
     const duration = await getDurationFromVideoFile(tempVideoPath)
 
     // Create video file object in database
     const videoFileData = {
       extname: extname(tempVideoPath),
       resolution: videoFileResolution,
-      size: stats.size,
+      size,
       fps,
       videoId: videoImport.videoId
     }
index 9d303eee256bcf85ab0c260c8e43e422e0cbd6ff..f3f257d573dbefb87457eefbb80efba0e2d1f8b4 100644 (file)
@@ -25,6 +25,7 @@ const customConfigUpdateValidator = [
   body('transcoding.resolutions.720p').isBoolean().withMessage('Should have a valid transcoding 720p resolution enabled boolean'),
   body('transcoding.resolutions.1080p').isBoolean().withMessage('Should have a valid transcoding 1080p resolution enabled boolean'),
   body('import.videos.http.enabled').isBoolean().withMessage('Should have a valid import video http enabled boolean'),
+  body('import.videos.torrent.enabled').isBoolean().withMessage('Should have a valid import video torrent enabled boolean'),
 
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
     logger.debug('Checking customConfigUpdateValidator parameters', { parameters: req.body })
index c03cf2e4d6c4146de56c797452941f02627ccbfc..9ac739101f2390ecb4a2984f35df9092a24d1d0e 100644 (file)
@@ -33,21 +33,28 @@ const videoImportAddValidator = getCommonVideoAttributes().concat([
     logger.debug('Checking videoImportAddValidator parameters', { parameters: req.body })
 
     const user = res.locals.oauth.token.User
+    const torrentFile = req.files && req.files['torrentfile'] ? req.files['torrentfile'][0] : undefined
 
     if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
 
-    if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) {
+    if (req.body.targetUrl && CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true) {
       cleanUpReqFiles(req)
       return res.status(409)
-        .json({ error: 'Import is not enabled on this instance.' })
+        .json({ error: 'HTTP import is not enabled on this instance.' })
         .end()
     }
 
+    if (CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED !== true && (req.body.magnetUri || torrentFile)) {
+      cleanUpReqFiles(req)
+      return res.status(409)
+                .json({ error: 'Torrent/magnet URI import is not enabled on this instance.' })
+                .end()
+    }
+
     if (!await isVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
 
     // Check we have at least 1 required param
-    const file = req.files['torrentfile'][0]
-    if (!req.body.targetUrl && !req.body.magnetUri && !file) {
+    if (!req.body.targetUrl && !req.body.magnetUri && !torrentFile) {
       cleanUpReqFiles(req)
 
       return res.status(400)
index 1165285ea318195200f818f4cf58af0b571eeb4a..1b1fc5ee80197a3d558b02320bc9b814c055f129 100644 (file)
@@ -295,7 +295,7 @@ export class UserModel extends Model<UserModel> {
     return json
   }
 
-  isAbleToUploadVideo (videoFile: Express.Multer.File) {
+  isAbleToUploadVideo (videoFile: { size: number }) {
     if (this.videoQuota === -1) return Promise.resolve(true)
 
     return UserModel.getOriginalVideoFileTotalFromUser(this)
index d6c02e5ac9af65dfded8a7b91dd2eb1e324e2a6d..b794d83244af4361e234673d822af2ea2b24b875 100644 (file)
@@ -15,34 +15,21 @@ import {
 } from 'sequelize-typescript'
 import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers'
 import { getSort, throwIfNotValid } from '../utils'
-import { VideoModel } from './video'
+import { ScopeNames as VideoModelScopeNames, VideoModel } from './video'
 import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports'
 import { VideoImport, VideoImportState } from '../../../shared'
-import { VideoChannelModel } from './video-channel'
-import { AccountModel } from '../account/account'
-import { TagModel } from './tag'
 import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos'
+import { UserModel } from '../account/user'
 
 @DefaultScope({
   include: [
     {
-      model: () => VideoModel,
-      required: false,
-      include: [
-        {
-          model: () => VideoChannelModel,
-          required: true,
-          include: [
-            {
-              model: () => AccountModel,
-              required: true
-            }
-          ]
-        },
-        {
-          model: () => TagModel
-        }
-      ]
+      model: () => UserModel.unscoped(),
+      required: true
+    },
+    {
+      model: () => VideoModel.scope([ VideoModelScopeNames.WITH_ACCOUNT_DETAILS, VideoModelScopeNames.WITH_TAGS]),
+      required: false
     }
   ]
 })
@@ -53,6 +40,9 @@ import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos'
     {
       fields: [ 'videoId' ],
       unique: true
+    },
+    {
+      fields: [ 'userId' ]
     }
   ]
 })
@@ -91,6 +81,18 @@ export class VideoImportModel extends Model<VideoImportModel> {
   @Column(DataType.TEXT)
   error: string
 
+  @ForeignKey(() => UserModel)
+  @Column
+  userId: number
+
+  @BelongsTo(() => UserModel, {
+    foreignKey: {
+      allowNull: false
+    },
+    onDelete: 'cascade'
+  })
+  User: UserModel
+
   @ForeignKey(() => VideoModel)
   @Column
   videoId: number
@@ -116,41 +118,24 @@ export class VideoImportModel extends Model<VideoImportModel> {
     return VideoImportModel.findById(id)
   }
 
-  static listUserVideoImportsForApi (accountId: number, start: number, count: number, sort: string) {
+  static listUserVideoImportsForApi (userId: number, start: number, count: number, sort: string) {
     const query = {
       distinct: true,
-      offset: start,
-      limit: count,
-      order: getSort(sort),
       include: [
         {
-          model: VideoModel,
-          required: false,
-          include: [
-            {
-              model: VideoChannelModel,
-              required: true,
-              include: [
-                {
-                  model: AccountModel,
-                  required: true,
-                  where: {
-                    id: accountId
-                  }
-                }
-              ]
-            },
-            {
-              model: TagModel,
-              required: false
-            }
-          ]
+          model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query
+          required: true
         }
-      ]
+      ],
+      offset: start,
+      limit: count,
+      order: getSort(sort),
+      where: {
+        userId
+      }
     }
 
-    return VideoImportModel.unscoped()
-                           .findAndCountAll(query)
+    return VideoImportModel.findAndCountAll(query)
                            .then(({ rows, count }) => {
                              return {
                                data: rows,
index 2742e26ded7944f3fa6774ef8215bb7780a8d77f..b26dfa25207629c54559ec1b91377d063a025cf3 100644 (file)
@@ -65,6 +65,9 @@ describe('Test config API validators', function () {
       videos: {
         http: {
           enabled: false
+        },
+        torrent: {
+          enabled: false
         }
       }
     }
index b65061a5dd21bcc35ce5a63bf6a5ff022b240115..f9805b6ea1f6e6b861d8604b74963579d6046fbc 100644 (file)
@@ -45,6 +45,7 @@ function checkInitialConfig (data: CustomConfig) {
   expect(data.transcoding.resolutions['720p']).to.be.true
   expect(data.transcoding.resolutions['1080p']).to.be.true
   expect(data.import.videos.http.enabled).to.be.true
+  expect(data.import.videos.torrent.enabled).to.be.true
 }
 
 function checkUpdatedConfig (data: CustomConfig) {
@@ -72,6 +73,7 @@ function checkUpdatedConfig (data: CustomConfig) {
   expect(data.transcoding.resolutions['720p']).to.be.false
   expect(data.transcoding.resolutions['1080p']).to.be.false
   expect(data.import.videos.http.enabled).to.be.false
+  expect(data.import.videos.torrent.enabled).to.be.false
 }
 
 describe('Test config', function () {
@@ -167,6 +169,9 @@ describe('Test config', function () {
         videos: {
           http: {
             enabled: false
+          },
+          torrent: {
+            enabled: false
           }
         }
       }
index e21614282b95ad04fd097f5ea0c44c70852fae60..d6ac3ef8acb88db10a56fb6fefdbc3bfd542b4bd 100644 (file)
@@ -97,6 +97,9 @@ function updateCustomSubConfig (url: string, token: string, newConfig: any) {
       videos: {
         http: {
           enabled: false
+        },
+        torrent: {
+          enabled: false
         }
       }
     }
index 46320435deb40fcfa662ba8abc5cf37102b2c917..d70c757b6c8edcf7051dc965b954e5e28ef0c4a9 100644 (file)
@@ -60,6 +60,9 @@ export interface CustomConfig {
     videos: {
       http: {
         enabled: boolean
+      },
+      torrent: {
+        enabled: boolean
       }
     }
   }
index 2cafedbbc7a59bf065088f0d716a37e14821c241..8cb08723407c684469c552c9bb90447160e0663f 100644 (file)
@@ -28,6 +28,9 @@ export interface ServerConfig {
       http: {
         enabled: boolean
       }
+      torrent: {
+        enabled: boolean
+      }
     }
   }