1 import * as express from 'express'
2 import * as magnetUtil from 'magnet-uri'
4 import { auditLoggerFactory, VideoImportAuditView } from '../../../helpers/audit-logger'
5 import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoImportAddValidator } from '../../../middlewares'
13 } from '../../../initializers'
14 import { getYoutubeDLInfo, YoutubeDLInfo } from '../../../helpers/youtube-dl'
15 import { createReqFiles } from '../../../helpers/express-utils'
16 import { logger } from '../../../helpers/logger'
17 import { VideoImportCreate, VideoImportState, VideoPrivacy, VideoState } from '../../../../shared'
18 import { VideoModel } from '../../../models/video/video'
19 import { getVideoActivityPubUrl } from '../../../lib/activitypub'
20 import { TagModel } from '../../../models/video/tag'
21 import { VideoImportModel } from '../../../models/video/video-import'
22 import { JobQueue } from '../../../lib/job-queue/job-queue'
23 import { processImage } from '../../../helpers/image-utils'
24 import { join } from 'path'
25 import { isArray } from '../../../helpers/custom-validators/misc'
26 import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
27 import { VideoChannelModel } from '../../../models/video/video-channel'
28 import * as Bluebird from 'bluebird'
29 import * as parseTorrent from 'parse-torrent'
30 import { readFileBufferPromise, renamePromise } from '../../../helpers/core-utils'
31 import { getSecureTorrentName } from '../../../helpers/utils'
33 const auditLogger = auditLoggerFactory('video-imports')
34 const videoImportsRouter = express.Router()
36 const reqVideoFileImport = createReqFiles(
37 [ 'thumbnailfile', 'previewfile', 'torrentfile' ],
38 Object.assign({}, TORRENT_MIMETYPE_EXT, IMAGE_MIMETYPE_EXT),
40 thumbnailfile: CONFIG.STORAGE.THUMBNAILS_DIR,
41 previewfile: CONFIG.STORAGE.PREVIEWS_DIR,
42 torrentfile: CONFIG.STORAGE.TORRENTS_DIR
46 videoImportsRouter.post('/imports',
49 asyncMiddleware(videoImportAddValidator),
50 asyncRetryTransactionMiddleware(addVideoImport)
53 // ---------------------------------------------------------------------------
59 // ---------------------------------------------------------------------------
61 function addVideoImport (req: express.Request, res: express.Response) {
62 if (req.body.targetUrl) return addYoutubeDLImport(req, res)
64 const file = req.files['torrentfile'][0]
65 if (req.body.magnetUri || file) return addTorrentImport(req, res, file)
68 async function addTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) {
69 const body: VideoImportCreate = req.body
72 let torrentName: string
76 torrentName = torrentfile.originalname
78 // Rename the torrent to a secured name
79 const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName))
80 await renamePromise(torrentfile.path, newTorrentPath)
81 torrentfile.path = newTorrentPath
83 const buf = await readFileBufferPromise(torrentfile.path)
84 const parsedTorrent = parseTorrent(buf)
86 videoName = isArray(parsedTorrent.name) ? parsedTorrent.name[ 0 ] : parsedTorrent.name as string
88 magnetUri = body.magnetUri
90 const parsed = magnetUtil.decode(magnetUri)
91 videoName = isArray(parsed.name) ? parsed.name[ 0 ] : parsed.name as string
94 const video = buildVideo(res.locals.videoChannel.id, body, { name: videoName })
96 await processThumbnail(req, video)
97 await processPreview(req, video)
100 const videoImportAttributes = {
103 state: VideoImportState.PENDING
105 const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
107 // Create job to import the video
109 type: torrentfile ? 'torrent-file' as 'torrent-file' : 'magnet-uri' as 'magnet-uri',
110 videoImportId: videoImport.id,
113 await JobQueue.Instance.createJob({ type: 'video-import', payload })
115 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
117 return res.json(videoImport.toFormattedJSON()).end()
120 async function addYoutubeDLImport (req: express.Request, res: express.Response) {
121 const body: VideoImportCreate = req.body
122 const targetUrl = body.targetUrl
124 let youtubeDLInfo: YoutubeDLInfo
126 youtubeDLInfo = await getYoutubeDLInfo(targetUrl)
128 logger.info('Cannot fetch information from import for URL %s.', targetUrl, { err })
130 return res.status(400).json({
131 error: 'Cannot fetch remote information of this URL.'
135 const video = buildVideo(res.locals.videoChannel.id, body, youtubeDLInfo)
137 const downloadThumbnail = !await processThumbnail(req, video)
138 const downloadPreview = !await processPreview(req, video)
140 const tags = body.tags || youtubeDLInfo.tags
141 const videoImportAttributes = {
143 state: VideoImportState.PENDING
145 const videoImport: VideoImportModel = await insertIntoDB(video, res.locals.videoChannel, tags, videoImportAttributes)
147 // Create job to import the video
149 type: 'youtube-dl' as 'youtube-dl',
150 videoImportId: videoImport.id,
151 thumbnailUrl: youtubeDLInfo.thumbnailUrl,
155 await JobQueue.Instance.createJob({ type: 'video-import', payload })
157 auditLogger.create(res.locals.oauth.token.User.Account.Actor.getIdentifier(), new VideoImportAuditView(videoImport.toFormattedJSON()))
159 return res.json(videoImport.toFormattedJSON()).end()
162 function buildVideo (channelId: number, body: VideoImportCreate, importData: YoutubeDLInfo) {
164 name: body.name || importData.name || 'Unknown name',
166 category: body.category || importData.category,
167 licence: body.licence || importData.licence,
168 language: body.language || undefined,
169 commentsEnabled: body.commentsEnabled || true,
170 waitTranscoding: body.waitTranscoding || false,
171 state: VideoState.TO_IMPORT,
172 nsfw: body.nsfw || importData.nsfw || false,
173 description: body.description || importData.description,
174 support: body.support || null,
175 privacy: body.privacy || VideoPrivacy.PRIVATE,
176 duration: 0, // duration will be set by the import job
179 const video = new VideoModel(videoData)
180 video.url = getVideoActivityPubUrl(video)
185 async function processThumbnail (req: express.Request, video: VideoModel) {
186 const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined
187 if (thumbnailField) {
188 const thumbnailPhysicalFile = thumbnailField[ 0 ]
189 await processImage(thumbnailPhysicalFile, join(CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()), THUMBNAILS_SIZE)
197 async function processPreview (req: express.Request, video: VideoModel) {
198 const previewField = req.files ? req.files['previewfile'] : undefined
200 const previewPhysicalFile = previewField[0]
201 await processImage(previewPhysicalFile, join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName()), PREVIEWS_SIZE)
209 function insertIntoDB (
211 videoChannel: VideoChannelModel,
213 videoImportAttributes: FilteredModelAttributes<VideoImportModel>
214 ): Bluebird<VideoImportModel> {
215 return sequelizeTypescript.transaction(async t => {
216 const sequelizeOptions = { transaction: t }
218 // Save video object in database
219 const videoCreated = await video.save(sequelizeOptions)
220 videoCreated.VideoChannel = videoChannel
222 // Set tags to the video
223 if (tags !== undefined) {
224 const tagInstances = await TagModel.findOrCreateTags(tags, t)
226 await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
227 videoCreated.Tags = tagInstances
230 // Create video import object in database
231 const videoImport = await VideoImportModel.create(
232 Object.assign({ videoId: videoCreated.id }, videoImportAttributes),
235 videoImport.Video = videoCreated