14 } from 'sequelize-typescript'
15 import { ActorModel } from '../activitypub/actor'
16 import { getVideoSort, throwIfNotValid } from '../utils'
17 import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc'
18 import { CONFIG, CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers'
19 import { VideoFileModel } from '../video/video-file'
20 import { getServerActor } from '../../helpers/utils'
21 import { VideoModel } from '../video/video'
22 import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
23 import { logger } from '../../helpers/logger'
24 import { CacheFileObject, VideoPrivacy } from '../../../shared'
25 import { VideoChannelModel } from '../video/video-channel'
26 import { ServerModel } from '../server/server'
27 import { sample } from 'lodash'
28 import { isTestInstance } from '../../helpers/core-utils'
29 import * as Bluebird from 'bluebird'
30 import * as Sequelize from 'sequelize'
31 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
33 export enum ScopeNames {
34 WITH_VIDEO = 'WITH_VIDEO'
38 [ ScopeNames.WITH_VIDEO ]: {
41 model: () => VideoFileModel,
45 model: () => VideoModel,
51 model: () => VideoStreamingPlaylistModel,
55 model: () => VideoModel,
65 tableName: 'videoRedundancy',
68 fields: [ 'videoFileId' ]
79 export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
92 @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl'))
93 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
97 @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url'))
98 @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max))
103 strategy: string // Only used by us
105 @ForeignKey(() => VideoFileModel)
109 @BelongsTo(() => VideoFileModel, {
115 VideoFile: VideoFileModel
117 @ForeignKey(() => VideoStreamingPlaylistModel)
119 videoStreamingPlaylistId: number
121 @BelongsTo(() => VideoStreamingPlaylistModel, {
127 VideoStreamingPlaylist: VideoStreamingPlaylistModel
129 @ForeignKey(() => ActorModel)
133 @BelongsTo(() => ActorModel, {
142 static async removeFile (instance: VideoRedundancyModel) {
143 if (!instance.isOwned()) return
145 if (instance.videoFileId) {
146 const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId)
148 const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}`
149 logger.info('Removing duplicated video file %s.', logIdentifier)
151 videoFile.Video.removeFile(videoFile, true)
152 .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err }))
155 if (instance.videoStreamingPlaylistId) {
156 const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId)
158 const videoUUID = videoStreamingPlaylist.Video.uuid
159 logger.info('Removing duplicated video streaming playlist %s.', videoUUID)
161 videoStreamingPlaylist.Video.removeStreamingPlaylist(true)
162 .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err }))
168 static async loadLocalByFileId (videoFileId: number) {
169 const actor = await getServerActor()
178 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
181 static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number) {
182 const actor = await getServerActor()
187 videoStreamingPlaylistId
191 return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
194 static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
202 return VideoRedundancyModel.findOne(query)
205 static async isLocalByVideoUUIDExists (uuid: string) {
206 const actor = await getServerActor()
210 attributes: [ 'id' ],
217 model: VideoFileModel,
233 return VideoRedundancyModel.findOne(query)
237 static async getVideoSample (p: Bluebird<VideoModel[]>) {
239 const ids = rows.map(r => r.id)
240 const id = sample(ids)
242 return VideoModel.loadWithFiles(id, undefined, !isTestInstance())
245 static async findMostViewToDuplicate (randomizedFactor: number) {
248 attributes: [ 'id', 'views' ],
249 limit: randomizedFactor,
250 order: getVideoSort('-views'),
252 privacy: VideoPrivacy.PUBLIC
255 await VideoRedundancyModel.buildVideoFileForDuplication(),
256 VideoRedundancyModel.buildServerRedundancyInclude()
260 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
263 static async findTrendingToDuplicate (randomizedFactor: number) {
266 attributes: [ 'id', 'views' ],
268 group: 'VideoModel.id',
269 limit: randomizedFactor,
270 order: getVideoSort('-trending'),
272 privacy: VideoPrivacy.PUBLIC
275 await VideoRedundancyModel.buildVideoFileForDuplication(),
276 VideoRedundancyModel.buildServerRedundancyInclude(),
278 VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
282 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
285 static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
288 attributes: [ 'id', 'publishedAt' ],
289 limit: randomizedFactor,
290 order: getVideoSort('-publishedAt'),
292 privacy: VideoPrivacy.PUBLIC,
294 [ Sequelize.Op.gte ]: minViews
298 await VideoRedundancyModel.buildVideoFileForDuplication(),
299 VideoRedundancyModel.buildServerRedundancyInclude()
303 return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
306 static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
307 const expiredDate = new Date()
308 expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
310 const actor = await getServerActor()
317 [ Sequelize.Op.lt ]: expiredDate
322 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
325 static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
326 const actor = await getServerActor()
332 model: VideoRedundancyModel,
342 return VideoFileModel.sum('size', options as any) // FIXME: typings
344 if (!v || isNaN(v)) return 0
350 static async listLocalExpired () {
351 const actor = await getServerActor()
357 [ Sequelize.Op.lt ]: new Date()
362 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
365 static async listRemoteExpired () {
366 const actor = await getServerActor()
371 [Sequelize.Op.ne]: actor.id
374 [ Sequelize.Op.lt ]: new Date()
379 return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
382 static async listLocalOfServer (serverId: number) {
383 const actor = await getServerActor()
384 const buildVideoInclude = () => ({
390 model: VideoChannelModel.unscoped(),
395 model: ActorModel.unscoped(),
412 model: VideoFileModel,
414 include: [ buildVideoInclude() ]
417 model: VideoStreamingPlaylistModel,
419 include: [ buildVideoInclude() ]
424 return VideoRedundancyModel.findAll(query)
427 static async getStats (strategy: VideoRedundancyStrategy) {
428 const actor = await getServerActor()
433 [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
434 [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', Sequelize.col('videoId'))), 'totalVideos' ],
435 [ Sequelize.fn('COUNT', Sequelize.col('videoFileId')), 'totalVideoFiles' ]
444 model: VideoFileModel,
450 return VideoRedundancyModel.findOne(query as any) // FIXME: typings
452 totalUsed: parseInt(r.totalUsed.toString(), 10),
453 totalVideos: r.totalVideos,
454 totalVideoFiles: r.totalVideoFiles
459 if (this.VideoFile) return this.VideoFile.Video
461 return this.VideoStreamingPlaylist.Video
465 return !!this.strategy
468 toActivityPubObject (): CacheFileObject {
469 if (this.VideoStreamingPlaylist) {
472 type: 'CacheFile' as 'CacheFile',
473 object: this.VideoStreamingPlaylist.Video.url,
474 expires: this.expiresOn.toISOString(),
477 mimeType: 'application/x-mpegURL',
478 mediaType: 'application/x-mpegURL',
486 type: 'CacheFile' as 'CacheFile',
487 object: this.VideoFile.Video.url,
488 expires: this.expiresOn.toISOString(),
491 mimeType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
492 mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[ this.VideoFile.extname ] as any,
494 height: this.VideoFile.resolution,
495 size: this.VideoFile.size,
496 fps: this.VideoFile.fps
501 // Don't include video files we already duplicated
502 private static async buildVideoFileForDuplication () {
503 const actor = await getServerActor()
505 const notIn = Sequelize.literal(
507 `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
513 model: VideoFileModel.unscoped(),
517 [ Sequelize.Op.notIn ]: notIn
523 private static buildServerRedundancyInclude () {
526 model: VideoChannelModel.unscoped(),
531 model: ActorModel.unscoped(),
536 model: ServerModel.unscoped(),
539 redundancyAllowed: true