server/lib/activitypub/playlist.ts

   1 import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object'
   2 import { crawlCollectionPage } from './crawl'
   3 import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
   4 import { isArray } from '../../helpers/custom-validators/misc'
   5 import { getOrCreateActorAndServerAndModel } from './actor'
   6 import { logger } from '../../helpers/logger'
   7 import { VideoPlaylistModel } from '../../models/video/video-playlist'
   8 import { doRequest } from '../../helpers/requests'
   9 import { checkUrlsSameHost } from '../../helpers/activitypub'
  10 import * as Bluebird from 'bluebird'
  11 import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object'
  12 import { getOrCreateVideoAndAccountAndChannel } from './videos'
  13 import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist'
  14 import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element'
  15 import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
  16 import { sequelizeTypescript } from '../../initializers/database'
  17 import { createPlaylistMiniatureFromUrl } from '../thumbnail'
  18 import { FilteredModelAttributes } from '../../typings/sequelize'
  19 import { MAccountDefault, MAccountId, MVideoId } from '../../typings/models'
  20 import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../typings/models/video/video-playlist'
  21
  22 function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  23   const privacy = to.includes(ACTIVITY_PUB.PUBLIC)
  24     ? VideoPlaylistPrivacy.PUBLIC
  25     : VideoPlaylistPrivacy.UNLISTED
  26
  27   return {
  28     name: playlistObject.name,
  29     description: playlistObject.content,
  30     privacy,
  31     url: playlistObject.id,
  32     uuid: playlistObject.uuid,
  33     ownerAccountId: byAccount.id,
  34     videoChannelId: null,
  35     createdAt: new Date(playlistObject.published),
  36     updatedAt: new Date(playlistObject.updated)
  37   }
  38 }
  39
  40 function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) {
  41   return {
  42     position: elementObject.position,
  43     url: elementObject.id,
  44     startTimestamp: elementObject.startTimestamp || null,
  45     stopTimestamp: elementObject.stopTimestamp || null,
  46     videoPlaylistId: videoPlaylist.id,
  47     videoId: video.id
  48   }
  49 }
  50
  51 async function createAccountPlaylists (playlistUrls: string[], account: MAccountDefault) {
  52   await Bluebird.map(playlistUrls, async playlistUrl => {
  53     try {
  54       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
  55       if (exists === true) return
  56
  57       // Fetch url
  58       const { body } = await doRequest<PlaylistObject>({
  59         uri: playlistUrl,
  60         json: true,
  61         activityPub: true
  62       })
  63
  64       if (!isPlaylistObjectValid(body)) {
  65         throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`)
  66       }
  67
  68       if (!isArray(body.to)) {
  69         throw new Error('Playlist does not have an audience.')
  70       }
  71
  72       return createOrUpdateVideoPlaylist(body, account, body.to)
  73     } catch (err) {
  74       logger.warn('Cannot add playlist element %s.', playlistUrl, { err })
  75     }
  76   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
  77 }
  78
  79 async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) {
  80   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, byAccount, to)
  81
  82   if (isArray(playlistObject.attributedTo) && playlistObject.attributedTo.length === 1) {
  83     const actor = await getOrCreateActorAndServerAndModel(playlistObject.attributedTo[0])
  84
  85     if (actor.VideoChannel) {
  86       playlistAttributes.videoChannelId = actor.VideoChannel.id
  87     } else {
  88       logger.warn('Attributed to of video playlist %s is not a video channel.', playlistObject.id, { playlistObject })
  89     }
  90   }
  91
  92   const [ playlist ] = await VideoPlaylistModel.upsert<MVideoPlaylist>(playlistAttributes, { returning: true })
  93
  94   let accItems: string[] = []
  95   await crawlCollectionPage<string>(playlistObject.id, items => {
  96     accItems = accItems.concat(items)
  97
  98     return Promise.resolve()
  99   })
 100
 101   const refreshedPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(playlist.id, null)
 102
 103   if (playlistObject.icon) {
 104     try {
 105       const thumbnailModel = await createPlaylistMiniatureFromUrl(playlistObject.icon.url, refreshedPlaylist)
 106       await refreshedPlaylist.setAndSaveThumbnail(thumbnailModel, undefined)
 107     } catch (err) {
 108       logger.warn('Cannot generate thumbnail of %s.', playlistObject.id, { err })
 109     }
 110   } else if (refreshedPlaylist.hasThumbnail()) {
 111     await refreshedPlaylist.Thumbnail.destroy()
 112     refreshedPlaylist.Thumbnail = null
 113   }
 114
 115   return resetVideoPlaylistElements(accItems, refreshedPlaylist)
 116 }
 117
 118 async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
 119   if (!videoPlaylist.isOutdated()) return videoPlaylist
 120
 121   try {
 122     const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
 123     if (statusCode === 404) {
 124       logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
 125
 126       await videoPlaylist.destroy()
 127       return undefined
 128     }
 129
 130     if (playlistObject === undefined) {
 131       logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
 132
 133       await videoPlaylist.setAsRefreshed()
 134       return videoPlaylist
 135     }
 136
 137     const byAccount = videoPlaylist.OwnerAccount
 138     await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
 139
 140     return videoPlaylist
 141   } catch (err) {
 142     logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
 143
 144     await videoPlaylist.setAsRefreshed()
 145     return videoPlaylist
 146   }
 147 }
 148
 149 // ---------------------------------------------------------------------------
 150
 151 export {
 152   createAccountPlaylists,
 153   playlistObjectToDBAttributes,
 154   playlistElementObjectToDBAttributes,
 155   createOrUpdateVideoPlaylist,
 156   refreshVideoPlaylistIfNeeded
 157 }
 158
 159 // ---------------------------------------------------------------------------
 160
 161 async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) {
 162   const elementsToCreate: FilteredModelAttributes<VideoPlaylistElementModel>[] = []
 163
 164   await Bluebird.map(elementUrls, async elementUrl => {
 165     try {
 166       // Fetch url
 167       const { body } = await doRequest<PlaylistElementObject>({
 168         uri: elementUrl,
 169         json: true,
 170         activityPub: true
 171       })
 172
 173       if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`)
 174
 175       if (checkUrlsSameHost(body.id, elementUrl) !== true) {
 176         throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`)
 177       }
 178
 179       const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: { id: body.url }, fetchType: 'only-video' })
 180
 181       elementsToCreate.push(playlistElementObjectToDBAttributes(body, playlist, video))
 182     } catch (err) {
 183       logger.warn('Cannot add playlist element %s.', elementUrl, { err })
 184     }
 185   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 186
 187   await sequelizeTypescript.transaction(async t => {
 188     await VideoPlaylistElementModel.deleteAllOf(playlist.id, t)
 189
 190     for (const element of elementsToCreate) {
 191       await VideoPlaylistElementModel.create(element, { transaction: t })
 192     }
 193   })
 194
 195   logger.info('Reset playlist %s with %s elements.', playlist.url, elementsToCreate.length)
 196
 197   return undefined
 198 }
 199
 200 async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
 201   const options = {
 202     uri: playlistUrl,
 203     method: 'GET',
 204     json: true,
 205     activityPub: true
 206   }
 207
 208   logger.info('Fetching remote playlist %s.', playlistUrl)
 209
 210   const { response, body } = await doRequest<any>(options)
 211
 212   if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
 213     logger.debug('Remote video playlist JSON is not valid.', { body })
 214     return { statusCode: response.statusCode, playlistObject: undefined }
 215   }
 216
 217   return { statusCode: response.statusCode, playlistObject: body }
 218 }