import { copy, pathExists } from 'fs-extra'
import { AccountModel } from '../../models/account/account'
import { VideoPlaylistReorder } from '../../../shared/models/videos/playlist/video-playlist-reorder.model'
+import { JobQueue } from '../../lib/job-queue'
const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR })
function getVideoPlaylist (req: express.Request, res: express.Response) {
const videoPlaylist = res.locals.videoPlaylist
+ if (videoPlaylist.isOutdated()) {
+ JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: videoPlaylist.url } })
+ .catch(err => logger.error('Cannot create AP refresher job for playlist %s.', videoPlaylist.url, { err }))
+ }
+
return res.json(videoPlaylist.toFormattedJSON())
}
},
MAX_RECURSION_COMMENTS: 100,
ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
- VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days
+ VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days
+ VIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days
}
const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = {
ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
+ ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL = 10 * 1000 // 10 seconds
CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB
}
if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) {
- throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id)
+ logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id)
+ return { result: undefined, statusCode: requestResult.response.statusCode }
}
const followersCount = await fetchActorTotalItems(actorJSON.followers)
return Promise.resolve()
})
- // Empty playlists generally do not have a miniature, so skip it
+ // Empty playlists generally do not have a miniature, so skip this
if (accItems.length !== 0) {
try {
await generateThumbnailFromUrl(playlist, playlistObject.icon)
return resetVideoPlaylistElements(accItems, playlist)
}
+async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise<VideoPlaylistModel> {
+ if (!videoPlaylist.isOutdated()) return videoPlaylist
+
+ try {
+ const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url)
+ if (statusCode === 404) {
+ logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url)
+
+ await videoPlaylist.destroy()
+ return undefined
+ }
+
+ if (playlistObject === undefined) {
+ logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url)
+
+ await videoPlaylist.setAsRefreshed()
+ return videoPlaylist
+ }
+
+ const byAccount = videoPlaylist.OwnerAccount
+ await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to)
+
+ return videoPlaylist
+ } catch (err) {
+ logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err })
+
+ await videoPlaylist.setAsRefreshed()
+ return videoPlaylist
+ }
+}
+
// ---------------------------------------------------------------------------
export {
createAccountPlaylists,
playlistObjectToDBAttributes,
playlistElementObjectToDBAttributes,
- createOrUpdateVideoPlaylist
+ createOrUpdateVideoPlaylist,
+ refreshVideoPlaylistIfNeeded
}
// ---------------------------------------------------------------------------
return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE)
}
+
+async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> {
+ const options = {
+ uri: playlistUrl,
+ method: 'GET',
+ json: true,
+ activityPub: true
+ }
+
+ logger.info('Fetching remote playlist %s.', playlistUrl)
+
+ const { response, body } = await doRequest(options)
+
+ if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) {
+ logger.debug('Remote video playlist JSON is not valid.', { body })
+ return { statusCode: response.statusCode, playlistObject: undefined }
+ }
+
+ return { statusCode: response.statusCode, playlistObject: body }
+}
import * as Bull from 'bull'
import { logger } from '../../../helpers/logger'
import { fetchVideoByUrl } from '../../../helpers/video'
-import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub'
+import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub'
import { ActorModel } from '../../../models/activitypub/actor'
+import { VideoPlaylistModel } from '../../../models/video/video-playlist'
export type RefreshPayload = {
- type: 'video' | 'actor'
+ type: 'video' | 'video-playlist' | 'actor'
url: string
}
logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url)
if (payload.type === 'video') return refreshVideo(payload.url)
+ if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url)
if (payload.type === 'actor') return refreshActor(payload.url)
}
// ---------------------------------------------------------------------------
export {
- refreshActor,
refreshAPObject
}
if (actor) {
await refreshActorIfNeeded(actor, fetchType)
}
+}
+
+async function refreshVideoPlaylist (playlistUrl: string) {
+ const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl)
+ if (playlist) {
+ await refreshVideoPlaylistIfNeeded(playlist)
+ }
}
import { CONSTRAINTS_FIELDS } from '../../../initializers'
import { isArrayOf, isIdOrUUIDValid, isIdValid, isUUIDValid, toIntArray, toValueOrNull } from '../../../helpers/custom-validators/misc'
import {
- isVideoPlaylistDescriptionValid,
doesVideoPlaylistExist,
+ isVideoPlaylistDescriptionValid,
isVideoPlaylistNameValid,
isVideoPlaylistPrivacyValid,
isVideoPlaylistTimestampValid,
import { cleanUpReqFiles } from '../../../helpers/express-utils'
import { doesVideoChannelIdExist } from '../../../helpers/custom-validators/video-channels'
import { VideoPlaylistElementModel } from '../../../models/video/video-playlist-element'
-import { VideoModel } from '../../../models/video/video'
import { authenticatePromiseIfNeeded } from '../../oauth'
import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model'
import { VideoPlaylistType } from '../../../../shared/models/videos/playlist/video-playlist-type.model'
import { AccountModel } from '../account/account'
import { AvatarModel } from '../avatar/avatar'
import { ServerModel } from '../server/server'
-import { throwIfNotValid } from '../utils'
+import { isOutdated, throwIfNotValid } from '../utils'
import { VideoChannelModel } from '../video/video-channel'
import { ActorFollowModel } from './actor-follow'
import { VideoModel } from '../video/video'
isOutdated () {
if (this.isOwned()) return false
- const now = Date.now()
- const createdAtTime = this.createdAt.getTime()
- const updatedAtTime = this.updatedAt.getTime()
-
- return (now - createdAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL &&
- (now - updatedAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL
+ return isOutdated(this, ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL)
}
}
import { Sequelize } from 'sequelize-typescript'
import * as validator from 'validator'
+import { ACTIVITY_PUB } from '../initializers'
type SortType = { sortModel: any, sortValue: string }
return [ firstSort, lastSort ]
}
+function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) {
+ const now = Date.now()
+ const createdAtTime = model.createdAt.getTime()
+ const updatedAtTime = model.updatedAt.getTime()
+
+ return (now - createdAtTime) > refreshInterval && (now - updatedAtTime) > refreshInterval
+}
+
function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value') {
if (validator(value) === false) {
throw new Error(`"${value}" is not a valid ${fieldName}.`)
throwIfNotValid,
buildServerIdsFollowedBy,
buildTrigramSearchIndex,
- buildWhereIdOrUUID
+ buildWhereIdOrUUID,
+ isOutdated
}
// ---------------------------------------------------------------------------
} from 'sequelize-typescript'
import * as Sequelize from 'sequelize'
import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model'
-import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, throwIfNotValid } from '../utils'
+import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, isOutdated, throwIfNotValid } from '../utils'
import {
isVideoPlaylistDescriptionValid,
isVideoPlaylistNameValid,
} from '../../helpers/custom-validators/video-playlists'
import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
import {
+ ACTIVITY_PUB,
CONFIG,
CONSTRAINTS_FIELDS,
STATIC_PATHS,
.catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err }))
}
+ setAsRefreshed () {
+ this.changed('updatedAt', true)
+
+ return this.save()
+ }
+
isOwned () {
return this.OwnerAccount.isOwned()
}
+ isOutdated () {
+ if (this.isOwned()) return false
+
+ return isOutdated(this, ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL)
+ }
+
toFormattedJSON (): VideoPlaylist {
return {
id: this.id,
buildTrigramSearchIndex,
buildWhereIdOrUUID,
createSimilarityAttribute,
- getVideoSort,
+ getVideoSort, isOutdated,
throwIfNotValid
} from '../utils'
import { TagModel } from './tag'
attributes: query.attributes,
order: [ // Keep original order
Sequelize.literal(
- ids.map(id => `"VideoModel".id = ${id} DESC`).join(', ')
+ ids.map(id => `"VideoModel".id = ${id}`).join(', ')
)
]
}
isOutdated () {
if (this.isOwned()) return false
- const now = Date.now()
- const createdAtTime = this.createdAt.getTime()
- const updatedAtTime = this.updatedAt.getTime()
-
- return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL &&
- (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL
+ return isOutdated(this, ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL)
}
setAsRefreshed () {
import 'mocha'
import {
+ createVideoPlaylist,
doubleFollow,
flushAndRunMultipleServers,
+ generateUserAccessToken,
getVideo,
+ getVideoPlaylist,
killallServers,
reRunServer,
ServerInfo,
setAccessTokensToServers,
+ setActorField,
+ setDefaultVideoChannel,
+ setPlaylistField,
+ setVideoField,
uploadVideo,
+ uploadVideoAndGetId,
wait,
- setVideoField,
waitJobs
} from '../../../../shared/utils'
+import { getAccount } from '../../../../shared/utils/users/accounts'
+import { VideoPlaylistPrivacy } from '../../../../shared/models/videos'
describe('Test AP refresher', function () {
let servers: ServerInfo[] = []
let videoUUID1: string
let videoUUID2: string
let videoUUID3: string
+ let playlistUUID1: string
+ let playlistUUID2: string
before(async function () {
this.timeout(60000)
- servers = await flushAndRunMultipleServers(2)
+ servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: false } })
// Get the access tokens
await setAccessTokensToServers(servers)
+ await setDefaultVideoChannel(servers)
+
+ {
+ videoUUID1 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video1' })).uuid
+ videoUUID2 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video2' })).uuid
+ videoUUID3 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video3' })).uuid
+ }
{
- const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video1' })
- videoUUID1 = res.body.video.uuid
+ const a1 = await generateUserAccessToken(servers[1], 'user1')
+ await uploadVideo(servers[1].url, a1, { name: 'video4' })
+
+ const a2 = await generateUserAccessToken(servers[1], 'user2')
+ await uploadVideo(servers[1].url, a2, { name: 'video5' })
}
{
- const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' })
- videoUUID2 = res.body.video.uuid
+ const playlistAttrs = { displayName: 'playlist1', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id }
+ const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs })
+ playlistUUID1 = res.body.videoPlaylist.uuid
}
{
- const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video3' })
- videoUUID3 = res.body.video.uuid
+ const playlistAttrs = { displayName: 'playlist2', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id }
+ const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs })
+ playlistUUID2 = res.body.videoPlaylist.uuid
}
await doubleFollow(servers[0], servers[1])
})
- it('Should remove a deleted remote video', async function () {
- this.timeout(60000)
+ describe('Videos refresher', function () {
+
+ it('Should remove a deleted remote video', async function () {
+ this.timeout(60000)
+
+ await wait(10000)
+
+ // Change UUID so the remote server returns a 404
+ await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f')
+
+ await getVideo(servers[ 0 ].url, videoUUID1)
+ await getVideo(servers[ 0 ].url, videoUUID2)
+
+ await waitJobs(servers)
- await wait(10000)
+ await getVideo(servers[ 0 ].url, videoUUID1, 404)
+ await getVideo(servers[ 0 ].url, videoUUID2, 200)
+ })
- // Change UUID so the remote server returns a 404
- await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f')
+ it('Should not update a remote video if the remote instance is down', async function () {
+ this.timeout(60000)
- await getVideo(servers[0].url, videoUUID1)
- await getVideo(servers[0].url, videoUUID2)
+ killallServers([ servers[ 1 ] ])
- await waitJobs(servers)
+ await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e')
- await getVideo(servers[0].url, videoUUID1, 404)
- await getVideo(servers[0].url, videoUUID2, 200)
+ // Video will need a refresh
+ await wait(10000)
+
+ await getVideo(servers[ 0 ].url, videoUUID3)
+ // The refresh should fail
+ await waitJobs([ servers[ 0 ] ])
+
+ await reRunServer(servers[ 1 ])
+
+ // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances)
+ await getVideo(servers[ 0 ].url, videoUUID3)
+ await waitJobs(servers)
+
+ await getVideo(servers[ 0 ].url, videoUUID3, 200)
+ })
})
- it('Should not update a remote video if the remote instance is down', async function () {
- this.timeout(60000)
+ describe('Actors refresher', function () {
+
+ it('Should remove a deleted actor', async function () {
+ this.timeout(60000)
+
+ await wait(10000)
+
+ // Change actor name so the remote server returns a 404
+ await setActorField(2, 'http://localhost:9002/accounts/user2', 'preferredUsername', 'toto')
+
+ await getAccount(servers[ 0 ].url, 'user1@localhost:9002')
+ await getAccount(servers[ 0 ].url, 'user2@localhost:9002')
+
+ await waitJobs(servers)
+
+ await getAccount(servers[ 0 ].url, 'user1@localhost:9002', 200)
+ await getAccount(servers[ 0 ].url, 'user2@localhost:9002', 404)
+ })
+ })
- killallServers([ servers[1] ])
+ describe('Playlist refresher', function () {
- await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e')
+ it('Should remove a deleted playlist', async function () {
+ this.timeout(60000)
- // Video will need a refresh
- await wait(10000)
+ await wait(10000)
- await getVideo(servers[0].url, videoUUID3)
- // The refresh should fail
- await waitJobs([ servers[0] ])
+ // Change UUID so the remote server returns a 404
+ await setPlaylistField(2, playlistUUID2, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b178e')
- await reRunServer(servers[1])
+ await getVideoPlaylist(servers[ 0 ].url, playlistUUID1)
+ await getVideoPlaylist(servers[ 0 ].url, playlistUUID2)
- // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances)
- await getVideo(servers[0].url, videoUUID3)
- await waitJobs(servers)
+ await waitJobs(servers)
- await getVideo(servers[0].url, videoUUID3, 200)
+ await getVideoPlaylist(servers[ 0 ].url, playlistUUID1, 200)
+ await getVideoPlaylist(servers[ 0 ].url, playlistUUID2, 404)
+ })
})
after(async function () {
return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
}
+function setPlaylistField (serverNumber: number, uuid: string, field: string, value: string) {
+ const seq = getSequelize(serverNumber)
+
+ const options = { type: Sequelize.QueryTypes.UPDATE }
+
+ return seq.query(`UPDATE "videoPlaylist" SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
+}
+
async function closeAllSequelize (servers: any[]) {
for (let i = 1; i <= servers.length; i++) {
if (sequelizes[ i ]) {
export {
setVideoField,
+ setPlaylistField,
setActorField,
closeAllSequelize
}
playlistAttrs: VideoPlaylistCreate,
expectedStatus?: number
}) {
- const path = '/api/v1/video-playlists/'
+ const path = '/api/v1/video-playlists'
const fields = omit(options.playlistAttrs, 'thumbnailfile')
projects / oweals / peertube.git / commitdiff