From: Chocobozzz Date: Tue, 19 Mar 2019 13:13:53 +0000 (+0100) Subject: Refresh playlists X-Git-Tag: v1.3.0-rc.1~127 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=9f79ade627f0044606a9fbbe16ca0154661d12b9;p=oweals%2Fpeertube.git Refresh playlists --- diff --git a/server/controllers/api/video-playlist.ts b/server/controllers/api/video-playlist.ts index 5b1601c4e..feba30564 100644 --- a/server/controllers/api/video-playlist.ts +++ b/server/controllers/api/video-playlist.ts @@ -40,6 +40,7 @@ import { VideoPlaylistElementUpdate } from '../../../shared/models/videos/playli import { copy, pathExists } from 'fs-extra' import { AccountModel } from '../../models/account/account' import { VideoPlaylistReorder } from '../../../shared/models/videos/playlist/video-playlist-reorder.model' +import { JobQueue } from '../../lib/job-queue' const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { thumbnailfile: CONFIG.STORAGE.TMP_DIR }) @@ -142,6 +143,11 @@ async function listVideoPlaylists (req: express.Request, res: express.Response) function getVideoPlaylist (req: express.Request, res: express.Response) { const videoPlaylist = res.locals.videoPlaylist + if (videoPlaylist.isOutdated()) { + JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: videoPlaylist.url } }) + .catch(err => logger.error('Cannot create AP refresher job for playlist %s.', videoPlaylist.url, { err })) + } + return res.json(videoPlaylist.toFormattedJSON()) } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 19e63d6fe..7fac8a4d6 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -584,7 +584,8 @@ const ACTIVITY_PUB = { }, MAX_RECURSION_COMMENTS: 100, ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days - VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days + VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days + VIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days } const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = { @@ -724,6 +725,7 @@ if (isTestInstance() === true) { ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds + ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL = 10 * 1000 // 10 seconds CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index f77df8b78..63e810642 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -375,7 +375,8 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe } if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { - throw new Error('Actor url ' + actorUrl + ' has not the same host than its AP id ' + actorJSON.id) + logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id) + return { result: undefined, statusCode: requestResult.response.statusCode } } const followersCount = await fetchActorTotalItems(actorJSON.followers) diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts index 70389044e..c4a8f12ec 100644 --- a/server/lib/activitypub/playlist.ts +++ b/server/lib/activitypub/playlist.ts @@ -95,7 +95,7 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc return Promise.resolve() }) - // Empty playlists generally do not have a miniature, so skip it + // Empty playlists generally do not have a miniature, so skip this if (accItems.length !== 0) { try { await generateThumbnailFromUrl(playlist, playlistObject.icon) @@ -107,13 +107,45 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, byAc return resetVideoPlaylistElements(accItems, playlist) } +async function refreshVideoPlaylistIfNeeded (videoPlaylist: VideoPlaylistModel): Promise { + if (!videoPlaylist.isOutdated()) return videoPlaylist + + try { + const { statusCode, playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) + if (statusCode === 404) { + logger.info('Cannot refresh remote video playlist %s: it does not exist anymore. Deleting it.', videoPlaylist.url) + + await videoPlaylist.destroy() + return undefined + } + + if (playlistObject === undefined) { + logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url) + + await videoPlaylist.setAsRefreshed() + return videoPlaylist + } + + const byAccount = videoPlaylist.OwnerAccount + await createOrUpdateVideoPlaylist(playlistObject, byAccount, playlistObject.to) + + return videoPlaylist + } catch (err) { + logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err }) + + await videoPlaylist.setAsRefreshed() + return videoPlaylist + } +} + // --------------------------------------------------------------------------- export { createAccountPlaylists, playlistObjectToDBAttributes, playlistElementObjectToDBAttributes, - createOrUpdateVideoPlaylist + createOrUpdateVideoPlaylist, + refreshVideoPlaylistIfNeeded } // --------------------------------------------------------------------------- @@ -162,3 +194,23 @@ function generateThumbnailFromUrl (playlist: VideoPlaylistModel, icon: ActivityI return downloadImage(icon.url, CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName, THUMBNAILS_SIZE) } + +async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { + const options = { + uri: playlistUrl, + method: 'GET', + json: true, + activityPub: true + } + + logger.info('Fetching remote playlist %s.', playlistUrl) + + const { response, body } = await doRequest(options) + + if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { + logger.debug('Remote video playlist JSON is not valid.', { body }) + return { statusCode: response.statusCode, playlistObject: undefined } + } + + return { statusCode: response.statusCode, playlistObject: body } +} diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 454b975fe..4d6c38cfa 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts @@ -1,11 +1,12 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' import { fetchVideoByUrl } from '../../../helpers/video' -import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' +import { refreshActorIfNeeded, refreshVideoIfNeeded, refreshVideoPlaylistIfNeeded } from '../../activitypub' import { ActorModel } from '../../../models/activitypub/actor' +import { VideoPlaylistModel } from '../../../models/video/video-playlist' export type RefreshPayload = { - type: 'video' | 'actor' + type: 'video' | 'video-playlist' | 'actor' url: string } @@ -15,13 +16,13 @@ async function refreshAPObject (job: Bull.Job) { logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) if (payload.type === 'video') return refreshVideo(payload.url) + if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url) if (payload.type === 'actor') return refreshActor(payload.url) } // --------------------------------------------------------------------------- export { - refreshActor, refreshAPObject } @@ -50,5 +51,12 @@ async function refreshActor (actorUrl: string) { if (actor) { await refreshActorIfNeeded(actor, fetchType) } +} + +async function refreshVideoPlaylist (playlistUrl: string) { + const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl) + if (playlist) { + await refreshVideoPlaylistIfNeeded(playlist) + } } diff --git a/server/middlewares/validators/videos/video-playlists.ts b/server/middlewares/validators/videos/video-playlists.ts index 55e09e354..6ba30fac9 100644 --- a/server/middlewares/validators/videos/video-playlists.ts +++ b/server/middlewares/validators/videos/video-playlists.ts @@ -8,8 +8,8 @@ import { doesVideoExist, isVideoImage } from '../../../helpers/custom-validators import { CONSTRAINTS_FIELDS } from '../../../initializers' import { isArrayOf, isIdOrUUIDValid, isIdValid, isUUIDValid, toIntArray, toValueOrNull } from '../../../helpers/custom-validators/misc' import { - isVideoPlaylistDescriptionValid, doesVideoPlaylistExist, + isVideoPlaylistDescriptionValid, isVideoPlaylistNameValid, isVideoPlaylistPrivacyValid, isVideoPlaylistTimestampValid, @@ -19,7 +19,6 @@ import { VideoPlaylistModel } from '../../../models/video/video-playlist' import { cleanUpReqFiles } from '../../../helpers/express-utils' import { doesVideoChannelIdExist } from '../../../helpers/custom-validators/video-channels' import { VideoPlaylistElementModel } from '../../../models/video/video-playlist-element' -import { VideoModel } from '../../../models/video/video' import { authenticatePromiseIfNeeded } from '../../oauth' import { VideoPlaylistPrivacy } from '../../../../shared/models/videos/playlist/video-playlist-privacy.model' import { VideoPlaylistType } from '../../../../shared/models/videos/playlist/video-playlist-type.model' diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index 2fceb21dd..7d91e8a4a 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -34,7 +34,7 @@ import { ACTIVITY_PUB, ACTIVITY_PUB_ACTOR_TYPES, CONFIG, CONSTRAINTS_FIELDS } fr import { AccountModel } from '../account/account' import { AvatarModel } from '../avatar/avatar' import { ServerModel } from '../server/server' -import { throwIfNotValid } from '../utils' +import { isOutdated, throwIfNotValid } from '../utils' import { VideoChannelModel } from '../video/video-channel' import { ActorFollowModel } from './actor-follow' import { VideoModel } from '../video/video' @@ -532,11 +532,6 @@ export class ActorModel extends Model { isOutdated () { if (this.isOwned()) return false - const now = Date.now() - const createdAtTime = this.createdAt.getTime() - const updatedAtTime = this.updatedAt.getTime() - - return (now - createdAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL && - (now - updatedAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL + return isOutdated(this, ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL) } } diff --git a/server/models/utils.ts b/server/models/utils.ts index 4ebd07dab..f8a71b270 100644 --- a/server/models/utils.ts +++ b/server/models/utils.ts @@ -1,5 +1,6 @@ import { Sequelize } from 'sequelize-typescript' import * as validator from 'validator' +import { ACTIVITY_PUB } from '../initializers' type SortType = { sortModel: any, sortValue: string } @@ -44,6 +45,14 @@ function getSortOnModel (model: any, value: string, lastSort: string[] = [ 'id', return [ firstSort, lastSort ] } +function isOutdated (model: { createdAt: Date, updatedAt: Date }, refreshInterval: number) { + const now = Date.now() + const createdAtTime = model.createdAt.getTime() + const updatedAtTime = model.updatedAt.getTime() + + return (now - createdAtTime) > refreshInterval && (now - updatedAtTime) > refreshInterval +} + function throwIfNotValid (value: any, validator: (value: any) => boolean, fieldName = 'value') { if (validator(value) === false) { throw new Error(`"${value}" is not a valid ${fieldName}.`) @@ -108,7 +117,8 @@ export { throwIfNotValid, buildServerIdsFollowedBy, buildTrigramSearchIndex, - buildWhereIdOrUUID + buildWhereIdOrUUID, + isOutdated } // --------------------------------------------------------------------------- diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts index 7dbe4ce8d..08e4d32c8 100644 --- a/server/models/video/video-playlist.ts +++ b/server/models/video/video-playlist.ts @@ -17,7 +17,7 @@ import { } from 'sequelize-typescript' import * as Sequelize from 'sequelize' import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' -import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, throwIfNotValid } from '../utils' +import { buildServerIdsFollowedBy, buildWhereIdOrUUID, getSort, isOutdated, throwIfNotValid } from '../utils' import { isVideoPlaylistDescriptionValid, isVideoPlaylistNameValid, @@ -25,6 +25,7 @@ import { } from '../../helpers/custom-validators/video-playlists' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { + ACTIVITY_PUB, CONFIG, CONSTRAINTS_FIELDS, STATIC_PATHS, @@ -429,10 +430,22 @@ export class VideoPlaylistModel extends Model { .catch(err => logger.warn('Cannot delete thumbnail %s.', thumbnailPath, { err })) } + setAsRefreshed () { + this.changed('updatedAt', true) + + return this.save() + } + isOwned () { return this.OwnerAccount.isOwned() } + isOutdated () { + if (this.isOwned()) return false + + return isOutdated(this, ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL) + } + toFormattedJSON (): VideoPlaylist { return { id: this.id, diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 946be6095..fb037c21a 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -77,7 +77,7 @@ import { buildTrigramSearchIndex, buildWhereIdOrUUID, createSimilarityAttribute, - getVideoSort, + getVideoSort, isOutdated, throwIfNotValid } from '../utils' import { TagModel } from './tag' @@ -1547,7 +1547,7 @@ export class VideoModel extends Model { attributes: query.attributes, order: [ // Keep original order Sequelize.literal( - ids.map(id => `"VideoModel".id = ${id} DESC`).join(', ') + ids.map(id => `"VideoModel".id = ${id}`).join(', ') ) ] } @@ -1767,12 +1767,7 @@ export class VideoModel extends Model { isOutdated () { if (this.isOwned()) return false - const now = Date.now() - const createdAtTime = this.createdAt.getTime() - const updatedAtTime = this.updatedAt.getTime() - - return (now - createdAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL && - (now - updatedAtTime) > ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL + return isOutdated(this, ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL) } setAsRefreshed () { diff --git a/server/tests/api/activitypub/refresher.ts b/server/tests/api/activitypub/refresher.ts index 62ad8a0b5..ae4859076 100644 --- a/server/tests/api/activitypub/refresher.ts +++ b/server/tests/api/activitypub/refresher.ts @@ -2,89 +2,155 @@ import 'mocha' import { + createVideoPlaylist, doubleFollow, flushAndRunMultipleServers, + generateUserAccessToken, getVideo, + getVideoPlaylist, killallServers, reRunServer, ServerInfo, setAccessTokensToServers, + setActorField, + setDefaultVideoChannel, + setPlaylistField, + setVideoField, uploadVideo, + uploadVideoAndGetId, wait, - setVideoField, waitJobs } from '../../../../shared/utils' +import { getAccount } from '../../../../shared/utils/users/accounts' +import { VideoPlaylistPrivacy } from '../../../../shared/models/videos' describe('Test AP refresher', function () { let servers: ServerInfo[] = [] let videoUUID1: string let videoUUID2: string let videoUUID3: string + let playlistUUID1: string + let playlistUUID2: string before(async function () { this.timeout(60000) - servers = await flushAndRunMultipleServers(2) + servers = await flushAndRunMultipleServers(2, { transcoding: { enabled: false } }) // Get the access tokens await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + { + videoUUID1 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video1' })).uuid + videoUUID2 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video2' })).uuid + videoUUID3 = (await uploadVideoAndGetId({ server: servers[ 1 ], videoName: 'video3' })).uuid + } { - const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video1' }) - videoUUID1 = res.body.video.uuid + const a1 = await generateUserAccessToken(servers[1], 'user1') + await uploadVideo(servers[1].url, a1, { name: 'video4' }) + + const a2 = await generateUserAccessToken(servers[1], 'user2') + await uploadVideo(servers[1].url, a2, { name: 'video5' }) } { - const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video2' }) - videoUUID2 = res.body.video.uuid + const playlistAttrs = { displayName: 'playlist1', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id } + const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs }) + playlistUUID1 = res.body.videoPlaylist.uuid } { - const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video3' }) - videoUUID3 = res.body.video.uuid + const playlistAttrs = { displayName: 'playlist2', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].videoChannel.id } + const res = await createVideoPlaylist({ url: servers[1].url, token: servers[1].accessToken, playlistAttrs }) + playlistUUID2 = res.body.videoPlaylist.uuid } await doubleFollow(servers[0], servers[1]) }) - it('Should remove a deleted remote video', async function () { - this.timeout(60000) + describe('Videos refresher', function () { + + it('Should remove a deleted remote video', async function () { + this.timeout(60000) + + await wait(10000) + + // Change UUID so the remote server returns a 404 + await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f') + + await getVideo(servers[ 0 ].url, videoUUID1) + await getVideo(servers[ 0 ].url, videoUUID2) + + await waitJobs(servers) - await wait(10000) + await getVideo(servers[ 0 ].url, videoUUID1, 404) + await getVideo(servers[ 0 ].url, videoUUID2, 200) + }) - // Change UUID so the remote server returns a 404 - await setVideoField(2, videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f') + it('Should not update a remote video if the remote instance is down', async function () { + this.timeout(60000) - await getVideo(servers[0].url, videoUUID1) - await getVideo(servers[0].url, videoUUID2) + killallServers([ servers[ 1 ] ]) - await waitJobs(servers) + await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e') - await getVideo(servers[0].url, videoUUID1, 404) - await getVideo(servers[0].url, videoUUID2, 200) + // Video will need a refresh + await wait(10000) + + await getVideo(servers[ 0 ].url, videoUUID3) + // The refresh should fail + await waitJobs([ servers[ 0 ] ]) + + await reRunServer(servers[ 1 ]) + + // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances) + await getVideo(servers[ 0 ].url, videoUUID3) + await waitJobs(servers) + + await getVideo(servers[ 0 ].url, videoUUID3, 200) + }) }) - it('Should not update a remote video if the remote instance is down', async function () { - this.timeout(60000) + describe('Actors refresher', function () { + + it('Should remove a deleted actor', async function () { + this.timeout(60000) + + await wait(10000) + + // Change actor name so the remote server returns a 404 + await setActorField(2, 'http://localhost:9002/accounts/user2', 'preferredUsername', 'toto') + + await getAccount(servers[ 0 ].url, 'user1@localhost:9002') + await getAccount(servers[ 0 ].url, 'user2@localhost:9002') + + await waitJobs(servers) + + await getAccount(servers[ 0 ].url, 'user1@localhost:9002', 200) + await getAccount(servers[ 0 ].url, 'user2@localhost:9002', 404) + }) + }) - killallServers([ servers[1] ]) + describe('Playlist refresher', function () { - await setVideoField(2, videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e') + it('Should remove a deleted playlist', async function () { + this.timeout(60000) - // Video will need a refresh - await wait(10000) + await wait(10000) - await getVideo(servers[0].url, videoUUID3) - // The refresh should fail - await waitJobs([ servers[0] ]) + // Change UUID so the remote server returns a 404 + await setPlaylistField(2, playlistUUID2, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b178e') - await reRunServer(servers[1]) + await getVideoPlaylist(servers[ 0 ].url, playlistUUID1) + await getVideoPlaylist(servers[ 0 ].url, playlistUUID2) - // Should not refresh the video, even if the last refresh failed (to avoir a loop on dead instances) - await getVideo(servers[0].url, videoUUID3) - await waitJobs(servers) + await waitJobs(servers) - await getVideo(servers[0].url, videoUUID3, 200) + await getVideoPlaylist(servers[ 0 ].url, playlistUUID1, 200) + await getVideoPlaylist(servers[ 0 ].url, playlistUUID2, 404) + }) }) after(async function () { diff --git a/shared/utils/miscs/sql.ts b/shared/utils/miscs/sql.ts index bb3f63837..1ce3d801a 100644 --- a/shared/utils/miscs/sql.ts +++ b/shared/utils/miscs/sql.ts @@ -40,6 +40,14 @@ function setVideoField (serverNumber: number, uuid: string, field: string, value return seq.query(`UPDATE video SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options) } +function setPlaylistField (serverNumber: number, uuid: string, field: string, value: string) { + const seq = getSequelize(serverNumber) + + const options = { type: Sequelize.QueryTypes.UPDATE } + + return seq.query(`UPDATE "videoPlaylist" SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options) +} + async function closeAllSequelize (servers: any[]) { for (let i = 1; i <= servers.length; i++) { if (sequelizes[ i ]) { @@ -51,6 +59,7 @@ async function closeAllSequelize (servers: any[]) { export { setVideoField, + setPlaylistField, setActorField, closeAllSequelize } diff --git a/shared/utils/videos/video-playlists.ts b/shared/utils/videos/video-playlists.ts index 7568852dc..4d110a131 100644 --- a/shared/utils/videos/video-playlists.ts +++ b/shared/utils/videos/video-playlists.ts @@ -127,7 +127,7 @@ function createVideoPlaylist (options: { playlistAttrs: VideoPlaylistCreate, expectedStatus?: number }) { - const path = '/api/v1/video-playlists/' + const path = '/api/v1/video-playlists' const fields = omit(options.playlistAttrs, 'thumbnailfile')