Fetch remote AP objects
authorChocobozzz <me@florianbigard.com>
Wed, 10 Jan 2018 16:18:12 +0000 (17:18 +0100)
committerChocobozzz <me@florianbigard.com>
Wed, 10 Jan 2018 16:19:14 +0000 (17:19 +0100)
22 files changed:
client/src/standalone/videos/embed.ts
server/controllers/activitypub/client.ts
server/helpers/custom-validators/activitypub/video-comments.ts
server/helpers/custom-validators/activitypub/videos.ts
server/initializers/constants.ts
server/initializers/installer.ts
server/initializers/migrator.ts
server/lib/activitypub/index.ts
server/lib/activitypub/process/misc.ts [deleted file]
server/lib/activitypub/process/process-announce.ts
server/lib/activitypub/process/process-create.ts
server/lib/activitypub/process/process-like.ts
server/lib/activitypub/process/process-undo.ts
server/lib/activitypub/process/process-update.ts
server/lib/activitypub/video-comments.ts [new file with mode: 0644]
server/lib/activitypub/video-rates.ts [new file with mode: 0644]
server/lib/activitypub/videos.ts
server/models/activitypub/actor.ts
server/models/video/video-comment.ts
server/models/video/video.ts
server/tests/api/server/handle-down.ts [new file with mode: 0644]
shared/models/activitypub/objects/video-torrent-object.ts

index e68ee193af38048385d33b83eaca6daffc442e9a..9c672529f35b3dcde7349eb36e3d01bd2faf125c 100644 (file)
@@ -1,6 +1,7 @@
 import './embed.scss'
 
 import * as videojs from 'video.js'
+import 'videojs-hotkeys'
 import '../../assets/player/peertube-videojs-plugin'
 import 'videojs-dock/dist/videojs-dock.es.js'
 import { VideoDetails } from '../../../../shared'
index d1a761724687e7939ddd24ffb987d639acb4ad65..ec3f72b64aa3695694cf2138296b4125a0b756a0 100644 (file)
@@ -1,9 +1,11 @@
 // Intercept ActivityPub client requests
 import * as express from 'express'
+import { VideoPrivacy } from '../../../shared/models/videos'
 import { activityPubCollectionPagination } from '../../helpers/activitypub'
 import { pageToStartAndCount } from '../../helpers/core-utils'
 import { ACTIVITY_PUB, CONFIG } from '../../initializers'
 import { buildVideoAnnounceToFollowers } from '../../lib/activitypub/send'
+import { audiencify, getAudience } from '../../lib/activitypub/send/misc'
 import { asyncMiddleware, executeIfActivityPub, localAccountValidator } from '../../middlewares'
 import { videoChannelsGetValidator, videosGetValidator, videosShareValidator } from '../../middlewares/validators'
 import { videoCommentGetValidator } from '../../middlewares/validators/video-comments'
@@ -95,7 +97,9 @@ async function videoController (req: express.Request, res: express.Response, nex
 
   // We need more attributes
   const videoAll = await VideoModel.loadAndPopulateAll(video.id)
-  return res.json(videoAll.toActivityPubObject())
+  const audience = await getAudience(video.VideoChannel.Account.Actor, undefined, video.privacy === VideoPrivacy.PUBLIC)
+
+  return res.json(audiencify(videoAll.toActivityPubObject(), audience))
 }
 
 async function videoAnnounceController (req: express.Request, res: express.Response, next: express.NextFunction) {
index ce1209035fa77cc9e41df0dcf03ea85250ff76bd..cbd4dac5c30cbd47ad79a9e428adce7126374234 100644 (file)
@@ -24,7 +24,8 @@ function isVideoCommentDeleteActivityValid (activity: any) {
 
 export {
   isVideoCommentCreateActivityValid,
-  isVideoCommentDeleteActivityValid
+  isVideoCommentDeleteActivityValid,
+  isVideoCommentObjectValid
 }
 
 // ---------------------------------------------------------------------------
index 37cd6965a454e653bcb29843a65885f68a70a909..fb1d2d094aa21f30ba3c1ea924c6c1893a4d9397 100644 (file)
@@ -70,7 +70,8 @@ export {
   isVideoTorrentCreateActivityValid,
   isVideoTorrentUpdateActivityValid,
   isVideoTorrentDeleteActivityValid,
-  isVideoFlagValid
+  isVideoFlagValid,
+  isVideoTorrentObjectValid
 }
 
 // ---------------------------------------------------------------------------
index c902e0cf69fdde0314a3c77a477cd31bd116385d..c735e6dafa8cca0783b512b98c65865b1fc63802 100644 (file)
@@ -279,6 +279,7 @@ const ACTIVITY_PUB = {
     TORRENT: [ 'application/x-bittorrent' ],
     MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ]
   },
+  MAX_RECURSION_COMMENTS: 100,
   ACTOR_REFRESH_INTERVAL: 3600 * 24 // 1 day
 }
 
index 58713c2c415da6aa81044d1b9a4f474fd5f40950..324a2c2e5bdd4eefa0fca36f6627c2804db696bf 100644 (file)
@@ -20,7 +20,7 @@ async function installApplication () {
     await createOAuthAdminIfNotExist()
   } catch (err) {
     logger.error('Cannot install application.', err)
-    throw err
+    process.exit(-1)
   }
 }
 
index 29310b91372a89a4d3b4b8bdbca50b0cb97f17e7..9ebc57f0764d3caf631b3433af60f315ae68c536 100644 (file)
@@ -44,7 +44,7 @@ async function migrate () {
       await executeMigration(actualVersion, migrationScript)
     } catch (err) {
       logger.error('Cannot execute migration %s.', migrationScript.version, err)
-      process.exit(0)
+      process.exit(-1)
     }
   }
 
@@ -92,7 +92,7 @@ async function executeMigration (actualVersion: number, entity: { version: strin
 
   const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName))
 
-  await sequelizeTypescript.transaction(async t => {
+  return sequelizeTypescript.transaction(async t => {
     const options = {
       transaction: t,
       queryInterface: sequelizeTypescript.getQueryInterface(),
index 94ed1edaa28b3c93d22bc78c3c36d7d8e45aafb7..0779d1e911b49f661f5b8fa958e0dbe63cab072f 100644 (file)
@@ -5,3 +5,8 @@ export * from './fetch'
 export * from './share'
 export * from './videos'
 export * from './url'
+export { videoCommentActivityObjectToDBAttributes } from './video-comments'
+export { addVideoComments } from './video-comments'
+export { addVideoComment } from './video-comments'
+export { sendVideoRateChangeToFollowers } from './video-rates'
+export { sendVideoRateChangeToOrigin } from './video-rates'
diff --git a/server/lib/activitypub/process/misc.ts b/server/lib/activitypub/process/misc.ts
deleted file mode 100644 (file)
index 461619e..0000000
+++ /dev/null
@@ -1,194 +0,0 @@
-import * as magnetUtil from 'magnet-uri'
-import { VideoTorrentObject } from '../../../../shared'
-import { VideoCommentObject } from '../../../../shared/models/activitypub/objects/video-comment-object'
-import { VideoPrivacy } from '../../../../shared/models/videos'
-import { isVideoFileInfoHashValid } from '../../../helpers/custom-validators/videos'
-import { logger } from '../../../helpers/logger'
-import { doRequest } from '../../../helpers/requests'
-import { ACTIVITY_PUB, VIDEO_MIMETYPE_EXT } from '../../../initializers'
-import { ActorModel } from '../../../models/activitypub/actor'
-import { VideoModel } from '../../../models/video/video'
-import { VideoChannelModel } from '../../../models/video/video-channel'
-import { VideoCommentModel } from '../../../models/video/video-comment'
-import { VideoShareModel } from '../../../models/video/video-share'
-import { getOrCreateActorAndServerAndModel } from '../actor'
-
-async function videoActivityObjectToDBAttributes (
-  videoChannel: VideoChannelModel,
-  videoObject: VideoTorrentObject,
-  to: string[] = [],
-  cc: string[] = []
-) {
-  let privacy = VideoPrivacy.PRIVATE
-  if (to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.PUBLIC
-  else if (cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.UNLISTED
-
-  const duration = videoObject.duration.replace(/[^\d]+/, '')
-  let language = null
-  if (videoObject.language) {
-    language = parseInt(videoObject.language.identifier, 10)
-  }
-
-  let category = null
-  if (videoObject.category) {
-    category = parseInt(videoObject.category.identifier, 10)
-  }
-
-  let licence = null
-  if (videoObject.licence) {
-    licence = parseInt(videoObject.licence.identifier, 10)
-  }
-
-  let description = null
-  if (videoObject.content) {
-    description = videoObject.content
-  }
-
-  return {
-    name: videoObject.name,
-    uuid: videoObject.uuid,
-    url: videoObject.id,
-    category,
-    licence,
-    language,
-    description,
-    nsfw: videoObject.nsfw,
-    commentsEnabled: videoObject.commentsEnabled,
-    channelId: videoChannel.id,
-    duration: parseInt(duration, 10),
-    createdAt: new Date(videoObject.published),
-    // FIXME: updatedAt does not seems to be considered by Sequelize
-    updatedAt: new Date(videoObject.updated),
-    views: videoObject.views,
-    likes: 0,
-    dislikes: 0,
-    remote: true,
-    privacy
-  }
-}
-
-function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
-  const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
-  const fileUrls = videoObject.url.filter(u => {
-    return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/')
-  })
-
-  if (fileUrls.length === 0) {
-    throw new Error('Cannot find video files for ' + videoCreated.url)
-  }
-
-  const attributes = []
-  for (const fileUrl of fileUrls) {
-    // Fetch associated magnet uri
-    const magnet = videoObject.url.find(u => {
-      return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === fileUrl.width
-    })
-
-    if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.url)
-
-    const parsed = magnetUtil.decode(magnet.url)
-    if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url)
-
-    const attribute = {
-      extname: VIDEO_MIMETYPE_EXT[fileUrl.mimeType],
-      infoHash: parsed.infoHash,
-      resolution: fileUrl.width,
-      size: fileUrl.size,
-      videoId: videoCreated.id
-    }
-    attributes.push(attribute)
-  }
-
-  return attributes
-}
-
-async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
-  let originCommentId: number = null
-  let inReplyToCommentId: number = null
-
-  // If this is not a reply to the video (thread), create or get the parent comment
-  if (video.url !== comment.inReplyTo) {
-    const [ parent ] = await addVideoComment(video, comment.inReplyTo)
-    if (!parent) {
-      logger.warn('Cannot fetch or get parent comment %s of comment %s.', comment.inReplyTo, comment.id)
-      return undefined
-    }
-
-    originCommentId = parent.originCommentId || parent.id
-    inReplyToCommentId = parent.id
-  }
-
-  return {
-    url: comment.url,
-    text: comment.content,
-    videoId: video.id,
-    accountId: actor.Account.id,
-    inReplyToCommentId,
-    originCommentId,
-    createdAt: new Date(comment.published),
-    updatedAt: new Date(comment.updated)
-  }
-}
-
-async function addVideoShares (instance: VideoModel, shareUrls: string[]) {
-  for (const shareUrl of shareUrls) {
-    // Fetch url
-    const { body } = await doRequest({
-      uri: shareUrl,
-      json: true,
-      activityPub: true
-    })
-    const actorUrl = body.actor
-    if (!actorUrl) continue
-
-    const actor = await getOrCreateActorAndServerAndModel(actorUrl)
-
-    const entry = {
-      actorId: actor.id,
-      videoId: instance.id
-    }
-
-    await VideoShareModel.findOrCreate({
-      where: entry,
-      defaults: entry
-    })
-  }
-}
-
-async function addVideoComments (instance: VideoModel, commentUrls: string[]) {
-  for (const commentUrl of commentUrls) {
-    await addVideoComment(instance, commentUrl)
-  }
-}
-
-async function addVideoComment (instance: VideoModel, commentUrl: string) {
-  // Fetch url
-  const { body } = await doRequest({
-    uri: commentUrl,
-    json: true,
-    activityPub: true
-  })
-
-  const actorUrl = body.attributedTo
-  if (!actorUrl) return []
-
-  const actor = await getOrCreateActorAndServerAndModel(actorUrl)
-  const entry = await videoCommentActivityObjectToDBAttributes(instance, actor, body)
-  if (!entry) return []
-
-  return VideoCommentModel.findOrCreate({
-    where: {
-      url: body.id
-    },
-    defaults: entry
-  })
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  videoFileActivityUrlToDBAttributes,
-  videoActivityObjectToDBAttributes,
-  addVideoShares,
-  addVideoComments
-}
index 9adb40e01c63f6087a15034e21bca920cfdc277e..bf7d7879db440b91556b01a164e495785dd32b9e 100644 (file)
@@ -7,6 +7,7 @@ import { VideoModel } from '../../../models/video/video'
 import { VideoShareModel } from '../../../models/video/video-share'
 import { getOrCreateActorAndServerAndModel } from '../actor'
 import { forwardActivity } from '../send/misc'
+import { getOrCreateAccountAndVideoAndChannel } from '../videos'
 import { processCreateActivity } from './process-create'
 
 async function processAnnounceActivity (activity: ActivityAnnounce) {
@@ -44,19 +45,19 @@ function processVideoShare (actorAnnouncer: ActorModel, activity: ActivityAnnoun
   return retryTransactionWrapper(shareVideo, options)
 }
 
-function shareVideo (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
+async function shareVideo (actorAnnouncer: ActorModel, activity: ActivityAnnounce) {
   const announced = activity.object
+  let video: VideoModel
+
+  if (typeof announced === 'string') {
+    const res = await getOrCreateAccountAndVideoAndChannel(announced)
+    video = res.video
+  } else {
+    video = await processCreateActivity(announced)
+  }
 
   return sequelizeTypescript.transaction(async t => {
     // Add share entry
-    let video: VideoModel
-
-    if (typeof announced === 'string') {
-      video = await VideoModel.loadByUrlAndPopulateAccount(announced)
-      if (!video) throw new Error('Unknown video to share ' + announced)
-    } else {
-      video = await processCreateActivity(announced)
-    }
 
     const share = {
       actorId: actorAnnouncer.id,
index e65b257c0cb13bd3dd575ef0caa4c2e0bb13dfcb..08d61996a2ff7bac2bfe08cb689b8b037f33c158 100644 (file)
@@ -8,15 +8,13 @@ import { logger } from '../../../helpers/logger'
 import { sequelizeTypescript } from '../../../initializers'
 import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
 import { ActorModel } from '../../../models/activitypub/actor'
-import { TagModel } from '../../../models/video/tag'
 import { VideoModel } from '../../../models/video/video'
 import { VideoAbuseModel } from '../../../models/video/video-abuse'
 import { VideoCommentModel } from '../../../models/video/video-comment'
-import { VideoFileModel } from '../../../models/video/video-file'
 import { getOrCreateActorAndServerAndModel } from '../actor'
 import { forwardActivity, getActorsInvolvedInVideo } from '../send/misc'
-import { generateThumbnailFromUrl } from '../videos'
-import { addVideoComments, addVideoShares, videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc'
+import { addVideoComments, resolveThread } from '../video-comments'
+import { addVideoShares, getOrCreateAccountAndVideoAndChannel } from '../videos'
 
 async function processCreateActivity (activity: ActivityCreate) {
   const activityObject = activity.object
@@ -53,17 +51,7 @@ async function processCreateVideo (
 ) {
   const videoToCreateData = activity.object as VideoTorrentObject
 
-  const channel = videoToCreateData.attributedTo.find(a => a.type === 'Group')
-  if (!channel) throw new Error('Cannot find associated video channel to video ' + videoToCreateData.url)
-
-  const channelActor = await getOrCreateActorAndServerAndModel(channel.id)
-
-  const options = {
-    arguments: [ actor, activity, videoToCreateData, channelActor ],
-    errorMessage: 'Cannot insert the remote video with many retries.'
-  }
-
-  const video = await retryTransactionWrapper(createRemoteVideo, options)
+  const { video } = await getOrCreateAccountAndVideoAndChannel(videoToCreateData, actor)
 
   // Process outside the transaction because we could fetch remote data
   if (videoToCreateData.likes && Array.isArray(videoToCreateData.likes.orderedItems)) {
@@ -89,48 +77,6 @@ async function processCreateVideo (
   return video
 }
 
-function createRemoteVideo (
-  account: ActorModel,
-  activity: ActivityCreate,
-  videoToCreateData: VideoTorrentObject,
-  channelActor: ActorModel
-) {
-  logger.debug('Adding remote video %s.', videoToCreateData.id)
-
-  return sequelizeTypescript.transaction(async t => {
-    const sequelizeOptions = {
-      transaction: t
-    }
-    const videoFromDatabase = await VideoModel.loadByUUIDOrURL(videoToCreateData.uuid, videoToCreateData.id, t)
-    if (videoFromDatabase) return videoFromDatabase
-
-    const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoToCreateData, activity.to, activity.cc)
-    const video = VideoModel.build(videoData)
-
-    // Don't block on request
-    generateThumbnailFromUrl(video, videoToCreateData.icon)
-      .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoToCreateData.id, err))
-
-    const videoCreated = await video.save(sequelizeOptions)
-
-    const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoToCreateData)
-    if (videoFileAttributes.length === 0) {
-      throw new Error('Cannot find valid files for video %s ' + videoToCreateData.url)
-    }
-
-    const tasks: Bluebird<any>[] = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
-    await Promise.all(tasks)
-
-    const tags = videoToCreateData.tag.map(t => t.name)
-    const tagInstances = await TagModel.findOrCreateTags(tags, t)
-    await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
-
-    logger.info('Remote video with uuid %s inserted.', videoToCreateData.uuid)
-
-    return videoCreated
-  })
-}
-
 async function createRates (actorUrls: string[], video: VideoModel, rate: VideoRateType) {
   let rateCounts = 0
   const tasks: Bluebird<any>[] = []
@@ -167,16 +113,15 @@ async function processCreateDislike (byActor: ActorModel, activity: ActivityCrea
   return retryTransactionWrapper(createVideoDislike, options)
 }
 
-function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) {
+async function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) {
   const dislike = activity.object as DislikeObject
   const byAccount = byActor.Account
 
   if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url)
 
-  return sequelizeTypescript.transaction(async t => {
-    const video = await VideoModel.loadByUrlAndPopulateAccount(dislike.object, t)
-    if (!video) throw new Error('Unknown video ' + dislike.object)
+  const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object)
 
+  return sequelizeTypescript.transaction(async t => {
     const rate = {
       type: 'dislike' as 'dislike',
       videoId: video.id,
@@ -200,9 +145,7 @@ function createVideoDislike (byActor: ActorModel, activity: ActivityCreate) {
 async function processCreateView (byActor: ActorModel, activity: ActivityCreate) {
   const view = activity.object as ViewObject
 
-  const video = await VideoModel.loadByUrlAndPopulateAccount(view.object)
-
-  if (!video) throw new Error('Unknown video ' + view.object)
+  const { video } = await getOrCreateAccountAndVideoAndChannel(view.object)
 
   const account = await ActorModel.loadByUrl(view.actor)
   if (!account) throw new Error('Unknown account ' + view.actor)
@@ -225,19 +168,15 @@ function processCreateVideoAbuse (actor: ActorModel, videoAbuseToCreateData: Vid
   return retryTransactionWrapper(addRemoteVideoAbuse, options)
 }
 
-function addRemoteVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
+async function addRemoteVideoAbuse (actor: ActorModel, videoAbuseToCreateData: VideoAbuseObject) {
   logger.debug('Reporting remote abuse for video %s.', videoAbuseToCreateData.object)
 
   const account = actor.Account
   if (!account) throw new Error('Cannot create dislike with the non account actor ' + actor.url)
 
-  return sequelizeTypescript.transaction(async t => {
-    const video = await VideoModel.loadByUrlAndPopulateAccount(videoAbuseToCreateData.object, t)
-    if (!video) {
-      logger.warn('Unknown video %s for remote video abuse.', videoAbuseToCreateData.object)
-      return undefined
-    }
+  const { video } = await getOrCreateAccountAndVideoAndChannel(videoAbuseToCreateData.object)
 
+  return sequelizeTypescript.transaction(async t => {
     const videoAbuseData = {
       reporterAccountId: account.id,
       reason: videoAbuseToCreateData.content,
@@ -259,41 +198,33 @@ function processCreateVideoComment (byActor: ActorModel, activity: ActivityCreat
   return retryTransactionWrapper(createVideoComment, options)
 }
 
-function createVideoComment (byActor: ActorModel, activity: ActivityCreate) {
+async function createVideoComment (byActor: ActorModel, activity: ActivityCreate) {
   const comment = activity.object as VideoCommentObject
   const byAccount = byActor.Account
 
   if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url)
 
+  const { video, parents } = await resolveThread(comment.inReplyTo)
+
   return sequelizeTypescript.transaction(async t => {
-    let video = await VideoModel.loadByUrlAndPopulateAccount(comment.inReplyTo, t)
-    let objectToCreate
+    let originCommentId = null
+    let inReplyToCommentId = null
+
+    if (parents.length !== 0) {
+      const parent = parents[0]
+
+      originCommentId = parent.getThreadId()
+      inReplyToCommentId = parent.id
+    }
 
     // This is a new thread
-    if (video) {
-      objectToCreate = {
-        url: comment.id,
-        text: comment.content,
-        originCommentId: null,
-        inReplyToComment: null,
-        videoId: video.id,
-        accountId: byAccount.id
-      }
-    } else {
-      const inReplyToComment = await VideoCommentModel.loadByUrl(comment.inReplyTo, t)
-      if (!inReplyToComment) throw new Error('Unknown replied comment ' + comment.inReplyTo)
-
-      video = await VideoModel.loadAndPopulateAccount(inReplyToComment.videoId)
-
-      const originCommentId = inReplyToComment.originCommentId || inReplyToComment.id
-      objectToCreate = {
-        url: comment.id,
-        text: comment.content,
-        originCommentId,
-        inReplyToCommentId: inReplyToComment.id,
-        videoId: video.id,
-        accountId: byAccount.id
-      }
+    const objectToCreate = {
+      url: comment.id,
+      text: comment.content,
+      originCommentId,
+      inReplyToCommentId,
+      videoId: video.id,
+      accountId: byAccount.id
     }
 
     const options = {
index 77fadabe149a7cd2582c48cea0a80b8ce34494c3..0d161b126d1389884b48cae346da36a4f5e9fee8 100644 (file)
@@ -3,9 +3,9 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
 import { sequelizeTypescript } from '../../../initializers'
 import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
 import { ActorModel } from '../../../models/activitypub/actor'
-import { VideoModel } from '../../../models/video/video'
 import { getOrCreateActorAndServerAndModel } from '../actor'
 import { forwardActivity } from '../send/misc'
+import { getOrCreateAccountAndVideoAndChannel } from '../videos'
 
 async function processLikeActivity (activity: ActivityLike) {
   const actor = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -30,17 +30,15 @@ async function processLikeVideo (actor: ActorModel, activity: ActivityLike) {
   return retryTransactionWrapper(createVideoLike, options)
 }
 
-function createVideoLike (byActor: ActorModel, activity: ActivityLike) {
+async function createVideoLike (byActor: ActorModel, activity: ActivityLike) {
   const videoUrl = activity.object
 
   const byAccount = byActor.Account
   if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url)
 
-  return sequelizeTypescript.transaction(async t => {
-    const video = await VideoModel.loadByUrlAndPopulateAccount(videoUrl)
-
-    if (!video) throw new Error('Unknown video ' + videoUrl)
+  const { video } = await getOrCreateAccountAndVideoAndChannel(videoUrl)
 
+  return sequelizeTypescript.transaction(async t => {
     const rate = {
       type: 'like' as 'like',
       videoId: video.id,
index 9cad592334b13bf3f9dd830fa0265910eff89b6d..5a770bb972948d6e94ec55adca670f0c6eac0020 100644 (file)
@@ -7,8 +7,8 @@ import { AccountModel } from '../../../models/account/account'
 import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
-import { VideoModel } from '../../../models/video/video'
 import { forwardActivity } from '../send/misc'
+import { getOrCreateAccountAndVideoAndChannel } from '../videos'
 
 async function processUndoActivity (activity: ActivityUndo) {
   const activityToUndo = activity.object
@@ -43,16 +43,15 @@ function processUndoLike (actorUrl: string, activity: ActivityUndo) {
   return retryTransactionWrapper(undoLike, options)
 }
 
-function undoLike (actorUrl: string, activity: ActivityUndo) {
+async function undoLike (actorUrl: string, activity: ActivityUndo) {
   const likeActivity = activity.object as ActivityLike
 
+  const { video } = await getOrCreateAccountAndVideoAndChannel(likeActivity.object)
+
   return sequelizeTypescript.transaction(async t => {
     const byAccount = await AccountModel.loadByUrl(actorUrl, t)
     if (!byAccount) throw new Error('Unknown account ' + actorUrl)
 
-    const video = await VideoModel.loadByUrlAndPopulateAccount(likeActivity.object, t)
-    if (!video) throw new Error('Unknown video ' + likeActivity.actor)
-
     const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t)
     if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`)
 
@@ -76,16 +75,15 @@ function processUndoDislike (actorUrl: string, activity: ActivityUndo) {
   return retryTransactionWrapper(undoDislike, options)
 }
 
-function undoDislike (actorUrl: string, activity: ActivityUndo) {
+async function undoDislike (actorUrl: string, activity: ActivityUndo) {
   const dislike = activity.object.object as DislikeObject
 
+  const { video } = await getOrCreateAccountAndVideoAndChannel(dislike.object)
+
   return sequelizeTypescript.transaction(async t => {
     const byAccount = await AccountModel.loadByUrl(actorUrl, t)
     if (!byAccount) throw new Error('Unknown account ' + actorUrl)
 
-    const video = await VideoModel.loadByUrlAndPopulateAccount(dislike.object, t)
-    if (!video) throw new Error('Unknown video ' + dislike.actor)
-
     const rate = await AccountVideoRateModel.load(byAccount.id, video.id, t)
     if (!rate) throw new Error(`Unknown rate by account ${byAccount.id} for video ${video.id}.`)
 
index 2c094f7ca05f1d46f789294c4f8c45399173dce9..a5431c76b93f7904c9932337a14ffa2933db47e1 100644 (file)
@@ -9,10 +9,9 @@ import { sequelizeTypescript } from '../../../initializers'
 import { AccountModel } from '../../../models/account/account'
 import { ActorModel } from '../../../models/activitypub/actor'
 import { TagModel } from '../../../models/video/tag'
-import { VideoModel } from '../../../models/video/video'
 import { VideoFileModel } from '../../../models/video/video-file'
 import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor'
-import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc'
+import { getOrCreateAccountAndVideoAndChannel, videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from '../videos'
 
 async function processUpdateActivity (activity: ActivityUpdate) {
   const actor = await getOrCreateActorAndServerAndModel(activity.actor)
@@ -46,8 +45,10 @@ function processUpdateVideo (actor: ActorModel, activity: ActivityUpdate) {
 async function updateRemoteVideo (actor: ActorModel, activity: ActivityUpdate) {
   const videoAttributesToUpdate = activity.object as VideoTorrentObject
 
+  const res = await getOrCreateAccountAndVideoAndChannel(videoAttributesToUpdate.id)
+
   logger.debug('Updating remote video "%s".', videoAttributesToUpdate.uuid)
-  let videoInstance: VideoModel
+  let videoInstance = res.video
   let videoFieldsSave: any
 
   try {
@@ -56,9 +57,6 @@ async function updateRemoteVideo (actor: ActorModel, activity: ActivityUpdate) {
         transaction: t
       }
 
-      const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(videoAttributesToUpdate.id, t)
-      if (!videoInstance) throw new Error('Video ' + videoAttributesToUpdate.id + ' not found.')
-
       videoFieldsSave = videoInstance.toJSON()
 
       const videoChannel = videoInstance.VideoChannel
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
new file mode 100644 (file)
index 0000000..17c86a3
--- /dev/null
@@ -0,0 +1,156 @@
+import { VideoCommentObject } from '../../../shared/models/activitypub/objects/video-comment-object'
+import { isVideoCommentObjectValid } from '../../helpers/custom-validators/activitypub/video-comments'
+import { logger } from '../../helpers/logger'
+import { doRequest } from '../../helpers/requests'
+import { ACTIVITY_PUB } from '../../initializers'
+import { ActorModel } from '../../models/activitypub/actor'
+import { VideoModel } from '../../models/video/video'
+import { VideoCommentModel } from '../../models/video/video-comment'
+import { getOrCreateActorAndServerAndModel } from './actor'
+import { getOrCreateAccountAndVideoAndChannel } from './videos'
+
+async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) {
+  let originCommentId: number = null
+  let inReplyToCommentId: number = null
+
+  // If this is not a reply to the video (thread), create or get the parent comment
+  if (video.url !== comment.inReplyTo) {
+    const [ parent ] = await addVideoComment(video, comment.inReplyTo)
+    if (!parent) {
+      logger.warn('Cannot fetch or get parent comment %s of comment %s.', comment.inReplyTo, comment.id)
+      return undefined
+    }
+
+    originCommentId = parent.originCommentId || parent.id
+    inReplyToCommentId = parent.id
+  }
+
+  return {
+    url: comment.url,
+    text: comment.content,
+    videoId: video.id,
+    accountId: actor.Account.id,
+    inReplyToCommentId,
+    originCommentId,
+    createdAt: new Date(comment.published),
+    updatedAt: new Date(comment.updated)
+  }
+}
+
+async function addVideoComments (instance: VideoModel, commentUrls: string[]) {
+  for (const commentUrl of commentUrls) {
+    await addVideoComment(instance, commentUrl)
+  }
+}
+
+async function addVideoComment (videoInstance: VideoModel, commentUrl: string) {
+  logger.info('Fetching remote video comment %s.', commentUrl)
+
+  const { body } = await doRequest({
+    uri: commentUrl,
+    json: true,
+    activityPub: true
+  })
+
+  if (isVideoCommentObjectValid(body) === false) {
+    logger.debug('Remote video comment JSON is not valid.', { body })
+    return undefined
+  }
+
+  const actorUrl = body.attributedTo
+  if (!actorUrl) return []
+
+  const actor = await getOrCreateActorAndServerAndModel(actorUrl)
+  const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body)
+  if (!entry) return []
+
+  return VideoCommentModel.findOrCreate({
+    where: {
+      url: body.id
+    },
+    defaults: entry
+  })
+}
+
+async function resolveThread (url: string, comments: VideoCommentModel[] = []) {
+   // Already have this comment?
+  const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideo(url)
+  if (commentFromDatabase) {
+    let parentComments = comments.concat([ commentFromDatabase ])
+
+    // Speed up things and resolve directly the thread
+    if (commentFromDatabase.InReplyToVideoComment) {
+      const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC')
+      console.log(data)
+
+      parentComments = parentComments.concat(data)
+    }
+
+    return resolveThread(commentFromDatabase.Video.url, parentComments)
+  }
+
+  try {
+    // Maybe it's a reply to a video?
+    const { video } = await getOrCreateAccountAndVideoAndChannel(url)
+
+    if (comments.length !== 0) {
+      const firstReply = comments[ comments.length - 1 ]
+      firstReply.inReplyToCommentId = null
+      firstReply.originCommentId = null
+      firstReply.videoId = video.id
+      comments[comments.length - 1] = await firstReply.save()
+
+      for (let i = comments.length - 2; i >= 0; i--) {
+        const comment = comments[ i ]
+        comment.originCommentId = firstReply.id
+        comment.inReplyToCommentId = comments[ i + 1 ].id
+        comment.videoId = video.id
+
+        comments[i] = await comment.save()
+      }
+    }
+
+    return { video, parents: comments }
+  } catch (err) {
+    logger.debug('Cannot get or create account and video and channel for reply %s, fetch comment', url, err)
+
+    if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) {
+      throw new Error('Recursion limit reached when resolving a thread')
+    }
+
+    const { body } = await doRequest({
+      uri: url,
+      json: true,
+      activityPub: true
+    })
+
+    if (isVideoCommentObjectValid(body) === false) {
+      throw new Error('Remote video comment JSON is not valid :' + JSON.stringify(body))
+    }
+
+    const actorUrl = body.attributedTo
+    if (!actorUrl) throw new Error('Miss attributed to in comment')
+
+    const actor = await getOrCreateActorAndServerAndModel(actorUrl)
+    const comment = new VideoCommentModel({
+      url: body.url,
+      text: body.content,
+      videoId: null,
+      accountId: actor.Account.id,
+      inReplyToCommentId: null,
+      originCommentId: null,
+      createdAt: new Date(body.published),
+      updatedAt: new Date(body.updated)
+    })
+
+    return resolveThread(body.inReplyTo, comments.concat([ comment ]))
+  }
+
+}
+
+export {
+  videoCommentActivityObjectToDBAttributes,
+  addVideoComments,
+  addVideoComment,
+  resolveThread
+}
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
new file mode 100644 (file)
index 0000000..1b2958c
--- /dev/null
@@ -0,0 +1,52 @@
+import { Transaction } from 'sequelize'
+import { AccountModel } from '../../models/account/account'
+import { VideoModel } from '../../models/video/video'
+import {
+  sendCreateDislikeToOrigin, sendCreateDislikeToVideoFollowers, sendLikeToOrigin, sendLikeToVideoFollowers, sendUndoDislikeToOrigin,
+  sendUndoDislikeToVideoFollowers, sendUndoLikeToOrigin, sendUndoLikeToVideoFollowers
+} from './send'
+
+async function sendVideoRateChangeToFollowers (account: AccountModel,
+                                               video: VideoModel,
+                                               likes: number,
+                                               dislikes: number,
+                                               t: Transaction) {
+  const actor = account.Actor
+
+  // Keep the order: first we undo and then we create
+
+  // Undo Like
+  if (likes < 0) await sendUndoLikeToVideoFollowers(actor, video, t)
+  // Undo Dislike
+  if (dislikes < 0) await sendUndoDislikeToVideoFollowers(actor, video, t)
+
+  // Like
+  if (likes > 0) await sendLikeToVideoFollowers(actor, video, t)
+  // Dislike
+  if (dislikes > 0) await sendCreateDislikeToVideoFollowers(actor, video, t)
+}
+
+async function sendVideoRateChangeToOrigin (account: AccountModel,
+                                            video: VideoModel,
+                                            likes: number,
+                                            dislikes: number,
+                                            t: Transaction) {
+  const actor = account.Actor
+
+  // Keep the order: first we undo and then we create
+
+  // Undo Like
+  if (likes < 0) await sendUndoLikeToOrigin(actor, video, t)
+  // Undo Dislike
+  if (dislikes < 0) await sendUndoDislikeToOrigin(actor, video, t)
+
+  // Like
+  if (likes > 0) await sendLikeToOrigin(actor, video, t)
+  // Dislike
+  if (dislikes > 0) await sendCreateDislikeToOrigin(actor, video, t)
+}
+
+export {
+  sendVideoRateChangeToFollowers,
+  sendVideoRateChangeToOrigin
+}
index 8bc928b935015cded14667fc998d4ef45e4e2e3d..708f4a8974b4acff8536245a2ecbe520f3f9404c 100644 (file)
@@ -1,15 +1,23 @@
+import * as Bluebird from 'bluebird'
+import * as magnetUtil from 'magnet-uri'
 import { join } from 'path'
 import * as request from 'request'
-import { Transaction } from 'sequelize'
 import { ActivityIconObject } from '../../../shared/index'
+import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
+import { VideoPrivacy } from '../../../shared/models/videos'
+import { isVideoTorrentObjectValid } from '../../helpers/custom-validators/activitypub/videos'
+import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos'
+import { retryTransactionWrapper } from '../../helpers/database-utils'
+import { logger } from '../../helpers/logger'
 import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests'
-import { CONFIG, REMOTE_SCHEME, STATIC_PATHS } from '../../initializers'
-import { AccountModel } from '../../models/account/account'
+import { ACTIVITY_PUB, CONFIG, REMOTE_SCHEME, sequelizeTypescript, STATIC_PATHS, VIDEO_MIMETYPE_EXT } from '../../initializers'
+import { ActorModel } from '../../models/activitypub/actor'
+import { TagModel } from '../../models/video/tag'
 import { VideoModel } from '../../models/video/video'
-import {
-  sendCreateDislikeToOrigin, sendCreateDislikeToVideoFollowers, sendLikeToOrigin, sendLikeToVideoFollowers, sendUndoDislikeToOrigin,
-  sendUndoDislikeToVideoFollowers, sendUndoLikeToOrigin, sendUndoLikeToVideoFollowers
-} from './send'
+import { VideoChannelModel } from '../../models/video/video-channel'
+import { VideoFileModel } from '../../models/video/video-file'
+import { VideoShareModel } from '../../models/video/video-share'
+import { getOrCreateActorAndServerAndModel } from './actor'
 
 function fetchRemoteVideoPreview (video: VideoModel, reject: Function) {
   // FIXME: use url
@@ -45,54 +53,221 @@ function generateThumbnailFromUrl (video: VideoModel, icon: ActivityIconObject)
   return doRequestAndSaveToFile(options, thumbnailPath)
 }
 
-async function sendVideoRateChangeToFollowers (
-  account: AccountModel,
-  video: VideoModel,
-  likes: number,
-  dislikes: number,
-  t: Transaction
-) {
-  const actor = account.Actor
-
-  // Keep the order: first we undo and then we create
-
-  // Undo Like
-  if (likes < 0) await sendUndoLikeToVideoFollowers(actor, video, t)
-  // Undo Dislike
-  if (dislikes < 0) await sendUndoDislikeToVideoFollowers(actor, video, t)
-
-  // Like
-  if (likes > 0) await sendLikeToVideoFollowers(actor, video, t)
-  // Dislike
-  if (dislikes > 0) await sendCreateDislikeToVideoFollowers(actor, video, t)
+async function videoActivityObjectToDBAttributes (videoChannel: VideoChannelModel,
+                                                  videoObject: VideoTorrentObject,
+                                                  to: string[] = [],
+                                                  cc: string[] = []) {
+  let privacy = VideoPrivacy.PRIVATE
+  if (to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.PUBLIC
+  else if (cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1) privacy = VideoPrivacy.UNLISTED
+
+  const duration = videoObject.duration.replace(/[^\d]+/, '')
+  let language = null
+  if (videoObject.language) {
+    language = parseInt(videoObject.language.identifier, 10)
+  }
+
+  let category = null
+  if (videoObject.category) {
+    category = parseInt(videoObject.category.identifier, 10)
+  }
+
+  let licence = null
+  if (videoObject.licence) {
+    licence = parseInt(videoObject.licence.identifier, 10)
+  }
+
+  let description = null
+  if (videoObject.content) {
+    description = videoObject.content
+  }
+
+  return {
+    name: videoObject.name,
+    uuid: videoObject.uuid,
+    url: videoObject.id,
+    category,
+    licence,
+    language,
+    description,
+    nsfw: videoObject.nsfw,
+    commentsEnabled: videoObject.commentsEnabled,
+    channelId: videoChannel.id,
+    duration: parseInt(duration, 10),
+    createdAt: new Date(videoObject.published),
+    // FIXME: updatedAt does not seems to be considered by Sequelize
+    updatedAt: new Date(videoObject.updated),
+    views: videoObject.views,
+    likes: 0,
+    dislikes: 0,
+    remote: true,
+    privacy
+  }
+}
+
+function videoFileActivityUrlToDBAttributes (videoCreated: VideoModel, videoObject: VideoTorrentObject) {
+  const mimeTypes = Object.keys(VIDEO_MIMETYPE_EXT)
+  const fileUrls = videoObject.url.filter(u => {
+    return mimeTypes.indexOf(u.mimeType) !== -1 && u.mimeType.startsWith('video/')
+  })
+
+  if (fileUrls.length === 0) {
+    throw new Error('Cannot find video files for ' + videoCreated.url)
+  }
+
+  const attributes = []
+  for (const fileUrl of fileUrls) {
+    // Fetch associated magnet uri
+    const magnet = videoObject.url.find(u => {
+      return u.mimeType === 'application/x-bittorrent;x-scheme-handler/magnet' && u.width === fileUrl.width
+    })
+
+    if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.url)
+
+    const parsed = magnetUtil.decode(magnet.url)
+    if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) throw new Error('Cannot parse magnet URI ' + magnet.url)
+
+    const attribute = {
+      extname: VIDEO_MIMETYPE_EXT[ fileUrl.mimeType ],
+      infoHash: parsed.infoHash,
+      resolution: fileUrl.width,
+      size: fileUrl.size,
+      videoId: videoCreated.id
+    }
+    attributes.push(attribute)
+  }
+
+  return attributes
+}
+
+async function getOrCreateVideo (videoObject: VideoTorrentObject, channelActor: ActorModel) {
+  logger.debug('Adding remote video %s.', videoObject.id)
+
+  return sequelizeTypescript.transaction(async t => {
+    const sequelizeOptions = {
+      transaction: t
+    }
+    const videoFromDatabase = await VideoModel.loadByUUIDOrURLAndPopulateAccount(videoObject.uuid, videoObject.id, t)
+    if (videoFromDatabase) return videoFromDatabase
+
+    const videoData = await videoActivityObjectToDBAttributes(channelActor.VideoChannel, videoObject, videoObject.to, videoObject.cc)
+    const video = VideoModel.build(videoData)
+
+    // Don't block on request
+    generateThumbnailFromUrl(video, videoObject.icon)
+      .catch(err => logger.warn('Cannot generate thumbnail of %s.', videoObject.id, err))
+
+    const videoCreated = await video.save(sequelizeOptions)
+
+    const videoFileAttributes = videoFileActivityUrlToDBAttributes(videoCreated, videoObject)
+    if (videoFileAttributes.length === 0) {
+      throw new Error('Cannot find valid files for video %s ' + videoObject.url)
+    }
+
+    const tasks: Bluebird<any>[] = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
+    await Promise.all(tasks)
+
+    const tags = videoObject.tag.map(t => t.name)
+    const tagInstances = await TagModel.findOrCreateTags(tags, t)
+    await videoCreated.$set('Tags', tagInstances, sequelizeOptions)
+
+    logger.info('Remote video with uuid %s inserted.', videoObject.uuid)
+
+    videoCreated.VideoChannel = channelActor.VideoChannel
+    return videoCreated
+  })
 }
 
-async function sendVideoRateChangeToOrigin (
-  account: AccountModel,
-  video: VideoModel,
-  likes: number,
-  dislikes: number,
-  t: Transaction
-) {
-  const actor = account.Actor
-
-  // Keep the order: first we undo and then we create
-
-  // Undo Like
-  if (likes < 0) await sendUndoLikeToOrigin(actor, video, t)
-  // Undo Dislike
-  if (dislikes < 0) await sendUndoDislikeToOrigin(actor, video, t)
-
-  // Like
-  if (likes > 0) await sendLikeToOrigin(actor, video, t)
-  // Dislike
-  if (dislikes > 0) await sendCreateDislikeToOrigin(actor, video, t)
+async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentObject | string, actor?: ActorModel) {
+  if (typeof videoObject === 'string') {
+    const videoFromDatabase = await VideoModel.loadByUrlAndPopulateAccount(videoObject)
+    if (videoFromDatabase) {
+      return {
+        video: videoFromDatabase,
+        actor: videoFromDatabase.VideoChannel.Account.Actor,
+        channelActor: videoFromDatabase.VideoChannel.Actor
+      }
+    }
+
+    videoObject = await fetchRemoteVideo(videoObject)
+    if (!videoObject) throw new Error('Cannot fetch remote video')
+  }
+
+  if (!actor) {
+    const actorObj = videoObject.attributedTo.find(a => a.type === 'Person')
+    if (!actorObj) throw new Error('Cannot find associated actor to video ' + videoObject.url)
+
+    actor = await getOrCreateActorAndServerAndModel(actorObj.id)
+  }
+
+  const channel = videoObject.attributedTo.find(a => a.type === 'Group')
+  if (!channel) throw new Error('Cannot find associated video channel to video ' + videoObject.url)
+
+  const channelActor = await getOrCreateActorAndServerAndModel(channel.id)
+
+  const options = {
+    arguments: [ videoObject, channelActor ],
+    errorMessage: 'Cannot insert the remote video with many retries.'
+  }
+
+  const video = await retryTransactionWrapper(getOrCreateVideo, options)
+
+  return { actor, channelActor, video }
+}
+
+async function addVideoShares (instance: VideoModel, shareUrls: string[]) {
+  for (const shareUrl of shareUrls) {
+    // Fetch url
+    const { body } = await doRequest({
+      uri: shareUrl,
+      json: true,
+      activityPub: true
+    })
+    const actorUrl = body.actor
+    if (!actorUrl) continue
+
+    const actor = await getOrCreateActorAndServerAndModel(actorUrl)
+
+    const entry = {
+      actorId: actor.id,
+      videoId: instance.id
+    }
+
+    await VideoShareModel.findOrCreate({
+      where: entry,
+      defaults: entry
+    })
+  }
 }
 
 export {
+  getOrCreateAccountAndVideoAndChannel,
   fetchRemoteVideoPreview,
   fetchRemoteVideoDescription,
   generateThumbnailFromUrl,
-  sendVideoRateChangeToFollowers,
-  sendVideoRateChangeToOrigin
+  videoActivityObjectToDBAttributes,
+  videoFileActivityUrlToDBAttributes,
+  getOrCreateVideo,
+  addVideoShares}
+
+// ---------------------------------------------------------------------------
+
+async function fetchRemoteVideo (videoUrl: string): Promise<VideoTorrentObject> {
+  const options = {
+    uri: videoUrl,
+    method: 'GET',
+    json: true,
+    activityPub: true
+  }
+
+  logger.info('Fetching remote video %s.', videoUrl)
+
+  const { body } = await doRequest(options)
+
+  if (isVideoTorrentObjectValid(body) === false) {
+    logger.debug('Remote video JSON is not valid.', { body })
+    return undefined
+  }
+
+  return body
 }
index 707f140af815900ed4f5e0670ed9092d89e7317b..b88e06b41a6eb2db3702cca9767db42df54132de 100644 (file)
@@ -62,6 +62,9 @@ enum ScopeNames {
 @Table({
   tableName: 'actor',
   indexes: [
+    {
+      fields: [ 'url' ]
+    },
     {
       fields: [ 'preferredUsername', 'serverId' ],
       unique: true
index dbb2fe42910f9cbffbff7ca66b9769bb5faa5c28..fffa4bb57d060848bc929beb396f52ad2dcfbefa 100644 (file)
@@ -208,7 +208,7 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
       .findOne(query)
   }
 
-  static loadByUrl (url: string, t?: Sequelize.Transaction) {
+  static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) {
     const query: IFindOptions<VideoCommentModel> = {
       where: {
         url
@@ -217,10 +217,10 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
 
     if (t !== undefined) query.transaction = t
 
-    return VideoCommentModel.findOne(query)
+    return VideoCommentModel.scope([ ScopeNames.WITH_ACCOUNT ]).findOne(query)
   }
 
-  static loadByUrlAndPopulateAccount (url: string, t?: Sequelize.Transaction) {
+  static loadByUrlAndPopulateReplyAndVideo (url: string, t?: Sequelize.Transaction) {
     const query: IFindOptions<VideoCommentModel> = {
       where: {
         url
@@ -229,7 +229,7 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
 
     if (t !== undefined) query.transaction = t
 
-    return VideoCommentModel.scope([ ScopeNames.WITH_ACCOUNT ]).findOne(query)
+    return VideoCommentModel.scope([ ScopeNames.WITH_IN_REPLY_TO, ScopeNames.WITH_VIDEO ]).findOne(query)
   }
 
   static listThreadsForApi (videoId: number, start: number, count: number, sort: string) {
@@ -271,9 +271,9 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
       })
   }
 
-  static listThreadParentComments (comment: VideoCommentModel, t: Sequelize.Transaction) {
+  static listThreadParentComments (comment: VideoCommentModel, t: Sequelize.Transaction, order: 'ASC' | 'DESC' = 'ASC') {
     const query = {
-      order: [ [ 'createdAt', 'ASC' ] ],
+      order: [ [ 'createdAt', order ] ],
       where: {
         [ Sequelize.Op.or ]: [
           { id: comment.getThreadId() },
@@ -281,6 +281,9 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
         ],
         id: {
           [ Sequelize.Op.ne ]: comment.id
+        },
+        createdAt: {
+          [ Sequelize.Op.lt ]: comment.createdAt
         }
       },
       transaction: t
index 9188929384f96a21308d0bff09f71acb3d7057d8..6b825bf931f39dc6141bec8249a23881c2a2dca3 100644 (file)
@@ -178,6 +178,10 @@ enum ScopeNames {
     },
     {
       fields: [ 'id', 'privacy' ]
+    },
+    {
+      fields: [ 'url'],
+      unique: true
     }
   ]
 })
@@ -535,7 +539,7 @@ export class VideoModel extends Model<VideoModel> {
     return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
   }
 
-  static loadByUUIDOrURL (uuid: string, url: string, t?: Sequelize.Transaction) {
+  static loadByUUIDOrURLAndPopulateAccount (uuid: string, url: string, t?: Sequelize.Transaction) {
     const query: IFindOptions<VideoModel> = {
       where: {
         [Sequelize.Op.or]: [
@@ -547,7 +551,7 @@ export class VideoModel extends Model<VideoModel> {
 
     if (t !== undefined) query.transaction = t
 
-    return VideoModel.scope(ScopeNames.WITH_FILES).findOne(query)
+    return VideoModel.scope([ ScopeNames.WITH_ACCOUNT_DETAILS, ScopeNames.WITH_FILES ]).findOne(query)
   }
 
   static loadAndPopulateAccountAndServerAndTags (id: number) {
@@ -983,6 +987,10 @@ export class VideoModel extends Model<VideoModel> {
         {
           type: 'Group',
           id: this.VideoChannel.Actor.url
+        },
+        {
+          type: 'Person',
+          id: this.VideoChannel.Account.Actor.url
         }
       ]
     }
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts
new file mode 100644 (file)
index 0000000..da134c0
--- /dev/null
@@ -0,0 +1,147 @@
+/* tslint:disable:no-unused-expression */
+
+import * as chai from 'chai'
+import 'mocha'
+import { VideoPrivacy } from '../../../../shared/models/videos'
+import { completeVideoCheck, runServer, viewVideo } from '../../utils'
+
+import {
+  flushAndRunMultipleServers, flushTests, getVideosList, killallServers, ServerInfo, setAccessTokensToServers, uploadVideo,
+  wait
+} from '../../utils/index'
+import { follow, getFollowersListPaginationAndSort } from '../../utils/server/follows'
+import { getJobsListPaginationAndSort } from '../../utils/server/jobs'
+
+const expect = chai.expect
+
+describe('Test handle downs', function () {
+  let servers: ServerInfo[] = []
+
+  const videoAttributes = {
+    name: 'my super name for server 1',
+    category: 5,
+    licence: 4,
+    language: 9,
+    nsfw: true,
+    description: 'my super description for server 1',
+    tags: [ 'tag1p1', 'tag2p1' ],
+    fixture: 'video_short1.webm'
+  }
+
+  const checkAttributes = {
+    name: 'my super name for server 1',
+    category: 5,
+    licence: 4,
+    language: 9,
+    nsfw: true,
+    description: 'my super description for server 1',
+    host: 'localhost:9001',
+    account: 'root',
+    isLocal: false,
+    duration: 10,
+    tags: [ 'tag1p1', 'tag2p1' ],
+    privacy: VideoPrivacy.PUBLIC,
+    commentsEnabled: true,
+    channel: {
+      name: 'Default root channel',
+      description: '',
+      isLocal: false
+    },
+    fixture: 'video_short1.webm',
+    files: [
+      {
+        resolution: 720,
+        size: 572456
+      }
+    ]
+  }
+
+  before(async function () {
+    this.timeout(20000)
+
+    servers = await flushAndRunMultipleServers(2)
+
+    // Get the access tokens
+    await setAccessTokensToServers(servers)
+  })
+
+  it('Should remove followers that are often down', async function () {
+    this.timeout(60000)
+
+    await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)
+
+    await wait(5000)
+
+    await uploadVideo(servers[0].url, servers[0].accessToken, videoAttributes)
+
+    await wait(5000)
+
+    for (const server of servers) {
+      const res = await getVideosList(server.url)
+      expect(res.body.data).to.be.an('array')
+      expect(res.body.data).to.have.lengthOf(1)
+    }
+
+    // Kill server 1
+    killallServers([ servers[1] ])
+
+    // Remove server 2 follower
+    for (let i = 0; i < 10; i++) {
+      await uploadVideo(servers[ 0 ].url, servers[ 0 ].accessToken, videoAttributes)
+    }
+
+    await wait(10000)
+
+    const res = await getFollowersListPaginationAndSort(servers[0].url, 0, 1, 'createdAt')
+    expect(res.body.data).to.be.an('array')
+    expect(res.body.data).to.have.lengthOf(0)
+  })
+
+  it('Should not have pending/processing jobs anymore', async function () {
+    const res = await getJobsListPaginationAndSort(servers[0].url, servers[0].accessToken, 0, 50, '-createdAt')
+    const jobs = res.body.data
+
+    for (const job of jobs) {
+      expect(job.state).not.to.equal('pending')
+      expect(job.state).not.to.equal('processing')
+    }
+  })
+
+  it('Should follow server 1', async function () {
+    servers[1] = await runServer(2)
+
+    await follow(servers[1].url, [ servers[0].url ], servers[1].accessToken)
+
+    await wait(5000)
+
+    const res = await getFollowersListPaginationAndSort(servers[0].url, 0, 1, 'createdAt')
+    expect(res.body.data).to.be.an('array')
+    expect(res.body.data).to.have.lengthOf(1)
+  })
+
+  it('Should send a view to server 2, and automatically fetch the video', async function () {
+    const resVideo = await getVideosList(servers[0].url)
+    const videoServer1 = resVideo.body.data[0]
+
+    await viewVideo(servers[0].url, videoServer1.uuid)
+
+    await wait(5000)
+
+    const res = await getVideosList(servers[1].url)
+    const videoServer2 = res.body.data.find(v => v.url === videoServer1.url)
+
+    expect(videoServer2).not.to.be.undefined
+
+    await completeVideoCheck(servers[1].url, videoServer2, checkAttributes)
+
+  })
+
+  after(async function () {
+    killallServers(servers)
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})
index cf0e0ba54240444a5322e9479a12a21846981df7..d3b5f7c263dbc07608576aa055619bcd9f3ee702 100644 (file)
@@ -30,4 +30,6 @@ export interface VideoTorrentObject {
   shares?: ActivityPubOrderedCollection<string>
   comments?: ActivityPubOrderedCollection<string>
   attributedTo: ActivityPubAttributedTo[]
+  to?: string[]
+  cc?: string[]
 }