Update P2P media loader peer version

diff --git a/server.ts b/server.ts
index df56bcd822ad3602a974a65821eb966fe2523144..dfaa8ad3df7a2575e9947a7abc22bc8b8ef62a61 100644 (file)
--- a/server.ts
+++ b/server.ts
@@ -103,6 +103,7 @@ import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-upd
 import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
 import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
 import { PeerTubeSocket } from './server/lib/peertube-socket'
+import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
 
 // ----------- Command line -----------
 
@@ -233,6 +234,9 @@ async function startApplication () {
 
   PeerTubeSocket.Instance.init(server)
 
+  updateStreamingPlaylistsInfohashesIfNeeded()
+    .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err }))
+
   // Make server listening
   server.listen(port, hostname, () => {
     logger.info('Server listening on %s:%d', hostname, port)

diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index f38b82d9744fa99110aaac6863ce6240be63f514..3f737c1d6ef94c67fd542756d78bf374ce1cef7e 100644 (file)
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -58,7 +58,7 @@ export function parseDuration (duration: number | string): number {
     }
   }
 
-  throw new Error('Duration could not be properly parsed')
+  throw new Error(`Duration ${duration} could not be properly parsed`)
 }
 
 export function parseBytes (value: string | number): number {

diff --git a/server/helpers/video.ts b/server/helpers/video.ts
index f6f51a297b6748579ab82cce910de1dd8c0f88e3..c90fe06c78e2729174f58f5dee75741d9f194cd5 100644 (file)
--- a/server/helpers/video.ts
+++ b/server/helpers/video.ts
@@ -1,7 +1,4 @@
-import { CONFIG } from '../initializers'
 import { VideoModel } from '../models/video/video'
-import { UserRight } from '../../shared'
-import { UserModel } from '../models/account/user'
 
 type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none'
 

diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index f59d3ef7a4430baa48fceb168b975f85b2a1ceb7..ac19231d0d3cc485458dde78e908a1ad253a2739 100644 (file)
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -18,7 +18,7 @@ let config: IConfig = require('config')
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 350
+const LAST_MIGRATION_VERSION = 355
 
 // ---------------------------------------------------------------------------
 
@@ -726,6 +726,8 @@ const TRACKER_RATE_LIMITS = {
   ANNOUNCES_PER_IP: 30 // maximum announces for all our torrents in the interval
 }
 
+const P2P_MEDIA_LOADER_PEER_VERSION = 2
+
 // ---------------------------------------------------------------------------
 
 // Special constants for a test instance
@@ -772,6 +774,7 @@ updateWebserverUrls()
 export {
   API_VERSION,
   HLS_REDUNDANCY_DIRECTORY,
+  P2P_MEDIA_LOADER_PEER_VERSION,
   AVATARS_SIZE,
   ACCEPT_HEADERS,
   BCRYPT_SALT_SIZE,

diff --git a/server/initializers/migrations/0355-p2p-peer-version.ts b/server/initializers/migrations/0355-p2p-peer-version.ts
new file mode 100644 (file)
index 0000000..18f23d9
--- /dev/null
+++ b/server/initializers/migrations/0355-p2p-peer-version.ts
@@ -0,0 +1,41 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction,
+  queryInterface: Sequelize.QueryInterface,
+  sequelize: Sequelize.Sequelize,
+  db: any
+}): Promise<void> {
+
+  {
+    const data = {
+      type: Sequelize.INTEGER,
+      allowNull: true,
+      defaultValue: null
+    }
+    await utils.queryInterface.addColumn('videoStreamingPlaylist', 'p2pMediaLoaderPeerVersion', data)
+  }
+
+  {
+    const query = `UPDATE "videoStreamingPlaylist" SET "p2pMediaLoaderPeerVersion" = 0;`
+    await utils.sequelize.query(query)
+  }
+
+  {
+    const data = {
+      type: Sequelize.INTEGER,
+      allowNull: false,
+      defaultValue: null
+    }
+    await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'p2pMediaLoaderPeerVersion', data)
+  }
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export {
+  up,
+  down
+}

diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index d935e3f90224a89a8668bc1d2e444073692a7f34..339f8e79701c1e1689a8e9502cc4e4c9c997cc60 100644 (file)
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -290,7 +290,11 @@ async function updateVideoFromAP (options: {
       }
 
       {
-        const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(options.video, options.videoObject)
+        const streamingPlaylistAttributes = streamingPlaylistActivityUrlToDBAttributes(
+          options.video,
+          options.videoObject,
+          options.video.VideoFiles
+        )
         const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a))
 
         // Remove video files that do not exist anymore
@@ -449,9 +453,9 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
     }
 
     const videoFilePromises = videoFileAttributes.map(f => VideoFileModel.create(f, { transaction: t }))
-    await Promise.all(videoFilePromises)
+    const videoFiles = await Promise.all(videoFilePromises)
 
-    const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject)
+    const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
     const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
     await Promise.all(playlistPromises)
 
@@ -575,20 +579,12 @@ function videoFileActivityUrlToDBAttributes (video: VideoModel, videoObject: Vid
   return attributes
 }
 
-function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject) {
+function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObject: VideoTorrentObject, videoFiles: VideoFileModel[]) {
   const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[]
   if (playlistUrls.length === 0) return []
 
   const attributes: FilteredModelAttributes<VideoStreamingPlaylistModel>[] = []
   for (const playlistUrlObject of playlistUrls) {
-    const p2pMediaLoaderInfohashes = playlistUrlObject.tag
-                                                      .filter(t => t.type === 'Infohash')
-                                                      .map(t => t.name)
-    if (p2pMediaLoaderInfohashes.length === 0) {
-      logger.warn('No infohashes found in AP playlist object.', { playlistUrl: playlistUrlObject })
-      continue
-    }
-
     const segmentsSha256UrlObject = playlistUrlObject.tag
                                                      .find(t => {
                                                        return isAPPlaylistSegmentHashesUrlObject(t)
@@ -602,7 +598,7 @@ function streamingPlaylistActivityUrlToDBAttributes (video: VideoModel, videoObj
       type: VideoStreamingPlaylistType.HLS,
       playlistUrl: playlistUrlObject.href,
       segmentsSha256Url: segmentsSha256UrlObject.href,
-      p2pMediaLoaderInfohashes,
+      p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, videoFiles),
       videoId: video.id
     }
 

diff --git a/server/lib/hls.ts b/server/lib/hls.ts
index 74ed25183bd0e5fb3e28863cc1b8229741dabac2..5a7d61dee6034ad180cb6fbcdf8671f3531c1506 100644 (file)
--- a/server/lib/hls.ts
+++ b/server/lib/hls.ts
@@ -1,6 +1,6 @@
 import { VideoModel } from '../models/video/video'
-import { basename, join, dirname } from 'path'
-import { CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY } from '../initializers'
+import { basename, dirname, join } from 'path'
+import { CONFIG, HLS_STREAMING_PLAYLIST_DIRECTORY, sequelizeTypescript } from '../initializers'
 import { close, ensureDir, move, open, outputJSON, pathExists, read, readFile, remove, writeFile } from 'fs-extra'
 import { getVideoFileSize } from '../helpers/ffmpeg-utils'
 import { sha256 } from '../helpers/core-utils'
@@ -9,6 +9,21 @@ import { logger } from '../helpers/logger'
 import { doRequest, doRequestAndSaveToFile } from '../helpers/requests'
 import { generateRandomString } from '../helpers/utils'
 import { flatten, uniq } from 'lodash'
+import { VideoFileModel } from '../models/video/video-file'
+
+async function updateStreamingPlaylistsInfohashesIfNeeded () {
+  const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion()
+
+  // Use separate SQL queries, because we could have many videos to update
+  for (const playlist of playlistsToUpdate) {
+    await sequelizeTypescript.transaction(async t => {
+      const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t)
+
+      playlist.p2pMediaLoaderInfohashes = await VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlist.playlistUrl, videoFiles)
+      await playlist.save({ transaction: t })
+    })
+  }
+}
 
 async function updateMasterHLSPlaylist (video: VideoModel) {
   const directory = join(HLS_STREAMING_PLAYLIST_DIRECTORY, video.uuid)
@@ -159,7 +174,8 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string,
 export {
   updateMasterHLSPlaylist,
   updateSha256Segments,
-  downloadPlaylistSegments
+  downloadPlaylistSegments,
+  updateStreamingPlaylistsInfohashesIfNeeded
 }
 
 // ---------------------------------------------------------------------------

diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index b861b0704c0b809f6e8ad5cf03fa0cda0ffc87f4..c14d96bc51f41eea49c15f57a5d13a252a66d8cb 100644 (file)
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -23,6 +23,7 @@ import { throwIfNotValid } from '../utils'
 import { VideoModel } from './video'
 import * as Sequelize from 'sequelize'
 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
+import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
 
 @Table({
   tableName: 'videoFile',
@@ -120,6 +121,29 @@ export class VideoFileModel extends Model<VideoFileModel> {
     return VideoFileModel.findByPk(id, options)
   }
 
+  static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Sequelize.Transaction) {
+    const query = {
+      include: [
+        {
+          model: VideoModel.unscoped(),
+          required: true,
+          include: [
+            {
+              model: VideoStreamingPlaylistModel.unscoped(),
+              required: true,
+              where: {
+                id: streamingPlaylistId
+              }
+            }
+          ]
+        }
+      ],
+      transaction
+    }
+
+    return VideoFileModel.findAll(query)
+  }
+
   static async getStats () {
     let totalLocalVideoFilesSize = await VideoFileModel.sum('size', {
       include: [

diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index b147aca3656ec5af2d964f29f1dfe0b8368c274a..0333755c5d9bb1e81cc92919ae34dc09db79949f 100644 (file)
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -6,7 +6,7 @@ import * as Sequelize from 'sequelize'
 import { VideoRedundancyModel } from '../redundancy/video-redundancy'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
 import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
-import { CONSTRAINTS_FIELDS, STATIC_PATHS } from '../../initializers'
+import { CONSTRAINTS_FIELDS, STATIC_PATHS, P2P_MEDIA_LOADER_PEER_VERSION } from '../../initializers'
 import { VideoFileModel } from './video-file'
 import { join } from 'path'
 import { sha1 } from '../../helpers/core-utils'
@@ -49,6 +49,10 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
   @Column(DataType.ARRAY(DataType.STRING))
   p2pMediaLoaderInfohashes: string[]
 
+  @AllowNull(false)
+  @Column
+  p2pMediaLoaderPeerVersion: number
+
   @AllowNull(false)
   @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url'))
   @Column
@@ -92,14 +96,26 @@ export class VideoStreamingPlaylistModel extends Model<VideoStreamingPlaylistMod
   static buildP2PMediaLoaderInfoHashes (playlistUrl: string, videoFiles: VideoFileModel[]) {
     const hashes: string[] = []
 
-    // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L97
+    // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L115
     for (let i = 0; i < videoFiles.length; i++) {
-      hashes.push(sha1(`1${playlistUrl}+V${i}`))
+      hashes.push(sha1(`${P2P_MEDIA_LOADER_PEER_VERSION}${playlistUrl}+V${i}`))
     }
 
     return hashes
   }
 
+  static listByIncorrectPeerVersion () {
+    const query = {
+      where: {
+        p2pMediaLoaderPeerVersion: {
+          [Sequelize.Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION
+        }
+      }
+    }
+
+    return VideoStreamingPlaylistModel.findAll(query)
+  }
+
   static loadWithVideo (id: number) {
     const options = {
       include: [