Add previews cache system between pods
authorChocobozzz <florian.bigard@gmail.com>
Wed, 12 Jul 2017 09:56:02 +0000 (11:56 +0200)
committerChocobozzz <florian.bigard@gmail.com>
Wed, 12 Jul 2017 09:56:02 +0000 (11:56 +0200)
27 files changed:
.gitignore
config/default.yaml
config/production.yaml.example
config/test-1.yaml
config/test-2.yaml
config/test-3.yaml
config/test-4.yaml
config/test-5.yaml
config/test-6.yaml
package.json
server.ts
server/controllers/static.ts
server/helpers/core-utils.ts
server/initializers/constants.ts
server/initializers/installer.ts
server/lib/cache/index.ts [new file with mode: 0644]
server/lib/cache/videos-preview-cache.ts [new file with mode: 0644]
server/lib/friends.ts
server/lib/index.ts
server/models/oauth/oauth-token-interface.ts
server/models/oauth/oauth-token.ts
server/models/video/video.ts
server/tests/api/fixtures/video_short1-preview.webm.jpg [new file with mode: 0644]
server/tests/api/multiple-pods.js
server/tests/utils/videos.js
shared/models/videos/video.model.ts
yarn.lock

index 6caee2e4c02a31624f29aebd25475efcca1758c7..169027c360b4cce6b7ff20ef41a66f4cd6e42676 100644 (file)
@@ -12,6 +12,7 @@
 /certs/
 /logs/
 /torrents/
+/cache/
 /config/production.yaml
 /ffmpeg/
 /*.sublime-project
index e03bf1aeaa9b6ab25d19e3fe18dc0791747a4b78..b4e7606cfb4556cde1d8e0294929eb2fa446e0dc 100644 (file)
@@ -22,6 +22,11 @@ storage:
   previews: 'previews/'
   thumbnails: 'thumbnails/'
   torrents: 'torrents/'
+  cache: 'cache/'
+
+cache:
+  previews:
+    size: 1 # Max number of previews you want to cache
 
 admin:
   email: 'admin@example.com'
index c18457df608213122dcb9dc4af66b269a97b44c8..0857aa3cab412c9b3ad3f2d995570f521e299c5f 100644 (file)
@@ -23,6 +23,7 @@ storage:
   previews: 'previews/'
   thumbnails: 'thumbnails/'
   torrents: 'torrents/'
+  cache: 'cache/'
 
 admin:
   email: 'admin@example.com'
index dbe408a8cbdfff7471e3502f8309106cb16f2e55..e244a8797cc99b7c8ad72a0ed75c97b98114df10 100644 (file)
@@ -13,8 +13,10 @@ storage:
   certs: 'test1/certs/'
   videos: 'test1/videos/'
   logs: 'test1/logs/'
+  previews: 'test1/previews/'
   thumbnails: 'test1/thumbnails/'
   torrents: 'test1/torrents/'
+  cache: 'test1/cache/'
 
 admin:
   email: 'admin1@example.com'
index c95b9c229440a7acc32db30eba70edc491c965c9..236dcb10d73d177dd3eb7fd0cbc768d615e24e09 100644 (file)
@@ -13,8 +13,10 @@ storage:
   certs: 'test2/certs/'
   videos: 'test2/videos/'
   logs: 'test2/logs/'
+  previews: 'test2/previews/'
   thumbnails: 'test2/thumbnails/'
   torrents: 'test2/torrents/'
+  cache: 'test2/cache/'
 
 admin:
   email: 'admin2@example.com'
index 2eb98469280638120b233229f451b6c0b95c6bfc..a29225a44bcf4986356e4f7e1e924bdae8cc659a 100644 (file)
@@ -13,8 +13,10 @@ storage:
   certs: 'test3/certs/'
   videos: 'test3/videos/'
   logs: 'test3/logs/'
+  previews: 'test3/previews/'
   thumbnails: 'test3/thumbnails/'
   torrents: 'test3/torrents/'
+  cache: 'test3/cache/'
 
 admin:
   email: 'admin3@example.com'
index a0a9bde21e819aa459b0576206467973f2c2c8cd..da93e128de3b2a6eae224c8516f9cebeed579006 100644 (file)
@@ -13,8 +13,10 @@ storage:
   certs: 'test4/certs/'
   videos: 'test4/videos/'
   logs: 'test4/logs/'
+  previews: 'test4/previews/'
   thumbnails: 'test4/thumbnails/'
   torrents: 'test4/torrents/'
+  cache: 'test4/cache/'
 
 admin:
   email: 'admin4@example.com'
index af8654e140362a798b44ea8bd0cd249c93c12ec7..f95e25eb8d3457e06f37da9b886d2678987342b7 100644 (file)
@@ -13,8 +13,10 @@ storage:
   certs: 'test5/certs/'
   videos: 'test5/videos/'
   logs: 'test5/logs/'
+  previews: 'test5/previews/'
   thumbnails: 'test5/thumbnails/'
   torrents: 'test5/torrents/'
+  cache: 'test5/cache/'
 
 admin:
   email: 'admin5@example.com'
index d74d3b05214214de7d7e797fe5fe70c3dcae36ad..87d0544392f277bc43d3fe90214f3a140ab0fd25 100644 (file)
@@ -13,8 +13,10 @@ storage:
   certs: 'test6/certs/'
   videos: 'test6/videos/'
   logs: 'test6/logs/'
+  previews: 'test6/previews/'
   thumbnails: 'test6/thumbnails/'
   torrents: 'test6/torrents/'
+  cache: 'test6/cache/'
 
 admin:
   email: 'admin6@example.com'
index b875f5c26f7f5cd6612b52b5c8fb3ee383c37ec3..d6da61975521f1d08f84812d19af1b81f35f604c 100644 (file)
@@ -47,6 +47,7 @@
   },
   "dependencies": {
     "async": "^2.0.0",
+    "async-lru": "^1.1.1",
     "bcrypt": "^1.0.2",
     "bittorrent-tracker": "^9.0.0",
     "bluebird": "^3.5.0",
index e7fa99c90d49f4965a1b315c3ba0438c156c1f58..a6a9fcb027e54b853e734c0c72a6e87c331c7e3b 100644 (file)
--- a/server.ts
+++ b/server.ts
@@ -47,7 +47,7 @@ if (errorMessage !== null) {
 
 // ----------- PeerTube modules -----------
 import { migrate, installApplication } from './server/initializers'
-import { JobScheduler, activateSchedulers } from './server/lib'
+import { JobScheduler, activateSchedulers, VideosPreviewCache } from './server/lib'
 import * as customValidators from './server/helpers/custom-validators'
 import { apiRouter, clientsRouter, staticRouter } from './server/controllers'
 
@@ -147,6 +147,8 @@ function onDatabaseInitDone () {
         // Activate job scheduler
         JobScheduler.Instance.activate()
 
+        VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE)
+
         logger.info('Server listening on port %d', port)
         logger.info('Webserver: %s', CONFIG.WEBSERVER.URL)
       })
index e65282339e9f2143484cbd6ad483be457166e00a..2fd14131e23e262101b1360ecbce968e4ad5f9dd 100644 (file)
@@ -6,6 +6,7 @@ import {
   STATIC_MAX_AGE,
   STATIC_PATHS
 } from '../initializers'
+import { VideosPreviewCache } from '../lib'
 
 const staticRouter = express.Router()
 
@@ -38,8 +39,8 @@ staticRouter.use(
 // Video previews path for express
 const previewsPhysicalPath = CONFIG.STORAGE.PREVIEWS_DIR
 staticRouter.use(
-  STATIC_PATHS.PREVIEWS,
-  express.static(previewsPhysicalPath, { maxAge: STATIC_MAX_AGE })
+  STATIC_PATHS.PREVIEWS + ':uuid.jpg',
+  getPreview
 )
 
 // ---------------------------------------------------------------------------
@@ -47,3 +48,14 @@ staticRouter.use(
 export {
   staticRouter
 }
+
+// ---------------------------------------------------------------------------
+
+function getPreview (req: express.Request, res: express.Response, next: express.NextFunction) {
+  VideosPreviewCache.Instance.getPreviewPath(req.params.uuid)
+    .then(path => {
+      if (!path) return res.sendStatus(404)
+
+      return res.sendFile(path, { maxAge: STATIC_MAX_AGE })
+    })
+}
index 1e92049f1b8e64dd40fb7e2a065842ce9adecba2..d28c97f092acfc306d93d48b944997bfcab9e89d 100644 (file)
@@ -16,6 +16,7 @@ import {
 import * as mkdirp from 'mkdirp'
 import * as bcrypt from 'bcrypt'
 import * as createTorrent from 'create-torrent'
+import * as rimraf from 'rimraf'
 import * as openssl from 'openssl-wrapper'
 import * as Promise from 'bluebird'
 
@@ -83,6 +84,7 @@ const bcryptComparePromise = promisify2<any, string, boolean>(bcrypt.compare)
 const bcryptGenSaltPromise = promisify1<number, string>(bcrypt.genSalt)
 const bcryptHashPromise = promisify2<any, string|number, string>(bcrypt.hash)
 const createTorrentPromise = promisify2<string, any, any>(createTorrent)
+const rimrafPromise = promisify1WithVoid<string>(rimraf)
 
 // ---------------------------------------------------------------------------
 
@@ -105,5 +107,6 @@ export {
   bcryptComparePromise,
   bcryptGenSaltPromise,
   bcryptHashPromise,
-  createTorrentPromise
+  createTorrentPromise,
+  rimrafPromise
 }
index f087b7476a95d8779fe2f6fa32abda5a3ea8760d..928a3f5704cea290dbf2c59288586250355543bf 100644 (file)
@@ -61,7 +61,8 @@ const CONFIG = {
     VIDEOS_DIR: join(root(), config.get<string>('storage.videos')),
     THUMBNAILS_DIR: join(root(), config.get<string>('storage.thumbnails')),
     PREVIEWS_DIR: join(root(), config.get<string>('storage.previews')),
-    TORRENTS_DIR: join(root(), config.get<string>('storage.torrents'))
+    TORRENTS_DIR: join(root(), config.get<string>('storage.torrents')),
+    CACHE_DIR: join(root(), config.get<string>('storage.cache'))
   },
   WEBSERVER: {
     SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http',
@@ -80,6 +81,11 @@ const CONFIG = {
   TRANSCODING: {
     ENABLED: config.get<boolean>('transcoding.enabled'),
     THREADS: config.get<number>('transcoding.threads')
+  },
+  CACHE: {
+    PREVIEWS: {
+      SIZE: config.get<number>('cache.previews.size')
+    }
   }
 }
 CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
@@ -278,6 +284,13 @@ let STATIC_MAX_AGE = '30d'
 const THUMBNAILS_SIZE = '200x110'
 const PREVIEWS_SIZE = '640x480'
 
+// Subfolders of cache directory
+const CACHE = {
+  DIRECTORIES: {
+    PREVIEWS: join(CONFIG.STORAGE.CACHE_DIR, 'previews')
+  }
+}
+
 // ---------------------------------------------------------------------------
 
 const USER_ROLES: { [ id: string ]: UserRole } = {
@@ -307,6 +320,7 @@ if (isTestInstance() === true) {
 export {
   API_VERSION,
   BCRYPT_SALT_SIZE,
+  CACHE,
   CONFIG,
   CONSTRAINTS_FIELDS,
   FRIEND_SCORE,
index 1ec24c4ade1555aeb1bb7da536906e76656669e3..3c5a77df9ac478e518b3b65f30a7fe6b4ca49db7 100644 (file)
@@ -4,12 +4,13 @@ import * as passwordGenerator from 'password-generator'
 import * as Promise from 'bluebird'
 
 import { database as db } from './database'
-import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION } from './constants'
+import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants'
 import { clientsExist, usersExist } from './checker'
-import { logger, createCertsIfNotExist, root, mkdirpPromise } from '../helpers'
+import { logger, createCertsIfNotExist, root, mkdirpPromise, rimrafPromise } from '../helpers'
 
 function installApplication () {
   return db.sequelize.sync()
+    .then(() => removeCacheDirectories())
     .then(() => createDirectoriesIfNotExist())
     .then(() => createCertsIfNotExist())
     .then(() => createOAuthClientIfNotExist())
@@ -24,13 +25,34 @@ export {
 
 // ---------------------------------------------------------------------------
 
+function removeCacheDirectories () {
+  const cacheDirectories = CACHE.DIRECTORIES
+
+  const tasks = []
+
+  // Cache directories
+  Object.keys(cacheDirectories).forEach(key => {
+    const dir = cacheDirectories[key]
+    tasks.push(rimrafPromise(dir))
+  })
+
+  return Promise.all(tasks)
+}
+
 function createDirectoriesIfNotExist () {
-  const storages = config.get('storage')
+  const storages = CONFIG.STORAGE
+  const cacheDirectories = CACHE.DIRECTORIES
 
   const tasks = []
   Object.keys(storages).forEach(key => {
     const dir = storages[key]
-    tasks.push(mkdirpPromise(join(root(), dir)))
+    tasks.push(mkdirpPromise(dir))
+  })
+
+  // Cache directories
+  Object.keys(cacheDirectories).forEach(key => {
+    const dir = cacheDirectories[key]
+    tasks.push(mkdirpPromise(dir))
   })
 
   return Promise.all(tasks)
diff --git a/server/lib/cache/index.ts b/server/lib/cache/index.ts
new file mode 100644 (file)
index 0000000..7bf6379
--- /dev/null
@@ -0,0 +1 @@
+export * from './videos-preview-cache'
diff --git a/server/lib/cache/videos-preview-cache.ts b/server/lib/cache/videos-preview-cache.ts
new file mode 100644 (file)
index 0000000..9d365e4
--- /dev/null
@@ -0,0 +1,74 @@
+import * as request from 'request'
+import * as asyncLRU from 'async-lru'
+import { join } from 'path'
+import { createWriteStream } from 'fs'
+import * as Promise from 'bluebird'
+
+import { database as db, CONFIG, CACHE } from '../../initializers'
+import { logger, writeFilePromise, unlinkPromise } from '../../helpers'
+import { VideoInstance } from '../../models'
+import { fetchRemotePreview } from '../../lib'
+
+class VideosPreviewCache {
+
+  private static instance: VideosPreviewCache
+
+  private lru
+
+  private constructor () { }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  init (max: number) {
+    this.lru = new asyncLRU({
+      max,
+      load: (key, cb) => {
+        this.loadPreviews(key)
+          .then(res => cb(null, res))
+          .catch(err => cb(err))
+      }
+    })
+
+    this.lru.on('evict', (obj: { key: string, value: string }) => {
+      unlinkPromise(obj.value).then(() => logger.debug('%s evicted from VideosPreviewCache', obj.value))
+    })
+  }
+
+  getPreviewPath (key: string) {
+    return new Promise<string>((res, rej) => {
+      this.lru.get(key, (err, value) => {
+        err ? rej(err) : res(value)
+      })
+    })
+  }
+
+  private loadPreviews (key: string) {
+    return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key)
+      .then(video => {
+        if (!video) return undefined
+
+        if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
+
+        return this.saveRemotePreviewAndReturnPath(video)
+      })
+  }
+
+  private saveRemotePreviewAndReturnPath (video: VideoInstance) {
+    const req = fetchRemotePreview(video.Author.Pod, video)
+
+    return new Promise<string>((res, rej) => {
+      const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName())
+      const stream = createWriteStream(path)
+
+      req.pipe(stream)
+         .on('finish', () => res(path))
+         .on('error', (err) => rej(err))
+    })
+  }
+}
+
+export {
+  VideosPreviewCache
+}
index 6ed0da013e45a123ac812f52cfdade5308f7a2f8..50355d5d17af216936ecc0f893167c4f74ff6deb 100644 (file)
@@ -1,6 +1,7 @@
 import * as request from 'request'
 import * as Sequelize from 'sequelize'
 import * as Promise from 'bluebird'
+import { join } from 'path'
 
 import { database as db } from '../initializers/database'
 import {
@@ -9,7 +10,8 @@ import {
   REQUESTS_IN_PARALLEL,
   REQUEST_ENDPOINTS,
   REQUEST_ENDPOINT_ACTIONS,
-  REMOTE_SCHEME
+  REMOTE_SCHEME,
+  STATIC_PATHS
 } from '../initializers'
 import {
   logger,
@@ -233,6 +235,13 @@ function sendOwnedVideosToPod (podId: number) {
     })
 }
 
+function fetchRemotePreview (pod: PodInstance, video: VideoInstance) {
+  const host = video.Author.Pod.host
+  const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
+
+  return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
+}
+
 function getRequestScheduler () {
   return requestScheduler
 }
@@ -263,7 +272,8 @@ export {
   sendOwnedVideosToPod,
   getRequestScheduler,
   getRequestVideoQaduScheduler,
-  getRequestVideoEventScheduler
+  getRequestVideoEventScheduler,
+  fetchRemotePreview
 }
 
 // ---------------------------------------------------------------------------
index b8697fb96e9c5893f3437328af6ae67badb01a52..8628da4dded5c4e18d1fd05f189b8dbb6cfc064e 100644 (file)
@@ -1,3 +1,4 @@
+export * from './cache'
 export * from './jobs'
 export * from './request'
 export * from './friends'
index f2ddafa5475368503a3f41b9fec43e04089dc30a..97af3c81520e26bfbd03a184f6c74b34321a1377 100644 (file)
@@ -35,6 +35,8 @@ export interface OAuthTokenAttributes {
   refreshToken: string
   refreshTokenExpiresAt: Date
 
+  userId?: number
+  oAuthClientId?: number
   User?: UserModel
 }
 
index 5c3781394fe107870628e19cbba7e847e31feb3a..e3de9468e1536738da90ecd827e638868350a7b4 100644 (file)
@@ -106,10 +106,10 @@ getByRefreshTokenAndPopulateClient = function (refreshToken: string) {
         refreshToken: token.refreshToken,
         refreshTokenExpiresAt: token.refreshTokenExpiresAt,
         client: {
-          id: token['client'].id
+          id: token.oAuthClientId
         },
         user: {
-          id: token['user']
+          id: token.userId
         }
       }
 
index 65002520557a3ba99eebe2ff596e1e85fdf5b69f..b7eb24c4a0f38559e25c49c280f569e1236506e8 100644 (file)
@@ -451,6 +451,7 @@ toFormatedJSON = function (this: VideoInstance) {
     dislikes: this.dislikes,
     tags: map<TagInstance, string>(this.Tags, 'name'),
     thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()),
+    previewPath: join(STATIC_PATHS.PREVIEWS, this.getPreviewName()),
     createdAt: this.createdAt,
     updatedAt: this.updatedAt
   }
diff --git a/server/tests/api/fixtures/video_short1-preview.webm.jpg b/server/tests/api/fixtures/video_short1-preview.webm.jpg
new file mode 100644 (file)
index 0000000..69c100c
Binary files /dev/null and b/server/tests/api/fixtures/video_short1-preview.webm.jpg differ
index 1bc6157e82923d0c871c4b5adffc4ed454b91626..7753e6f2dc8dc075830fbe73ed86ff7a8ed7ceb5 100644 (file)
@@ -747,7 +747,7 @@ describe('Test multiple pods', function () {
           expect(videos[0].name).not.to.equal(toRemove[1].name)
           expect(videos[1].name).not.to.equal(toRemove[1].name)
 
-          videoUUID = videos[0].uuid
+          videoUUID = videos.find(video => video.name === 'my super name for pod 1').uuid
 
           callback()
         })
@@ -781,6 +781,23 @@ describe('Test multiple pods', function () {
         })
       }, done)
     })
+
+    it('Should get the preview from each pod', function (done) {
+      each(servers, function (server, callback) {
+        videosUtils.getVideo(server.url, videoUUID, function (err, res) {
+          if (err) throw err
+
+          const video = res.body
+
+          videosUtils.testVideoImage(server.url, 'video_short1-preview.webm', video.previewPath, function (err, test) {
+            if (err) throw err
+            expect(test).to.equal(true)
+
+            callback()
+          })
+        })
+      }, done)
+    })
   })
 
   after(function (done) {
index 6e7aabc5df921be2a84d7675a4221fceb6e60373..cb3be6897be461e8190cbe499b4a30364ea92a8d 100644 (file)
@@ -195,7 +195,7 @@ function searchVideoWithSort (url, search, sort, end) {
     .end(end)
 }
 
-function testVideoImage (url, videoName, imagePath, callback) {
+function testVideoImage (url, imageName, imagePath, callback) {
   // Don't test images if the node env is not set
   // Because we need a special ffmpeg version for this test
   if (process.env.NODE_TEST_IMAGE) {
@@ -205,7 +205,7 @@ function testVideoImage (url, videoName, imagePath, callback) {
       .end(function (err, res) {
         if (err) return callback(err)
 
-        fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', videoName + '.jpg'), function (err, data) {
+        fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', imageName + '.jpg'), function (err, data) {
           if (err) return callback(err)
 
           callback(null, data.equals(res.body))
index d472cc8fbf32a19d7feaa4d15a4024bdf91ac00e..8aa8ee448b953faa0b58fa32369fa4454deed651 100644 (file)
@@ -17,6 +17,7 @@ export interface Video {
   podHost: string
   tags: string[]
   thumbnailPath: string
+  previewPath: string
   views: number
   likes: number
   dislikes: number
index 5636db49479645b6ff4b1db7afc05d9c5eab192c..68187f6844aa37f4600bcefad60fc89a6882eb8d 100644 (file)
--- a/yarn.lock
+++ b/yarn.lock
@@ -285,6 +285,12 @@ async-each@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d"
 
+async-lru@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/async-lru/-/async-lru-1.1.1.tgz#3edbf7e96484d5c2dd852a8bf9794fc07f5e7274"
+  dependencies:
+    lru "^3.1.0"
+
 async@>=0.2.9, async@^2.0.0:
   version "2.4.1"
   resolved "https://registry.yarnpkg.com/async/-/async-2.4.1.tgz#62a56b279c98a11d0987096a01cc3eeb8eb7bbd7"