/certs/
/logs/
/torrents/
+/cache/
/config/production.yaml
/ffmpeg/
/*.sublime-project
previews: 'previews/'
thumbnails: 'thumbnails/'
torrents: 'torrents/'
+ cache: 'cache/'
+
+cache:
+ previews:
+ size: 1 # Max number of previews you want to cache
admin:
email: 'admin@example.com'
previews: 'previews/'
thumbnails: 'thumbnails/'
torrents: 'torrents/'
+ cache: 'cache/'
admin:
email: 'admin@example.com'
certs: 'test1/certs/'
videos: 'test1/videos/'
logs: 'test1/logs/'
+ previews: 'test1/previews/'
thumbnails: 'test1/thumbnails/'
torrents: 'test1/torrents/'
+ cache: 'test1/cache/'
admin:
email: 'admin1@example.com'
certs: 'test2/certs/'
videos: 'test2/videos/'
logs: 'test2/logs/'
+ previews: 'test2/previews/'
thumbnails: 'test2/thumbnails/'
torrents: 'test2/torrents/'
+ cache: 'test2/cache/'
admin:
email: 'admin2@example.com'
certs: 'test3/certs/'
videos: 'test3/videos/'
logs: 'test3/logs/'
+ previews: 'test3/previews/'
thumbnails: 'test3/thumbnails/'
torrents: 'test3/torrents/'
+ cache: 'test3/cache/'
admin:
email: 'admin3@example.com'
certs: 'test4/certs/'
videos: 'test4/videos/'
logs: 'test4/logs/'
+ previews: 'test4/previews/'
thumbnails: 'test4/thumbnails/'
torrents: 'test4/torrents/'
+ cache: 'test4/cache/'
admin:
email: 'admin4@example.com'
certs: 'test5/certs/'
videos: 'test5/videos/'
logs: 'test5/logs/'
+ previews: 'test5/previews/'
thumbnails: 'test5/thumbnails/'
torrents: 'test5/torrents/'
+ cache: 'test5/cache/'
admin:
email: 'admin5@example.com'
certs: 'test6/certs/'
videos: 'test6/videos/'
logs: 'test6/logs/'
+ previews: 'test6/previews/'
thumbnails: 'test6/thumbnails/'
torrents: 'test6/torrents/'
+ cache: 'test6/cache/'
admin:
email: 'admin6@example.com'
},
"dependencies": {
"async": "^2.0.0",
+ "async-lru": "^1.1.1",
"bcrypt": "^1.0.2",
"bittorrent-tracker": "^9.0.0",
"bluebird": "^3.5.0",
// ----------- PeerTube modules -----------
import { migrate, installApplication } from './server/initializers'
-import { JobScheduler, activateSchedulers } from './server/lib'
+import { JobScheduler, activateSchedulers, VideosPreviewCache } from './server/lib'
import * as customValidators from './server/helpers/custom-validators'
import { apiRouter, clientsRouter, staticRouter } from './server/controllers'
// Activate job scheduler
JobScheduler.Instance.activate()
+ VideosPreviewCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE)
+
logger.info('Server listening on port %d', port)
logger.info('Webserver: %s', CONFIG.WEBSERVER.URL)
})
STATIC_MAX_AGE,
STATIC_PATHS
} from '../initializers'
+import { VideosPreviewCache } from '../lib'
const staticRouter = express.Router()
// Video previews path for express
const previewsPhysicalPath = CONFIG.STORAGE.PREVIEWS_DIR
staticRouter.use(
- STATIC_PATHS.PREVIEWS,
- express.static(previewsPhysicalPath, { maxAge: STATIC_MAX_AGE })
+ STATIC_PATHS.PREVIEWS + ':uuid.jpg',
+ getPreview
)
// ---------------------------------------------------------------------------
export {
staticRouter
}
+
+// ---------------------------------------------------------------------------
+
+function getPreview (req: express.Request, res: express.Response, next: express.NextFunction) {
+ VideosPreviewCache.Instance.getPreviewPath(req.params.uuid)
+ .then(path => {
+ if (!path) return res.sendStatus(404)
+
+ return res.sendFile(path, { maxAge: STATIC_MAX_AGE })
+ })
+}
import * as mkdirp from 'mkdirp'
import * as bcrypt from 'bcrypt'
import * as createTorrent from 'create-torrent'
+import * as rimraf from 'rimraf'
import * as openssl from 'openssl-wrapper'
import * as Promise from 'bluebird'
const bcryptGenSaltPromise = promisify1<number, string>(bcrypt.genSalt)
const bcryptHashPromise = promisify2<any, string|number, string>(bcrypt.hash)
const createTorrentPromise = promisify2<string, any, any>(createTorrent)
+const rimrafPromise = promisify1WithVoid<string>(rimraf)
// ---------------------------------------------------------------------------
bcryptComparePromise,
bcryptGenSaltPromise,
bcryptHashPromise,
- createTorrentPromise
+ createTorrentPromise,
+ rimrafPromise
}
VIDEOS_DIR: join(root(), config.get<string>('storage.videos')),
THUMBNAILS_DIR: join(root(), config.get<string>('storage.thumbnails')),
PREVIEWS_DIR: join(root(), config.get<string>('storage.previews')),
- TORRENTS_DIR: join(root(), config.get<string>('storage.torrents'))
+ TORRENTS_DIR: join(root(), config.get<string>('storage.torrents')),
+ CACHE_DIR: join(root(), config.get<string>('storage.cache'))
},
WEBSERVER: {
SCHEME: config.get<boolean>('webserver.https') === true ? 'https' : 'http',
TRANSCODING: {
ENABLED: config.get<boolean>('transcoding.enabled'),
THREADS: config.get<number>('transcoding.threads')
+ },
+ CACHE: {
+ PREVIEWS: {
+ SIZE: config.get<number>('cache.previews.size')
+ }
}
}
CONFIG.WEBSERVER.URL = CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
const THUMBNAILS_SIZE = '200x110'
const PREVIEWS_SIZE = '640x480'
+// Subfolders of cache directory
+const CACHE = {
+ DIRECTORIES: {
+ PREVIEWS: join(CONFIG.STORAGE.CACHE_DIR, 'previews')
+ }
+}
+
// ---------------------------------------------------------------------------
const USER_ROLES: { [ id: string ]: UserRole } = {
export {
API_VERSION,
BCRYPT_SALT_SIZE,
+ CACHE,
CONFIG,
CONSTRAINTS_FIELDS,
FRIEND_SCORE,
import * as Promise from 'bluebird'
import { database as db } from './database'
-import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION } from './constants'
+import { USER_ROLES, CONFIG, LAST_MIGRATION_VERSION, CACHE } from './constants'
import { clientsExist, usersExist } from './checker'
-import { logger, createCertsIfNotExist, root, mkdirpPromise } from '../helpers'
+import { logger, createCertsIfNotExist, root, mkdirpPromise, rimrafPromise } from '../helpers'
function installApplication () {
return db.sequelize.sync()
+ .then(() => removeCacheDirectories())
.then(() => createDirectoriesIfNotExist())
.then(() => createCertsIfNotExist())
.then(() => createOAuthClientIfNotExist())
// ---------------------------------------------------------------------------
+function removeCacheDirectories () {
+ const cacheDirectories = CACHE.DIRECTORIES
+
+ const tasks = []
+
+ // Cache directories
+ Object.keys(cacheDirectories).forEach(key => {
+ const dir = cacheDirectories[key]
+ tasks.push(rimrafPromise(dir))
+ })
+
+ return Promise.all(tasks)
+}
+
function createDirectoriesIfNotExist () {
- const storages = config.get('storage')
+ const storages = CONFIG.STORAGE
+ const cacheDirectories = CACHE.DIRECTORIES
const tasks = []
Object.keys(storages).forEach(key => {
const dir = storages[key]
- tasks.push(mkdirpPromise(join(root(), dir)))
+ tasks.push(mkdirpPromise(dir))
+ })
+
+ // Cache directories
+ Object.keys(cacheDirectories).forEach(key => {
+ const dir = cacheDirectories[key]
+ tasks.push(mkdirpPromise(dir))
})
return Promise.all(tasks)
--- /dev/null
+export * from './videos-preview-cache'
--- /dev/null
+import * as request from 'request'
+import * as asyncLRU from 'async-lru'
+import { join } from 'path'
+import { createWriteStream } from 'fs'
+import * as Promise from 'bluebird'
+
+import { database as db, CONFIG, CACHE } from '../../initializers'
+import { logger, writeFilePromise, unlinkPromise } from '../../helpers'
+import { VideoInstance } from '../../models'
+import { fetchRemotePreview } from '../../lib'
+
+class VideosPreviewCache {
+
+ private static instance: VideosPreviewCache
+
+ private lru
+
+ private constructor () { }
+
+ static get Instance () {
+ return this.instance || (this.instance = new this())
+ }
+
+ init (max: number) {
+ this.lru = new asyncLRU({
+ max,
+ load: (key, cb) => {
+ this.loadPreviews(key)
+ .then(res => cb(null, res))
+ .catch(err => cb(err))
+ }
+ })
+
+ this.lru.on('evict', (obj: { key: string, value: string }) => {
+ unlinkPromise(obj.value).then(() => logger.debug('%s evicted from VideosPreviewCache', obj.value))
+ })
+ }
+
+ getPreviewPath (key: string) {
+ return new Promise<string>((res, rej) => {
+ this.lru.get(key, (err, value) => {
+ err ? rej(err) : res(value)
+ })
+ })
+ }
+
+ private loadPreviews (key: string) {
+ return db.Video.loadByUUIDAndPopulateAuthorAndPodAndTags(key)
+ .then(video => {
+ if (!video) return undefined
+
+ if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
+
+ return this.saveRemotePreviewAndReturnPath(video)
+ })
+ }
+
+ private saveRemotePreviewAndReturnPath (video: VideoInstance) {
+ const req = fetchRemotePreview(video.Author.Pod, video)
+
+ return new Promise<string>((res, rej) => {
+ const path = join(CACHE.DIRECTORIES.PREVIEWS, video.getPreviewName())
+ const stream = createWriteStream(path)
+
+ req.pipe(stream)
+ .on('finish', () => res(path))
+ .on('error', (err) => rej(err))
+ })
+ }
+}
+
+export {
+ VideosPreviewCache
+}
import * as request from 'request'
import * as Sequelize from 'sequelize'
import * as Promise from 'bluebird'
+import { join } from 'path'
import { database as db } from '../initializers/database'
import {
REQUESTS_IN_PARALLEL,
REQUEST_ENDPOINTS,
REQUEST_ENDPOINT_ACTIONS,
- REMOTE_SCHEME
+ REMOTE_SCHEME,
+ STATIC_PATHS
} from '../initializers'
import {
logger,
})
}
+function fetchRemotePreview (pod: PodInstance, video: VideoInstance) {
+ const host = video.Author.Pod.host
+ const path = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
+
+ return request.get(REMOTE_SCHEME.HTTP + '://' + host + path)
+}
+
function getRequestScheduler () {
return requestScheduler
}
sendOwnedVideosToPod,
getRequestScheduler,
getRequestVideoQaduScheduler,
- getRequestVideoEventScheduler
+ getRequestVideoEventScheduler,
+ fetchRemotePreview
}
// ---------------------------------------------------------------------------
+export * from './cache'
export * from './jobs'
export * from './request'
export * from './friends'
refreshToken: string
refreshTokenExpiresAt: Date
+ userId?: number
+ oAuthClientId?: number
User?: UserModel
}
refreshToken: token.refreshToken,
refreshTokenExpiresAt: token.refreshTokenExpiresAt,
client: {
- id: token['client'].id
+ id: token.oAuthClientId
},
user: {
- id: token['user']
+ id: token.userId
}
}
dislikes: this.dislikes,
tags: map<TagInstance, string>(this.Tags, 'name'),
thumbnailPath: join(STATIC_PATHS.THUMBNAILS, this.getThumbnailName()),
+ previewPath: join(STATIC_PATHS.PREVIEWS, this.getPreviewName()),
createdAt: this.createdAt,
updatedAt: this.updatedAt
}
expect(videos[0].name).not.to.equal(toRemove[1].name)
expect(videos[1].name).not.to.equal(toRemove[1].name)
- videoUUID = videos[0].uuid
+ videoUUID = videos.find(video => video.name === 'my super name for pod 1').uuid
callback()
})
})
}, done)
})
+
+ it('Should get the preview from each pod', function (done) {
+ each(servers, function (server, callback) {
+ videosUtils.getVideo(server.url, videoUUID, function (err, res) {
+ if (err) throw err
+
+ const video = res.body
+
+ videosUtils.testVideoImage(server.url, 'video_short1-preview.webm', video.previewPath, function (err, test) {
+ if (err) throw err
+ expect(test).to.equal(true)
+
+ callback()
+ })
+ })
+ }, done)
+ })
})
after(function (done) {
.end(end)
}
-function testVideoImage (url, videoName, imagePath, callback) {
+function testVideoImage (url, imageName, imagePath, callback) {
// Don't test images if the node env is not set
// Because we need a special ffmpeg version for this test
if (process.env.NODE_TEST_IMAGE) {
.end(function (err, res) {
if (err) return callback(err)
- fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', videoName + '.jpg'), function (err, data) {
+ fs.readFile(pathUtils.join(__dirname, '..', 'api', 'fixtures', imageName + '.jpg'), function (err, data) {
if (err) return callback(err)
callback(null, data.equals(res.body))
podHost: string
tags: string[]
thumbnailPath: string
+ previewPath: string
views: number
likes: number
dislikes: number
version "1.0.1"
resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d"
+async-lru@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/async-lru/-/async-lru-1.1.1.tgz#3edbf7e96484d5c2dd852a8bf9794fc07f5e7274"
+ dependencies:
+ lru "^3.1.0"
+
async@>=0.2.9, async@^2.0.0:
version "2.4.1"
resolved "https://registry.yarnpkg.com/async/-/async-2.4.1.tgz#62a56b279c98a11d0987096a01cc3eeb8eb7bbd7"