From: Chocobozzz Date: Sat, 30 Jan 2016 16:05:22 +0000 (+0100) Subject: New directory organization X-Git-Tag: v0.0.1-alpha~1050 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=cda021079ff455cc0fd0eb95a5395fa808ab63d1;p=oweals%2Fpeertube.git New directory organization --- diff --git a/Gruntfile.js b/Gruntfile.js index a8fcbb609..6df0c023a 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -8,8 +8,7 @@ module.exports = function (grunt) { scss: 'public/stylesheets/application.scss', vendor: 'public/stylesheets/vendor', js: 'public/javascripts/*.js', - src: 'src/*.js', - routes: 'routes/**/*.js', + routes: 'controllers/**/*.js', main: './server.js', browserified: 'public/javascripts/bundle.js', img: 'public/images/*.{png,jpg,jpeg,gif,webp,svg}', diff --git a/controllers/api/v1/index.js b/controllers/api/v1/index.js new file mode 100644 index 000000000..f5504ad85 --- /dev/null +++ b/controllers/api/v1/index.js @@ -0,0 +1,12 @@ +;(function () { + 'use strict' + + var express = require('express') + var router = express.Router() + + router.use('/videos', require('./videos')) + router.use('/remotevideos', require('./remoteVideos')) + router.use('/pods', require('./pods')) + + module.exports = router +})() diff --git a/controllers/api/v1/pods.js b/controllers/api/v1/pods.js new file mode 100644 index 000000000..30385bd5a --- /dev/null +++ b/controllers/api/v1/pods.js @@ -0,0 +1,69 @@ +;(function () { + 'use strict' + + var express = require('express') + var router = express.Router() + var middleware = require('../../../middlewares') + var miscMiddleware = middleware.misc + var reqValidator = middleware.reqValidators.pods + var secureRequest = middleware.reqValidators.remote.secureRequest + var pods = require('../../../models/pods') + + function listPods (req, res, next) { + pods.list(function (err, pods_list) { + if (err) return next(err) + + res.json(pods_list) + }) + } + + function addPods (req, res, next) { + pods.add(req.body.data, function (err, json) { + if (err) return next(err) + + res.json(json) + }) + } + + function removePods (req, res, next) { + pods.remove(req.body.signature.url, function (err) { + if (err) return next(err) + + res.sendStatus(204) + }) + } + + function makeFriends (req, res, next) { + pods.hasFriends(function (err, has_friends) { + if (err) return next(err) + + if (has_friends === true) { + // We need to quit our friends before make new ones + res.sendStatus(409) + } else { + pods.makeFriends(function (err) { + if (err) return next(err) + + res.sendStatus(204) + }) + } + }) + } + + function quitFriends (req, res, next) { + pods.quitFriends(function (err) { + if (err) return next(err) + + res.sendStatus(204) + }) + } + + router.get('/', miscMiddleware.cache(false), listPods) + router.get('/makefriends', miscMiddleware.cache(false), makeFriends) + router.get('/quitfriends', miscMiddleware.cache(false), quitFriends) + router.post('/', reqValidator.podsAdd, miscMiddleware.cache(false), addPods) + // Post because this is a secured request + router.post('/remove', secureRequest, miscMiddleware.decryptBody, removePods) + + module.exports = router +})() diff --git a/controllers/api/v1/remoteVideos.js b/controllers/api/v1/remoteVideos.js new file mode 100644 index 000000000..d534d6792 --- /dev/null +++ b/controllers/api/v1/remoteVideos.js @@ -0,0 +1,33 @@ +;(function () { + 'use strict' + + var express = require('express') + var router = express.Router() + var pluck = require('lodash-node/compat/collection/pluck') + + var middleware = require('../../../middlewares') + var miscMiddleware = middleware.misc + var reqValidator = middleware.reqValidators.remote + var videos = require('../../../models/videos') + + function addRemoteVideos (req, res, next) { + videos.addRemotes(req.body.data, function (err, videos) { + if (err) return next(err) + + res.json(videos) + }) + } + + function removeRemoteVideo (req, res, next) { + videos.removeRemotes(req.body.signature.url, pluck(req.body.data, 'magnetUri'), function (err) { + if (err) return next(err) + + res.sendStatus(204) + }) + } + + router.post('/add', reqValidator.secureRequest, miscMiddleware.decryptBody, reqValidator.remoteVideosAdd, miscMiddleware.cache(false), addRemoteVideos) + router.post('/remove', reqValidator.secureRequest, miscMiddleware.decryptBody, reqValidator.remoteVideosRemove, miscMiddleware.cache(false), removeRemoteVideo) + + module.exports = router +})() diff --git a/controllers/api/v1/videos.js b/controllers/api/v1/videos.js new file mode 100644 index 000000000..aa8cb466b --- /dev/null +++ b/controllers/api/v1/videos.js @@ -0,0 +1,88 @@ +;(function () { + 'use strict' + + var express = require('express') + var config = require('config') + var crypto = require('crypto') + var multer = require('multer') + var router = express.Router() + + var middleware = require('../../../middlewares') + var miscMiddleware = middleware.misc + var reqValidator = middleware.reqValidators.videos + var videos = require('../../../models/videos') + + var uploads = config.get('storage.uploads') + + function listVideos (req, res, next) { + videos.list(function (err, videos_list) { + if (err) return next(err) + + res.json(videos_list) + }) + } + + function searchVideos (req, res, next) { + videos.search(req.params.name, function (err, videos_list) { + if (err) return next(err) + + res.json(videos_list) + }) + } + + function addVideos (req, res, next) { + videos.add({ video: req.files.input_video[0], data: req.body }, function (err) { + if (err) return next(err) + + // TODO : include Location of the new video + res.sendStatus(201) + }) + } + + function getVideos (req, res, next) { + videos.get(req.params.id, function (err, video) { + if (err) return next(err) + + if (video === null) { + return res.sendStatus(404) + } + + res.json(video) + }) + } + + function removeVideo (req, res, next) { + videos.remove(req.params.id, function (err) { + if (err) return next(err) + + res.sendStatus(204) + }) + } + + // multer configuration + var storage = multer.diskStorage({ + destination: function (req, file, cb) { + cb(null, uploads) + }, + + filename: function (req, file, cb) { + var extension = '' + if (file.mimetype === 'video/webm') extension = 'webm' + else if (file.mimetype === 'video/mp4') extension = 'mp4' + else if (file.mimetype === 'video/ogg') extension = 'ogv' + crypto.pseudoRandomBytes(16, function (err, raw) { + var fieldname = err ? undefined : raw.toString('hex') + cb(null, fieldname + '.' + extension) + }) + } + }) + var reqFiles = multer({ storage: storage }).fields([{ name: 'input_video', maxCount: 1 }]) + + router.get('/', miscMiddleware.cache(false), listVideos) + router.post('/', reqFiles, reqValidator.videosAdd, miscMiddleware.cache(false), addVideos) + router.get('/search/:name', reqValidator.videosSearch, miscMiddleware.cache(false), searchVideos) + router.get('/:id', reqValidator.videosGet, miscMiddleware.cache(false), getVideos) + router.delete('/:id', reqValidator.videosRemove, miscMiddleware.cache(false), removeVideo) + + module.exports = router +})() diff --git a/controllers/index.js b/controllers/index.js new file mode 100644 index 000000000..7dca002ff --- /dev/null +++ b/controllers/index.js @@ -0,0 +1,12 @@ +;(function () { + 'use strict' + + var constants = require('../initializers/constants') + + var routes = { + api: require('./api/' + constants.API_VERSION), + views: require('./views') + } + + module.exports = routes +})() diff --git a/controllers/views.js b/controllers/views.js new file mode 100644 index 000000000..ebd97380e --- /dev/null +++ b/controllers/views.js @@ -0,0 +1,24 @@ +;(function () { + 'use strict' + + function getPartial (req, res) { + var directory = req.params.directory + var name = req.params.name + + res.render('partials/' + directory + '/' + name) + } + + function getIndex (req, res) { + res.render('index') + } + + var express = require('express') + var middleware = require('../middlewares').misc + + var router = express.Router() + + router.get('/partials/:directory/:name', middleware.cache(), getPartial) + router.get(/^\/(index)?$/, middleware.cache(), getIndex) + + module.exports = router +})() diff --git a/helpers/customValidators.js b/helpers/customValidators.js new file mode 100644 index 000000000..73c2f8461 --- /dev/null +++ b/helpers/customValidators.js @@ -0,0 +1,29 @@ +;(function () { + 'use strict' + + var validator = require('validator') + + var customValidators = {} + + customValidators.eachIsRemoteVideosAddValid = function (values) { + return values.every(function (val) { + return validator.isLength(val.name, 1, 50) && + validator.isLength(val.description, 1, 50) && + validator.isLength(val.magnetUri, 10) && + validator.isURL(val.podUrl) + }) + } + + customValidators.eachIsRemoteVideosRemoveValid = function (values) { + return values.every(function (val) { + return validator.isLength(val.magnetUri, 10) + }) + } + + customValidators.isArray = function (value) { + return Array.isArray(value) + } + + // ----------- Export ----------- + module.exports = customValidators +})() diff --git a/helpers/logger.js b/helpers/logger.js new file mode 100644 index 000000000..850af10cb --- /dev/null +++ b/helpers/logger.js @@ -0,0 +1,40 @@ +;(function () { + // Thanks http://tostring.it/2014/06/23/advanced-logging-with-nodejs/ + 'use strict' + + var config = require('config') + var winston = require('winston') + + var logDir = __dirname + '/../' + config.get('storage.logs') + + winston.emitErrs = true + + var logger = new winston.Logger({ + transports: [ + new winston.transports.File({ + level: 'debug', + filename: logDir + '/all-logs.log', + handleExceptions: true, + json: true, + maxsize: 5242880, + maxFiles: 5, + colorize: false + }), + new winston.transports.Console({ + level: 'debug', + handleExceptions: true, + humanReadableUnhandledException: true, + json: false, + colorize: true + }) + ], + exitOnError: true + }) + + module.exports = logger + module.exports.stream = { + write: function (message, encoding) { + logger.info(message) + } + } +})() diff --git a/helpers/utils.js b/helpers/utils.js new file mode 100644 index 000000000..7cdb2600d --- /dev/null +++ b/helpers/utils.js @@ -0,0 +1,202 @@ +;(function () { + 'use strict' + + var async = require('async') + var config = require('config') + var crypto = require('crypto') + var fs = require('fs') + var openssl = require('openssl-wrapper') + var request = require('request') + var replay = require('request-replay') + var ursa = require('ursa') + + var constants = require('../initializers/constants') + var logger = require('./logger') + + var utils = {} + + var http = config.get('webserver.https') ? 'https' : 'http' + var host = config.get('webserver.host') + var port = config.get('webserver.port') + var algorithm = 'aes-256-ctr' + + // ----------- Private functions ---------- + + function makeRetryRequest (params, from_url, to_pod, signature, callbackEach) { + // Append the signature + if (signature) { + params.json.signature = { + url: from_url, + signature: signature + } + } + + logger.debug('Make retry requests to %s.', to_pod.url) + + replay( + request.post(params, function (err, response, body) { + callbackEach(err, response, body, params.url, to_pod) + }), + { + retries: constants.REQUEST_RETRIES, + factor: 3, + maxTimeout: Infinity, + errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ] + } + ).on('replay', function (replay) { + logger.info('Replaying request to %s. Request failed: %d %s. Replay number: #%d. Will retry in: %d ms.', + params.url, replay.error.code, replay.error.message, replay.number, replay.delay) + }) + } + + // ----------- Public attributes ---------- + utils.certDir = __dirname + '/../' + config.get('storage.certs') + + // { path, data } + utils.makeMultipleRetryRequest = function (all_data, pods, callbackEach, callback) { + if (!callback) { + callback = callbackEach + callbackEach = null + } + + var url = http + '://' + host + ':' + port + var signature + + // Add signature if it is specified in the params + if (all_data.method === 'POST' && all_data.data && all_data.sign === true) { + var myKey = ursa.createPrivateKey(fs.readFileSync(utils.certDir + 'peertube.key.pem')) + signature = myKey.hashAndSign('sha256', url, 'utf8', 'hex') + } + + // Make a request for each pod + async.each(pods, function (pod, callback_each_async) { + function callbackEachRetryRequest (err, response, body, url, pod) { + if (callbackEach !== null) { + callbackEach(err, response, body, url, pod, function () { + callback_each_async() + }) + } else { + callback_each_async() + } + } + + var params = { + url: pod.url + all_data.path, + method: all_data.method + } + + // Add data with POST requst ? + if (all_data.method === 'POST' && all_data.data) { + // Encrypt data ? + if (all_data.encrypt === true) { + var crt = ursa.createPublicKey(pod.publicKey) + + // TODO: ES6 with let + ;(function (crt_copy, copy_params, copy_url, copy_pod, copy_signature) { + utils.symetricEncrypt(JSON.stringify(all_data.data), function (err, dataEncrypted) { + if (err) throw err + + var passwordEncrypted = crt_copy.encrypt(dataEncrypted.password, 'utf8', 'hex') + copy_params.json = { + data: dataEncrypted.crypted, + key: passwordEncrypted + } + + makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEachRetryRequest) + }) + })(crt, params, url, pod, signature) + } else { + params.json = { data: all_data.data } + makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) + } + } else { + makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) + } + }, callback) + } + + utils.certsExist = function (callback) { + fs.exists(utils.certDir + 'peertube.key.pem', function (exists) { + return callback(exists) + }) + } + + utils.createCerts = function (callback) { + utils.certsExist(function (exist) { + if (exist === true) { + var string = 'Certs already exist.' + logger.warning(string) + return callback(new Error(string)) + } + + logger.info('Generating a RSA key...') + openssl.exec('genrsa', { 'out': utils.certDir + 'peertube.key.pem', '2048': false }, function (err) { + if (err) { + logger.error('Cannot create private key on this pod.', { error: err }) + return callback(err) + } + logger.info('RSA key generated.') + + logger.info('Manage public key...') + openssl.exec('rsa', { 'in': utils.certDir + 'peertube.key.pem', 'pubout': true, 'out': utils.certDir + 'peertube.pub' }, function (err) { + if (err) { + logger.error('Cannot create public key on this pod .', { error: err }) + return callback(err) + } + + logger.info('Public key managed.') + return callback(null) + }) + }) + }) + } + + utils.createCertsIfNotExist = function (callback) { + utils.certsExist(function (exist) { + if (exist === true) { + return callback(null) + } + + utils.createCerts(function (err) { + return callback(err) + }) + }) + } + + utils.generatePassword = function (callback) { + crypto.randomBytes(32, function (err, buf) { + if (err) { + return callback(err) + } + + callback(null, buf.toString('utf8')) + }) + } + + utils.symetricEncrypt = function (text, callback) { + utils.generatePassword(function (err, password) { + if (err) { + return callback(err) + } + + var cipher = crypto.createCipher(algorithm, password) + var crypted = cipher.update(text, 'utf8', 'hex') + crypted += cipher.final('hex') + callback(null, { crypted: crypted, password: password }) + }) + } + + utils.symetricDecrypt = function (text, password) { + var decipher = crypto.createDecipher(algorithm, password) + var dec = decipher.update(text, 'hex', 'utf8') + dec += decipher.final('utf8') + return dec + } + + utils.cleanForExit = function (webtorrent_process) { + logger.info('Gracefully exiting') + process.kill(-webtorrent_process.pid) + } + + module.exports = utils +})() diff --git a/initializers/checker.js b/initializers/checker.js new file mode 100644 index 000000000..7a3a53616 --- /dev/null +++ b/initializers/checker.js @@ -0,0 +1,45 @@ +;(function () { + 'use strict' + + var config = require('config') + var mkdirp = require('mkdirp') + + var checker = {} + + // Check the config files + checker.checkConfig = function () { + var required = [ 'listen.port', + 'webserver.https', 'webserver.host', 'webserver.port', + 'database.host', 'database.port', 'database.suffix', + 'storage.certs', 'storage.uploads', 'storage.logs', + 'network.friends' ] + var miss = [] + + for (var key of required) { + if (!config.has(key)) { + miss.push(key) + } + } + + return miss + } + + // Create directories for the storage if it doesn't exist + checker.createDirectoriesIfNotExist = function () { + var storages = config.get('storage') + + for (var key of Object.keys(storages)) { + var path = storages[key] + try { + mkdirp.sync(__dirname + '/../' + path) + } catch (error) { + // Do not use logger + console.error('Cannot create ' + path + ':' + error) + process.exit(0) + } + } + } + + // ----------- Export ----------- + module.exports = checker +})() diff --git a/initializers/constants.js b/initializers/constants.js new file mode 100644 index 000000000..00b713961 --- /dev/null +++ b/initializers/constants.js @@ -0,0 +1,37 @@ +;(function () { + 'use strict' + + var constants = {} + + function isTestInstance () { + return (process.env.NODE_ENV === 'test') + } + + // API version of our pod + constants.API_VERSION = 'v1' + + // Score a pod has when we create it as a friend + constants.FRIEND_BASE_SCORE = 100 + + // Time to wait between requests to the friends + constants.INTERVAL = 60000 + + // Number of points we add/remove from a friend after a successful/bad request + constants.PODS_SCORE = { + MALUS: -10, + BONUS: 10 + } + + // Number of retries we make for the make retry requests (to friends...) + constants.REQUEST_RETRIES = 10 + + // Special constants for a test instance + if (isTestInstance() === true) { + constants.FRIEND_BASE_SCORE = 20 + constants.INTERVAL = 10000 + constants.REQUEST_RETRIES = 2 + } + + // ----------- Export ----------- + module.exports = constants +})() diff --git a/initializers/database.js b/initializers/database.js new file mode 100644 index 000000000..4570d3739 --- /dev/null +++ b/initializers/database.js @@ -0,0 +1,61 @@ +;(function () { + 'use strict' + + var config = require('config') + var mongoose = require('mongoose') + + var constants = require('./constants') + var logger = require('../helpers/logger') + + var dbname = 'peertube' + config.get('database.suffix') + var host = config.get('database.host') + var port = config.get('database.port') + + // ----------- Videos ----------- + var videosSchema = mongoose.Schema({ + name: String, + namePath: String, + description: String, + magnetUri: String, + podUrl: String + }) + + var VideosDB = mongoose.model('videos', videosSchema) + + // ----------- Pods ----------- + var podsSchema = mongoose.Schema({ + url: String, + publicKey: String, + score: { type: Number, max: constants.FRIEND_BASE_SCORE } + }) + + var PodsDB = mongoose.model('pods', podsSchema) + + // ----------- PoolRequests ----------- + var poolRequestsSchema = mongoose.Schema({ + type: String, + id: String, // Special id to find duplicates (video created we want to remove...) + request: mongoose.Schema.Types.Mixed + }) + + var PoolRequestsDB = mongoose.model('poolRequests', poolRequestsSchema) + + // ----------- Connection ----------- + + mongoose.connect('mongodb://' + host + ':' + port + '/' + dbname) + mongoose.connection.on('error', function () { + logger.error('Mongodb connection error.') + process.exit(0) + }) + + mongoose.connection.on('open', function () { + logger.info('Connected to mongodb.') + }) + + // ----------- Export ----------- + module.exports = { + VideosDB: VideosDB, + PodsDB: PodsDB, + PoolRequestsDB: PoolRequestsDB + } +})() diff --git a/lib/poolRequests.js b/lib/poolRequests.js new file mode 100644 index 000000000..9c7f3238b --- /dev/null +++ b/lib/poolRequests.js @@ -0,0 +1,206 @@ +;(function () { + 'use strict' + + var async = require('async') + + var constants = require('../initializers/constants') + var logger = require('../helpers/logger') + var database = require('../initializers/database') + var pluck = require('lodash-node/compat/collection/pluck') + var PoolRequestsDB = database.PoolRequestsDB + var PodsDB = database.PodsDB + var utils = require('../helpers/utils') + var VideosDB = database.VideosDB + + var poolRequests = {} + + // ----------- Private ----------- + var timer = null + + function removePoolRequestsFromDB (ids) { + PoolRequestsDB.remove({ _id: { $in: ids } }, function (err) { + if (err) { + logger.error('Cannot remove requests from the pool requests database.', { error: err }) + return + } + + logger.info('Pool requests flushed.') + }) + } + + function makePoolRequests () { + logger.info('Making pool requests to friends.') + + PoolRequestsDB.find({}, { _id: 1, type: 1, request: 1 }, function (err, pool_requests) { + if (err) throw err + + if (pool_requests.length === 0) return + + var requests = { + add: { + ids: [], + requests: [] + }, + remove: { + ids: [], + requests: [] + } + } + + async.each(pool_requests, function (pool_request, callback_each) { + if (pool_request.type === 'add') { + requests.add.requests.push(pool_request.request) + requests.add.ids.push(pool_request._id) + } else if (pool_request.type === 'remove') { + requests.remove.requests.push(pool_request.request) + requests.remove.ids.push(pool_request._id) + } else { + throw new Error('Unkown pool request type.') + } + + callback_each() + }, function () { + // Send the add requests + if (requests.add.requests.length !== 0) { + makePoolRequest('add', requests.add.requests, function (err) { + if (err) logger.error('Errors when sent add pool requests.', { error: err }) + + removePoolRequestsFromDB(requests.add.ids) + }) + } + + // Send the remove requests + if (requests.remove.requests.length !== 0) { + makePoolRequest('remove', requests.remove.requests, function (err) { + if (err) logger.error('Errors when sent remove pool requests.', { error: err }) + + removePoolRequestsFromDB(requests.remove.ids) + }) + } + }) + }) + } + + function updatePodsScore (good_pods, bad_pods) { + logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length) + + PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: constants.PODS_SCORE.BONUS } }, { multi: true }).exec() + PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: constants.PODS_SCORE.MALUS } }, { multi: true }, function (err) { + if (err) throw err + removeBadPods() + }) + } + + function removeBadPods () { + PodsDB.find({ score: 0 }, { _id: 1, url: 1 }, function (err, pods) { + if (err) throw err + + if (pods.length === 0) return + + var urls = pluck(pods, 'url') + var ids = pluck(pods, '_id') + + VideosDB.remove({ podUrl: { $in: urls } }, function (err, r) { + if (err) logger.error('Cannot remove videos from a pod that we removing.', { error: err }) + var videos_removed = r.result.n + logger.info('Removed %d videos.', videos_removed) + + PodsDB.remove({ _id: { $in: ids } }, function (err, r) { + if (err) logger.error('Cannot remove bad pods.', { error: err }) + + var pods_removed = r.result.n + logger.info('Removed %d pods.', pods_removed) + }) + }) + }) + } + + function makePoolRequest (type, requests, callback) { + if (!callback) callback = function () {} + + PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) { + if (err) throw err + + var params = { + encrypt: true, + sign: true, + method: 'POST', + path: null, + data: requests + } + + if (type === 'add') { + params.path = '/api/' + constants.API_VERSION + '/remotevideos/add' + } else if (type === 'remove') { + params.path = '/api/' + constants.API_VERSION + '/remotevideos/remove' + } else { + throw new Error('Unkown pool request type.') + } + + var bad_pods = [] + var good_pods = [] + + utils.makeMultipleRetryRequest(params, pods, callbackEachPodFinished, callbackAllPodsFinished) + + function callbackEachPodFinished (err, response, body, url, pod, callback_each_pod_finished) { + if (err || (response.statusCode !== 200 && response.statusCode !== 204)) { + bad_pods.push(pod._id) + logger.error('Error sending secure request to %s pod.', url, { error: err || new Error('Status code not 20x') }) + } else { + good_pods.push(pod._id) + } + + return callback_each_pod_finished() + } + + function callbackAllPodsFinished (err) { + if (err) return callback(err) + + updatePodsScore(good_pods, bad_pods) + callback(null) + } + }) + } + + // ----------- Public ----------- + poolRequests.activate = function () { + logger.info('Pool requests activated.') + timer = setInterval(makePoolRequests, constants.INTERVAL) + } + + poolRequests.addToPoolRequests = function (id, type, request) { + logger.debug('Add request to the pool requests.', { id: id, type: type, request: request }) + + PoolRequestsDB.findOne({ id: id }, function (err, entity) { + if (err) logger.error(err) + + if (entity) { + if (entity.type === type) { + logger.error(new Error('Cannot insert two same requests.')) + return + } + + // Remove the request of the other type + PoolRequestsDB.remove({ id: id }, function (err) { + if (err) logger.error(err) + }) + } else { + PoolRequestsDB.create({ id: id, type: type, request: request }, function (err) { + if (err) logger.error(err) + }) + } + }) + } + + poolRequests.deactivate = function () { + logger.info('Pool requests deactivated.') + clearInterval(timer) + } + + poolRequests.forceSend = function () { + logger.info('Force pool requests sending.') + makePoolRequests() + } + + module.exports = poolRequests +})() diff --git a/lib/webTorrentNode.js b/lib/webTorrentNode.js new file mode 100644 index 000000000..8827c68c5 --- /dev/null +++ b/lib/webTorrentNode.js @@ -0,0 +1,160 @@ +;(function () { + 'use strict' + + var config = require('config') + var ipc = require('node-ipc') + var pathUtils = require('path') + var spawn = require('electron-spawn') + + var logger = require('../helpers/logger') + + var host = config.get('webserver.host') + var port = config.get('webserver.port') + + var nodeKey = 'webtorrentnode' + port + var processKey = 'webtorrent' + port + + ipc.config.silent = true + ipc.config.id = nodeKey + + var webtorrentnode = {} + + // Useful for beautiful tests + webtorrentnode.silent = false + + // Useful to kill it + webtorrentnode.app = null + + webtorrentnode.create = function (options, callback) { + if (typeof options === 'function') { + callback = options + options = {} + } + + // Override options + if (options.host) host = options.host + if (options.port) { + port = options.port + nodeKey = 'webtorrentnode' + port + processKey = 'webtorrent' + port + ipc.config.id = nodeKey + } + + ipc.serve(function () { + if (!webtorrentnode.silent) logger.info('IPC server ready.') + + // Run a timeout of 30s after which we exit the process + var timeout_webtorrent_process = setTimeout(function () { + logger.error('Timeout : cannot run the webtorrent process. Please ensure you have electron-prebuilt npm package installed with xvfb-run.') + process.exit() + }, 30000) + + ipc.server.on(processKey + '.ready', function () { + if (!webtorrentnode.silent) logger.info('Webtorrent process ready.') + clearTimeout(timeout_webtorrent_process) + callback() + }) + + ipc.server.on(processKey + '.exception', function (data) { + logger.error('Received exception error from webtorrent process.', { exception: data.exception }) + process.exit() + }) + + var webtorrent_process = spawn(__dirname + '/webtorrent.js', host, port, { detached: true }) + webtorrent_process.stderr.on('data', function (data) { + // logger.debug('Webtorrent process stderr: ', data.toString()) + }) + + webtorrent_process.stdout.on('data', function (data) { + // logger.debug('Webtorrent process:', data.toString()) + }) + + webtorrentnode.app = webtorrent_process + }) + + ipc.server.start() + } + + webtorrentnode.seed = function (path, callback) { + var extension = pathUtils.extname(path) + var basename = pathUtils.basename(path, extension) + var data = { + _id: basename, + args: { + path: path + } + } + + if (!webtorrentnode.silent) logger.debug('Node wants to seed %s.', data._id) + + // Finish signal + var event_key = nodeKey + '.seedDone.' + data._id + ipc.server.on(event_key, function listener (received) { + if (!webtorrentnode.silent) logger.debug('Process seeded torrent %s.', received.magnetUri) + + // This is a fake object, we just use the magnetUri in this project + var torrent = { + magnetURI: received.magnetUri + } + + ipc.server.off(event_key) + callback(torrent) + }) + + ipc.server.broadcast(processKey + '.seed', data) + } + + webtorrentnode.add = function (magnetUri, callback) { + var data = { + _id: magnetUri, + args: { + magnetUri: magnetUri + } + } + + if (!webtorrentnode.silent) logger.debug('Node wants to add ' + data._id) + + // Finish signal + var event_key = nodeKey + '.addDone.' + data._id + ipc.server.on(event_key, function (received) { + if (!webtorrentnode.silent) logger.debug('Process added torrent.') + + // This is a fake object, we just use the magnetUri in this project + var torrent = { + files: received.files + } + + ipc.server.off(event_key) + callback(torrent) + }) + + ipc.server.broadcast(processKey + '.add', data) + } + + webtorrentnode.remove = function (magnetUri, callback) { + var data = { + _id: magnetUri, + args: { + magnetUri: magnetUri + } + } + + if (!webtorrentnode.silent) logger.debug('Node wants to stop seeding %s.', data._id) + + // Finish signal + var event_key = nodeKey + '.removeDone.' + data._id + ipc.server.on(event_key, function (received) { + if (!webtorrentnode.silent) logger.debug('Process removed torrent %s.', data._id) + + var err = null + if (received.err) err = received.err + + ipc.server.off(event_key) + callback(err) + }) + + ipc.server.broadcast(processKey + '.remove', data) + } + + module.exports = webtorrentnode +})() diff --git a/lib/webtorrent.js b/lib/webtorrent.js new file mode 100644 index 000000000..b72bc500d --- /dev/null +++ b/lib/webtorrent.js @@ -0,0 +1,91 @@ +;(function () { + 'use strict' + + module.exports = function (args) { + var WebTorrent = require('webtorrent') + var ipc = require('node-ipc') + + if (args.length !== 3) { + console.log('Wrong arguments number: ' + args.length + '/3') + process.exit(-1) + } + + var host = args[1] + var port = args[2] + var nodeKey = 'webtorrentnode' + port + var processKey = 'webtorrent' + port + + ipc.config.silent = true + ipc.config.id = processKey + + if (host === 'client' && port === '1') global.WEBTORRENT_ANNOUNCE = [] + else global.WEBTORRENT_ANNOUNCE = 'ws://' + host + ':' + port + '/tracker/socket' + var wt = new WebTorrent({ dht: false }) + + function seed (data) { + var args = data.args + var path = args.path + var _id = data._id + + wt.seed(path, { announceList: '' }, function (torrent) { + var to_send = { + magnetUri: torrent.magnetURI + } + + ipc.of[nodeKey].emit(nodeKey + '.seedDone.' + _id, to_send) + }) + } + + function add (data) { + var args = data.args + var magnetUri = args.magnetUri + var _id = data._id + + wt.add(magnetUri, function (torrent) { + var to_send = { + files: [] + } + + torrent.files.forEach(function (file) { + to_send.files.push({ path: file.path }) + }) + + ipc.of[nodeKey].emit(nodeKey + '.addDone.' + _id, to_send) + }) + } + + function remove (data) { + var args = data.args + var magnetUri = args.magnetUri + var _id = data._id + + try { + wt.remove(magnetUri, callback) + } catch (err) { + console.log('Cannot remove the torrent from WebTorrent', { err: err }) + return callback(null) + } + + function callback () { + var to_send = {} + ipc.of[nodeKey].emit(nodeKey + '.removeDone.' + _id, to_send) + } + } + + console.log('Configuration: ' + host + ':' + port) + console.log('Connecting to IPC...') + + ipc.connectTo(nodeKey, function () { + ipc.of[nodeKey].on(processKey + '.seed', seed) + ipc.of[nodeKey].on(processKey + '.add', add) + ipc.of[nodeKey].on(processKey + '.remove', remove) + + ipc.of[nodeKey].emit(processKey + '.ready') + console.log('Ready.') + }) + + process.on('uncaughtException', function (e) { + ipc.of[nodeKey].emit(processKey + '.exception', { exception: e }) + }) + } +})() diff --git a/middlewares/misc.js b/middlewares/misc.js index c10b0792a..f814acd9f 100644 --- a/middlewares/misc.js +++ b/middlewares/misc.js @@ -4,9 +4,9 @@ var ursa = require('ursa') var fs = require('fs') - var logger = require('../src/logger') - var utils = require('../src/utils') - var PodsDB = require('../src/database').PodsDB + var logger = require('../helpers/logger') + var utils = require('../helpers/utils') + var PodsDB = require('../initializers/database').PodsDB var misc = {} diff --git a/middlewares/reqValidators/pods.js b/middlewares/reqValidators/pods.js index 31eaf8449..0d023842d 100644 --- a/middlewares/reqValidators/pods.js +++ b/middlewares/reqValidators/pods.js @@ -2,7 +2,7 @@ 'use strict' var checkErrors = require('./utils').checkErrors - var logger = require('../../src/logger') + var logger = require('../../helpers/logger') var pods = {} diff --git a/middlewares/reqValidators/remote.js b/middlewares/reqValidators/remote.js index a9d2cdf20..4b161e292 100644 --- a/middlewares/reqValidators/remote.js +++ b/middlewares/reqValidators/remote.js @@ -2,7 +2,7 @@ 'use strict' var checkErrors = require('./utils').checkErrors - var logger = require('../../src/logger') + var logger = require('../../helpers/logger') var remote = {} diff --git a/middlewares/reqValidators/utils.js b/middlewares/reqValidators/utils.js index 91ead27a5..5bc9f4f0b 100644 --- a/middlewares/reqValidators/utils.js +++ b/middlewares/reqValidators/utils.js @@ -2,7 +2,7 @@ 'use strict' var util = require('util') - var logger = require('../../src/logger') + var logger = require('../../helpers/logger') var utils = {} diff --git a/middlewares/reqValidators/videos.js b/middlewares/reqValidators/videos.js index 8c4e23b6b..a34445f7a 100644 --- a/middlewares/reqValidators/videos.js +++ b/middlewares/reqValidators/videos.js @@ -2,8 +2,8 @@ 'use strict' var checkErrors = require('./utils').checkErrors - var VideosDB = require('../../src/database').VideosDB - var logger = require('../../src/logger') + var VideosDB = require('../../initializers/database').VideosDB + var logger = require('../../helpers/logger') var videos = {} diff --git a/models/pods.js b/models/pods.js new file mode 100644 index 000000000..c8d08b26f --- /dev/null +++ b/models/pods.js @@ -0,0 +1,274 @@ +;(function () { + 'use strict' + + var async = require('async') + var config = require('config') + var fs = require('fs') + var request = require('request') + + var constants = require('../initializers/constants') + var logger = require('../helpers/logger') + var PodsDB = require('../initializers/database').PodsDB + var poolRequests = require('../lib/poolRequests') + var utils = require('../helpers/utils') + + var pods = {} + + var http = config.get('webserver.https') ? 'https' : 'http' + var host = config.get('webserver.host') + var port = config.get('webserver.port') + + // ----------- Private functions ----------- + + function getForeignPodsList (url, callback) { + var path = '/api/' + constants.API_VERSION + '/pods' + + request.get(url + path, function (err, response, body) { + if (err) throw err + callback(JSON.parse(body)) + }) + } + + // ----------- Public functions ----------- + + pods.list = function (callback) { + PodsDB.find(function (err, pods_list) { + if (err) { + logger.error('Cannot get the list of the pods.', { error: err }) + return callback(err) + } + + return callback(null, pods_list) + }) + } + + // { url } + // TODO: check if the pod is not already a friend + pods.add = function (data, callback) { + var videos = require('./videos') + logger.info('Adding pod: %s', data.url) + + var params = { + url: data.url, + publicKey: data.publicKey, + score: constants.FRIEND_BASE_SCORE + } + + PodsDB.create(params, function (err, pod) { + if (err) { + logger.error('Cannot insert the pod.', { error: err }) + return callback(err) + } + + videos.addRemotes(data.videos) + + fs.readFile(utils.certDir + 'peertube.pub', 'utf8', function (err, cert) { + if (err) { + logger.error('Cannot read cert file.', { error: err }) + return callback(err) + } + + videos.listOwned(function (err, videos_list) { + if (err) { + logger.error('Cannot get the list of owned videos.', { error: err }) + return callback(err) + } + + return callback(null, { cert: cert, videos: videos_list }) + }) + }) + }) + } + + pods.remove = function (url, callback) { + var videos = require('./videos') + logger.info('Removing %s pod.', url) + + videos.removeAllRemotesOf(url, function (err) { + if (err) logger.error('Cannot remove all remote videos of %s.', url) + + PodsDB.remove({ url: url }, function (err) { + if (err) return callback(err) + + logger.info('%s pod removed.', url) + callback(null) + }) + }) + } + + pods.addVideoToFriends = function (video) { + // To avoid duplicates + var id = video.name + video.magnetUri + poolRequests.addToPoolRequests(id, 'add', video) + } + + pods.removeVideoToFriends = function (video) { + // To avoid duplicates + var id = video.name + video.magnetUri + poolRequests.addToPoolRequests(id, 'remove', video) + } + + pods.makeFriends = function (callback) { + var videos = require('./videos') + var pods_score = {} + + logger.info('Make friends!') + fs.readFile(utils.certDir + 'peertube.pub', 'utf8', function (err, cert) { + if (err) { + logger.error('Cannot read public cert.', { error: err }) + return callback(err) + } + + var urls = config.get('network.friends') + + async.each(urls, computeForeignPodsList, function () { + logger.debug('Pods scores computed.', { pods_score: pods_score }) + var pods_list = computeWinningPods(urls, pods_score) + logger.debug('Pods that we keep computed.', { pods_to_keep: pods_list }) + + makeRequestsToWinningPods(cert, pods_list) + }) + }) + + // ----------------------------------------------------------------------- + + function computeForeignPodsList (url, callback) { + // Let's give 1 point to the pod we ask the friends list + pods_score[url] = 1 + + getForeignPodsList(url, function (foreign_pods_list) { + if (foreign_pods_list.length === 0) return callback() + + async.each(foreign_pods_list, function (foreign_pod, callback_each) { + var foreign_url = foreign_pod.url + + if (pods_score[foreign_url]) pods_score[foreign_url]++ + else pods_score[foreign_url] = 1 + + callback_each() + }, function () { + callback() + }) + }) + } + + function computeWinningPods (urls, pods_score) { + // Build the list of pods to add + // Only add a pod if it exists in more than a half base pods + var pods_list = [] + var base_score = urls.length / 2 + Object.keys(pods_score).forEach(function (pod) { + if (pods_score[pod] > base_score) pods_list.push({ url: pod }) + }) + + return pods_list + } + + function makeRequestsToWinningPods (cert, pods_list) { + // Stop pool requests + poolRequests.deactivate() + // Flush pool requests + poolRequests.forceSend() + + // Get the list of our videos to send to our new friends + videos.listOwned(function (err, videos_list) { + if (err) throw err + + var data = { + url: http + '://' + host + ':' + port, + publicKey: cert, + videos: videos_list + } + + utils.makeMultipleRetryRequest( + { method: 'POST', path: '/api/' + constants.API_VERSION + '/pods/', data: data }, + + pods_list, + + function eachRequest (err, response, body, url, pod, callback_each_request) { + // We add the pod if it responded correctly with its public certificate + if (!err && response.statusCode === 200) { + pods.add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err) { + if (err) logger.error('Error with adding %s pod.', pod.url, { error: err }) + + videos.addRemotes(body.videos, function (err) { + if (err) logger.error('Error with adding videos of pod.', pod.url, { error: err }) + + logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos }) + return callback_each_request() + }) + }) + } else { + logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') }) + return callback_each_request() + } + }, + + function endRequests (err) { + // Now we made new friends, we can re activate the pool of requests + poolRequests.activate() + + if (err) { + logger.error('There was some errors when we wanted to make friends.', { error: err }) + return callback(err) + } + + logger.debug('makeRequestsToWinningPods finished.') + return callback(null) + } + ) + }) + } + } + + pods.quitFriends = function (callback) { + // Stop pool requests + poolRequests.deactivate() + // Flush pool requests + poolRequests.forceSend() + + PodsDB.find(function (err, pods) { + if (err) return callback(err) + + var request = { + method: 'POST', + path: '/api/' + constants.API_VERSION + '/pods/remove', + sign: true, + encrypt: true, + data: { + url: 'me' // Fake data + } + } + + // Announce we quit them + utils.makeMultipleRetryRequest(request, pods, function () { + PodsDB.remove(function (err) { + poolRequests.activate() + + if (err) return callback(err) + + logger.info('Broke friends, so sad :(') + + var videos = require('./videos') + videos.removeAllRemotes(function (err) { + if (err) return callback(err) + + logger.info('Removed all remote videos.') + callback(null) + }) + }) + }) + }) + } + + pods.hasFriends = function (callback) { + PodsDB.count(function (err, count) { + if (err) return callback(err) + + var has_friends = (count !== 0) + callback(null, has_friends) + }) + } + + module.exports = pods +})() diff --git a/models/videos.js b/models/videos.js new file mode 100644 index 000000000..626c55819 --- /dev/null +++ b/models/videos.js @@ -0,0 +1,272 @@ +;(function () { + 'use strict' + + var async = require('async') + var config = require('config') + var dz = require('dezalgo') + var fs = require('fs') + var webtorrent = require('../lib/webTorrentNode') + + var logger = require('../helpers/logger') + var pods = require('./pods') + var VideosDB = require('../initializers/database').VideosDB + + var videos = {} + + var http = config.get('webserver.https') === true ? 'https' : 'http' + var host = config.get('webserver.host') + var port = config.get('webserver.port') + + // ----------- Private functions ----------- + function seedVideo (path, callback) { + logger.info('Seeding %s...', path) + + webtorrent.seed(path, function (torrent) { + logger.info('%s seeded (%s).', path, torrent.magnetURI) + + return callback(null, torrent) + }) + } + + // ----------- Public attributes ---------- + videos.uploadDir = __dirname + '/../' + config.get('storage.uploads') + + // ----------- Public functions ----------- + videos.list = function (callback) { + VideosDB.find(function (err, videos_list) { + if (err) { + logger.error('Cannot get list of the videos.', { error: err }) + return callback(err) + } + + return callback(null, videos_list) + }) + } + + videos.listOwned = function (callback) { + // If namePath is not null this is *our* video + VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) { + if (err) { + logger.error('Cannot get list of the videos.', { error: err }) + return callback(err) + } + + return callback(null, videos_list) + }) + } + + videos.add = function (data, callback) { + var video_file = data.video + var video_data = data.data + + logger.info('Adding %s video.', video_file.path) + seedVideo(video_file.path, function (err, torrent) { + if (err) { + logger.error('Cannot seed this video.', { error: err }) + return callback(err) + } + + var params = { + name: video_data.name, + namePath: video_file.filename, + description: video_data.description, + magnetUri: torrent.magnetURI, + podUrl: http + '://' + host + ':' + port + } + + VideosDB.create(params, function (err, video) { + if (err) { + logger.error('Cannot insert this video.', { error: err }) + return callback(err) + } + + // Now we'll add the video's meta data to our friends + params.namePath = null + + pods.addVideoToFriends(params) + callback(null) + }) + }) + } + + videos.remove = function (id, callback) { + // Maybe the torrent is not seeded, but we catch the error to don't stop the removing process + function removeTorrent (magnetUri, callback) { + try { + webtorrent.remove(magnetUri, callback) + } catch (err) { + logger.warn('Cannot remove the torrent from WebTorrent', { err: err }) + return callback(null) + } + } + + VideosDB.findById(id, function (err, video) { + if (err || !video) { + if (!err) err = new Error('Cannot find this video.') + logger.error('Cannot find this video.', { error: err }) + return callback(err) + } + + if (video.namePath === null) { + var error_string = 'Cannot remove the video of another pod.' + logger.error(error_string) + return callback(new Error(error_string)) + } + + logger.info('Removing %s video', video.name) + + removeTorrent(video.magnetUri, function () { + VideosDB.findByIdAndRemove(id, function (err) { + if (err) { + logger.error('Cannot remove the torrent.', { error: err }) + return callback(err) + } + + fs.unlink(videos.uploadDir + video.namePath, function (err) { + if (err) { + logger.error('Cannot remove this video file.', { error: err }) + return callback(err) + } + + var params = { + name: video.name, + magnetUri: video.magnetUri + } + + pods.removeVideoToFriends(params) + callback(null) + }) + }) + }) + }) + } + + // Use the magnet Uri because the _id field is not the same on different servers + videos.removeRemotes = function (fromUrl, magnetUris, callback) { + if (callback === undefined) callback = function () {} + + VideosDB.find({ magnetUri: { $in: magnetUris } }, function (err, videos) { + if (err || !videos) { + logger.error('Cannot find the torrent URI of these remote videos.') + return callback(err) + } + + var to_remove = [] + async.each(videos, function (video, callback_async) { + callback_async = dz(callback_async) + + if (video.podUrl !== fromUrl) { + logger.error('The pod %s has not the rights on the video of %s.', fromUrl, video.podUrl) + } else { + to_remove.push(video._id) + } + + callback_async() + }, function () { + VideosDB.remove({ _id: { $in: to_remove } }, function (err) { + if (err) { + logger.error('Cannot remove the remote videos.') + return callback(err) + } + + logger.info('Removed remote videos from %s.', fromUrl) + callback(null) + }) + }) + }) + } + + videos.removeAllRemotes = function (callback) { + VideosDB.remove({ namePath: null }, function (err) { + if (err) return callback(err) + + callback(null) + }) + } + + videos.removeAllRemotesOf = function (fromUrl, callback) { + VideosDB.remove({ podUrl: fromUrl }, function (err) { + if (err) return callback(err) + + callback(null) + }) + } + + // { name, magnetUri, podUrl } + // TODO: avoid doublons + videos.addRemotes = function (videos, callback) { + if (callback === undefined) callback = function () {} + + var to_add = [] + + async.each(videos, function (video, callback_each) { + callback_each = dz(callback_each) + logger.debug('Add remote video from pod: %s', video.podUrl) + + var params = { + name: video.name, + namePath: null, + description: video.description, + magnetUri: video.magnetUri, + podUrl: video.podUrl + } + + to_add.push(params) + + callback_each() + }, function () { + VideosDB.create(to_add, function (err, videos) { + if (err) { + logger.error('Cannot insert this remote video.', { error: err }) + return callback(err) + } + + return callback(null, videos) + }) + }) + } + + videos.get = function (id, callback) { + VideosDB.findById(id, function (err, video) { + if (err) { + logger.error('Cannot get this video.', { error: err }) + return callback(err) + } + + return callback(null, video) + }) + } + + videos.search = function (name, callback) { + VideosDB.find({ name: new RegExp(name) }, function (err, videos) { + if (err) { + logger.error('Cannot search the videos.', { error: err }) + return callback(err) + } + + return callback(null, videos) + }) + } + + videos.seedAll = function (callback) { + VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) { + if (err) { + logger.error('Cannot get list of the videos to seed.', { error: err }) + return callback(err) + } + + async.each(videos_list, function (video, each_callback) { + seedVideo(videos.uploadDir + video.namePath, function (err) { + if (err) { + logger.error('Cannot seed this video.', { error: err }) + return callback(err) + } + + each_callback(null) + }) + }, callback) + }) + } + + module.exports = videos +})() diff --git a/package.json b/package.json index f04d97423..cd6e36369 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ }, "scripts": { "start": "grunt dev", - "test": "grunt build && standard && mocha test" + "test": "grunt build && standard && mocha tests" }, "dependencies": { "async": "^1.2.1", diff --git a/routes/api/v1/index.js b/routes/api/v1/index.js deleted file mode 100644 index f5504ad85..000000000 --- a/routes/api/v1/index.js +++ /dev/null @@ -1,12 +0,0 @@ -;(function () { - 'use strict' - - var express = require('express') - var router = express.Router() - - router.use('/videos', require('./videos')) - router.use('/remotevideos', require('./remoteVideos')) - router.use('/pods', require('./pods')) - - module.exports = router -})() diff --git a/routes/api/v1/pods.js b/routes/api/v1/pods.js deleted file mode 100644 index 2430b0d7e..000000000 --- a/routes/api/v1/pods.js +++ /dev/null @@ -1,69 +0,0 @@ -;(function () { - 'use strict' - - var express = require('express') - var router = express.Router() - var middleware = require('../../../middlewares') - var miscMiddleware = middleware.misc - var reqValidator = middleware.reqValidators.pods - var secureRequest = middleware.reqValidators.remote.secureRequest - var pods = require('../../../src/pods') - - function listPods (req, res, next) { - pods.list(function (err, pods_list) { - if (err) return next(err) - - res.json(pods_list) - }) - } - - function addPods (req, res, next) { - pods.add(req.body.data, function (err, json) { - if (err) return next(err) - - res.json(json) - }) - } - - function removePods (req, res, next) { - pods.remove(req.body.signature.url, function (err) { - if (err) return next(err) - - res.sendStatus(204) - }) - } - - function makeFriends (req, res, next) { - pods.hasFriends(function (err, has_friends) { - if (err) return next(err) - - if (has_friends === true) { - // We need to quit our friends before make new ones - res.sendStatus(409) - } else { - pods.makeFriends(function (err) { - if (err) return next(err) - - res.sendStatus(204) - }) - } - }) - } - - function quitFriends (req, res, next) { - pods.quitFriends(function (err) { - if (err) return next(err) - - res.sendStatus(204) - }) - } - - router.get('/', miscMiddleware.cache(false), listPods) - router.get('/makefriends', miscMiddleware.cache(false), makeFriends) - router.get('/quitfriends', miscMiddleware.cache(false), quitFriends) - router.post('/', reqValidator.podsAdd, miscMiddleware.cache(false), addPods) - // Post because this is a secured request - router.post('/remove', secureRequest, miscMiddleware.decryptBody, removePods) - - module.exports = router -})() diff --git a/routes/api/v1/remoteVideos.js b/routes/api/v1/remoteVideos.js deleted file mode 100644 index 6ba6ce17b..000000000 --- a/routes/api/v1/remoteVideos.js +++ /dev/null @@ -1,33 +0,0 @@ -;(function () { - 'use strict' - - var express = require('express') - var router = express.Router() - var pluck = require('lodash-node/compat/collection/pluck') - - var middleware = require('../../../middlewares') - var miscMiddleware = middleware.misc - var reqValidator = middleware.reqValidators.remote - var videos = require('../../../src/videos') - - function addRemoteVideos (req, res, next) { - videos.addRemotes(req.body.data, function (err, videos) { - if (err) return next(err) - - res.json(videos) - }) - } - - function removeRemoteVideo (req, res, next) { - videos.removeRemotes(req.body.signature.url, pluck(req.body.data, 'magnetUri'), function (err) { - if (err) return next(err) - - res.sendStatus(204) - }) - } - - router.post('/add', reqValidator.secureRequest, miscMiddleware.decryptBody, reqValidator.remoteVideosAdd, miscMiddleware.cache(false), addRemoteVideos) - router.post('/remove', reqValidator.secureRequest, miscMiddleware.decryptBody, reqValidator.remoteVideosRemove, miscMiddleware.cache(false), removeRemoteVideo) - - module.exports = router -})() diff --git a/routes/api/v1/videos.js b/routes/api/v1/videos.js deleted file mode 100644 index 95c1dffc7..000000000 --- a/routes/api/v1/videos.js +++ /dev/null @@ -1,88 +0,0 @@ -;(function () { - 'use strict' - - var express = require('express') - var config = require('config') - var crypto = require('crypto') - var multer = require('multer') - var router = express.Router() - - var middleware = require('../../../middlewares') - var miscMiddleware = middleware.misc - var reqValidator = middleware.reqValidators.videos - var videos = require('../../../src/videos') - - var uploads = config.get('storage.uploads') - - function listVideos (req, res, next) { - videos.list(function (err, videos_list) { - if (err) return next(err) - - res.json(videos_list) - }) - } - - function searchVideos (req, res, next) { - videos.search(req.params.name, function (err, videos_list) { - if (err) return next(err) - - res.json(videos_list) - }) - } - - function addVideos (req, res, next) { - videos.add({ video: req.files.input_video[0], data: req.body }, function (err) { - if (err) return next(err) - - // TODO : include Location of the new video - res.sendStatus(201) - }) - } - - function getVideos (req, res, next) { - videos.get(req.params.id, function (err, video) { - if (err) return next(err) - - if (video === null) { - return res.sendStatus(404) - } - - res.json(video) - }) - } - - function removeVideo (req, res, next) { - videos.remove(req.params.id, function (err) { - if (err) return next(err) - - res.sendStatus(204) - }) - } - - // multer configuration - var storage = multer.diskStorage({ - destination: function (req, file, cb) { - cb(null, uploads) - }, - - filename: function (req, file, cb) { - var extension = '' - if (file.mimetype === 'video/webm') extension = 'webm' - else if (file.mimetype === 'video/mp4') extension = 'mp4' - else if (file.mimetype === 'video/ogg') extension = 'ogv' - crypto.pseudoRandomBytes(16, function (err, raw) { - var fieldname = err ? undefined : raw.toString('hex') - cb(null, fieldname + '.' + extension) - }) - } - }) - var reqFiles = multer({ storage: storage }).fields([{ name: 'input_video', maxCount: 1 }]) - - router.get('/', miscMiddleware.cache(false), listVideos) - router.post('/', reqFiles, reqValidator.videosAdd, miscMiddleware.cache(false), addVideos) - router.get('/search/:name', reqValidator.videosSearch, miscMiddleware.cache(false), searchVideos) - router.get('/:id', reqValidator.videosGet, miscMiddleware.cache(false), getVideos) - router.delete('/:id', reqValidator.videosRemove, miscMiddleware.cache(false), removeVideo) - - module.exports = router -})() diff --git a/routes/index.js b/routes/index.js deleted file mode 100644 index f45aa7ba5..000000000 --- a/routes/index.js +++ /dev/null @@ -1,12 +0,0 @@ -;(function () { - 'use strict' - - var constants = require('../src/constants') - - var routes = { - api: require('./api/' + constants.API_VERSION), - views: require('./views') - } - - module.exports = routes -})() diff --git a/routes/views.js b/routes/views.js deleted file mode 100644 index ebd97380e..000000000 --- a/routes/views.js +++ /dev/null @@ -1,24 +0,0 @@ -;(function () { - 'use strict' - - function getPartial (req, res) { - var directory = req.params.directory - var name = req.params.name - - res.render('partials/' + directory + '/' + name) - } - - function getIndex (req, res) { - res.render('index') - } - - var express = require('express') - var middleware = require('../middlewares').misc - - var router = express.Router() - - router.get('/partials/:directory/:name', middleware.cache(), getPartial) - router.get(/^\/(index)?$/, middleware.cache(), getIndex) - - module.exports = router -})() diff --git a/server.js b/server.js index ad57649b2..1e0222f4f 100644 --- a/server.js +++ b/server.js @@ -15,7 +15,7 @@ var app = express() // ----------- Checker ----------- - var checker = require('./src/checker') + var checker = require('./initializers/checker') var miss = checker.checkConfig() if (miss.length !== 0) { @@ -28,14 +28,14 @@ // ----------- PeerTube modules ----------- var config = require('config') - var constants = require('./src/constants') - var customValidators = require('./src/customValidators') - var logger = require('./src/logger') - var poolRequests = require('./src/poolRequests') - var routes = require('./routes') - var utils = require('./src/utils') - var videos = require('./src/videos') - var webtorrent = require('./src/webTorrentNode') + var constants = require('./initializers/constants') + var customValidators = require('./helpers/customValidators') + var logger = require('./helpers/logger') + var poolRequests = require('./lib/poolRequests') + var routes = require('./controllers') + var utils = require('./helpers/utils') + var videos = require('./models/videos') + var webtorrent = require('./lib/webTorrentNode') // Get configurations var port = config.get('listen.port') diff --git a/src/checker.js b/src/checker.js deleted file mode 100644 index 7a3a53616..000000000 --- a/src/checker.js +++ /dev/null @@ -1,45 +0,0 @@ -;(function () { - 'use strict' - - var config = require('config') - var mkdirp = require('mkdirp') - - var checker = {} - - // Check the config files - checker.checkConfig = function () { - var required = [ 'listen.port', - 'webserver.https', 'webserver.host', 'webserver.port', - 'database.host', 'database.port', 'database.suffix', - 'storage.certs', 'storage.uploads', 'storage.logs', - 'network.friends' ] - var miss = [] - - for (var key of required) { - if (!config.has(key)) { - miss.push(key) - } - } - - return miss - } - - // Create directories for the storage if it doesn't exist - checker.createDirectoriesIfNotExist = function () { - var storages = config.get('storage') - - for (var key of Object.keys(storages)) { - var path = storages[key] - try { - mkdirp.sync(__dirname + '/../' + path) - } catch (error) { - // Do not use logger - console.error('Cannot create ' + path + ':' + error) - process.exit(0) - } - } - } - - // ----------- Export ----------- - module.exports = checker -})() diff --git a/src/constants.js b/src/constants.js deleted file mode 100644 index 00b713961..000000000 --- a/src/constants.js +++ /dev/null @@ -1,37 +0,0 @@ -;(function () { - 'use strict' - - var constants = {} - - function isTestInstance () { - return (process.env.NODE_ENV === 'test') - } - - // API version of our pod - constants.API_VERSION = 'v1' - - // Score a pod has when we create it as a friend - constants.FRIEND_BASE_SCORE = 100 - - // Time to wait between requests to the friends - constants.INTERVAL = 60000 - - // Number of points we add/remove from a friend after a successful/bad request - constants.PODS_SCORE = { - MALUS: -10, - BONUS: 10 - } - - // Number of retries we make for the make retry requests (to friends...) - constants.REQUEST_RETRIES = 10 - - // Special constants for a test instance - if (isTestInstance() === true) { - constants.FRIEND_BASE_SCORE = 20 - constants.INTERVAL = 10000 - constants.REQUEST_RETRIES = 2 - } - - // ----------- Export ----------- - module.exports = constants -})() diff --git a/src/customValidators.js b/src/customValidators.js deleted file mode 100644 index 73c2f8461..000000000 --- a/src/customValidators.js +++ /dev/null @@ -1,29 +0,0 @@ -;(function () { - 'use strict' - - var validator = require('validator') - - var customValidators = {} - - customValidators.eachIsRemoteVideosAddValid = function (values) { - return values.every(function (val) { - return validator.isLength(val.name, 1, 50) && - validator.isLength(val.description, 1, 50) && - validator.isLength(val.magnetUri, 10) && - validator.isURL(val.podUrl) - }) - } - - customValidators.eachIsRemoteVideosRemoveValid = function (values) { - return values.every(function (val) { - return validator.isLength(val.magnetUri, 10) - }) - } - - customValidators.isArray = function (value) { - return Array.isArray(value) - } - - // ----------- Export ----------- - module.exports = customValidators -})() diff --git a/src/database.js b/src/database.js deleted file mode 100644 index e03176b31..000000000 --- a/src/database.js +++ /dev/null @@ -1,61 +0,0 @@ -;(function () { - 'use strict' - - var config = require('config') - var mongoose = require('mongoose') - - var constants = require('./constants') - var logger = require('./logger') - - var dbname = 'peertube' + config.get('database.suffix') - var host = config.get('database.host') - var port = config.get('database.port') - - // ----------- Videos ----------- - var videosSchema = mongoose.Schema({ - name: String, - namePath: String, - description: String, - magnetUri: String, - podUrl: String - }) - - var VideosDB = mongoose.model('videos', videosSchema) - - // ----------- Pods ----------- - var podsSchema = mongoose.Schema({ - url: String, - publicKey: String, - score: { type: Number, max: constants.FRIEND_BASE_SCORE } - }) - - var PodsDB = mongoose.model('pods', podsSchema) - - // ----------- PoolRequests ----------- - var poolRequestsSchema = mongoose.Schema({ - type: String, - id: String, // Special id to find duplicates (video created we want to remove...) - request: mongoose.Schema.Types.Mixed - }) - - var PoolRequestsDB = mongoose.model('poolRequests', poolRequestsSchema) - - // ----------- Connection ----------- - - mongoose.connect('mongodb://' + host + ':' + port + '/' + dbname) - mongoose.connection.on('error', function () { - logger.error('Mongodb connection error.') - process.exit(0) - }) - - mongoose.connection.on('open', function () { - logger.info('Connected to mongodb.') - }) - - // ----------- Export ----------- - module.exports = { - VideosDB: VideosDB, - PodsDB: PodsDB, - PoolRequestsDB: PoolRequestsDB - } -})() diff --git a/src/logger.js b/src/logger.js deleted file mode 100644 index 850af10cb..000000000 --- a/src/logger.js +++ /dev/null @@ -1,40 +0,0 @@ -;(function () { - // Thanks http://tostring.it/2014/06/23/advanced-logging-with-nodejs/ - 'use strict' - - var config = require('config') - var winston = require('winston') - - var logDir = __dirname + '/../' + config.get('storage.logs') - - winston.emitErrs = true - - var logger = new winston.Logger({ - transports: [ - new winston.transports.File({ - level: 'debug', - filename: logDir + '/all-logs.log', - handleExceptions: true, - json: true, - maxsize: 5242880, - maxFiles: 5, - colorize: false - }), - new winston.transports.Console({ - level: 'debug', - handleExceptions: true, - humanReadableUnhandledException: true, - json: false, - colorize: true - }) - ], - exitOnError: true - }) - - module.exports = logger - module.exports.stream = { - write: function (message, encoding) { - logger.info(message) - } - } -})() diff --git a/src/pods.js b/src/pods.js deleted file mode 100644 index defa9b1c1..000000000 --- a/src/pods.js +++ /dev/null @@ -1,274 +0,0 @@ -;(function () { - 'use strict' - - var async = require('async') - var config = require('config') - var fs = require('fs') - var request = require('request') - - var constants = require('./constants') - var logger = require('./logger') - var PodsDB = require('./database').PodsDB - var poolRequests = require('./poolRequests') - var utils = require('./utils') - - var pods = {} - - var http = config.get('webserver.https') ? 'https' : 'http' - var host = config.get('webserver.host') - var port = config.get('webserver.port') - - // ----------- Private functions ----------- - - function getForeignPodsList (url, callback) { - var path = '/api/' + constants.API_VERSION + '/pods' - - request.get(url + path, function (err, response, body) { - if (err) throw err - callback(JSON.parse(body)) - }) - } - - // ----------- Public functions ----------- - - pods.list = function (callback) { - PodsDB.find(function (err, pods_list) { - if (err) { - logger.error('Cannot get the list of the pods.', { error: err }) - return callback(err) - } - - return callback(null, pods_list) - }) - } - - // { url } - // TODO: check if the pod is not already a friend - pods.add = function (data, callback) { - var videos = require('./videos') - logger.info('Adding pod: %s', data.url) - - var params = { - url: data.url, - publicKey: data.publicKey, - score: constants.FRIEND_BASE_SCORE - } - - PodsDB.create(params, function (err, pod) { - if (err) { - logger.error('Cannot insert the pod.', { error: err }) - return callback(err) - } - - videos.addRemotes(data.videos) - - fs.readFile(utils.certDir + 'peertube.pub', 'utf8', function (err, cert) { - if (err) { - logger.error('Cannot read cert file.', { error: err }) - return callback(err) - } - - videos.listOwned(function (err, videos_list) { - if (err) { - logger.error('Cannot get the list of owned videos.', { error: err }) - return callback(err) - } - - return callback(null, { cert: cert, videos: videos_list }) - }) - }) - }) - } - - pods.remove = function (url, callback) { - var videos = require('./videos') - logger.info('Removing %s pod.', url) - - videos.removeAllRemotesOf(url, function (err) { - if (err) logger.error('Cannot remove all remote videos of %s.', url) - - PodsDB.remove({ url: url }, function (err) { - if (err) return callback(err) - - logger.info('%s pod removed.', url) - callback(null) - }) - }) - } - - pods.addVideoToFriends = function (video) { - // To avoid duplicates - var id = video.name + video.magnetUri - poolRequests.addToPoolRequests(id, 'add', video) - } - - pods.removeVideoToFriends = function (video) { - // To avoid duplicates - var id = video.name + video.magnetUri - poolRequests.addToPoolRequests(id, 'remove', video) - } - - pods.makeFriends = function (callback) { - var videos = require('./videos') - var pods_score = {} - - logger.info('Make friends!') - fs.readFile(utils.certDir + 'peertube.pub', 'utf8', function (err, cert) { - if (err) { - logger.error('Cannot read public cert.', { error: err }) - return callback(err) - } - - var urls = config.get('network.friends') - - async.each(urls, computeForeignPodsList, function () { - logger.debug('Pods scores computed.', { pods_score: pods_score }) - var pods_list = computeWinningPods(urls, pods_score) - logger.debug('Pods that we keep computed.', { pods_to_keep: pods_list }) - - makeRequestsToWinningPods(cert, pods_list) - }) - }) - - // ----------------------------------------------------------------------- - - function computeForeignPodsList (url, callback) { - // Let's give 1 point to the pod we ask the friends list - pods_score[url] = 1 - - getForeignPodsList(url, function (foreign_pods_list) { - if (foreign_pods_list.length === 0) return callback() - - async.each(foreign_pods_list, function (foreign_pod, callback_each) { - var foreign_url = foreign_pod.url - - if (pods_score[foreign_url]) pods_score[foreign_url]++ - else pods_score[foreign_url] = 1 - - callback_each() - }, function () { - callback() - }) - }) - } - - function computeWinningPods (urls, pods_score) { - // Build the list of pods to add - // Only add a pod if it exists in more than a half base pods - var pods_list = [] - var base_score = urls.length / 2 - Object.keys(pods_score).forEach(function (pod) { - if (pods_score[pod] > base_score) pods_list.push({ url: pod }) - }) - - return pods_list - } - - function makeRequestsToWinningPods (cert, pods_list) { - // Stop pool requests - poolRequests.deactivate() - // Flush pool requests - poolRequests.forceSend() - - // Get the list of our videos to send to our new friends - videos.listOwned(function (err, videos_list) { - if (err) throw err - - var data = { - url: http + '://' + host + ':' + port, - publicKey: cert, - videos: videos_list - } - - utils.makeMultipleRetryRequest( - { method: 'POST', path: '/api/' + constants.API_VERSION + '/pods/', data: data }, - - pods_list, - - function eachRequest (err, response, body, url, pod, callback_each_request) { - // We add the pod if it responded correctly with its public certificate - if (!err && response.statusCode === 200) { - pods.add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err) { - if (err) logger.error('Error with adding %s pod.', pod.url, { error: err }) - - videos.addRemotes(body.videos, function (err) { - if (err) logger.error('Error with adding videos of pod.', pod.url, { error: err }) - - logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos }) - return callback_each_request() - }) - }) - } else { - logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') }) - return callback_each_request() - } - }, - - function endRequests (err) { - // Now we made new friends, we can re activate the pool of requests - poolRequests.activate() - - if (err) { - logger.error('There was some errors when we wanted to make friends.', { error: err }) - return callback(err) - } - - logger.debug('makeRequestsToWinningPods finished.') - return callback(null) - } - ) - }) - } - } - - pods.quitFriends = function (callback) { - // Stop pool requests - poolRequests.deactivate() - // Flush pool requests - poolRequests.forceSend() - - PodsDB.find(function (err, pods) { - if (err) return callback(err) - - var request = { - method: 'POST', - path: '/api/' + constants.API_VERSION + '/pods/remove', - sign: true, - encrypt: true, - data: { - url: 'me' // Fake data - } - } - - // Announce we quit them - utils.makeMultipleRetryRequest(request, pods, function () { - PodsDB.remove(function (err) { - poolRequests.activate() - - if (err) return callback(err) - - logger.info('Broke friends, so sad :(') - - var videos = require('./videos') - videos.removeAllRemotes(function (err) { - if (err) return callback(err) - - logger.info('Removed all remote videos.') - callback(null) - }) - }) - }) - }) - } - - pods.hasFriends = function (callback) { - PodsDB.count(function (err, count) { - if (err) return callback(err) - - var has_friends = (count !== 0) - callback(null, has_friends) - }) - } - - module.exports = pods -})() diff --git a/src/poolRequests.js b/src/poolRequests.js deleted file mode 100644 index 7f422f372..000000000 --- a/src/poolRequests.js +++ /dev/null @@ -1,206 +0,0 @@ -;(function () { - 'use strict' - - var async = require('async') - - var constants = require('./constants') - var logger = require('./logger') - var database = require('./database') - var pluck = require('lodash-node/compat/collection/pluck') - var PoolRequestsDB = database.PoolRequestsDB - var PodsDB = database.PodsDB - var utils = require('./utils') - var VideosDB = database.VideosDB - - var poolRequests = {} - - // ----------- Private ----------- - var timer = null - - function removePoolRequestsFromDB (ids) { - PoolRequestsDB.remove({ _id: { $in: ids } }, function (err) { - if (err) { - logger.error('Cannot remove requests from the pool requests database.', { error: err }) - return - } - - logger.info('Pool requests flushed.') - }) - } - - function makePoolRequests () { - logger.info('Making pool requests to friends.') - - PoolRequestsDB.find({}, { _id: 1, type: 1, request: 1 }, function (err, pool_requests) { - if (err) throw err - - if (pool_requests.length === 0) return - - var requests = { - add: { - ids: [], - requests: [] - }, - remove: { - ids: [], - requests: [] - } - } - - async.each(pool_requests, function (pool_request, callback_each) { - if (pool_request.type === 'add') { - requests.add.requests.push(pool_request.request) - requests.add.ids.push(pool_request._id) - } else if (pool_request.type === 'remove') { - requests.remove.requests.push(pool_request.request) - requests.remove.ids.push(pool_request._id) - } else { - throw new Error('Unkown pool request type.') - } - - callback_each() - }, function () { - // Send the add requests - if (requests.add.requests.length !== 0) { - makePoolRequest('add', requests.add.requests, function (err) { - if (err) logger.error('Errors when sent add pool requests.', { error: err }) - - removePoolRequestsFromDB(requests.add.ids) - }) - } - - // Send the remove requests - if (requests.remove.requests.length !== 0) { - makePoolRequest('remove', requests.remove.requests, function (err) { - if (err) logger.error('Errors when sent remove pool requests.', { error: err }) - - removePoolRequestsFromDB(requests.remove.ids) - }) - } - }) - }) - } - - function updatePodsScore (good_pods, bad_pods) { - logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length) - - PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: constants.PODS_SCORE.BONUS } }, { multi: true }).exec() - PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: constants.PODS_SCORE.MALUS } }, { multi: true }, function (err) { - if (err) throw err - removeBadPods() - }) - } - - function removeBadPods () { - PodsDB.find({ score: 0 }, { _id: 1, url: 1 }, function (err, pods) { - if (err) throw err - - if (pods.length === 0) return - - var urls = pluck(pods, 'url') - var ids = pluck(pods, '_id') - - VideosDB.remove({ podUrl: { $in: urls } }, function (err, r) { - if (err) logger.error('Cannot remove videos from a pod that we removing.', { error: err }) - var videos_removed = r.result.n - logger.info('Removed %d videos.', videos_removed) - - PodsDB.remove({ _id: { $in: ids } }, function (err, r) { - if (err) logger.error('Cannot remove bad pods.', { error: err }) - - var pods_removed = r.result.n - logger.info('Removed %d pods.', pods_removed) - }) - }) - }) - } - - function makePoolRequest (type, requests, callback) { - if (!callback) callback = function () {} - - PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) { - if (err) throw err - - var params = { - encrypt: true, - sign: true, - method: 'POST', - path: null, - data: requests - } - - if (type === 'add') { - params.path = '/api/' + constants.API_VERSION + '/remotevideos/add' - } else if (type === 'remove') { - params.path = '/api/' + constants.API_VERSION + '/remotevideos/remove' - } else { - throw new Error('Unkown pool request type.') - } - - var bad_pods = [] - var good_pods = [] - - utils.makeMultipleRetryRequest(params, pods, callbackEachPodFinished, callbackAllPodsFinished) - - function callbackEachPodFinished (err, response, body, url, pod, callback_each_pod_finished) { - if (err || (response.statusCode !== 200 && response.statusCode !== 204)) { - bad_pods.push(pod._id) - logger.error('Error sending secure request to %s pod.', url, { error: err || new Error('Status code not 20x') }) - } else { - good_pods.push(pod._id) - } - - return callback_each_pod_finished() - } - - function callbackAllPodsFinished (err) { - if (err) return callback(err) - - updatePodsScore(good_pods, bad_pods) - callback(null) - } - }) - } - - // ----------- Public ----------- - poolRequests.activate = function () { - logger.info('Pool requests activated.') - timer = setInterval(makePoolRequests, constants.INTERVAL) - } - - poolRequests.addToPoolRequests = function (id, type, request) { - logger.debug('Add request to the pool requests.', { id: id, type: type, request: request }) - - PoolRequestsDB.findOne({ id: id }, function (err, entity) { - if (err) logger.error(err) - - if (entity) { - if (entity.type === type) { - logger.error(new Error('Cannot insert two same requests.')) - return - } - - // Remove the request of the other type - PoolRequestsDB.remove({ id: id }, function (err) { - if (err) logger.error(err) - }) - } else { - PoolRequestsDB.create({ id: id, type: type, request: request }, function (err) { - if (err) logger.error(err) - }) - } - }) - } - - poolRequests.deactivate = function () { - logger.info('Pool requests deactivated.') - clearInterval(timer) - } - - poolRequests.forceSend = function () { - logger.info('Force pool requests sending.') - makePoolRequests() - } - - module.exports = poolRequests -})() diff --git a/src/utils.js b/src/utils.js deleted file mode 100644 index 176648a31..000000000 --- a/src/utils.js +++ /dev/null @@ -1,202 +0,0 @@ -;(function () { - 'use strict' - - var async = require('async') - var config = require('config') - var crypto = require('crypto') - var fs = require('fs') - var openssl = require('openssl-wrapper') - var request = require('request') - var replay = require('request-replay') - var ursa = require('ursa') - - var constants = require('./constants') - var logger = require('./logger') - - var utils = {} - - var http = config.get('webserver.https') ? 'https' : 'http' - var host = config.get('webserver.host') - var port = config.get('webserver.port') - var algorithm = 'aes-256-ctr' - - // ----------- Private functions ---------- - - function makeRetryRequest (params, from_url, to_pod, signature, callbackEach) { - // Append the signature - if (signature) { - params.json.signature = { - url: from_url, - signature: signature - } - } - - logger.debug('Make retry requests to %s.', to_pod.url) - - replay( - request.post(params, function (err, response, body) { - callbackEach(err, response, body, params.url, to_pod) - }), - { - retries: constants.REQUEST_RETRIES, - factor: 3, - maxTimeout: Infinity, - errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ] - } - ).on('replay', function (replay) { - logger.info('Replaying request to %s. Request failed: %d %s. Replay number: #%d. Will retry in: %d ms.', - params.url, replay.error.code, replay.error.message, replay.number, replay.delay) - }) - } - - // ----------- Public attributes ---------- - utils.certDir = __dirname + '/../' + config.get('storage.certs') - - // { path, data } - utils.makeMultipleRetryRequest = function (all_data, pods, callbackEach, callback) { - if (!callback) { - callback = callbackEach - callbackEach = null - } - - var url = http + '://' + host + ':' + port - var signature - - // Add signature if it is specified in the params - if (all_data.method === 'POST' && all_data.data && all_data.sign === true) { - var myKey = ursa.createPrivateKey(fs.readFileSync(utils.certDir + 'peertube.key.pem')) - signature = myKey.hashAndSign('sha256', url, 'utf8', 'hex') - } - - // Make a request for each pod - async.each(pods, function (pod, callback_each_async) { - function callbackEachRetryRequest (err, response, body, url, pod) { - if (callbackEach !== null) { - callbackEach(err, response, body, url, pod, function () { - callback_each_async() - }) - } else { - callback_each_async() - } - } - - var params = { - url: pod.url + all_data.path, - method: all_data.method - } - - // Add data with POST requst ? - if (all_data.method === 'POST' && all_data.data) { - // Encrypt data ? - if (all_data.encrypt === true) { - var crt = ursa.createPublicKey(pod.publicKey) - - // TODO: ES6 with let - ;(function (crt_copy, copy_params, copy_url, copy_pod, copy_signature) { - utils.symetricEncrypt(JSON.stringify(all_data.data), function (err, dataEncrypted) { - if (err) throw err - - var passwordEncrypted = crt_copy.encrypt(dataEncrypted.password, 'utf8', 'hex') - copy_params.json = { - data: dataEncrypted.crypted, - key: passwordEncrypted - } - - makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEachRetryRequest) - }) - })(crt, params, url, pod, signature) - } else { - params.json = { data: all_data.data } - makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) - } - } else { - makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) - } - }, callback) - } - - utils.certsExist = function (callback) { - fs.exists(utils.certDir + 'peertube.key.pem', function (exists) { - return callback(exists) - }) - } - - utils.createCerts = function (callback) { - utils.certsExist(function (exist) { - if (exist === true) { - var string = 'Certs already exist.' - logger.warning(string) - return callback(new Error(string)) - } - - logger.info('Generating a RSA key...') - openssl.exec('genrsa', { 'out': utils.certDir + 'peertube.key.pem', '2048': false }, function (err) { - if (err) { - logger.error('Cannot create private key on this pod.', { error: err }) - return callback(err) - } - logger.info('RSA key generated.') - - logger.info('Manage public key...') - openssl.exec('rsa', { 'in': utils.certDir + 'peertube.key.pem', 'pubout': true, 'out': utils.certDir + 'peertube.pub' }, function (err) { - if (err) { - logger.error('Cannot create public key on this pod .', { error: err }) - return callback(err) - } - - logger.info('Public key managed.') - return callback(null) - }) - }) - }) - } - - utils.createCertsIfNotExist = function (callback) { - utils.certsExist(function (exist) { - if (exist === true) { - return callback(null) - } - - utils.createCerts(function (err) { - return callback(err) - }) - }) - } - - utils.generatePassword = function (callback) { - crypto.randomBytes(32, function (err, buf) { - if (err) { - return callback(err) - } - - callback(null, buf.toString('utf8')) - }) - } - - utils.symetricEncrypt = function (text, callback) { - utils.generatePassword(function (err, password) { - if (err) { - return callback(err) - } - - var cipher = crypto.createCipher(algorithm, password) - var crypted = cipher.update(text, 'utf8', 'hex') - crypted += cipher.final('hex') - callback(null, { crypted: crypted, password: password }) - }) - } - - utils.symetricDecrypt = function (text, password) { - var decipher = crypto.createDecipher(algorithm, password) - var dec = decipher.update(text, 'hex', 'utf8') - dec += decipher.final('utf8') - return dec - } - - utils.cleanForExit = function (webtorrent_process) { - logger.info('Gracefully exiting') - process.kill(-webtorrent_process.pid) - } - - module.exports = utils -})() diff --git a/src/videos.js b/src/videos.js deleted file mode 100644 index 90821fdf6..000000000 --- a/src/videos.js +++ /dev/null @@ -1,272 +0,0 @@ -;(function () { - 'use strict' - - var async = require('async') - var config = require('config') - var dz = require('dezalgo') - var fs = require('fs') - var webtorrent = require('./webTorrentNode') - - var logger = require('./logger') - var pods = require('./pods') - var VideosDB = require('./database').VideosDB - - var videos = {} - - var http = config.get('webserver.https') === true ? 'https' : 'http' - var host = config.get('webserver.host') - var port = config.get('webserver.port') - - // ----------- Private functions ----------- - function seedVideo (path, callback) { - logger.info('Seeding %s...', path) - - webtorrent.seed(path, function (torrent) { - logger.info('%s seeded (%s).', path, torrent.magnetURI) - - return callback(null, torrent) - }) - } - - // ----------- Public attributes ---------- - videos.uploadDir = __dirname + '/../' + config.get('storage.uploads') - - // ----------- Public functions ----------- - videos.list = function (callback) { - VideosDB.find(function (err, videos_list) { - if (err) { - logger.error('Cannot get list of the videos.', { error: err }) - return callback(err) - } - - return callback(null, videos_list) - }) - } - - videos.listOwned = function (callback) { - // If namePath is not null this is *our* video - VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) { - if (err) { - logger.error('Cannot get list of the videos.', { error: err }) - return callback(err) - } - - return callback(null, videos_list) - }) - } - - videos.add = function (data, callback) { - var video_file = data.video - var video_data = data.data - - logger.info('Adding %s video.', video_file.path) - seedVideo(video_file.path, function (err, torrent) { - if (err) { - logger.error('Cannot seed this video.', { error: err }) - return callback(err) - } - - var params = { - name: video_data.name, - namePath: video_file.filename, - description: video_data.description, - magnetUri: torrent.magnetURI, - podUrl: http + '://' + host + ':' + port - } - - VideosDB.create(params, function (err, video) { - if (err) { - logger.error('Cannot insert this video.', { error: err }) - return callback(err) - } - - // Now we'll add the video's meta data to our friends - params.namePath = null - - pods.addVideoToFriends(params) - callback(null) - }) - }) - } - - videos.remove = function (id, callback) { - // Maybe the torrent is not seeded, but we catch the error to don't stop the removing process - function removeTorrent (magnetUri, callback) { - try { - webtorrent.remove(magnetUri, callback) - } catch (err) { - logger.warn('Cannot remove the torrent from WebTorrent', { err: err }) - return callback(null) - } - } - - VideosDB.findById(id, function (err, video) { - if (err || !video) { - if (!err) err = new Error('Cannot find this video.') - logger.error('Cannot find this video.', { error: err }) - return callback(err) - } - - if (video.namePath === null) { - var error_string = 'Cannot remove the video of another pod.' - logger.error(error_string) - return callback(new Error(error_string)) - } - - logger.info('Removing %s video', video.name) - - removeTorrent(video.magnetUri, function () { - VideosDB.findByIdAndRemove(id, function (err) { - if (err) { - logger.error('Cannot remove the torrent.', { error: err }) - return callback(err) - } - - fs.unlink(videos.uploadDir + video.namePath, function (err) { - if (err) { - logger.error('Cannot remove this video file.', { error: err }) - return callback(err) - } - - var params = { - name: video.name, - magnetUri: video.magnetUri - } - - pods.removeVideoToFriends(params) - callback(null) - }) - }) - }) - }) - } - - // Use the magnet Uri because the _id field is not the same on different servers - videos.removeRemotes = function (fromUrl, magnetUris, callback) { - if (callback === undefined) callback = function () {} - - VideosDB.find({ magnetUri: { $in: magnetUris } }, function (err, videos) { - if (err || !videos) { - logger.error('Cannot find the torrent URI of these remote videos.') - return callback(err) - } - - var to_remove = [] - async.each(videos, function (video, callback_async) { - callback_async = dz(callback_async) - - if (video.podUrl !== fromUrl) { - logger.error('The pod %s has not the rights on the video of %s.', fromUrl, video.podUrl) - } else { - to_remove.push(video._id) - } - - callback_async() - }, function () { - VideosDB.remove({ _id: { $in: to_remove } }, function (err) { - if (err) { - logger.error('Cannot remove the remote videos.') - return callback(err) - } - - logger.info('Removed remote videos from %s.', fromUrl) - callback(null) - }) - }) - }) - } - - videos.removeAllRemotes = function (callback) { - VideosDB.remove({ namePath: null }, function (err) { - if (err) return callback(err) - - callback(null) - }) - } - - videos.removeAllRemotesOf = function (fromUrl, callback) { - VideosDB.remove({ podUrl: fromUrl }, function (err) { - if (err) return callback(err) - - callback(null) - }) - } - - // { name, magnetUri, podUrl } - // TODO: avoid doublons - videos.addRemotes = function (videos, callback) { - if (callback === undefined) callback = function () {} - - var to_add = [] - - async.each(videos, function (video, callback_each) { - callback_each = dz(callback_each) - logger.debug('Add remote video from pod: %s', video.podUrl) - - var params = { - name: video.name, - namePath: null, - description: video.description, - magnetUri: video.magnetUri, - podUrl: video.podUrl - } - - to_add.push(params) - - callback_each() - }, function () { - VideosDB.create(to_add, function (err, videos) { - if (err) { - logger.error('Cannot insert this remote video.', { error: err }) - return callback(err) - } - - return callback(null, videos) - }) - }) - } - - videos.get = function (id, callback) { - VideosDB.findById(id, function (err, video) { - if (err) { - logger.error('Cannot get this video.', { error: err }) - return callback(err) - } - - return callback(null, video) - }) - } - - videos.search = function (name, callback) { - VideosDB.find({ name: new RegExp(name) }, function (err, videos) { - if (err) { - logger.error('Cannot search the videos.', { error: err }) - return callback(err) - } - - return callback(null, videos) - }) - } - - videos.seedAll = function (callback) { - VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) { - if (err) { - logger.error('Cannot get list of the videos to seed.', { error: err }) - return callback(err) - } - - async.each(videos_list, function (video, each_callback) { - seedVideo(videos.uploadDir + video.namePath, function (err) { - if (err) { - logger.error('Cannot seed this video.', { error: err }) - return callback(err) - } - - each_callback(null) - }) - }, callback) - }) - } - - module.exports = videos -})() diff --git a/src/webTorrentNode.js b/src/webTorrentNode.js deleted file mode 100644 index d6801d0fb..000000000 --- a/src/webTorrentNode.js +++ /dev/null @@ -1,160 +0,0 @@ -;(function () { - 'use strict' - - var config = require('config') - var ipc = require('node-ipc') - var pathUtils = require('path') - var spawn = require('electron-spawn') - - var logger = require('./logger') - - var host = config.get('webserver.host') - var port = config.get('webserver.port') - - var nodeKey = 'webtorrentnode' + port - var processKey = 'webtorrent' + port - - ipc.config.silent = true - ipc.config.id = nodeKey - - var webtorrentnode = {} - - // Useful for beautiful tests - webtorrentnode.silent = false - - // Useful to kill it - webtorrentnode.app = null - - webtorrentnode.create = function (options, callback) { - if (typeof options === 'function') { - callback = options - options = {} - } - - // Override options - if (options.host) host = options.host - if (options.port) { - port = options.port - nodeKey = 'webtorrentnode' + port - processKey = 'webtorrent' + port - ipc.config.id = nodeKey - } - - ipc.serve(function () { - if (!webtorrentnode.silent) logger.info('IPC server ready.') - - // Run a timeout of 30s after which we exit the process - var timeout_webtorrent_process = setTimeout(function () { - logger.error('Timeout : cannot run the webtorrent process. Please ensure you have electron-prebuilt npm package installed with xvfb-run.') - process.exit() - }, 30000) - - ipc.server.on(processKey + '.ready', function () { - if (!webtorrentnode.silent) logger.info('Webtorrent process ready.') - clearTimeout(timeout_webtorrent_process) - callback() - }) - - ipc.server.on(processKey + '.exception', function (data) { - logger.error('Received exception error from webtorrent process.', { exception: data.exception }) - process.exit() - }) - - var webtorrent_process = spawn(__dirname + '/webtorrent.js', host, port, { detached: true }) - webtorrent_process.stderr.on('data', function (data) { - // logger.debug('Webtorrent process stderr: ', data.toString()) - }) - - webtorrent_process.stdout.on('data', function (data) { - // logger.debug('Webtorrent process:', data.toString()) - }) - - webtorrentnode.app = webtorrent_process - }) - - ipc.server.start() - } - - webtorrentnode.seed = function (path, callback) { - var extension = pathUtils.extname(path) - var basename = pathUtils.basename(path, extension) - var data = { - _id: basename, - args: { - path: path - } - } - - if (!webtorrentnode.silent) logger.debug('Node wants to seed %s.', data._id) - - // Finish signal - var event_key = nodeKey + '.seedDone.' + data._id - ipc.server.on(event_key, function listener (received) { - if (!webtorrentnode.silent) logger.debug('Process seeded torrent %s.', received.magnetUri) - - // This is a fake object, we just use the magnetUri in this project - var torrent = { - magnetURI: received.magnetUri - } - - ipc.server.off(event_key) - callback(torrent) - }) - - ipc.server.broadcast(processKey + '.seed', data) - } - - webtorrentnode.add = function (magnetUri, callback) { - var data = { - _id: magnetUri, - args: { - magnetUri: magnetUri - } - } - - if (!webtorrentnode.silent) logger.debug('Node wants to add ' + data._id) - - // Finish signal - var event_key = nodeKey + '.addDone.' + data._id - ipc.server.on(event_key, function (received) { - if (!webtorrentnode.silent) logger.debug('Process added torrent.') - - // This is a fake object, we just use the magnetUri in this project - var torrent = { - files: received.files - } - - ipc.server.off(event_key) - callback(torrent) - }) - - ipc.server.broadcast(processKey + '.add', data) - } - - webtorrentnode.remove = function (magnetUri, callback) { - var data = { - _id: magnetUri, - args: { - magnetUri: magnetUri - } - } - - if (!webtorrentnode.silent) logger.debug('Node wants to stop seeding %s.', data._id) - - // Finish signal - var event_key = nodeKey + '.removeDone.' + data._id - ipc.server.on(event_key, function (received) { - if (!webtorrentnode.silent) logger.debug('Process removed torrent %s.', data._id) - - var err = null - if (received.err) err = received.err - - ipc.server.off(event_key) - callback(err) - }) - - ipc.server.broadcast(processKey + '.remove', data) - } - - module.exports = webtorrentnode -})() diff --git a/src/webtorrent.js b/src/webtorrent.js deleted file mode 100644 index b72bc500d..000000000 --- a/src/webtorrent.js +++ /dev/null @@ -1,91 +0,0 @@ -;(function () { - 'use strict' - - module.exports = function (args) { - var WebTorrent = require('webtorrent') - var ipc = require('node-ipc') - - if (args.length !== 3) { - console.log('Wrong arguments number: ' + args.length + '/3') - process.exit(-1) - } - - var host = args[1] - var port = args[2] - var nodeKey = 'webtorrentnode' + port - var processKey = 'webtorrent' + port - - ipc.config.silent = true - ipc.config.id = processKey - - if (host === 'client' && port === '1') global.WEBTORRENT_ANNOUNCE = [] - else global.WEBTORRENT_ANNOUNCE = 'ws://' + host + ':' + port + '/tracker/socket' - var wt = new WebTorrent({ dht: false }) - - function seed (data) { - var args = data.args - var path = args.path - var _id = data._id - - wt.seed(path, { announceList: '' }, function (torrent) { - var to_send = { - magnetUri: torrent.magnetURI - } - - ipc.of[nodeKey].emit(nodeKey + '.seedDone.' + _id, to_send) - }) - } - - function add (data) { - var args = data.args - var magnetUri = args.magnetUri - var _id = data._id - - wt.add(magnetUri, function (torrent) { - var to_send = { - files: [] - } - - torrent.files.forEach(function (file) { - to_send.files.push({ path: file.path }) - }) - - ipc.of[nodeKey].emit(nodeKey + '.addDone.' + _id, to_send) - }) - } - - function remove (data) { - var args = data.args - var magnetUri = args.magnetUri - var _id = data._id - - try { - wt.remove(magnetUri, callback) - } catch (err) { - console.log('Cannot remove the torrent from WebTorrent', { err: err }) - return callback(null) - } - - function callback () { - var to_send = {} - ipc.of[nodeKey].emit(nodeKey + '.removeDone.' + _id, to_send) - } - } - - console.log('Configuration: ' + host + ':' + port) - console.log('Connecting to IPC...') - - ipc.connectTo(nodeKey, function () { - ipc.of[nodeKey].on(processKey + '.seed', seed) - ipc.of[nodeKey].on(processKey + '.add', add) - ipc.of[nodeKey].on(processKey + '.remove', remove) - - ipc.of[nodeKey].emit(processKey + '.ready') - console.log('Ready.') - }) - - process.on('uncaughtException', function (e) { - ipc.of[nodeKey].emit(processKey + '.exception', { exception: e }) - }) - } -})() diff --git a/test/api/checkParams.js b/test/api/checkParams.js deleted file mode 100644 index 11fc68ff9..000000000 --- a/test/api/checkParams.js +++ /dev/null @@ -1,301 +0,0 @@ -;(function () { - 'use strict' - - var async = require('async') - var chai = require('chai') - var expect = chai.expect - var request = require('supertest') - - var utils = require('./utils') - - describe('Test parameters validator', function () { - var app = null - var url = '' - - function makePostRequest (path, fields, attach, done, fail) { - var status_code = 400 - if (fail !== undefined && fail === false) status_code = 200 - - var req = request(url) - .post(path) - .set('Accept', 'application/json') - - Object.keys(fields).forEach(function (field) { - var value = fields[field] - req.field(field, value) - }) - - req.expect(status_code, done) - } - - function makePostBodyRequest (path, fields, done, fail) { - var status_code = 400 - if (fail !== undefined && fail === false) status_code = 200 - - request(url) - .post(path) - .set('Accept', 'application/json') - .send(fields) - .expect(status_code, done) - } - - // --------------------------------------------------------------- - - before(function (done) { - this.timeout(20000) - - async.series([ - function (next) { - utils.flushTests(next) - }, - function (next) { - utils.runServer(1, function (app1, url1) { - app = app1 - url = url1 - next() - }) - } - ], done) - }) - - describe('Of the pods API', function () { - var path = '/api/v1/pods/' - - describe('When adding a pod', function () { - it('Should fail with nothing', function (done) { - var data = {} - makePostBodyRequest(path, data, done) - }) - - it('Should fail without public key', function (done) { - var data = { - data: { - url: 'http://coucou.com' - } - } - makePostBodyRequest(path, data, done) - }) - - it('Should fail without an url', function (done) { - var data = { - data: { - publicKey: 'mysuperpublickey' - } - } - makePostBodyRequest(path, data, done) - }) - - it('Should fail with an incorrect url', function (done) { - var data = { - data: { - url: 'coucou.com', - publicKey: 'mysuperpublickey' - } - } - makePostBodyRequest(path, data, function () { - data.data.url = 'http://coucou' - makePostBodyRequest(path, data, function () { - data.data.url = 'coucou' - makePostBodyRequest(path, data, done) - }) - }) - }) - - it('Should succeed with the correct parameters', function (done) { - var data = { - data: { - url: 'http://coucou.com', - publicKey: 'mysuperpublickey' - } - } - makePostBodyRequest(path, data, done, false) - }) - }) - }) - - describe('Of the videos API', function () { - var path = '/api/v1/videos/' - - describe('When searching a video', function () { - it('Should fail with nothing', function (done) { - request(url) - .get(path + '/search/') - .set('Accept', 'application/json') - .expect(400, done) - }) - }) - - describe('When adding a video', function () { - it('Should fail with nothing', function (done) { - var data = {} - var attach = {} - makePostRequest(path, data, attach, done) - }) - - it('Should fail without name', function (done) { - var data = { - description: 'my super description' - } - var attach = { - 'input_video': __dirname + '/fixtures/video_short.webm' - } - makePostRequest(path, data, attach, done) - }) - - it('Should fail with a long name', function (done) { - var data = { - name: 'My very very very very very very very very very very very very very very very very long name', - description: 'my super description' - } - var attach = { - 'input_video': __dirname + '/fixtures/video_short.webm' - } - makePostRequest(path, data, attach, done) - }) - - it('Should fail without description', function (done) { - var data = { - name: 'my super name' - } - var attach = { - 'input_video': __dirname + '/fixtures/video_short.webm' - } - makePostRequest(path, data, attach, done) - }) - - it('Should fail with a long description', function (done) { - var data = { - name: 'my super name', - description: 'my super description which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very long' - } - var attach = { - 'input_video': __dirname + '/fixtures/video_short.webm' - } - makePostRequest(path, data, attach, done) - }) - - it('Should fail without an input file', function (done) { - var data = { - name: 'my super name', - description: 'my super description' - } - var attach = {} - makePostRequest(path, data, attach, done) - }) - - it('Should fail without an incorrect input file', function (done) { - var data = { - name: 'my super name', - description: 'my super description' - } - var attach = { - 'input_video': __dirname + '/../fixtures/video_short_fake.webm' - } - makePostRequest(path, data, attach, done) - }) - - it('Should succeed with the correct parameters', function (done) { - var data = { - name: 'my super name', - description: 'my super description' - } - var attach = { - 'input_video': __dirname + '/fixtures/video_short.webm' - } - makePostRequest(path, data, attach, function () { - attach.input_video = __dirname + '/fixtures/video_short.mp4' - makePostRequest(path, data, attach, function () { - attach.input_video = __dirname + '/fixtures/video_short.ogv' - makePostRequest(path, data, attach, done, true) - }, true) - }, true) - }) - }) - - describe('When getting a video', function () { - it('Should return the list of the videos with nothing', function (done) { - request(url) - .get(path) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(function (err, res) { - if (err) throw err - - expect(res.body).to.be.an('array') - expect(res.body.length).to.equal(0) - - done() - }) - }) - - it('Should fail without a mongodb id', function (done) { - request(url) - .get(path + 'coucou') - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should return 404 with an incorrect video', function (done) { - request(url) - .get(path + '123456789012345678901234') - .set('Accept', 'application/json') - .expect(404, done) - }) - - it('Should succeed with the correct parameters') - }) - - describe('When removing a video', function () { - it('Should have 404 with nothing', function (done) { - request(url) - .delete(path) - .expect(404, done) - }) - - it('Should fail without a mongodb id', function (done) { - request(url) - .delete(path + 'hello') - .expect(400, done) - }) - - it('Should fail with a video which does not exist', function (done) { - request(url) - .delete(path + '123456789012345678901234') - .expect(404, done) - }) - - it('Should fail with a video of another pod') - - it('Should succeed with the correct parameters') - }) - }) - - describe('Of the remote videos API', function () { - describe('When making a secure request', function () { - it('Should check a secure request') - }) - - describe('When adding a video', function () { - it('Should check when adding a video') - }) - - describe('When removing a video', function () { - it('Should check when removing a video') - }) - }) - - after(function (done) { - process.kill(-app.pid) - - // Keep the logs if the test failed - if (this.ok) { - utils.flushTests(done) - } else { - done() - } - }) - }) -})() diff --git a/test/api/fixtures/video_short.mp4 b/test/api/fixtures/video_short.mp4 deleted file mode 100644 index 35678362b..000000000 Binary files a/test/api/fixtures/video_short.mp4 and /dev/null differ diff --git a/test/api/fixtures/video_short.ogv b/test/api/fixtures/video_short.ogv deleted file mode 100644 index 9e253da82..000000000 Binary files a/test/api/fixtures/video_short.ogv and /dev/null differ diff --git a/test/api/fixtures/video_short.webm b/test/api/fixtures/video_short.webm deleted file mode 100644 index bf4b0ab6c..000000000 Binary files a/test/api/fixtures/video_short.webm and /dev/null differ diff --git a/test/api/fixtures/video_short1.webm b/test/api/fixtures/video_short1.webm deleted file mode 100644 index 70ac0c644..000000000 Binary files a/test/api/fixtures/video_short1.webm and /dev/null differ diff --git a/test/api/fixtures/video_short2.webm b/test/api/fixtures/video_short2.webm deleted file mode 100644 index 13d72dff7..000000000 Binary files a/test/api/fixtures/video_short2.webm and /dev/null differ diff --git a/test/api/fixtures/video_short3.webm b/test/api/fixtures/video_short3.webm deleted file mode 100644 index cde5dcd58..000000000 Binary files a/test/api/fixtures/video_short3.webm and /dev/null differ diff --git a/test/api/fixtures/video_short_fake.webm b/test/api/fixtures/video_short_fake.webm deleted file mode 100644 index d85290ae5..000000000 --- a/test/api/fixtures/video_short_fake.webm +++ /dev/null @@ -1 +0,0 @@ -this is a fake video mouahahah diff --git a/test/api/friendsAdvanced.js b/test/api/friendsAdvanced.js deleted file mode 100644 index 61483bee6..000000000 --- a/test/api/friendsAdvanced.js +++ /dev/null @@ -1,252 +0,0 @@ -;(function () { - 'use strict' - - var async = require('async') - var chai = require('chai') - var expect = chai.expect - - var utils = require('./utils') - - describe('Test advanced friends', function () { - var apps = [] - var urls = [] - - function makeFriends (pod_number, callback) { - return utils.makeFriends(urls[pod_number - 1], callback) - } - - function quitFriends (pod_number, callback) { - return utils.quitFriends(urls[pod_number - 1], callback) - } - - function getFriendsList (pod_number, end) { - return utils.getFriendsList(urls[pod_number - 1], end) - } - - function uploadVideo (pod_number, callback) { - var name = 'my super video' - var description = 'my super description' - var fixture = 'video_short.webm' - - return utils.uploadVideo(urls[pod_number - 1], name, description, fixture, callback) - } - - function getVideos (pod_number, callback) { - return utils.getVideosList(urls[pod_number - 1], callback) - } - - // --------------------------------------------------------------- - - before(function (done) { - this.timeout(30000) - utils.flushAndRunMultipleServers(6, function (apps_run, urls_run) { - apps = apps_run - urls = urls_run - done() - }) - }) - - it('Should make friends with two pod each in a different group', function (done) { - this.timeout(20000) - - async.series([ - // Pod 3 makes friend with the first one - function (next) { - makeFriends(3, next) - }, - // Pod 4 makes friend with the second one - function (next) { - makeFriends(4, next) - }, - // Now if the fifth wants to make friends with the third et the first - function (next) { - makeFriends(5, next) - }, - function (next) { - setTimeout(next, 11000) - }], - function (err) { - if (err) throw err - - // It should have 0 friends - getFriendsList(5, function (err, res) { - if (err) throw err - - expect(res.body.length).to.equal(0) - - done() - }) - } - ) - }) - - it('Should quit all friends', function (done) { - this.timeout(10000) - - async.series([ - function (next) { - quitFriends(1, next) - }, - function (next) { - quitFriends(2, next) - }], - function (err) { - if (err) throw err - - async.each([ 1, 2, 3, 4, 5, 6 ], function (i, callback) { - getFriendsList(i, function (err, res) { - if (err) throw err - - expect(res.body.length).to.equal(0) - - callback() - }) - }, done) - } - ) - }) - - it('Should make friends with the pods 1, 2, 3', function (done) { - this.timeout(150000) - - async.series([ - // Pods 1, 2, 3 and 4 become friends - function (next) { - makeFriends(2, next) - }, - function (next) { - makeFriends(1, next) - }, - function (next) { - makeFriends(4, next) - }, - // Kill pod 4 - function (next) { - apps[3].kill() - next() - }, - // Expulse pod 4 from pod 1 and 2 - function (next) { - uploadVideo(1, next) - }, - function (next) { - uploadVideo(2, next) - }, - function (next) { - setTimeout(next, 11000) - }, - function (next) { - uploadVideo(1, next) - }, - function (next) { - uploadVideo(2, next) - }, - function (next) { - setTimeout(next, 20000) - }, - // Rerun server 4 - function (next) { - utils.runServer(4, function (app, url) { - apps[3] = app - next() - }) - }, - function (next) { - getFriendsList(4, function (err, res) { - if (err) throw err - - // Pod 4 didn't know pod 1 and 2 removed it - expect(res.body.length).to.equal(3) - - next() - }) - }, - // Pod 6 ask pod 1, 2 and 3 - function (next) { - makeFriends(6, next) - }], - function (err) { - if (err) throw err - - getFriendsList(6, function (err, res) { - if (err) throw err - - // Pod 4 should not be our friend - var result = res.body - expect(result.length).to.equal(3) - for (var pod of result) { - expect(pod.url).not.equal(urls[3]) - } - - done() - }) - } - ) - }) - - it('Should pod 1 quit friends', function (done) { - this.timeout(25000) - - async.series([ - // Upload a video on server 3 for aditionnal tests - function (next) { - uploadVideo(3, next) - }, - function (next) { - setTimeout(next, 15000) - }, - function (next) { - quitFriends(1, next) - }, - // Remove pod 1 from pod 2 - function (next) { - getVideos(1, function (err, res) { - if (err) throw err - expect(res.body).to.be.an('array') - expect(res.body.length).to.equal(2) - - next() - }) - }], - function (err) { - if (err) throw err - - getVideos(2, function (err, res) { - if (err) throw err - expect(res.body).to.be.an('array') - expect(res.body.length).to.equal(3) - done() - }) - } - ) - }) - - it('Should make friends between pod 1 and 2 and exchange their videos', function (done) { - this.timeout(20000) - makeFriends(1, function () { - setTimeout(function () { - getVideos(1, function (err, res) { - if (err) throw err - - expect(res.body).to.be.an('array') - expect(res.body.length).to.equal(5) - - done() - }) - }, 5000) - }) - }) - - after(function (done) { - apps.forEach(function (app) { - process.kill(-app.pid) - }) - - if (this.ok) { - utils.flushTests(done) - } else { - done() - } - }) - }) -})() diff --git a/test/api/friendsBasic.js b/test/api/friendsBasic.js deleted file mode 100644 index dbc918383..000000000 --- a/test/api/friendsBasic.js +++ /dev/null @@ -1,187 +0,0 @@ -;(function () { - 'use strict' - - var async = require('async') - var chai = require('chai') - var expect = chai.expect - var request = require('supertest') - - var utils = require('./utils') - - describe('Test basic friends', function () { - var apps = [] - var urls = [] - - function testMadeFriends (urls, url_to_test, callback) { - var friends = [] - for (var i = 0; i < urls.length; i++) { - if (urls[i] === url_to_test) continue - friends.push(urls[i]) - } - - utils.getFriendsList(url_to_test, function (err, res) { - if (err) throw err - - var result = res.body - var result_urls = [ result[0].url, result[1].url ] - expect(result).to.be.an('array') - expect(result.length).to.equal(2) - expect(result_urls[0]).to.not.equal(result_urls[1]) - - var error_string = 'Friends url do not correspond for ' + url_to_test - expect(friends).to.contain(result_urls[0], error_string) - expect(friends).to.contain(result_urls[1], error_string) - callback() - }) - } - - // --------------------------------------------------------------- - - before(function (done) { - this.timeout(20000) - utils.flushAndRunMultipleServers(3, function (apps_run, urls_run) { - apps = apps_run - urls = urls_run - done() - }) - }) - - it('Should not have friends', function (done) { - async.each(urls, function (url, callback) { - utils.getFriendsList(url, function (err, res) { - if (err) throw err - - var result = res.body - expect(result).to.be.an('array') - expect(result.length).to.equal(0) - callback() - }) - }, done) - }) - - it('Should make friends', function (done) { - this.timeout(10000) - - var path = '/api/v1/pods/makefriends' - - async.series([ - // The second pod make friend with the third - function (next) { - request(urls[1]) - .get(path) - .set('Accept', 'application/json') - .expect(204) - .end(next) - }, - // Wait for the request between pods - function (next) { - setTimeout(next, 1000) - }, - // The second pod should have the third as a friend - function (next) { - utils.getFriendsList(urls[1], function (err, res) { - if (err) throw err - - var result = res.body - expect(result).to.be.an('array') - expect(result.length).to.equal(1) - expect(result[0].url).to.be.equal(urls[2]) - - next() - }) - }, - // Same here, the third pod should have the second pod as a friend - function (next) { - utils.getFriendsList(urls[2], function (err, res) { - if (err) throw err - - var result = res.body - expect(result).to.be.an('array') - expect(result.length).to.equal(1) - expect(result[0].url).to.be.equal(urls[1]) - - next() - }) - }, - // Finally the first pod make friend with the second pod - function (next) { - request(urls[0]) - .get(path) - .set('Accept', 'application/json') - .expect(204) - .end(next) - }, - // Wait for the request between pods - function (next) { - setTimeout(next, 1000) - } - ], - // Now each pod should be friend with the other ones - function (err) { - if (err) throw err - async.each(urls, function (url, callback) { - testMadeFriends(urls, url, callback) - }, done) - }) - }) - - it('Should not be allowed to make friend again', function (done) { - utils.makeFriends(urls[1], 409, done) - }) - - it('Should quit friends of pod 2', function (done) { - async.series([ - // Pod 1 quit friends - function (next) { - utils.quitFriends(urls[1], next) - }, - // Pod 1 should not have friends anymore - function (next) { - utils.getFriendsList(urls[1], function (err, res) { - if (err) throw err - - var result = res.body - expect(result).to.be.an('array') - expect(result.length).to.equal(0) - - next() - }) - }, - // Other pods shouldn't have pod 1 too - function (next) { - async.each([ urls[0], urls[2] ], function (url, callback) { - utils.getFriendsList(url, function (err, res) { - if (err) throw err - - var result = res.body - expect(result).to.be.an('array') - expect(result.length).to.equal(1) - expect(result[0].url).not.to.be.equal(urls[1]) - callback() - }) - }, next) - } - ], done) - }) - - it('Should allow pod 2 to make friend again', function (done) { - utils.makeFriends(urls[1], function () { - async.each(urls, function (url, callback) { - testMadeFriends(urls, url, callback) - }, done) - }) - }) - - after(function (done) { - apps.forEach(function (app) { - process.kill(-app.pid) - }) - - if (this.ok) { - utils.flushTests(done) - } else { - done() - } - }) - }) -})() diff --git a/test/api/index.js b/test/api/index.js deleted file mode 100644 index 3bdcdae2d..000000000 --- a/test/api/index.js +++ /dev/null @@ -1,10 +0,0 @@ -;(function () { - 'use strict' - - // Order of the tests we want to execute - require('./checkParams') - require('./friendsBasic') - require('./singlePod') - require('./multiplePods') - require('./friendsAdvanced') -})() diff --git a/test/api/multiplePods.js b/test/api/multiplePods.js deleted file mode 100644 index b579e5e32..000000000 --- a/test/api/multiplePods.js +++ /dev/null @@ -1,329 +0,0 @@ -;(function () { - 'use strict' - - var async = require('async') - var chai = require('chai') - var expect = chai.expect - - var utils = require('./utils') - var webtorrent = require(__dirname + '/../../src/webTorrentNode') - webtorrent.silent = true - - describe('Test multiple pods', function () { - var apps = [] - var urls = [] - var to_remove = [] - - before(function (done) { - this.timeout(30000) - - async.series([ - // Run servers - function (next) { - utils.flushAndRunMultipleServers(3, function (apps_run, urls_run) { - apps = apps_run - urls = urls_run - next() - }) - }, - // The second pod make friend with the third - function (next) { - utils.makeFriends(urls[1], next) - }, - // Wait for the request between pods - function (next) { - setTimeout(next, 10000) - }, - // Pod 1 make friends too - function (next) { - utils.makeFriends(urls[0], next) - }, - function (next) { - webtorrent.create({ host: 'client', port: '1' }, next) - } - ], done) - }) - - it('Should not have videos for all pods', function (done) { - async.each(urls, function (url, callback) { - utils.getVideosList(url, function (err, res) { - if (err) throw err - - expect(res.body).to.be.an('array') - expect(res.body.length).to.equal(0) - - callback() - }) - }, done) - }) - - describe('Should upload the video and propagate on each pod', function () { - it('Should upload the video on pod 1 and propagate on each pod', function (done) { - this.timeout(15000) - - async.series([ - function (next) { - utils.uploadVideo(urls[0], 'my super name for pod 1', 'my super description for pod 1', 'video_short1.webm', next) - }, - function (next) { - setTimeout(next, 11000) - }], - // All pods should have this video - function (err) { - if (err) throw err - - async.each(urls, function (url, callback) { - var base_magnet = null - - utils.getVideosList(url, function (err, res) { - if (err) throw err - - var videos = res.body - expect(videos).to.be.an('array') - expect(videos.length).to.equal(1) - var video = videos[0] - expect(video.name).to.equal('my super name for pod 1') - expect(video.description).to.equal('my super description for pod 1') - expect(video.podUrl).to.equal('http://localhost:9001') - expect(video.magnetUri).to.exist - - // All pods should have the same magnet Uri - if (base_magnet === null) { - base_magnet = video.magnetUri - } else { - expect(video.magnetUri).to.equal.magnetUri - } - - callback() - }) - }, done) - } - ) - }) - - it('Should upload the video on pod 2 and propagate on each pod', function (done) { - this.timeout(15000) - - async.series([ - function (next) { - utils.uploadVideo(urls[1], 'my super name for pod 2', 'my super description for pod 2', 'video_short2.webm', next) - }, - function (next) { - setTimeout(next, 11000) - }], - // All pods should have this video - function (err) { - if (err) throw err - - async.each(urls, function (url, callback) { - var base_magnet = null - - utils.getVideosList(url, function (err, res) { - if (err) throw err - - var videos = res.body - expect(videos).to.be.an('array') - expect(videos.length).to.equal(2) - var video = videos[1] - expect(video.name).to.equal('my super name for pod 2') - expect(video.description).to.equal('my super description for pod 2') - expect(video.podUrl).to.equal('http://localhost:9002') - expect(video.magnetUri).to.exist - - // All pods should have the same magnet Uri - if (base_magnet === null) { - base_magnet = video.magnetUri - } else { - expect(video.magnetUri).to.equal.magnetUri - } - - callback() - }) - }, done) - } - ) - }) - - it('Should upload two videos on pod 3 and propagate on each pod', function (done) { - this.timeout(30000) - - async.series([ - function (next) { - utils.uploadVideo(urls[2], 'my super name for pod 3', 'my super description for pod 3', 'video_short3.webm', next) - }, - function (next) { - utils.uploadVideo(urls[2], 'my super name for pod 3-2', 'my super description for pod 3-2', 'video_short.webm', next) - }, - function (next) { - setTimeout(next, 22000) - }], - function (err) { - if (err) throw err - - var base_magnet = null - // All pods should have this video - async.each(urls, function (url, callback) { - utils.getVideosList(url, function (err, res) { - if (err) throw err - - var videos = res.body - expect(videos).to.be.an('array') - expect(videos.length).to.equal(4) - var video = videos[2] - expect(video.name).to.equal('my super name for pod 3') - expect(video.description).to.equal('my super description for pod 3') - expect(video.podUrl).to.equal('http://localhost:9003') - expect(video.magnetUri).to.exist - - video = videos[3] - expect(video.name).to.equal('my super name for pod 3-2') - expect(video.description).to.equal('my super description for pod 3-2') - expect(video.podUrl).to.equal('http://localhost:9003') - expect(video.magnetUri).to.exist - - // All pods should have the same magnet Uri - if (base_magnet === null) { - base_magnet = video.magnetUri - } else { - expect(video.magnetUri).to.equal.magnetUri - } - - callback() - }) - }, done) - } - ) - }) - }) - - describe('Should seed the uploaded video', function () { - it('Should add the file 1 by asking pod 3', function (done) { - // Yes, this could be long - this.timeout(200000) - - utils.getVideosList(urls[2], function (err, res) { - if (err) throw err - - var video = res.body[0] - to_remove.push(res.body[2]._id) - to_remove.push(res.body[3]._id) - - webtorrent.add(video.magnetUri, function (torrent) { - expect(torrent.files).to.exist - expect(torrent.files.length).to.equal(1) - expect(torrent.files[0].path).to.exist.and.to.not.equal('') - - done() - }) - }) - }) - - it('Should add the file 2 by asking pod 1', function (done) { - // Yes, this could be long - this.timeout(200000) - - utils.getVideosList(urls[0], function (err, res) { - if (err) throw err - - var video = res.body[1] - - webtorrent.add(video.magnetUri, function (torrent) { - expect(torrent.files).to.exist - expect(torrent.files.length).to.equal(1) - expect(torrent.files[0].path).to.exist.and.to.not.equal('') - - done() - }) - }) - }) - - it('Should add the file 3 by asking pod 2', function (done) { - // Yes, this could be long - this.timeout(200000) - - utils.getVideosList(urls[1], function (err, res) { - if (err) throw err - - var video = res.body[2] - - webtorrent.add(video.magnetUri, function (torrent) { - expect(torrent.files).to.exist - expect(torrent.files.length).to.equal(1) - expect(torrent.files[0].path).to.exist.and.to.not.equal('') - - done() - }) - }) - }) - - it('Should add the file 3-2 by asking pod 1', function (done) { - // Yes, this could be long - this.timeout(200000) - - utils.getVideosList(urls[0], function (err, res) { - if (err) throw err - - var video = res.body[3] - - webtorrent.add(video.magnetUri, function (torrent) { - expect(torrent.files).to.exist - expect(torrent.files.length).to.equal(1) - expect(torrent.files[0].path).to.exist.and.to.not.equal('') - - done() - }) - }) - }) - - it('Should remove the file 3 and 3-2 by asking pod 3', function (done) { - this.timeout(15000) - - async.series([ - function (next) { - utils.removeVideo(urls[2], to_remove[0], next) - }, - function (next) { - utils.removeVideo(urls[2], to_remove[1], next) - }], - function (err) { - if (err) throw err - setTimeout(done, 11000) - } - ) - }) - - it('Should have videos 1 and 3 on each pod', function (done) { - async.each(urls, function (url, callback) { - utils.getVideosList(url, function (err, res) { - if (err) throw err - - var videos = res.body - expect(videos).to.be.an('array') - expect(videos.length).to.equal(2) - expect(videos[0]._id).not.to.equal(videos[1]._id) - expect(videos[0]._id).not.to.equal(to_remove[0]) - expect(videos[1]._id).not.to.equal(to_remove[0]) - expect(videos[0]._id).not.to.equal(to_remove[1]) - expect(videos[1]._id).not.to.equal(to_remove[1]) - - callback() - }) - }, done) - }) - }) - - after(function (done) { - apps.forEach(function (app) { - process.kill(-app.pid) - }) - process.kill(-webtorrent.app.pid) - - // Keep the logs if the test failed - if (this.ok) { - utils.flushTests(done) - } else { - done() - } - }) - }) -})() diff --git a/test/api/singlePod.js b/test/api/singlePod.js deleted file mode 100644 index a8ae43aee..000000000 --- a/test/api/singlePod.js +++ /dev/null @@ -1,147 +0,0 @@ -;(function () { - 'use strict' - - var async = require('async') - var chai = require('chai') - var fs = require('fs') - var expect = chai.expect - - var webtorrent = require(__dirname + '/../../src/webTorrentNode') - webtorrent.silent = true - - var utils = require('./utils') - - describe('Test a single pod', function () { - var app = null - var url = '' - var video_id = -1 - - before(function (done) { - this.timeout(20000) - - async.series([ - function (next) { - utils.flushTests(next) - }, - function (next) { - utils.runServer(1, function (app1, url1) { - app = app1 - url = url1 - next() - }) - }, - function (next) { - webtorrent.create({ host: 'client', port: '1' }, next) - } - ], done) - }) - - it('Should not have videos', function (done) { - utils.getVideosList(url, function (err, res) { - if (err) throw err - - expect(res.body).to.be.an('array') - expect(res.body.length).to.equal(0) - - done() - }) - }) - - it('Should upload the video', function (done) { - this.timeout(5000) - utils.uploadVideo(url, 'my super name', 'my super description', 'video_short.webm', done) - }) - - it('Should seed the uploaded video', function (done) { - // Yes, this could be long - this.timeout(60000) - - utils.getVideosList(url, function (err, res) { - if (err) throw err - - expect(res.body).to.be.an('array') - expect(res.body.length).to.equal(1) - - var video = res.body[0] - expect(video.name).to.equal('my super name') - expect(video.description).to.equal('my super description') - expect(video.podUrl).to.equal('http://localhost:9001') - expect(video.magnetUri).to.exist - - video_id = video._id - - webtorrent.add(video.magnetUri, function (torrent) { - expect(torrent.files).to.exist - expect(torrent.files.length).to.equal(1) - expect(torrent.files[0].path).to.exist.and.to.not.equal('') - - done() - }) - }) - }) - - it('Should search the video', function (done) { - utils.searchVideo(url, 'my', function (err, res) { - if (err) throw err - - expect(res.body).to.be.an('array') - expect(res.body.length).to.equal(1) - - var video = res.body[0] - expect(video.name).to.equal('my super name') - expect(video.description).to.equal('my super description') - expect(video.podUrl).to.equal('http://localhost:9001') - expect(video.magnetUri).to.exist - - done() - }) - }) - - it('Should not find a search', function (done) { - utils.searchVideo(url, 'hello', function (err, res) { - if (err) throw err - - expect(res.body).to.be.an('array') - expect(res.body.length).to.equal(0) - - done() - }) - }) - - it('Should remove the video', function (done) { - utils.removeVideo(url, video_id, function (err) { - if (err) throw err - - fs.readdir(__dirname + '/../../test1/uploads/', function (err, files) { - if (err) throw err - - expect(files.length).to.equal(0) - done() - }) - }) - }) - - it('Should not have videos', function (done) { - utils.getVideosList(url, function (err, res) { - if (err) throw err - - expect(res.body).to.be.an('array') - expect(res.body.length).to.equal(0) - - done() - }) - }) - - after(function (done) { - process.kill(-app.pid) - process.kill(-webtorrent.app.pid) - - // Keep the logs if the test failed - if (this.ok) { - utils.flushTests(done) - } else { - done() - } - }) - }) -})() diff --git a/test/api/utils.js b/test/api/utils.js deleted file mode 100644 index afb0abb33..000000000 --- a/test/api/utils.js +++ /dev/null @@ -1,182 +0,0 @@ -;(function () { - 'use strict' - - var child_process = require('child_process') - var exec = child_process.exec - var fork = child_process.fork - var request = require('supertest') - - module.exports = { - flushTests: flushTests, - getFriendsList: getFriendsList, - getVideosList: getVideosList, - makeFriends: makeFriends, - quitFriends: quitFriends, - removeVideo: removeVideo, - flushAndRunMultipleServers: flushAndRunMultipleServers, - runServer: runServer, - searchVideo: searchVideo, - uploadVideo: uploadVideo - } - - // ---------------------- Export functions -------------------- - - function flushTests (callback) { - exec(__dirname + '/../../scripts/clean_test.sh', callback) - } - - function getFriendsList (url, end) { - var path = '/api/v1/pods/' - - request(url) - .get(path) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(end) - } - - function getVideosList (url, end) { - var path = '/api/v1/videos' - - request(url) - .get(path) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(end) - } - - function makeFriends (url, expected_status, callback) { - if (!callback) { - callback = expected_status - expected_status = 204 - } - - var path = '/api/v1/pods/makefriends' - - // The first pod make friend with the third - request(url) - .get(path) - .set('Accept', 'application/json') - .expect(expected_status) - .end(function (err, res) { - if (err) throw err - - // Wait for the request between pods - setTimeout(callback, 1000) - }) - } - - function quitFriends (url, callback) { - var path = '/api/v1/pods/quitfriends' - - // The first pod make friend with the third - request(url) - .get(path) - .set('Accept', 'application/json') - .expect(204) - .end(function (err, res) { - if (err) throw err - - // Wait for the request between pods - setTimeout(callback, 1000) - }) - } - - function removeVideo (url, id, end) { - var path = '/api/v1/videos' - - request(url) - .delete(path + '/' + id) - .set('Accept', 'application/json') - .expect(204) - .end(end) - } - - function flushAndRunMultipleServers (total_servers, serversRun) { - var apps = [] - var urls = [] - var i = 0 - - function anotherServerDone (number, app, url) { - apps[number - 1] = app - urls[number - 1] = url - i++ - if (i === total_servers) { - serversRun(apps, urls) - } - } - - flushTests(function () { - for (var j = 1; j <= total_servers; j++) { - (function (k) { // TODO: ES6 with let - // For the virtual buffer - setTimeout(function () { - runServer(k, function (app, url) { - anotherServerDone(k, app, url) - }) - }, 1000 * k) - })(j) - } - }) - } - - function runServer (number, callback) { - var port = 9000 + number - var server_run_string = { - 'Connected to mongodb': false, - 'Server listening on port': false - } - - // Share the environment - var env = Object.create(process.env) - env.NODE_ENV = 'test' - env.NODE_APP_INSTANCE = number - var options = { - silent: true, - env: env, - detached: true - } - - var app = fork(__dirname + '/../../server.js', [], options) - app.stdout.on('data', function onStdout (data) { - var dont_continue = false - // Check if all required sentences are here - for (var key of Object.keys(server_run_string)) { - if (data.toString().indexOf(key) !== -1) server_run_string[key] = true - if (server_run_string[key] === false) dont_continue = true - } - - // If no, there is maybe one thing not already initialized (mongodb...) - if (dont_continue === true) return - - app.stdout.removeListener('data', onStdout) - callback(app, 'http://localhost:' + port) - }) - } - - function searchVideo (url, search, end) { - var path = '/api/v1/videos' - - request(url) - .get(path + '/search/' + search) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(end) - } - - function uploadVideo (url, name, description, fixture, end) { - var path = '/api/v1/videos' - - request(url) - .post(path) - .set('Accept', 'application/json') - .field('name', name) - .field('description', description) - .attach('input_video', __dirname + '/fixtures/' + fixture) - .expect(201) - .end(end) - } -})() diff --git a/test/index.js b/test/index.js deleted file mode 100644 index ccebbfe51..000000000 --- a/test/index.js +++ /dev/null @@ -1,6 +0,0 @@ -;(function () { - 'use strict' - - // Order of the tests we want to execute - require('./api/') -})() diff --git a/tests/api/checkParams.js b/tests/api/checkParams.js new file mode 100644 index 000000000..11fc68ff9 --- /dev/null +++ b/tests/api/checkParams.js @@ -0,0 +1,301 @@ +;(function () { + 'use strict' + + var async = require('async') + var chai = require('chai') + var expect = chai.expect + var request = require('supertest') + + var utils = require('./utils') + + describe('Test parameters validator', function () { + var app = null + var url = '' + + function makePostRequest (path, fields, attach, done, fail) { + var status_code = 400 + if (fail !== undefined && fail === false) status_code = 200 + + var req = request(url) + .post(path) + .set('Accept', 'application/json') + + Object.keys(fields).forEach(function (field) { + var value = fields[field] + req.field(field, value) + }) + + req.expect(status_code, done) + } + + function makePostBodyRequest (path, fields, done, fail) { + var status_code = 400 + if (fail !== undefined && fail === false) status_code = 200 + + request(url) + .post(path) + .set('Accept', 'application/json') + .send(fields) + .expect(status_code, done) + } + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + async.series([ + function (next) { + utils.flushTests(next) + }, + function (next) { + utils.runServer(1, function (app1, url1) { + app = app1 + url = url1 + next() + }) + } + ], done) + }) + + describe('Of the pods API', function () { + var path = '/api/v1/pods/' + + describe('When adding a pod', function () { + it('Should fail with nothing', function (done) { + var data = {} + makePostBodyRequest(path, data, done) + }) + + it('Should fail without public key', function (done) { + var data = { + data: { + url: 'http://coucou.com' + } + } + makePostBodyRequest(path, data, done) + }) + + it('Should fail without an url', function (done) { + var data = { + data: { + publicKey: 'mysuperpublickey' + } + } + makePostBodyRequest(path, data, done) + }) + + it('Should fail with an incorrect url', function (done) { + var data = { + data: { + url: 'coucou.com', + publicKey: 'mysuperpublickey' + } + } + makePostBodyRequest(path, data, function () { + data.data.url = 'http://coucou' + makePostBodyRequest(path, data, function () { + data.data.url = 'coucou' + makePostBodyRequest(path, data, done) + }) + }) + }) + + it('Should succeed with the correct parameters', function (done) { + var data = { + data: { + url: 'http://coucou.com', + publicKey: 'mysuperpublickey' + } + } + makePostBodyRequest(path, data, done, false) + }) + }) + }) + + describe('Of the videos API', function () { + var path = '/api/v1/videos/' + + describe('When searching a video', function () { + it('Should fail with nothing', function (done) { + request(url) + .get(path + '/search/') + .set('Accept', 'application/json') + .expect(400, done) + }) + }) + + describe('When adding a video', function () { + it('Should fail with nothing', function (done) { + var data = {} + var attach = {} + makePostRequest(path, data, attach, done) + }) + + it('Should fail without name', function (done) { + var data = { + description: 'my super description' + } + var attach = { + 'input_video': __dirname + '/fixtures/video_short.webm' + } + makePostRequest(path, data, attach, done) + }) + + it('Should fail with a long name', function (done) { + var data = { + name: 'My very very very very very very very very very very very very very very very very long name', + description: 'my super description' + } + var attach = { + 'input_video': __dirname + '/fixtures/video_short.webm' + } + makePostRequest(path, data, attach, done) + }) + + it('Should fail without description', function (done) { + var data = { + name: 'my super name' + } + var attach = { + 'input_video': __dirname + '/fixtures/video_short.webm' + } + makePostRequest(path, data, attach, done) + }) + + it('Should fail with a long description', function (done) { + var data = { + name: 'my super name', + description: 'my super description which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very long' + } + var attach = { + 'input_video': __dirname + '/fixtures/video_short.webm' + } + makePostRequest(path, data, attach, done) + }) + + it('Should fail without an input file', function (done) { + var data = { + name: 'my super name', + description: 'my super description' + } + var attach = {} + makePostRequest(path, data, attach, done) + }) + + it('Should fail without an incorrect input file', function (done) { + var data = { + name: 'my super name', + description: 'my super description' + } + var attach = { + 'input_video': __dirname + '/../fixtures/video_short_fake.webm' + } + makePostRequest(path, data, attach, done) + }) + + it('Should succeed with the correct parameters', function (done) { + var data = { + name: 'my super name', + description: 'my super description' + } + var attach = { + 'input_video': __dirname + '/fixtures/video_short.webm' + } + makePostRequest(path, data, attach, function () { + attach.input_video = __dirname + '/fixtures/video_short.mp4' + makePostRequest(path, data, attach, function () { + attach.input_video = __dirname + '/fixtures/video_short.ogv' + makePostRequest(path, data, attach, done, true) + }, true) + }, true) + }) + }) + + describe('When getting a video', function () { + it('Should return the list of the videos with nothing', function (done) { + request(url) + .get(path) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(function (err, res) { + if (err) throw err + + expect(res.body).to.be.an('array') + expect(res.body.length).to.equal(0) + + done() + }) + }) + + it('Should fail without a mongodb id', function (done) { + request(url) + .get(path + 'coucou') + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should return 404 with an incorrect video', function (done) { + request(url) + .get(path + '123456789012345678901234') + .set('Accept', 'application/json') + .expect(404, done) + }) + + it('Should succeed with the correct parameters') + }) + + describe('When removing a video', function () { + it('Should have 404 with nothing', function (done) { + request(url) + .delete(path) + .expect(404, done) + }) + + it('Should fail without a mongodb id', function (done) { + request(url) + .delete(path + 'hello') + .expect(400, done) + }) + + it('Should fail with a video which does not exist', function (done) { + request(url) + .delete(path + '123456789012345678901234') + .expect(404, done) + }) + + it('Should fail with a video of another pod') + + it('Should succeed with the correct parameters') + }) + }) + + describe('Of the remote videos API', function () { + describe('When making a secure request', function () { + it('Should check a secure request') + }) + + describe('When adding a video', function () { + it('Should check when adding a video') + }) + + describe('When removing a video', function () { + it('Should check when removing a video') + }) + }) + + after(function (done) { + process.kill(-app.pid) + + // Keep the logs if the test failed + if (this.ok) { + utils.flushTests(done) + } else { + done() + } + }) + }) +})() diff --git a/tests/api/fixtures/video_short.mp4 b/tests/api/fixtures/video_short.mp4 new file mode 100644 index 000000000..35678362b Binary files /dev/null and b/tests/api/fixtures/video_short.mp4 differ diff --git a/tests/api/fixtures/video_short.ogv b/tests/api/fixtures/video_short.ogv new file mode 100644 index 000000000..9e253da82 Binary files /dev/null and b/tests/api/fixtures/video_short.ogv differ diff --git a/tests/api/fixtures/video_short.webm b/tests/api/fixtures/video_short.webm new file mode 100644 index 000000000..bf4b0ab6c Binary files /dev/null and b/tests/api/fixtures/video_short.webm differ diff --git a/tests/api/fixtures/video_short1.webm b/tests/api/fixtures/video_short1.webm new file mode 100644 index 000000000..70ac0c644 Binary files /dev/null and b/tests/api/fixtures/video_short1.webm differ diff --git a/tests/api/fixtures/video_short2.webm b/tests/api/fixtures/video_short2.webm new file mode 100644 index 000000000..13d72dff7 Binary files /dev/null and b/tests/api/fixtures/video_short2.webm differ diff --git a/tests/api/fixtures/video_short3.webm b/tests/api/fixtures/video_short3.webm new file mode 100644 index 000000000..cde5dcd58 Binary files /dev/null and b/tests/api/fixtures/video_short3.webm differ diff --git a/tests/api/fixtures/video_short_fake.webm b/tests/api/fixtures/video_short_fake.webm new file mode 100644 index 000000000..d85290ae5 --- /dev/null +++ b/tests/api/fixtures/video_short_fake.webm @@ -0,0 +1 @@ +this is a fake video mouahahah diff --git a/tests/api/friendsAdvanced.js b/tests/api/friendsAdvanced.js new file mode 100644 index 000000000..61483bee6 --- /dev/null +++ b/tests/api/friendsAdvanced.js @@ -0,0 +1,252 @@ +;(function () { + 'use strict' + + var async = require('async') + var chai = require('chai') + var expect = chai.expect + + var utils = require('./utils') + + describe('Test advanced friends', function () { + var apps = [] + var urls = [] + + function makeFriends (pod_number, callback) { + return utils.makeFriends(urls[pod_number - 1], callback) + } + + function quitFriends (pod_number, callback) { + return utils.quitFriends(urls[pod_number - 1], callback) + } + + function getFriendsList (pod_number, end) { + return utils.getFriendsList(urls[pod_number - 1], end) + } + + function uploadVideo (pod_number, callback) { + var name = 'my super video' + var description = 'my super description' + var fixture = 'video_short.webm' + + return utils.uploadVideo(urls[pod_number - 1], name, description, fixture, callback) + } + + function getVideos (pod_number, callback) { + return utils.getVideosList(urls[pod_number - 1], callback) + } + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(30000) + utils.flushAndRunMultipleServers(6, function (apps_run, urls_run) { + apps = apps_run + urls = urls_run + done() + }) + }) + + it('Should make friends with two pod each in a different group', function (done) { + this.timeout(20000) + + async.series([ + // Pod 3 makes friend with the first one + function (next) { + makeFriends(3, next) + }, + // Pod 4 makes friend with the second one + function (next) { + makeFriends(4, next) + }, + // Now if the fifth wants to make friends with the third et the first + function (next) { + makeFriends(5, next) + }, + function (next) { + setTimeout(next, 11000) + }], + function (err) { + if (err) throw err + + // It should have 0 friends + getFriendsList(5, function (err, res) { + if (err) throw err + + expect(res.body.length).to.equal(0) + + done() + }) + } + ) + }) + + it('Should quit all friends', function (done) { + this.timeout(10000) + + async.series([ + function (next) { + quitFriends(1, next) + }, + function (next) { + quitFriends(2, next) + }], + function (err) { + if (err) throw err + + async.each([ 1, 2, 3, 4, 5, 6 ], function (i, callback) { + getFriendsList(i, function (err, res) { + if (err) throw err + + expect(res.body.length).to.equal(0) + + callback() + }) + }, done) + } + ) + }) + + it('Should make friends with the pods 1, 2, 3', function (done) { + this.timeout(150000) + + async.series([ + // Pods 1, 2, 3 and 4 become friends + function (next) { + makeFriends(2, next) + }, + function (next) { + makeFriends(1, next) + }, + function (next) { + makeFriends(4, next) + }, + // Kill pod 4 + function (next) { + apps[3].kill() + next() + }, + // Expulse pod 4 from pod 1 and 2 + function (next) { + uploadVideo(1, next) + }, + function (next) { + uploadVideo(2, next) + }, + function (next) { + setTimeout(next, 11000) + }, + function (next) { + uploadVideo(1, next) + }, + function (next) { + uploadVideo(2, next) + }, + function (next) { + setTimeout(next, 20000) + }, + // Rerun server 4 + function (next) { + utils.runServer(4, function (app, url) { + apps[3] = app + next() + }) + }, + function (next) { + getFriendsList(4, function (err, res) { + if (err) throw err + + // Pod 4 didn't know pod 1 and 2 removed it + expect(res.body.length).to.equal(3) + + next() + }) + }, + // Pod 6 ask pod 1, 2 and 3 + function (next) { + makeFriends(6, next) + }], + function (err) { + if (err) throw err + + getFriendsList(6, function (err, res) { + if (err) throw err + + // Pod 4 should not be our friend + var result = res.body + expect(result.length).to.equal(3) + for (var pod of result) { + expect(pod.url).not.equal(urls[3]) + } + + done() + }) + } + ) + }) + + it('Should pod 1 quit friends', function (done) { + this.timeout(25000) + + async.series([ + // Upload a video on server 3 for aditionnal tests + function (next) { + uploadVideo(3, next) + }, + function (next) { + setTimeout(next, 15000) + }, + function (next) { + quitFriends(1, next) + }, + // Remove pod 1 from pod 2 + function (next) { + getVideos(1, function (err, res) { + if (err) throw err + expect(res.body).to.be.an('array') + expect(res.body.length).to.equal(2) + + next() + }) + }], + function (err) { + if (err) throw err + + getVideos(2, function (err, res) { + if (err) throw err + expect(res.body).to.be.an('array') + expect(res.body.length).to.equal(3) + done() + }) + } + ) + }) + + it('Should make friends between pod 1 and 2 and exchange their videos', function (done) { + this.timeout(20000) + makeFriends(1, function () { + setTimeout(function () { + getVideos(1, function (err, res) { + if (err) throw err + + expect(res.body).to.be.an('array') + expect(res.body.length).to.equal(5) + + done() + }) + }, 5000) + }) + }) + + after(function (done) { + apps.forEach(function (app) { + process.kill(-app.pid) + }) + + if (this.ok) { + utils.flushTests(done) + } else { + done() + } + }) + }) +})() diff --git a/tests/api/friendsBasic.js b/tests/api/friendsBasic.js new file mode 100644 index 000000000..dbc918383 --- /dev/null +++ b/tests/api/friendsBasic.js @@ -0,0 +1,187 @@ +;(function () { + 'use strict' + + var async = require('async') + var chai = require('chai') + var expect = chai.expect + var request = require('supertest') + + var utils = require('./utils') + + describe('Test basic friends', function () { + var apps = [] + var urls = [] + + function testMadeFriends (urls, url_to_test, callback) { + var friends = [] + for (var i = 0; i < urls.length; i++) { + if (urls[i] === url_to_test) continue + friends.push(urls[i]) + } + + utils.getFriendsList(url_to_test, function (err, res) { + if (err) throw err + + var result = res.body + var result_urls = [ result[0].url, result[1].url ] + expect(result).to.be.an('array') + expect(result.length).to.equal(2) + expect(result_urls[0]).to.not.equal(result_urls[1]) + + var error_string = 'Friends url do not correspond for ' + url_to_test + expect(friends).to.contain(result_urls[0], error_string) + expect(friends).to.contain(result_urls[1], error_string) + callback() + }) + } + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + utils.flushAndRunMultipleServers(3, function (apps_run, urls_run) { + apps = apps_run + urls = urls_run + done() + }) + }) + + it('Should not have friends', function (done) { + async.each(urls, function (url, callback) { + utils.getFriendsList(url, function (err, res) { + if (err) throw err + + var result = res.body + expect(result).to.be.an('array') + expect(result.length).to.equal(0) + callback() + }) + }, done) + }) + + it('Should make friends', function (done) { + this.timeout(10000) + + var path = '/api/v1/pods/makefriends' + + async.series([ + // The second pod make friend with the third + function (next) { + request(urls[1]) + .get(path) + .set('Accept', 'application/json') + .expect(204) + .end(next) + }, + // Wait for the request between pods + function (next) { + setTimeout(next, 1000) + }, + // The second pod should have the third as a friend + function (next) { + utils.getFriendsList(urls[1], function (err, res) { + if (err) throw err + + var result = res.body + expect(result).to.be.an('array') + expect(result.length).to.equal(1) + expect(result[0].url).to.be.equal(urls[2]) + + next() + }) + }, + // Same here, the third pod should have the second pod as a friend + function (next) { + utils.getFriendsList(urls[2], function (err, res) { + if (err) throw err + + var result = res.body + expect(result).to.be.an('array') + expect(result.length).to.equal(1) + expect(result[0].url).to.be.equal(urls[1]) + + next() + }) + }, + // Finally the first pod make friend with the second pod + function (next) { + request(urls[0]) + .get(path) + .set('Accept', 'application/json') + .expect(204) + .end(next) + }, + // Wait for the request between pods + function (next) { + setTimeout(next, 1000) + } + ], + // Now each pod should be friend with the other ones + function (err) { + if (err) throw err + async.each(urls, function (url, callback) { + testMadeFriends(urls, url, callback) + }, done) + }) + }) + + it('Should not be allowed to make friend again', function (done) { + utils.makeFriends(urls[1], 409, done) + }) + + it('Should quit friends of pod 2', function (done) { + async.series([ + // Pod 1 quit friends + function (next) { + utils.quitFriends(urls[1], next) + }, + // Pod 1 should not have friends anymore + function (next) { + utils.getFriendsList(urls[1], function (err, res) { + if (err) throw err + + var result = res.body + expect(result).to.be.an('array') + expect(result.length).to.equal(0) + + next() + }) + }, + // Other pods shouldn't have pod 1 too + function (next) { + async.each([ urls[0], urls[2] ], function (url, callback) { + utils.getFriendsList(url, function (err, res) { + if (err) throw err + + var result = res.body + expect(result).to.be.an('array') + expect(result.length).to.equal(1) + expect(result[0].url).not.to.be.equal(urls[1]) + callback() + }) + }, next) + } + ], done) + }) + + it('Should allow pod 2 to make friend again', function (done) { + utils.makeFriends(urls[1], function () { + async.each(urls, function (url, callback) { + testMadeFriends(urls, url, callback) + }, done) + }) + }) + + after(function (done) { + apps.forEach(function (app) { + process.kill(-app.pid) + }) + + if (this.ok) { + utils.flushTests(done) + } else { + done() + } + }) + }) +})() diff --git a/tests/api/index.js b/tests/api/index.js new file mode 100644 index 000000000..3bdcdae2d --- /dev/null +++ b/tests/api/index.js @@ -0,0 +1,10 @@ +;(function () { + 'use strict' + + // Order of the tests we want to execute + require('./checkParams') + require('./friendsBasic') + require('./singlePod') + require('./multiplePods') + require('./friendsAdvanced') +})() diff --git a/tests/api/multiplePods.js b/tests/api/multiplePods.js new file mode 100644 index 000000000..c27f7121e --- /dev/null +++ b/tests/api/multiplePods.js @@ -0,0 +1,329 @@ +;(function () { + 'use strict' + + var async = require('async') + var chai = require('chai') + var expect = chai.expect + + var utils = require('./utils') + var webtorrent = require(__dirname + '/../../lib/webTorrentNode') + webtorrent.silent = true + + describe('Test multiple pods', function () { + var apps = [] + var urls = [] + var to_remove = [] + + before(function (done) { + this.timeout(30000) + + async.series([ + // Run servers + function (next) { + utils.flushAndRunMultipleServers(3, function (apps_run, urls_run) { + apps = apps_run + urls = urls_run + next() + }) + }, + // The second pod make friend with the third + function (next) { + utils.makeFriends(urls[1], next) + }, + // Wait for the request between pods + function (next) { + setTimeout(next, 10000) + }, + // Pod 1 make friends too + function (next) { + utils.makeFriends(urls[0], next) + }, + function (next) { + webtorrent.create({ host: 'client', port: '1' }, next) + } + ], done) + }) + + it('Should not have videos for all pods', function (done) { + async.each(urls, function (url, callback) { + utils.getVideosList(url, function (err, res) { + if (err) throw err + + expect(res.body).to.be.an('array') + expect(res.body.length).to.equal(0) + + callback() + }) + }, done) + }) + + describe('Should upload the video and propagate on each pod', function () { + it('Should upload the video on pod 1 and propagate on each pod', function (done) { + this.timeout(15000) + + async.series([ + function (next) { + utils.uploadVideo(urls[0], 'my super name for pod 1', 'my super description for pod 1', 'video_short1.webm', next) + }, + function (next) { + setTimeout(next, 11000) + }], + // All pods should have this video + function (err) { + if (err) throw err + + async.each(urls, function (url, callback) { + var base_magnet = null + + utils.getVideosList(url, function (err, res) { + if (err) throw err + + var videos = res.body + expect(videos).to.be.an('array') + expect(videos.length).to.equal(1) + var video = videos[0] + expect(video.name).to.equal('my super name for pod 1') + expect(video.description).to.equal('my super description for pod 1') + expect(video.podUrl).to.equal('http://localhost:9001') + expect(video.magnetUri).to.exist + + // All pods should have the same magnet Uri + if (base_magnet === null) { + base_magnet = video.magnetUri + } else { + expect(video.magnetUri).to.equal.magnetUri + } + + callback() + }) + }, done) + } + ) + }) + + it('Should upload the video on pod 2 and propagate on each pod', function (done) { + this.timeout(15000) + + async.series([ + function (next) { + utils.uploadVideo(urls[1], 'my super name for pod 2', 'my super description for pod 2', 'video_short2.webm', next) + }, + function (next) { + setTimeout(next, 11000) + }], + // All pods should have this video + function (err) { + if (err) throw err + + async.each(urls, function (url, callback) { + var base_magnet = null + + utils.getVideosList(url, function (err, res) { + if (err) throw err + + var videos = res.body + expect(videos).to.be.an('array') + expect(videos.length).to.equal(2) + var video = videos[1] + expect(video.name).to.equal('my super name for pod 2') + expect(video.description).to.equal('my super description for pod 2') + expect(video.podUrl).to.equal('http://localhost:9002') + expect(video.magnetUri).to.exist + + // All pods should have the same magnet Uri + if (base_magnet === null) { + base_magnet = video.magnetUri + } else { + expect(video.magnetUri).to.equal.magnetUri + } + + callback() + }) + }, done) + } + ) + }) + + it('Should upload two videos on pod 3 and propagate on each pod', function (done) { + this.timeout(30000) + + async.series([ + function (next) { + utils.uploadVideo(urls[2], 'my super name for pod 3', 'my super description for pod 3', 'video_short3.webm', next) + }, + function (next) { + utils.uploadVideo(urls[2], 'my super name for pod 3-2', 'my super description for pod 3-2', 'video_short.webm', next) + }, + function (next) { + setTimeout(next, 22000) + }], + function (err) { + if (err) throw err + + var base_magnet = null + // All pods should have this video + async.each(urls, function (url, callback) { + utils.getVideosList(url, function (err, res) { + if (err) throw err + + var videos = res.body + expect(videos).to.be.an('array') + expect(videos.length).to.equal(4) + var video = videos[2] + expect(video.name).to.equal('my super name for pod 3') + expect(video.description).to.equal('my super description for pod 3') + expect(video.podUrl).to.equal('http://localhost:9003') + expect(video.magnetUri).to.exist + + video = videos[3] + expect(video.name).to.equal('my super name for pod 3-2') + expect(video.description).to.equal('my super description for pod 3-2') + expect(video.podUrl).to.equal('http://localhost:9003') + expect(video.magnetUri).to.exist + + // All pods should have the same magnet Uri + if (base_magnet === null) { + base_magnet = video.magnetUri + } else { + expect(video.magnetUri).to.equal.magnetUri + } + + callback() + }) + }, done) + } + ) + }) + }) + + describe('Should seed the uploaded video', function () { + it('Should add the file 1 by asking pod 3', function (done) { + // Yes, this could be long + this.timeout(200000) + + utils.getVideosList(urls[2], function (err, res) { + if (err) throw err + + var video = res.body[0] + to_remove.push(res.body[2]._id) + to_remove.push(res.body[3]._id) + + webtorrent.add(video.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + done() + }) + }) + }) + + it('Should add the file 2 by asking pod 1', function (done) { + // Yes, this could be long + this.timeout(200000) + + utils.getVideosList(urls[0], function (err, res) { + if (err) throw err + + var video = res.body[1] + + webtorrent.add(video.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + done() + }) + }) + }) + + it('Should add the file 3 by asking pod 2', function (done) { + // Yes, this could be long + this.timeout(200000) + + utils.getVideosList(urls[1], function (err, res) { + if (err) throw err + + var video = res.body[2] + + webtorrent.add(video.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + done() + }) + }) + }) + + it('Should add the file 3-2 by asking pod 1', function (done) { + // Yes, this could be long + this.timeout(200000) + + utils.getVideosList(urls[0], function (err, res) { + if (err) throw err + + var video = res.body[3] + + webtorrent.add(video.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + done() + }) + }) + }) + + it('Should remove the file 3 and 3-2 by asking pod 3', function (done) { + this.timeout(15000) + + async.series([ + function (next) { + utils.removeVideo(urls[2], to_remove[0], next) + }, + function (next) { + utils.removeVideo(urls[2], to_remove[1], next) + }], + function (err) { + if (err) throw err + setTimeout(done, 11000) + } + ) + }) + + it('Should have videos 1 and 3 on each pod', function (done) { + async.each(urls, function (url, callback) { + utils.getVideosList(url, function (err, res) { + if (err) throw err + + var videos = res.body + expect(videos).to.be.an('array') + expect(videos.length).to.equal(2) + expect(videos[0]._id).not.to.equal(videos[1]._id) + expect(videos[0]._id).not.to.equal(to_remove[0]) + expect(videos[1]._id).not.to.equal(to_remove[0]) + expect(videos[0]._id).not.to.equal(to_remove[1]) + expect(videos[1]._id).not.to.equal(to_remove[1]) + + callback() + }) + }, done) + }) + }) + + after(function (done) { + apps.forEach(function (app) { + process.kill(-app.pid) + }) + process.kill(-webtorrent.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + utils.flushTests(done) + } else { + done() + } + }) + }) +})() diff --git a/tests/api/singlePod.js b/tests/api/singlePod.js new file mode 100644 index 000000000..ce3ca80f2 --- /dev/null +++ b/tests/api/singlePod.js @@ -0,0 +1,147 @@ +;(function () { + 'use strict' + + var async = require('async') + var chai = require('chai') + var fs = require('fs') + var expect = chai.expect + + var webtorrent = require(__dirname + '/../../lib/webTorrentNode') + webtorrent.silent = true + + var utils = require('./utils') + + describe('Test a single pod', function () { + var app = null + var url = '' + var video_id = -1 + + before(function (done) { + this.timeout(20000) + + async.series([ + function (next) { + utils.flushTests(next) + }, + function (next) { + utils.runServer(1, function (app1, url1) { + app = app1 + url = url1 + next() + }) + }, + function (next) { + webtorrent.create({ host: 'client', port: '1' }, next) + } + ], done) + }) + + it('Should not have videos', function (done) { + utils.getVideosList(url, function (err, res) { + if (err) throw err + + expect(res.body).to.be.an('array') + expect(res.body.length).to.equal(0) + + done() + }) + }) + + it('Should upload the video', function (done) { + this.timeout(5000) + utils.uploadVideo(url, 'my super name', 'my super description', 'video_short.webm', done) + }) + + it('Should seed the uploaded video', function (done) { + // Yes, this could be long + this.timeout(60000) + + utils.getVideosList(url, function (err, res) { + if (err) throw err + + expect(res.body).to.be.an('array') + expect(res.body.length).to.equal(1) + + var video = res.body[0] + expect(video.name).to.equal('my super name') + expect(video.description).to.equal('my super description') + expect(video.podUrl).to.equal('http://localhost:9001') + expect(video.magnetUri).to.exist + + video_id = video._id + + webtorrent.add(video.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + done() + }) + }) + }) + + it('Should search the video', function (done) { + utils.searchVideo(url, 'my', function (err, res) { + if (err) throw err + + expect(res.body).to.be.an('array') + expect(res.body.length).to.equal(1) + + var video = res.body[0] + expect(video.name).to.equal('my super name') + expect(video.description).to.equal('my super description') + expect(video.podUrl).to.equal('http://localhost:9001') + expect(video.magnetUri).to.exist + + done() + }) + }) + + it('Should not find a search', function (done) { + utils.searchVideo(url, 'hello', function (err, res) { + if (err) throw err + + expect(res.body).to.be.an('array') + expect(res.body.length).to.equal(0) + + done() + }) + }) + + it('Should remove the video', function (done) { + utils.removeVideo(url, video_id, function (err) { + if (err) throw err + + fs.readdir(__dirname + '/../../test1/uploads/', function (err, files) { + if (err) throw err + + expect(files.length).to.equal(0) + done() + }) + }) + }) + + it('Should not have videos', function (done) { + utils.getVideosList(url, function (err, res) { + if (err) throw err + + expect(res.body).to.be.an('array') + expect(res.body.length).to.equal(0) + + done() + }) + }) + + after(function (done) { + process.kill(-app.pid) + process.kill(-webtorrent.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + utils.flushTests(done) + } else { + done() + } + }) + }) +})() diff --git a/tests/api/utils.js b/tests/api/utils.js new file mode 100644 index 000000000..afb0abb33 --- /dev/null +++ b/tests/api/utils.js @@ -0,0 +1,182 @@ +;(function () { + 'use strict' + + var child_process = require('child_process') + var exec = child_process.exec + var fork = child_process.fork + var request = require('supertest') + + module.exports = { + flushTests: flushTests, + getFriendsList: getFriendsList, + getVideosList: getVideosList, + makeFriends: makeFriends, + quitFriends: quitFriends, + removeVideo: removeVideo, + flushAndRunMultipleServers: flushAndRunMultipleServers, + runServer: runServer, + searchVideo: searchVideo, + uploadVideo: uploadVideo + } + + // ---------------------- Export functions -------------------- + + function flushTests (callback) { + exec(__dirname + '/../../scripts/clean_test.sh', callback) + } + + function getFriendsList (url, end) { + var path = '/api/v1/pods/' + + request(url) + .get(path) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(end) + } + + function getVideosList (url, end) { + var path = '/api/v1/videos' + + request(url) + .get(path) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(end) + } + + function makeFriends (url, expected_status, callback) { + if (!callback) { + callback = expected_status + expected_status = 204 + } + + var path = '/api/v1/pods/makefriends' + + // The first pod make friend with the third + request(url) + .get(path) + .set('Accept', 'application/json') + .expect(expected_status) + .end(function (err, res) { + if (err) throw err + + // Wait for the request between pods + setTimeout(callback, 1000) + }) + } + + function quitFriends (url, callback) { + var path = '/api/v1/pods/quitfriends' + + // The first pod make friend with the third + request(url) + .get(path) + .set('Accept', 'application/json') + .expect(204) + .end(function (err, res) { + if (err) throw err + + // Wait for the request between pods + setTimeout(callback, 1000) + }) + } + + function removeVideo (url, id, end) { + var path = '/api/v1/videos' + + request(url) + .delete(path + '/' + id) + .set('Accept', 'application/json') + .expect(204) + .end(end) + } + + function flushAndRunMultipleServers (total_servers, serversRun) { + var apps = [] + var urls = [] + var i = 0 + + function anotherServerDone (number, app, url) { + apps[number - 1] = app + urls[number - 1] = url + i++ + if (i === total_servers) { + serversRun(apps, urls) + } + } + + flushTests(function () { + for (var j = 1; j <= total_servers; j++) { + (function (k) { // TODO: ES6 with let + // For the virtual buffer + setTimeout(function () { + runServer(k, function (app, url) { + anotherServerDone(k, app, url) + }) + }, 1000 * k) + })(j) + } + }) + } + + function runServer (number, callback) { + var port = 9000 + number + var server_run_string = { + 'Connected to mongodb': false, + 'Server listening on port': false + } + + // Share the environment + var env = Object.create(process.env) + env.NODE_ENV = 'test' + env.NODE_APP_INSTANCE = number + var options = { + silent: true, + env: env, + detached: true + } + + var app = fork(__dirname + '/../../server.js', [], options) + app.stdout.on('data', function onStdout (data) { + var dont_continue = false + // Check if all required sentences are here + for (var key of Object.keys(server_run_string)) { + if (data.toString().indexOf(key) !== -1) server_run_string[key] = true + if (server_run_string[key] === false) dont_continue = true + } + + // If no, there is maybe one thing not already initialized (mongodb...) + if (dont_continue === true) return + + app.stdout.removeListener('data', onStdout) + callback(app, 'http://localhost:' + port) + }) + } + + function searchVideo (url, search, end) { + var path = '/api/v1/videos' + + request(url) + .get(path + '/search/' + search) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(end) + } + + function uploadVideo (url, name, description, fixture, end) { + var path = '/api/v1/videos' + + request(url) + .post(path) + .set('Accept', 'application/json') + .field('name', name) + .field('description', description) + .attach('input_video', __dirname + '/fixtures/' + fixture) + .expect(201) + .end(end) + } +})() diff --git a/tests/index.js b/tests/index.js new file mode 100644 index 000000000..ccebbfe51 --- /dev/null +++ b/tests/index.js @@ -0,0 +1,6 @@ +;(function () { + 'use strict' + + // Order of the tests we want to execute + require('./api/') +})()