- [ ] Inscription
- [ ] Connection
- [ ] Account rights (upload...)
-- [ ] Make the network auto sufficient (eject bad pods etc)
+- [X] Make the network auto sufficient (eject bad pods etc)
- [ ] Manage API breaks
- [ ] Add "DDOS" security (check if a pod don't send too many requests for example)
--- /dev/null
+listen:
+ port: 9004
+
+webserver:
+ host: 'localhost'
+ port: 9004
+
+database:
+ suffix: '-test4'
+
+# From the project root directory
+storage:
+ certs: 'test4/certs/'
+ uploads: 'test4/uploads/'
+ logs: 'test4/logs/'
+
+network:
+ friends:
+ - 'http://localhost:9002'
--- /dev/null
+listen:
+ port: 9005
+
+webserver:
+ host: 'localhost'
+ port: 9005
+
+database:
+ suffix: '-test5'
+
+# From the project root directory
+storage:
+ certs: 'test5/certs/'
+ uploads: 'test5/uploads/'
+ logs: 'test5/logs/'
+
+network:
+ friends:
+ - 'http://localhost:9001'
+ - 'http://localhost:9004'
--- /dev/null
+listen:
+ port: 9006
+
+webserver:
+ host: 'localhost'
+ port: 9006
+
+database:
+ suffix: '-test6'
+
+# From the project root directory
+storage:
+ certs: 'test6/certs/'
+ uploads: 'test6/uploads/'
+ logs: 'test6/logs/'
+
+network:
+ friends:
+ - 'http://localhost:9001'
+ - 'http://localhost:9002'
+ - 'http://localhost:9003'
"confirm",
"it",
"after",
+ "afterEach",
"before",
+ "beforeEach",
"describe"
]
}
#!/bin/bash
-printf "use peertube-test1;\ndb.dropDatabase();\nuse peertube-test2;\ndb.dropDatabase();\nuse peertube-test3;\ndb.dropDatabase();" | mongo
-
-rm -rf ./test1 ./test2 ./test3
+for i in $(seq 1 6); do
+ printf "use peertube-test%s;\ndb.dropDatabase();" "$i" | mongo
+ rm -rf "./test$i"
+done
;(function () {
'use strict'
- // ----------- Constants -----------
- global.API_VERSION = 'v1'
-
// ----------- Node modules -----------
var bodyParser = require('body-parser')
var express = require('express')
checker.createDirectoriesIfNotExist()
+ // ----------- Constants -----------
+ var utils = require('./src/utils')
+
+ global.API_VERSION = 'v1'
+ global.FRIEND_BASE_SCORE = utils.isTestInstance() ? 20 : 100
+
// ----------- PeerTube modules -----------
var config = require('config')
var logger = require('./src/logger')
var routes = require('./routes')
- var utils = require('./src/utils')
var videos = require('./src/videos')
var webtorrent = require('./src/webTorrentNode')
// ----------- Pods -----------
var podsSchema = mongoose.Schema({
url: String,
- publicKey: String
+ publicKey: String,
+ score: { type: Number, max: global.FRIEND_BASE_SCORE }
})
var PodsDB = mongoose.model('pods', podsSchema)
var host = config.get('webserver.host')
var port = config.get('webserver.port')
+ // ----------- Constants -----------
+
+ var PODS_SCORE = {
+ MALUS: -10,
+ BONUS: 10
+ }
+
// ----------- Private functions -----------
function getForeignPodsList (url, callback) {
})
}
+ function updatePodsScore (good_pods, bad_pods) {
+ logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length)
+
+ PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: PODS_SCORE.BONUS } }, { multi: true }).exec()
+ PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: PODS_SCORE.MALUS } }, { multi: true }, function (err) {
+ if (err) throw err
+ removeBadPods()
+ })
+ }
+
+ function removeBadPods () {
+ PodsDB.remove({ score: 0 }, function (err, result) {
+ if (err) throw err
+
+ var number_removed = result.result.n
+ if (number_removed !== 0) logger.info('Removed %d pod.', number_removed)
+ })
+ }
+
// ----------- Public functions -----------
pods.list = function (callback) {
var params = {
url: data.url,
- publicKey: data.publicKey
+ publicKey: data.publicKey,
+ score: global.FRIEND_BASE_SCORE
}
PodsDB.create(params, function (err, pod) {
// { path, data }
pods.makeSecureRequest = function (data, callback) {
- PodsDB.find({}, { url: 1, publicKey: 1 }).exec(function (err, urls) {
+ if (callback === undefined) callback = function () {}
+
+ PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) {
if (err) {
logger.error('Cannot get the list of the pods.', { error: err })
return callback(err)
data: data.data
}
+ var bad_pods = []
+ var good_pods = []
+
utils.makeMultipleRetryRequest(
params,
- urls,
+ pods,
- function callbackEachPodFinished (err, response, body, url) {
+ function callbackEachPodFinished (err, response, body, pod, callback_each_pod_finished) {
if (err || response.statusCode !== 200) {
- logger.error('Error sending secure request to %s/%s pod.', url, data.path, { error: err })
+ bad_pods.push(pod._id)
+ logger.error('Error sending secure request to %s/%s pod.', pod.url, data.path, { error: err })
+ } else {
+ good_pods.push(pod._id)
}
+
+ return callback_each_pod_finished()
},
function callbackAllPodsFinished (err) {
}
logger.debug('Finished')
+
+ updatePodsScore(good_pods, bad_pods)
callback(null)
}
)
// -----------------------------------------------------------------------
function computeForeignPodsList (url, callback) {
- // Always add a trust pod
- pods_score[url] = Infinity
+ // Let's give 1 point to the pod we ask the friends list
+ pods_score[url] = 1
getForeignPodsList(url, function (foreign_pods_list) {
if (foreign_pods_list.length === 0) return callback()
pods_list,
- function eachRequest (err, response, body, url) {
+ function eachRequest (err, response, body, pod, callback_each_request) {
// We add the pod if it responded correctly with its public certificate
if (!err && response.statusCode === 200) {
- pods.add({ url: url, publicKey: body.cert }, function (err) {
+ pods.add({ url: pod.url, publicKey: body.cert, score: global.FRIEND_BASE_SCORE }, function (err) {
if (err) {
- logger.error('Error with adding %s pod.', url, { error: err })
+ logger.error('Error with adding %s pod.', pod.url, { error: err })
}
+
+ return callback_each_request()
})
} else {
- logger.error('Error with adding %s pod.', url, { error: err || new Error('Status not 200') })
+ logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') })
+ return callback_each_request()
}
},
;(function () {
'use strict'
+ var async = require('async')
var config = require('config')
var crypto = require('crypto')
var fs = require('fs')
}
logger.debug('Sending informations to %s.', to_pod.url, { params: params })
+ // Default 10 but in tests we want to be faster
+ var retries = utils.isTestInstance() ? 2 : 10
- // Replay 15 times, with factor 3
replay(
request.post(params, function (err, response, body) {
- callbackEach(err, response, body, to_pod.url)
+ callbackEach(err, response, body, to_pod)
}),
{
- retries: 10,
+ retries: retries,
factor: 3,
maxTimeout: Infinity,
errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ]
}
// Make a request for each pod
- for (var pod of pods) {
+ async.each(pods, function (pod, callback_each_async) {
+ function callbackEachRetryRequest (err, response, body, pod) {
+ callbackEach(err, response, body, pod, function () {
+ callback_each_async()
+ })
+ }
+
var params = {
url: pod.url + all_data.path,
method: all_data.method
key: passwordEncrypted
}
- makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEach)
+ makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEachRetryRequest)
})
})(crt, params, url, pod, signature)
} else {
params.json = { data: all_data.data }
- makeRetryRequest(params, url, pod, signature, callbackEach)
+ makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest)
}
} else {
logger.debug('Make a GET/DELETE request')
- makeRetryRequest(params, url, pod, signature, callbackEach)
+ makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest)
}
- }
-
- return callback()
+ }, callback)
}
utils.certsExist = function (callback) {
process.kill(-webtorrent_process.pid)
}
+ utils.isTestInstance = function () {
+ return (process.env.NODE_ENV === 'test')
+ }
+
module.exports = utils
})()
data: params
}
- pods.makeSecureRequest(data, function (err) {
- if (err) {
- logger.error('Somes issues when sending this video to friends.', { error: err })
- return callback(err)
- }
-
- return callback(null)
- })
+ // Do not wait the secure requests
+ pods.makeSecureRequest(data)
+ callback(null)
})
})
}
}
// Yes this is a POST request because we add some informations in the body (signature, encrypt etc)
- pods.makeSecureRequest(data, function (err) {
- if (err) {
- logger.error('Somes issues when sending we want to remove the video to friends.', { error: err })
- return callback(err)
- }
-
- callback(null)
- })
+ pods.makeSecureRequest(data)
+ callback(null)
})
})
})
+++ /dev/null
-;(function () {
- 'use strict'
-
- var request = require('supertest')
- var chai = require('chai')
- var expect = chai.expect
- var async = require('async')
-
- var utils = require('../utils')
-
- function getFriendsList (url, end) {
- var path = '/api/v1/pods/'
-
- request(url)
- .get(path)
- .set('Accept', 'application/json')
- .expect(200)
- .expect('Content-Type', /json/)
- .end(end)
- }
-
- describe('Test friends', function () {
- var apps = []
- var urls = []
-
- before(function (done) {
- this.timeout(20000)
- utils.runMultipleServers(3, function (apps_run, urls_run) {
- apps = apps_run
- urls = urls_run
- done()
- })
- })
-
- it('Should not have friends', function (done) {
- async.each(urls, function (url, callback) {
- getFriendsList(url, function (err, res) {
- if (err) throw err
-
- var result = res.body
- expect(result).to.be.an('array')
- expect(result.length).to.equal(0)
- callback()
- })
- }, function (err) {
- if (err) throw err
-
- done()
- })
- })
-
- it('Should make friends', function (done) {
- this.timeout(10000)
-
- function testMadeFriends (urls, url_to_test, callback) {
- var friends = []
- for (var i = 0; i < urls.length; i++) {
- if (urls[i] === url_to_test) continue
- friends.push(urls[i])
- }
-
- getFriendsList(url_to_test, function (err, res) {
- if (err) throw err
-
- var result = res.body
- var result_urls = [ result[0].url, result[1].url ]
- expect(result).to.be.an('array')
- expect(result.length).to.equal(2)
- expect(result_urls[0]).to.not.equal(result_urls[1])
-
- var error_string = 'Friends url do not correspond for ' + url_to_test
- expect(friends).to.contain(result_urls[0], error_string)
- expect(friends).to.contain(result_urls[1], error_string)
- callback()
- })
- }
-
- var path = '/api/v1/pods/makefriends'
-
- // The second pod make friend with the third
- request(urls[1])
- .get(path)
- .set('Accept', 'application/json')
- .expect(204)
- .end(function (err, res) {
- if (err) throw err
-
- // Wait for the request between pods
- setTimeout(function () {
- // The second pod should have the third as a friend
- getFriendsList(urls[1], function (err, res) {
- if (err) throw err
-
- var result = res.body
- expect(result).to.be.an('array')
- expect(result.length).to.equal(1)
- expect(result[0].url).to.be.equal(urls[2])
-
- // Same here, the third pod should have the second pod as a friend
- getFriendsList(urls[2], function (err, res) {
- if (err) throw err
-
- var result = res.body
- expect(result).to.be.an('array')
- expect(result.length).to.equal(1)
- expect(result[0].url).to.be.equal(urls[1])
-
- // Finally the first pod make friend with the second pod
- request(urls[0])
- .get(path)
- .set('Accept', 'application/json')
- .expect(204)
- .end(function (err, res) {
- if (err) throw err
-
- setTimeout(function () {
- // Now each pod should be friend with the other ones
- async.each(urls, function (url, callback) {
- testMadeFriends(urls, url, callback)
- }, function (err) {
- if (err) throw err
- done()
- })
- }, 1000)
- })
- })
- })
- }, 1000)
- })
- })
-
- // TODO
- it('Should not be able to make friends again')
-
- after(function (done) {
- apps.forEach(function (app) {
- process.kill(-app.pid)
- })
-
- if (this.ok) {
- utils.flushTests(function () {
- done()
- })
- } else {
- done()
- }
- })
- })
-})()
--- /dev/null
+;(function () {
+ 'use strict'
+
+ var request = require('supertest')
+ var chai = require('chai')
+ var expect = chai.expect
+
+ var utils = require('../utils')
+
+ describe('Test advanced friends', function () {
+ var path = '/api/v1/pods/makefriends'
+ var apps = []
+ var urls = []
+
+ function makeFriend (pod_number, callback) {
+ // The first pod make friend with the third
+ request(urls[pod_number - 1])
+ .get(path)
+ .set('Accept', 'application/json')
+ .expect(204)
+ .end(function (err, res) {
+ if (err) throw err
+
+ // Wait for the request between pods
+ setTimeout(function () {
+ callback()
+ }, 1000)
+ })
+ }
+
+ function getFriendsList (pod_number, end) {
+ var path = '/api/v1/pods/'
+
+ request(urls[pod_number - 1])
+ .get(path)
+ .set('Accept', 'application/json')
+ .expect(200)
+ .expect('Content-Type', /json/)
+ .end(end)
+ }
+
+ function uploadVideo (pod_number, callback) {
+ var path = '/api/v1/videos'
+
+ request(urls[pod_number - 1])
+ .post(path)
+ .set('Accept', 'application/json')
+ .field('name', 'my super video')
+ .field('description', 'my super description')
+ .attach('input_video', __dirname + '/../fixtures/video_short.webm')
+ .expect(201)
+ .end(function (err) {
+ if (err) throw err
+
+ // Wait for the retry requests
+ setTimeout(callback, 10000)
+ })
+ }
+
+ beforeEach(function (done) {
+ this.timeout(30000)
+ utils.runMultipleServers(6, function (apps_run, urls_run) {
+ apps = apps_run
+ urls = urls_run
+ done()
+ })
+ })
+
+ afterEach(function (done) {
+ apps.forEach(function (app) {
+ process.kill(-app.pid)
+ })
+
+ if (this.ok) {
+ utils.flushTests(function () {
+ done()
+ })
+ } else {
+ done()
+ }
+ })
+
+ it('Should make friends with two pod each in a different group', function (done) {
+ this.timeout(10000)
+
+ // Pod 3 makes friend with the first one
+ makeFriend(3, function () {
+ // Pod 4 makes friend with the second one
+ makeFriend(4, function () {
+ // Now if the fifth wants to make friends with the third et the first
+ makeFriend(5, function () {
+ // It should have 0 friends
+ getFriendsList(5, function (err, res) {
+ if (err) throw err
+
+ expect(res.body.length).to.equal(0)
+
+ done()
+ })
+ })
+ })
+ })
+ })
+
+ it('Should make friends with the pods 1, 2, 3', function (done) {
+ this.timeout(100000)
+
+ // Pods 1, 2, 3 and 4 become friends
+ makeFriend(2, function () {
+ makeFriend(1, function () {
+ makeFriend(4, function () {
+ // Kill the server 4
+ apps[3].kill()
+
+ // Expulse pod 4 from pod 1 and 2
+ uploadVideo(1, function () {
+ uploadVideo(1, function () {
+ uploadVideo(2, function () {
+ uploadVideo(2, function () {
+ // Rerun server 4
+ utils.runServer(4, function (app, url) {
+ apps[3] = app
+ getFriendsList(4, function (err, res) {
+ if (err) throw err
+ // Pod 4 didn't know pod 1 and 2 removed it
+ expect(res.body.length).to.equal(3)
+
+ // Pod 6 ask pod 1, 2 and 3
+ makeFriend(6, function () {
+ getFriendsList(6, function (err, res) {
+ if (err) throw err
+
+ // Pod 4 should not be our friend
+ var result = res.body
+ expect(result.length).to.equal(3)
+ for (var pod of result) {
+ expect(pod.url).not.equal(urls[3])
+ }
+
+ done()
+ })
+ })
+ })
+ })
+ })
+ })
+ })
+ })
+ })
+ })
+ })
+ })
+ })
+})()
--- /dev/null
+;(function () {
+ 'use strict'
+
+ var request = require('supertest')
+ var chai = require('chai')
+ var expect = chai.expect
+ var async = require('async')
+
+ var utils = require('../utils')
+
+ function getFriendsList (url, end) {
+ var path = '/api/v1/pods/'
+
+ request(url)
+ .get(path)
+ .set('Accept', 'application/json')
+ .expect(200)
+ .expect('Content-Type', /json/)
+ .end(end)
+ }
+
+ describe('Test basic friends', function () {
+ var apps = []
+ var urls = []
+
+ before(function (done) {
+ this.timeout(20000)
+ utils.runMultipleServers(3, function (apps_run, urls_run) {
+ apps = apps_run
+ urls = urls_run
+ done()
+ })
+ })
+
+ it('Should not have friends', function (done) {
+ async.each(urls, function (url, callback) {
+ getFriendsList(url, function (err, res) {
+ if (err) throw err
+
+ var result = res.body
+ expect(result).to.be.an('array')
+ expect(result.length).to.equal(0)
+ callback()
+ })
+ }, function (err) {
+ if (err) throw err
+
+ done()
+ })
+ })
+
+ it('Should make friends', function (done) {
+ this.timeout(10000)
+
+ function testMadeFriends (urls, url_to_test, callback) {
+ var friends = []
+ for (var i = 0; i < urls.length; i++) {
+ if (urls[i] === url_to_test) continue
+ friends.push(urls[i])
+ }
+
+ getFriendsList(url_to_test, function (err, res) {
+ if (err) throw err
+
+ var result = res.body
+ var result_urls = [ result[0].url, result[1].url ]
+ expect(result).to.be.an('array')
+ expect(result.length).to.equal(2)
+ expect(result_urls[0]).to.not.equal(result_urls[1])
+
+ var error_string = 'Friends url do not correspond for ' + url_to_test
+ expect(friends).to.contain(result_urls[0], error_string)
+ expect(friends).to.contain(result_urls[1], error_string)
+ callback()
+ })
+ }
+
+ var path = '/api/v1/pods/makefriends'
+
+ // The second pod make friend with the third
+ request(urls[1])
+ .get(path)
+ .set('Accept', 'application/json')
+ .expect(204)
+ .end(function (err, res) {
+ if (err) throw err
+
+ // Wait for the request between pods
+ setTimeout(function () {
+ // The second pod should have the third as a friend
+ getFriendsList(urls[1], function (err, res) {
+ if (err) throw err
+
+ var result = res.body
+ expect(result).to.be.an('array')
+ expect(result.length).to.equal(1)
+ expect(result[0].url).to.be.equal(urls[2])
+
+ // Same here, the third pod should have the second pod as a friend
+ getFriendsList(urls[2], function (err, res) {
+ if (err) throw err
+
+ var result = res.body
+ expect(result).to.be.an('array')
+ expect(result.length).to.equal(1)
+ expect(result[0].url).to.be.equal(urls[1])
+
+ // Finally the first pod make friend with the second pod
+ request(urls[0])
+ .get(path)
+ .set('Accept', 'application/json')
+ .expect(204)
+ .end(function (err, res) {
+ if (err) throw err
+
+ setTimeout(function () {
+ // Now each pod should be friend with the other ones
+ async.each(urls, function (url, callback) {
+ testMadeFriends(urls, url, callback)
+ }, function (err) {
+ if (err) throw err
+ done()
+ })
+ }, 1000)
+ })
+ })
+ })
+ }, 1000)
+ })
+ })
+
+ // TODO
+ it('Should not be able to make friends again')
+
+ after(function (done) {
+ apps.forEach(function (app) {
+ process.kill(-app.pid)
+ })
+
+ if (this.ok) {
+ utils.flushTests(function () {
+ done()
+ })
+ } else {
+ done()
+ }
+ })
+ })
+})()
}
module.exports = {
+ flushTests: flushTests,
runMultipleServers: runMultipleServers,
- runServer: runServer,
- flushTests: flushTests
+ runServer: runServer
}
})()
projects / oweals / peertube.git / commitdiff